Isolate LocalStack fixtures and keep default tests service-free

README.md

@@ -301,20 +301,36 @@ These are independent - joining does not monitor events, and monitoring does not
 
 ## Running Tests
 
-### Unit Tests (with LocalStack)
+### Default suite (no LocalStack or cloud credentials)
+
+```bash
+mix test
+```
+
+The default suite includes backend-independent tests and local EKV integration
+tests. To run only the local EKV lane, use `mix test --only ekv`.
+
+### LocalStack suite
 
 Start LocalStack for S3-compatible storage:
 
 ```bash
 docker run -d --name localstack -p 4566:4566 localstack/localstack
 ```
 
-Run the tests:
+Run the default suite plus LocalStack cases, including mirror end-to-end cases:
 
 ```bash
-mix test
+mix test --include localstack
 ```
 
+Use `mix test --only localstack` for only this lane. CI should run both the
+default suite and the LocalStack lane; a default-only run does not cover S3.
+Set `DURABLE_TEST_S3_ENDPOINT` to use a different local endpoint.
+
+Each test run allocates its own bucket and cleans up only that bucket. Selecting
+a backend-independent test never creates or clears storage.
+
 ### Integration Tests (with Tigris)
 
 Set the required environment variables:

test/durable_server/lifecycle_test.exs

@@ -1,5 +1,5 @@
 defmodule DurableServer.LifecycleTest do
-  use ExUnit.Case, async: true
+  use DurableServer.LocalStackCase, async: true
   import DurableServer.TestHelper
 
   alias DurableServer
@@ -709,49 +709,31 @@ defmodule DurableServer.LifecycleTest do
       })
 
     object_store = test_object_store()
-    test_bucket_name = "durable-test-lifecycle-#{DurableServer.UUID.uuid4()}"
-
-    case ObjectStore.create_bucket_with_credentials(object_store, test_bucket_name) do
-      {:ok, %ObjectStore{} = store} ->
-        on_exit(fn ->
-          try do
-            ObjectStore.delete_bucket(store, test_bucket_name)
-          catch
-            _, _ -> :ok
-          end
-        end)
-
-        supervisor_config = DurableServer.Supervisor.__get_config__(supervisor_name)
-        circuit_breaker = supervisor_config.circuit_breaker
+    supervisor_config = DurableServer.Supervisor.__get_config__(supervisor_name)
+    circuit_breaker = supervisor_config.circuit_breaker
 
-        # Create test config that mimics what supervisor provides
-        test_config = %{
-          name: supervisor_name,
-          prefix: prefix,
-          object_store: object_store,
-          discovery_interval_ms: 60_000,
-          heartbeat_interval_ms: 10_000,
-          graceful_shutdown_timeout_ms: 30_000,
-          dead_node_threshold_ms: 24 * 60 * 60 * 1000,
-          crash_threshold_count: 5,
-          crash_threshold_window_ms: 60 * 60 * 1000,
-          module_circuit_breaker_count: 50,
-          module_circuit_breaker_window_ms: 5 * 60 * 1000,
-          module_circuit_breaker_cooldown_ms: 30 * 60 * 1000,
-          ets_table: supervisor_config.ets_table
-        }
-
-        {:ok,
-         test_bucket: test_bucket_name,
-         store: store,
-         supervisor_name: supervisor_name,
-         prefix: prefix,
-         config: test_config,
-         circuit_breaker: circuit_breaker}
+    # Create test config that mimics what supervisor provides.
+    test_config = %{
+      name: supervisor_name,
+      prefix: prefix,
+      object_store: object_store,
+      discovery_interval_ms: 60_000,
+      heartbeat_interval_ms: 10_000,
+      graceful_shutdown_timeout_ms: 30_000,
+      dead_node_threshold_ms: 24 * 60 * 60 * 1000,
+      crash_threshold_count: 5,
+      crash_threshold_window_ms: 60 * 60 * 1000,
+      module_circuit_breaker_count: 50,
+      module_circuit_breaker_window_ms: 5 * 60 * 1000,
+      module_circuit_breaker_cooldown_ms: 30 * 60 * 1000,
+      ets_table: supervisor_config.ets_table
+    }
 
-      {:error, reason} ->
-        {:skip, "Failed to create test bucket: #{inspect(reason)}"}
-    end
+    {:ok,
+     supervisor_name: supervisor_name,
+     prefix: prefix,
+     config: test_config,
+     circuit_breaker: circuit_breaker}
   end
 
   describe "stop modes" do

test/durable_server/localstack_isolation_test.exs

@@ -0,0 +1,19 @@
+defmodule DurableServer.LocalStackIsolationTest do
+  use DurableServer.LocalStackCase, async: true
+
+  alias DurableServer.{ObjectStore, TestHelper}
+
+  test "cleaning one run leaves another run's objects intact" do
+    first = TestHelper.test_object_store(bucket: TestHelper.new_test_bucket())
+    second = TestHelper.test_object_store(bucket: TestHelper.new_test_bucket())
+    :ok = ObjectStore.ensure_bucket_exists(first)
+    :ok = ObjectStore.ensure_bucket_exists(second)
+    on_exit(fn -> TestHelper.cleanup_bucket!(second) end)
+
+    assert {:ok, _} = ObjectStore.put_object(first, "same-key", "first-run")
+    assert {:ok, _} = ObjectStore.put_object(second, "same-key", "second-run")
+
+    assert :ok = TestHelper.cleanup_bucket!(first)
+    assert {:ok, %{body: "second-run"}} = ObjectStore.get_object(second, "same-key")
+  end
+end

test/durable_server/remote_placement_test.exs

@@ -1,5 +1,5 @@
 defmodule DurableServer.RemotePlacementTest do
-  use ExUnit.Case, async: false
+  use DurableServer.LocalStackCase, async: false
   import DurableServer.TestHelper
   alias DurableServer
   alias DurableServer.{LifecycleManager, Meta, StoredState}

test/durable_server/sticky_placement_test.exs

@@ -1,5 +1,5 @@
 defmodule DurableServer.StickyPlacementTest do
-  use ExUnit.Case, async: false
+  use DurableServer.LocalStackCase, async: false
   import DurableServer.TestHelper
   alias DurableServer
   alias DurableServer.{LifecycleManager, Meta, StoredState}

test/durable_server/test_helper_test.exs

@@ -0,0 +1,27 @@
+defmodule DurableServer.TestHelperTest do
+  use ExUnit.Case, async: true
+
+  alias DurableServer.TestHelper
+
+  test "a run reuses its namespace without requiring a storage connection" do
+    first = TestHelper.test_object_store_opts()
+    second = TestHelper.test_object_store_opts()
+    assert first[:bucket] == second[:bucket]
+    assert first[:bucket] =~ ~r/^durable-test-[a-f0-9-]{36}$/
+    refute first[:bucket] == "durable-test-bucket"
+  end
+
+  test "independent runs allocate different bucket names" do
+    buckets = for _ <- 1..100, do: TestHelper.new_test_bucket()
+    assert length(Enum.uniq(buckets)) == 100
+  end
+
+  test "explicit storage options override only the requested defaults" do
+    opts =
+      TestHelper.test_object_store_opts(bucket: "explicit", s3_endpoint: "http://localhost:1")
+
+    assert opts[:bucket] == "explicit"
+    assert opts[:s3_endpoint] == "http://localhost:1"
+    assert opts[:access_key_id] == "test"
+  end
+end

test/durable_server/watermark_test.exs

@@ -1,5 +1,5 @@
 defmodule DurableServer.WatermarkTest do
-  use ExUnit.Case, async: false
+  use DurableServer.LocalStackCase, async: false
   import DurableServer.TestHelper
   alias DurableServer
 

test/durable_server_test.exs

@@ -1,5 +1,5 @@
 defmodule DurableServerTest do
-  use ExUnit.Case, async: true
+  use DurableServer.LocalStackCase, async: true
   import ExUnit.CaptureLog
   import DurableServer.TestHelper
 
@@ -937,34 +937,8 @@ defmodule DurableServerTest do
   end
 
   setup do
-    # Create a test bucket
-    test_bucket_name =
-      "durable-test-durable-#{DurableServer.UUID.uuid4()}"
-
-    case ObjectStore.create_bucket_with_credentials(test_object_store(), test_bucket_name) do
-      {:ok, %ObjectStore{} = store} ->
-        on_exit(fn ->
-          # Clean up bucket on test completion
-          try do
-            ObjectStore.delete_bucket(store, test_bucket_name)
-          catch
-            _, _ -> :ok
-          end
-        end)
-
-        # Start a DurableServer.Supervisor for tests that need one
-        {supervisor_name, supervisor_pid, prefix} = start_test_supervisor()
-
-        {:ok,
-         test_bucket: test_bucket_name,
-         store: store,
-         supervisor_name: supervisor_name,
-         supervisor_pid: supervisor_pid,
-         prefix: prefix}
-
-      {:error, reason} ->
-        {:skip, "Failed to create test bucket: #{inspect(reason)}"}
-    end
+    {supervisor_name, supervisor_pid, prefix} = start_test_supervisor()
+    {:ok, supervisor_name: supervisor_name, supervisor_pid: supervisor_pid, prefix: prefix}
   end
 
   describe "ConsistencyProbeBackend atomic storage contract" do

test/ekv_integration_test.exs

@@ -7,7 +7,7 @@
   alias DurableServer.TestCounterServer, as: CounterServer
   alias DurableServer.TestTemporalServer
 
-  @moduletag :integration
+  @moduletag :ekv
   @moduletag :capture_log
 
   setup do
@@ -209,7 +209,7 @@
              )
   end
 
   test "EKVStore client backend can update an existing remote key without etag" do
     ensure_distributed_node!()
 
     unique_id = System.unique_integer([:positive, :monotonic])
@@ -527,7 +527,7 @@
     assert {"#{prefix}b", 2} in listed_objects
   end
 
   test "lifecycle manager discovers and restarts a seeded permanent object via shared EKV" do
     ensure_distributed_node!()
 
     unique_id = System.unique_integer([:positive, :monotonic])

test/group_test.exs

@@ -1,5 +1,5 @@
 defmodule GroupTest do
-  use ExUnit.Case, async: true
+  use DurableServer.LocalStackCase, async: true
   import DurableServer.TestHelper
 
   @moduletag :capture_log

test/mirror_backend_e2e_test.exs

@@ -1,5 +1,5 @@
 defmodule DurableServer.MirrorBackendE2ETest do
-  use ExUnit.Case, async: false
+  use DurableServer.LocalStackCase, async: false
 
   import DurableServer.TestHelper
 
@@ -8,7 +8,6 @@ defmodule DurableServer.MirrorBackendE2ETest do
   alias DurableServer.StorageBackend
   alias DurableServer.TestCounterServer, as: CounterServer
 
-  @moduletag :integration
   @moduletag :capture_log
 
   setup do

test/mirror_backend_integration_test.exs

@@ -56,7 +56,7 @@
       do: StorageBackend.unsubscribe(delegate, subscription_ref)
   end
 
-  @moduletag :integration
+  @moduletag :ekv
   @moduletag :capture_log
 
   setup do
@@ -161,7 +161,7 @@
              StorageBackend.get_object(read_only_mirror, key)
   end
 
   test "required mirror failure does not hide a committed authoritative write", %{
     primary: primary,
     secondary: secondary
   } do

test/support/local_stack_case.ex

@@ -0,0 +1,17 @@
+defmodule DurableServer.LocalStackCase do
+  @moduledoc """
+  Opt-in LocalStack fixtures. Storage is created only for selected test modules,
+  and the suite owns a unique bucket rather than clearing a shared bucket.
+  """
+  use ExUnit.CaseTemplate
+
+  using do
+    quote do
+      @moduletag :localstack
+    end
+  end
+
+  setup_all do
+    DurableServer.TestHelper.ensure_localstack!()
+  end
+end

test/support/test_helper.ex

@@ -5,18 +5,52 @@ defmodule DurableServer.TestHelper do
 
   alias DurableServer.ObjectStore
 
+  def init_test_run do
+    Application.put_env(:durable_server, :test_bucket, new_test_bucket())
+    Application.delete_env(:durable_server, :created_test_store)
+    :ok
+  end
+
+  def new_test_bucket, do: "durable-test-#{DurableServer.UUID.uuid4()}"
+
+  def ensure_localstack! do
+    store = test_object_store()
+    :ok = ObjectStore.ensure_bucket_exists(store)
+    Application.put_env(:durable_server, :created_test_store, store)
+    :ok
+  end
+
+  def cleanup_test_run! do
+    if store = Application.get_env(:durable_server, :created_test_store) do
+      cleanup_bucket!(store)
+      Application.delete_env(:durable_server, :created_test_store)
+    end
+
+    :ok
+  end
+
+  def cleanup_bucket!(%ObjectStore{} = store) do
+    for obj <- ObjectStore.list_all_objects_stream(store, "") do
+      :ok = ObjectStore.delete_object(store, obj.key)
+    end
+
+    :ok = ObjectStore.delete_bucket(store, store.bucket)
+  end
+
   @doc """
   Returns the default object store config for testing as a keyword list.
   """
   def test_object_store_opts(opts \\ []) do
+    endpoint = System.get_env("DURABLE_TEST_S3_ENDPOINT", "http://localhost:4566")
+
     Keyword.merge(
       [
         access_key_id: "test",
         secret_access_key: "test",
-        s3_endpoint: "http://localhost:4566",
-        iam_endpoint: "http://localhost:4566",
+        s3_endpoint: endpoint,
+        iam_endpoint: endpoint,
         default_region: "us-east-1",
-        bucket: "durable-test-bucket"
+        bucket: Application.fetch_env!(:durable_server, :test_bucket)
       ],
       opts
     )

test/test_helper.exs

@@ -14,18 +14,9 @@ case File.read(".env") do
     :noop
 end
 
-# Exclude integration tests by default (they require real credentials)
-ExUnit.configure(exclude: [:integration, :stress])
-
-alias DurableServer.ObjectStore
-import DurableServer.TestHelper
-
-# Clear object store (local stack) for this run
-store = test_object_store()
-:ok = ObjectStore.ensure_bucket_exists(store)
-
-for obj <- ObjectStore.list_all_objects_stream(store, "") do
-  :ok = ObjectStore.delete_object(store, obj.key)
-end
+# Allocate a namespace without contacting storage. Only LocalStackCase creates it.
+DurableServer.TestHelper.init_test_run()
+ExUnit.configure(exclude: [:localstack, :integration, :stress])
+ExUnit.after_suite(fn _ -> DurableServer.TestHelper.cleanup_test_run!() end)
 
 ExUnit.start()