From 6f0950fbec974105b5183dae50e556880f80572f Mon Sep 17 00:00:00 2001 From: Jason Stiebs Date: Wed, 9 Sep 2026 15:07:51 -0500 Subject: [PATCH] Make LocalStack fixtures opt-in and isolate each test run --- README.md | 22 ++++++- test/durable_server/lifecycle_test.exs | 66 +++++++------------ .../localstack_isolation_test.exs | 19 ++++++ test/durable_server/remote_placement_test.exs | 2 +- test/durable_server/sticky_placement_test.exs | 2 +- test/durable_server/test_helper_test.exs | 27 ++++++++ test/durable_server/watermark_test.exs | 2 +- test/durable_server_test.exs | 32 +-------- test/ekv_integration_test.exs | 2 +- test/group_test.exs | 2 +- test/mirror_backend_e2e_test.exs | 3 +- test/mirror_backend_integration_test.exs | 2 +- test/support/local_stack_case.ex | 17 +++++ test/support/test_helper.ex | 40 ++++++++++- test/test_helper.exs | 17 ++--- 15 files changed, 157 insertions(+), 98 deletions(-) create mode 100644 test/durable_server/localstack_isolation_test.exs create mode 100644 test/durable_server/test_helper_test.exs create mode 100644 test/support/local_stack_case.ex diff --git a/README.md b/README.md index 789c27b..dce6557 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,16 @@ These are independent - joining does not monitor events, and monitoring does not ## Running Tests -### Unit Tests (with LocalStack) +### Default suite (no LocalStack or cloud credentials) + +```bash +mix test +``` + +The default suite includes backend-independent tests and local EKV integration +tests. To run only the local EKV lane, use `mix test --only ekv`. + +### LocalStack suite Start LocalStack for S3-compatible storage: @@ -309,12 +318,19 @@ Start LocalStack for S3-compatible storage: docker run -d --name localstack -p 4566:4566 localstack/localstack ``` -Run the tests: +Run the default suite plus LocalStack cases, including mirror end-to-end cases: ```bash -mix test +mix test --include localstack ``` +Use `mix test --only localstack` for only this lane. CI should run both the +default suite and the LocalStack lane; a default-only run does not cover S3. +Set `DURABLE_TEST_S3_ENDPOINT` to use a different local endpoint. + +Each test run allocates its own bucket and cleans up only that bucket. Selecting +a backend-independent test never creates or clears storage. + ### Integration Tests (with Tigris) Set the required environment variables: diff --git a/test/durable_server/lifecycle_test.exs b/test/durable_server/lifecycle_test.exs index fadda4d..37b9b73 100644 --- a/test/durable_server/lifecycle_test.exs +++ b/test/durable_server/lifecycle_test.exs @@ -1,5 +1,5 @@ defmodule DurableServer.LifecycleTest do - use ExUnit.Case, async: true + use DurableServer.LocalStackCase, async: true import DurableServer.TestHelper alias DurableServer @@ -709,49 +709,31 @@ defmodule DurableServer.LifecycleTest do }) object_store = test_object_store() - test_bucket_name = "durable-test-lifecycle-#{DurableServer.UUID.uuid4()}" - - case ObjectStore.create_bucket_with_credentials(object_store, test_bucket_name) do - {:ok, %ObjectStore{} = store} -> - on_exit(fn -> - try do - ObjectStore.delete_bucket(store, test_bucket_name) - catch - _, _ -> :ok - end - end) - - supervisor_config = DurableServer.Supervisor.__get_config__(supervisor_name) - circuit_breaker = supervisor_config.circuit_breaker + supervisor_config = DurableServer.Supervisor.__get_config__(supervisor_name) + circuit_breaker = supervisor_config.circuit_breaker - # Create test config that mimics what supervisor provides - test_config = %{ - name: supervisor_name, - prefix: prefix, - object_store: object_store, - discovery_interval_ms: 60_000, - heartbeat_interval_ms: 10_000, - graceful_shutdown_timeout_ms: 30_000, - dead_node_threshold_ms: 24 * 60 * 60 * 1000, - crash_threshold_count: 5, - crash_threshold_window_ms: 60 * 60 * 1000, - module_circuit_breaker_count: 50, - module_circuit_breaker_window_ms: 5 * 60 * 1000, - module_circuit_breaker_cooldown_ms: 30 * 60 * 1000, - ets_table: supervisor_config.ets_table - } - - {:ok, - test_bucket: test_bucket_name, - store: store, - supervisor_name: supervisor_name, - prefix: prefix, - config: test_config, - circuit_breaker: circuit_breaker} + # Create test config that mimics what supervisor provides. + test_config = %{ + name: supervisor_name, + prefix: prefix, + object_store: object_store, + discovery_interval_ms: 60_000, + heartbeat_interval_ms: 10_000, + graceful_shutdown_timeout_ms: 30_000, + dead_node_threshold_ms: 24 * 60 * 60 * 1000, + crash_threshold_count: 5, + crash_threshold_window_ms: 60 * 60 * 1000, + module_circuit_breaker_count: 50, + module_circuit_breaker_window_ms: 5 * 60 * 1000, + module_circuit_breaker_cooldown_ms: 30 * 60 * 1000, + ets_table: supervisor_config.ets_table + } - {:error, reason} -> - {:skip, "Failed to create test bucket: #{inspect(reason)}"} - end + {:ok, + supervisor_name: supervisor_name, + prefix: prefix, + config: test_config, + circuit_breaker: circuit_breaker} end describe "stop modes" do diff --git a/test/durable_server/localstack_isolation_test.exs b/test/durable_server/localstack_isolation_test.exs new file mode 100644 index 0000000..4d85aef --- /dev/null +++ b/test/durable_server/localstack_isolation_test.exs @@ -0,0 +1,19 @@ +defmodule DurableServer.LocalStackIsolationTest do + use DurableServer.LocalStackCase, async: true + + alias DurableServer.{ObjectStore, TestHelper} + + test "cleaning one run leaves another run's objects intact" do + first = TestHelper.test_object_store(bucket: TestHelper.new_test_bucket()) + second = TestHelper.test_object_store(bucket: TestHelper.new_test_bucket()) + :ok = ObjectStore.ensure_bucket_exists(first) + :ok = ObjectStore.ensure_bucket_exists(second) + on_exit(fn -> TestHelper.cleanup_bucket!(second) end) + + assert {:ok, _} = ObjectStore.put_object(first, "same-key", "first-run") + assert {:ok, _} = ObjectStore.put_object(second, "same-key", "second-run") + + assert :ok = TestHelper.cleanup_bucket!(first) + assert {:ok, %{body: "second-run"}} = ObjectStore.get_object(second, "same-key") + end +end diff --git a/test/durable_server/remote_placement_test.exs b/test/durable_server/remote_placement_test.exs index 364b1c8..e989c48 100644 --- a/test/durable_server/remote_placement_test.exs +++ b/test/durable_server/remote_placement_test.exs @@ -1,5 +1,5 @@ defmodule DurableServer.RemotePlacementTest do - use ExUnit.Case, async: false + use DurableServer.LocalStackCase, async: false import DurableServer.TestHelper alias DurableServer alias DurableServer.{LifecycleManager, Meta, StoredState} diff --git a/test/durable_server/sticky_placement_test.exs b/test/durable_server/sticky_placement_test.exs index 53a1459..e19e6d0 100644 --- a/test/durable_server/sticky_placement_test.exs +++ b/test/durable_server/sticky_placement_test.exs @@ -1,5 +1,5 @@ defmodule DurableServer.StickyPlacementTest do - use ExUnit.Case, async: false + use DurableServer.LocalStackCase, async: false import DurableServer.TestHelper alias DurableServer alias DurableServer.{LifecycleManager, Meta, StoredState} diff --git a/test/durable_server/test_helper_test.exs b/test/durable_server/test_helper_test.exs new file mode 100644 index 0000000..8f12f3c --- /dev/null +++ b/test/durable_server/test_helper_test.exs @@ -0,0 +1,27 @@ +defmodule DurableServer.TestHelperTest do + use ExUnit.Case, async: true + + alias DurableServer.TestHelper + + test "a run reuses its namespace without requiring a storage connection" do + first = TestHelper.test_object_store_opts() + second = TestHelper.test_object_store_opts() + assert first[:bucket] == second[:bucket] + assert first[:bucket] =~ ~r/^durable-test-[a-f0-9-]{36}$/ + refute first[:bucket] == "durable-test-bucket" + end + + test "independent runs allocate different bucket names" do + buckets = for _ <- 1..100, do: TestHelper.new_test_bucket() + assert length(Enum.uniq(buckets)) == 100 + end + + test "explicit storage options override only the requested defaults" do + opts = + TestHelper.test_object_store_opts(bucket: "explicit", s3_endpoint: "http://localhost:1") + + assert opts[:bucket] == "explicit" + assert opts[:s3_endpoint] == "http://localhost:1" + assert opts[:access_key_id] == "test" + end +end diff --git a/test/durable_server/watermark_test.exs b/test/durable_server/watermark_test.exs index a8a6f7a..9421381 100644 --- a/test/durable_server/watermark_test.exs +++ b/test/durable_server/watermark_test.exs @@ -1,5 +1,5 @@ defmodule DurableServer.WatermarkTest do - use ExUnit.Case, async: false + use DurableServer.LocalStackCase, async: false import DurableServer.TestHelper alias DurableServer diff --git a/test/durable_server_test.exs b/test/durable_server_test.exs index 3ef61ce..acccaa2 100644 --- a/test/durable_server_test.exs +++ b/test/durable_server_test.exs @@ -1,5 +1,5 @@ defmodule DurableServerTest do - use ExUnit.Case, async: true + use DurableServer.LocalStackCase, async: true import ExUnit.CaptureLog import DurableServer.TestHelper @@ -937,34 +937,8 @@ defmodule DurableServerTest do end setup do - # Create a test bucket - test_bucket_name = - "durable-test-durable-#{DurableServer.UUID.uuid4()}" - - case ObjectStore.create_bucket_with_credentials(test_object_store(), test_bucket_name) do - {:ok, %ObjectStore{} = store} -> - on_exit(fn -> - # Clean up bucket on test completion - try do - ObjectStore.delete_bucket(store, test_bucket_name) - catch - _, _ -> :ok - end - end) - - # Start a DurableServer.Supervisor for tests that need one - {supervisor_name, supervisor_pid, prefix} = start_test_supervisor() - - {:ok, - test_bucket: test_bucket_name, - store: store, - supervisor_name: supervisor_name, - supervisor_pid: supervisor_pid, - prefix: prefix} - - {:error, reason} -> - {:skip, "Failed to create test bucket: #{inspect(reason)}"} - end + {supervisor_name, supervisor_pid, prefix} = start_test_supervisor() + {:ok, supervisor_name: supervisor_name, supervisor_pid: supervisor_pid, prefix: prefix} end describe "ConsistencyProbeBackend atomic storage contract" do diff --git a/test/ekv_integration_test.exs b/test/ekv_integration_test.exs index 36549e1..fa5d83f 100644 --- a/test/ekv_integration_test.exs +++ b/test/ekv_integration_test.exs @@ -7,7 +7,7 @@ defmodule DurableServer.EKVIntegrationTest do alias DurableServer.TestCounterServer, as: CounterServer alias DurableServer.TestTemporalServer - @moduletag :integration + @moduletag :ekv @moduletag :capture_log setup do diff --git a/test/group_test.exs b/test/group_test.exs index 04d7551..7d9f6be 100644 --- a/test/group_test.exs +++ b/test/group_test.exs @@ -1,5 +1,5 @@ defmodule GroupTest do - use ExUnit.Case, async: true + use DurableServer.LocalStackCase, async: true import DurableServer.TestHelper @moduletag :capture_log diff --git a/test/mirror_backend_e2e_test.exs b/test/mirror_backend_e2e_test.exs index 4eb56a1..c7e07d0 100644 --- a/test/mirror_backend_e2e_test.exs +++ b/test/mirror_backend_e2e_test.exs @@ -1,5 +1,5 @@ defmodule DurableServer.MirrorBackendE2ETest do - use ExUnit.Case, async: false + use DurableServer.LocalStackCase, async: false import DurableServer.TestHelper @@ -8,7 +8,6 @@ defmodule DurableServer.MirrorBackendE2ETest do alias DurableServer.StorageBackend alias DurableServer.TestCounterServer, as: CounterServer - @moduletag :integration @moduletag :capture_log setup do diff --git a/test/mirror_backend_integration_test.exs b/test/mirror_backend_integration_test.exs index 86c4e70..ab39fe2 100644 --- a/test/mirror_backend_integration_test.exs +++ b/test/mirror_backend_integration_test.exs @@ -56,7 +56,7 @@ defmodule DurableServer.MirrorBackendIntegrationTest do do: StorageBackend.unsubscribe(delegate, subscription_ref) end - @moduletag :integration + @moduletag :ekv @moduletag :capture_log setup do diff --git a/test/support/local_stack_case.ex b/test/support/local_stack_case.ex new file mode 100644 index 0000000..bcd63b4 --- /dev/null +++ b/test/support/local_stack_case.ex @@ -0,0 +1,17 @@ +defmodule DurableServer.LocalStackCase do + @moduledoc """ + Opt-in LocalStack fixtures. Storage is created only for selected test modules, + and the suite owns a unique bucket rather than clearing a shared bucket. + """ + use ExUnit.CaseTemplate + + using do + quote do + @moduletag :localstack + end + end + + setup_all do + DurableServer.TestHelper.ensure_localstack!() + end +end diff --git a/test/support/test_helper.ex b/test/support/test_helper.ex index 4054343..5ddc398 100644 --- a/test/support/test_helper.ex +++ b/test/support/test_helper.ex @@ -5,18 +5,52 @@ defmodule DurableServer.TestHelper do alias DurableServer.ObjectStore + def init_test_run do + Application.put_env(:durable_server, :test_bucket, new_test_bucket()) + Application.delete_env(:durable_server, :created_test_store) + :ok + end + + def new_test_bucket, do: "durable-test-#{DurableServer.UUID.uuid4()}" + + def ensure_localstack! do + store = test_object_store() + :ok = ObjectStore.ensure_bucket_exists(store) + Application.put_env(:durable_server, :created_test_store, store) + :ok + end + + def cleanup_test_run! do + if store = Application.get_env(:durable_server, :created_test_store) do + cleanup_bucket!(store) + Application.delete_env(:durable_server, :created_test_store) + end + + :ok + end + + def cleanup_bucket!(%ObjectStore{} = store) do + for obj <- ObjectStore.list_all_objects_stream(store, "") do + :ok = ObjectStore.delete_object(store, obj.key) + end + + :ok = ObjectStore.delete_bucket(store, store.bucket) + end + @doc """ Returns the default object store config for testing as a keyword list. """ def test_object_store_opts(opts \\ []) do + endpoint = System.get_env("DURABLE_TEST_S3_ENDPOINT", "http://localhost:4566") + Keyword.merge( [ access_key_id: "test", secret_access_key: "test", - s3_endpoint: "http://localhost:4566", - iam_endpoint: "http://localhost:4566", + s3_endpoint: endpoint, + iam_endpoint: endpoint, default_region: "us-east-1", - bucket: "durable-test-bucket" + bucket: Application.fetch_env!(:durable_server, :test_bucket) ], opts ) diff --git a/test/test_helper.exs b/test/test_helper.exs index c438299..81365d2 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -14,18 +14,9 @@ case File.read(".env") do :noop end -# Exclude integration tests by default (they require real credentials) -ExUnit.configure(exclude: [:integration, :stress]) - -alias DurableServer.ObjectStore -import DurableServer.TestHelper - -# Clear object store (local stack) for this run -store = test_object_store() -:ok = ObjectStore.ensure_bucket_exists(store) - -for obj <- ObjectStore.list_all_objects_stream(store, "") do - :ok = ObjectStore.delete_object(store, obj.key) -end +# Allocate a namespace without contacting storage. Only LocalStackCase creates it. +DurableServer.TestHelper.init_test_run() +ExUnit.configure(exclude: [:localstack, :integration, :stress]) +ExUnit.after_suite(fn _ -> DurableServer.TestHelper.cleanup_test_run!() end) ExUnit.start()