diff --git a/.github/workflows/testing-tool-ci.yml b/.github/workflows/testing-tool-ci.yml new file mode 100644 index 0000000000..3e8f1af917 --- /dev/null +++ b/.github/workflows/testing-tool-ci.yml @@ -0,0 +1,52 @@ +name: Testing Tool CI + +on: + push: + branches: + - master + - release-* + paths: + - 'tools/testing-tool/**' + - '.github/workflows/testing-tool-ci.yml' + pull_request: + paths: + - 'tools/testing-tool/**' + - '.github/workflows/testing-tool-ci.yml' + workflow_dispatch: + +env: + DOTNET_NOLOGO: true + +defaults: + run: + shell: pwsh + +jobs: + build: + name: Build solution + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v6.0.0 + with: + global-json-file: global.json + + - name: Build + run: dotnet build tools/testing-tool/TestingTool.slnx --configuration Release + + - name: Build Aspire AppHost + run: dotnet build tools/testing-tool/aspire/AppHost.cs --configuration Release + + - name: Build container image + uses: docker/build-push-action@v7.2.0 + with: + context: . + file: tools/testing-tool/Dockerfile + push: false + load: true + tags: particular/testing-tool:ci diff --git a/tools/testing-tool/Directory.Build.props b/tools/testing-tool/Directory.Build.props new file mode 100644 index 0000000000..a8f0c98b2f --- /dev/null +++ b/tools/testing-tool/Directory.Build.props @@ -0,0 +1,24 @@ + + + + + net10.0 + enable + enable + true + true + true + low + all + + + + true + + + diff --git a/tools/testing-tool/Directory.Packages.props b/tools/testing-tool/Directory.Packages.props new file mode 100644 index 0000000000..eb5823bbfc --- /dev/null +++ b/tools/testing-tool/Directory.Packages.props @@ -0,0 +1,25 @@ + + + + + true + true + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tools/testing-tool/Dockerfile b/tools/testing-tool/Dockerfile new file mode 100644 index 0000000000..f1d23a07ba --- /dev/null +++ b/tools/testing-tool/Dockerfile @@ -0,0 +1,38 @@ +# Multi-stage build for the ServiceControl testing tool. +# Build context is the repository root so global.json + nuget.config are available. +ARG RUNTIMEVERSION=10.0 + +# --- build stage --- +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:10.0 AS build +ARG TARGETARCH +WORKDIR /src +ENV CI=true + +# global.json pins the SDK; nuget.config adds the Particular feed for later NServiceBus deps. +COPY global.json nuget.config ./ +COPY tools/testing-tool/ ./tools/testing-tool/ + +# Publish the runnable host; its ProjectReferences pull in Contracts + Scenarios. +RUN dotnet publish tools/testing-tool/TestingTool/TestingTool.csproj \ + --configuration Release \ + --arch $TARGETARCH \ + --output /app \ + /p:UseAppHost=false + +# --- runtime stage --- +# Uses the same chiseled composite base as the rest of the ServiceControl images so that +# globalization/ICU dependencies are present. +FROM mcr.microsoft.com/dotnet/aspnet:$RUNTIMEVERSION-noble-chiseled-composite-extra +WORKDIR /app + +COPY --from=build /app ./ + +ENV ASPNETCORE_URLS=http://+:8080 +EXPOSE 8080 + +# Health probes are served by the /health/live and /health/ready HTTP endpoints (added in +# Phase 6). Kubernetes uses HTTP GET probes directly; the chiseled base ships no shell/curl so +# exec-based probes are not supported. + +USER $APP_UID +ENTRYPOINT ["dotnet", "TestingTool.dll"] diff --git a/tools/testing-tool/README.md b/tools/testing-tool/README.md new file mode 100644 index 0000000000..5083ce0122 --- /dev/null +++ b/tools/testing-tool/README.md @@ -0,0 +1,232 @@ +# ServiceControl Testing Tool + +A stateless, horizontally-scalable .NET 10 service that generates error load and real-world failure +scenarios against a test ServiceControl instance, with OpenTelemetry observability and a simple web +UI for manual scenario control. + +See [requirements-test-tool-plan.md](./requirements-test-tool-plan.md) for the full plan. + +## Status + +All phases complete (0–7): project bootstrap, OTel foundation (traces + metrics + logs), +NServiceBus error path (handler + direct error-queue bypass writer), scenarios, background jobs, +web UI, containerization & scaling, Aspire AppHost, observability stack (OTel Collector → +Jaeger + Prometheus + Grafana with prebuilt dashboard), and smoke tests. + +## What it does + +The tool runs an NServiceBus endpoint (`TestingTool.Load`) that sends messages through a handler +which fails based on the active scenario. Failed messages are routed to the `error` queue for +ServiceControl to ingest. Five scenarios are built in, each producing naturally-grouped errors: + +| Scenario | Category | Failure shape | +|---|---|---| +| `third-party-outage` | Outage | 100% fail for 20s bursts, 30s cooldown — grouped by downstream host | +| `timeout-spike` | Timeout | Oscillating 10–70% fail rate — grouped by 5-min batch bucket | +| `poison-message` | Poison | 15% deterministic always-fail messages — retry storm | +| `deserialization-failure` | Deserialization | 100% fail — grouped by message type (bad deployment) | +| `background-noise` | Noise | ~3% always-on baseline — rotates through exception types | + +Recoverability/search jobs are controllable from the web UI (no longer hidden config-gated + timers). They run a cycle on a configurable interval until stopped: +- **Retry** — fetches error groups from ServiceControl and retries each group +- **Archive** — fetches error groups from ServiceControl and archives each group +- **Search** — runs canned FTS queries to exercise the ServiceControl search index + +Jobs do not auto-start; start them from the UI (or `/api/jobs`) when needed. Control via: +- `GET /api/jobs` — list jobs with live status +- `POST /api/jobs/{name}/start` — `{ "intervalSeconds": 120 }` (omit for the job default) +- `POST /api/jobs/{name}/stop` +- `POST /api/jobs/stop-all` + +All telemetry is exported via OTLP (traces + metrics + logs) and a Prometheus `/metrics` endpoint. + +### Direct error-queue bypass writer + +In addition to the handler path, the tool can write failed-message envelopes directly to the +ServiceControl error queue, bypassing the handler entirely for high-throughput error load. +Each message carries standard NServiceBus failure headers (`NServiceBus.ExceptionInfo.*`, +`NServiceBus.FailedQ`) so ServiceControl ingests it as a genuine failed message. Control via: +- `POST /api/bypass/start` — `{ "scenario": "third-party-outage", "rate": 100, "durationSeconds": 60 }` +- `POST /api/bypass/stop` +- `GET /api/bypass/status` + +### Release-test scenario presets + +The tool ships with presets mapped from `docs/testing-scenarios.md` so release-test scenarios can +be kicked off manually by name: +- `GET /api/release-tests` — list all presets +- `POST /api/release-tests/{name}/start` — start a preset (e.g. `retry-message-group`, `ingestion-load`) + +## Layout + +``` +tools/testing-tool/ + TestingTool.slnx + Directory.Build.props # repo-style conventions (warnings-as-errors, nullable, analyzers) + Directory.Packages.props # central package management (OTel + NServiceBus pinned) + Dockerfile + global.json + TestingTool/ # ASP.NET Core host: Program.cs, web UI, services + TestingTool.csproj + Program.cs # OTel wiring, NServiceBus endpoint, DI, API endpoints + appsettings.json # base config (TestingTool section, overridable by env vars) + appsettings.Development.json # Development overrides + wwwroot/index.html # single-page web UI (vanilla JS, no build step) + ScenarioRunner.cs # start/stop, rate control, per-scenario error counting + DirectErrorQueueWriter.cs # bypass path: writes failed-message envelopes directly to error queue + Jobs/ # UI-controllable recoverability/search jobs (retry, archive, search) + JobBase.cs # periodic job base class (start/stop, cycle counters) + JobRunner.cs # manages job lifecycle, exposes /api/jobs + RetryJob.cs # retries all error groups each cycle + ArchiveJob.cs # archives all error groups each cycle + SearchJob.cs # canned FTS queries each cycle + FailingMessageHandler.cs # NServiceBus handler that throws per scenario logic + ReleaseTestScenarios.cs # release-test preset mappings (Phase 5) + ServiceControlClient.cs # REST API client (error groups, retry, archive, search) + TelemetrySetup.cs # OTel traces + metrics + logs + OTLP/Prometheus exporters + NServiceBusSetup.cs # endpoint config (Learning transport, error queue routing) + TestingToolOptions.cs # config (SC URL, retry/archive/search intervals, error queue name) + TestingToolMetrics.cs # shared live counters for /api/status + ShardIdResolver.cs # shard id from env var, StatefulSet ordinal, or hostname + IScenarioRegistry.cs # scenario registry abstraction (DI) + ScenarioRegistry.cs # default scenario registry implementation + TestingTool.Scenarios/ # IScenario contract + 5 scenario implementations + TestingTool.Contracts/ # shared DTOs (ScenarioInfo, TestingToolStatus, BypassStatus, etc.) + TestingTool.SmokeTests/ # xunit smoke tests (requires running SC + tool) + TestingTool.AppHost/ # Aspire AppHost project (platform + tool + observability stack) + AppHost.cs # top-level orchestration (platform, observability, testing tool) + HostBuilderExtensions.cs # persistence-type extensions (RavenDB / SQL Server / PostgreSQL) + ObservabilityExtensions.cs # AddObservabilityStack() — OTel Collector + Jaeger + Prometheus + Grafana + PersistenceType.cs # persistence enum + obs/ # observability config (collector, Prometheus, Grafana provisioning + dashboard) + otel-collector-config.yaml # collector pipeline: traces → Jaeger, metrics → Prometheus exporter + prometheus.yml # scrape config (targets the collector's metrics exporter) + grafana/provisioning/ # auto-provisioned data sources (Prometheus + Jaeger) and dashboard provider + grafana/dashboards/ # prebuilt "Testing Tool" Grafana dashboard JSON +``` + +## Run locally + +```bash +dotnet build tools/testing-tool/TestingTool.slnx --configuration Release +dotnet run --project tools/testing-tool/TestingTool --configuration Release +``` + +Open http://localhost:5290 (or the port shown in the console). + +## Run with Aspire + +The Aspire AppHost orchestrates the testing tool together with the full Particular platform +(ServiceControl + Learning transport + RavenDB + ServicePulse) and a complete observability +stack (OTel Collector, Jaeger, Prometheus, Grafana), so a single command brings up the whole +system locally: + +```bash +aspire run tools/testing-tool/TestingTool.AppHost/TestingTool.AppHost.csproj +``` + +To test a specific ServiceControl image tag (e.g. a PR-based prerelease tag): + +```bash +aspire run tools/testing-tool/TestingTool.AppHost/TestingTool.AppHost.csproj -- --tag pr-1234 +``` + +To select a persistence backend for the ServiceControl error instance (`RavenDb`, +`SqlServer`, or `PostgreSql`; defaults to `PostgreSql`): + +```bash +aspire run tools/testing-tool/TestingTool.AppHost/TestingTool.AppHost.csproj -- --persistence:RavenDb +``` + +`--persistence RavenDb` (space separator) is accepted too. Both flags may be combined: + +```bash +aspire run tools/testing-tool/TestingTool.AppHost/TestingTool.AppHost.csproj -- --tag pr-1234 --persistence:SqlServer +``` + +The Aspire dashboard provides allocated ports for each service. The testing tool automatically +connects to ServiceControl via the platform's transport and REST API URL, and sends its OTLP +telemetry to the OTel Collector, which fans out traces to Jaeger and metrics to Prometheus. +Grafana (auto-provisioned with Prometheus + Jaeger data sources) provides a prebuilt dashboard +at the allocated port — log in with `admin`/`admin` or browse anonymously as Viewer. + +### Observability stack + +| Service | Image | Purpose | +|---|---|---| +| OTel Collector | `otel/opentelemetry-collector-contrib` | Receives OTLP, fans out traces → Jaeger, metrics → Prometheus exporter | +| Jaeger | `jaegertracing/all-in-one` | Distributed-trace UI — purpose-built trace analysis richer than the Aspire dashboard | +| Prometheus | `prom/prometheus` | Scrapes the collector's metrics exporter | +| Grafana | `grafana/grafana-oss` | Dashboards with auto-provisioned Prometheus + Jaeger data sources | + +The stack is wired via `AddObservabilityStack()` in `ObservabilityExtensions.cs` so `AppHost.cs` +stays clean. Config files live under `obs/` next to the AppHost project. The prebuilt Grafana +dashboard ("Testing Tool — Error Load & Observability") shows errors/sec by scenario (handler +and bypass paths emitted separately and combined into the raised total), search latency p95, +replay/archive rates, and — using ServiceControl's own OTel ingestion metrics +(`sc.error.ingestion.*`) — side-by-side comparison of errors raised vs errors ingested (rate +and cumulative), ingestion duration p95, and ingestion outcome by result. + +## Run smoke tests + +The smoke tests require a running ServiceControl + testing tool (e.g. via the Aspire AppHost above, +or `dotnet run` against an existing ServiceControl): + +```bash +# Start the stack first (see Run with Aspire) +dotnet test tools/testing-tool/TestingTool.SmokeTests +``` + +The test URLs default to `http://localhost:8080` (tool) and `http://localhost:33333` (ServiceControl). +Override them to match your run — Aspire assigns dynamic ports, shown in the Aspire dashboard: +```bash +TESTING_TOOL_URL=http://localhost: SERVICECONTROL_URL=http://localhost: \ + dotnet test tools/testing-tool/TestingTool.SmokeTests +``` + +## Horizontal scaling + +The tool is **stateless** — all state is in-memory per replica. The repo no longer ships +docker-compose or Kubernetes manifests; run a single instance via `dotnet run` or the Aspire +AppHost. For multi-replica deployments, bring your own orchestration and give each replica a +distinct shard id so deterministic failure decisions don't overlap: + +| Shard id source | When | +|---|---| +| `SHARD_ID` env var | Explicit override — recommended for any custom deployment | +| Hostname trailing ordinal (e.g. `testing-tool-2` → `2`) | StatefulSet-style ordered hostnames | +| `MachineName` | Fallback — unique per host/pod | + +To achieve a target aggregate rate of R msg/s across N replicas, set each replica's scenario rate +to R/N. The web UI and `/api/status` endpoint report per-replica counters; aggregate across +replicas via Prometheus queries or the OTLP backend. + +## Configuration + +All configuration is via environment variables (no files, no database). Settings are in +`appsettings.json` under the `TestingTool` section, overridable by environment variables using +`__` as the section separator (e.g. `TestingTool__ServiceControlApiUrl`): + +| Setting | Default | Description | +|---|---|---| +| `TestingTool__ServiceControlApiUrl` | `http://localhost:33333` | ServiceControl REST API base URL | +| `TestingTool__ReplayInterval` | `00:02:00` | Default interval for the retry job | +| `TestingTool__ReplayMinGroupSize` | `1` | Min messages in a group before retrying | +| `TestingTool__SearchInterval` | `00:01:00` | Default interval for the search job | +| `TestingTool__ArchiveInterval` | `00:02:00` | Default interval for the archive job | +| `TestingTool__ArchiveMinGroupSize` | `1` | Min messages in a group before archiving | +| `TestingTool__ErrorQueueName` | `error` | NServiceBus error queue (ServiceControl monitors this) | +| `TestingTool__AutoStartBackgroundNoise` | `false` | Auto-start the background-noise scenario on startup | +| `SHARD_ID` (env) | *(auto: hostname ordinal or machine name)* | Shard id for disjoint scenario slices when scaled | +| `OTEL_EXPORTER_OTLP_ENDPOINT` (env) | `http://localhost:4317` | OTLP collector endpoint | +| `OTEL_SERVICE_NAME` (env) | `testing-tool` | OTel service name | + +## Health checks + +| Endpoint | Purpose | +|---|---| +| `GET /health/live` | Liveness — process is alive | +| `GET /health/ready` | Readiness — app is ready to serve requests | +| `GET /api/status` | Full status snapshot (counters, shard, uptime) | +| `GET /metrics` | Prometheus scraping endpoint | \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.AppHost/AppHost.cs b/tools/testing-tool/TestingTool.AppHost/AppHost.cs new file mode 100644 index 0000000000..6122cdd703 --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/AppHost.cs @@ -0,0 +1,47 @@ +using Particular.Aspire.Hosting.ServicePlatform.Platform; +using Particular.Aspire.Hosting.ServicePlatform.Transport; +using TestingTool.AppHost; + +var options = CliOptions.Parse(args); +var persistenceType = options.GetEnumOrDefault("persistence", PersistenceType.PostgreSql); +Console.WriteLine($"Using persistence type: {persistenceType}"); + +var builder = DistributedApplication.CreateBuilder(args); + +// --- Observability stack (OTel Collector → Jaeger + Prometheus + Grafana) --- +var observability = builder.AddObservabilityStack(); + +// --- Particular Platform (ServiceControl + RabbitMQ transport) --- +var transportUserName = builder.AddParameter("transportUserName", "guest", secret: true); +var transportPassword = builder.AddParameter("transportPassword", "guest", secret: true); +var transport = builder.AddRabbitMQ("transport", transportUserName, transportPassword) + .WithManagementPlugin(15672) + .WithUrlForEndpoint("management", url => url.DisplayText = "RabbitMQ Management"); + +var platform = builder + .AddParticularPlatform("particular") + .WithTransportRabbitMQ(RabbitMqRouting.QuorumConventionalRouting, transport); + +var raven = platform.AddPersistenceRavenDb("raven"); + +var errorInstance = platform + .AddServiceControlErrorInstance("error", raven) + .WithEnvironment("OTEL_EXPORTER_OTLP_ENDPOINT", observability.Collector.GetEndpoint("otlp-grpc")) + .WithPersistenceType(persistenceType) + .WithRunMode(PlatformRunMode.SetupAndRun); + +platform.AddServicePulse("pulse", errorInstance); + +// --- Testing tool --- +builder.AddProject("testing-tool") + .WithParticularPlatform(platform) + .WithEnvironment("TestingTool__ServiceControlApiUrl", errorInstance.GetEndpoint("http")) + .WithEnvironment("TestingTool__AutoStartBackgroundNoise", "true") + .WithEnvironment("OTEL_EXPORTER_OTLP_ENDPOINT", observability.Collector.GetEndpoint("otlp-grpc")) + .WaitFor(errorInstance) + .WaitFor(observability.Collector); + +// --- Optional: override ServiceControl image tag for prerelease testing --- +builder.UseServiceControlImageTag(options.GetValueOrDefault("tag")); + +builder.Build().Run(); \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.AppHost/CliOptions.cs b/tools/testing-tool/TestingTool.AppHost/CliOptions.cs new file mode 100644 index 0000000000..56f2f1fffe --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/CliOptions.cs @@ -0,0 +1,53 @@ +using System.Runtime.CompilerServices; + +namespace TestingTool.AppHost; + +/// +/// This is a bare-bones parser for commandline options. +/// +public class CliOptions(IEnumerable> values) : Dictionary(values) +{ + public T GetEnumOrDefault(string key, T defaultValue) where T : struct => ContainsKey(key) ? Enum.Parse(this[key]) : defaultValue; + + public static CliOptions Parse(params string[] args) + { + return new CliOptions(Scan()); + + IEnumerable> Scan() + { + for (var i = 0; i < args.Length; i++) + { + var arg = args[i]; + + //allow bare arguments to be passed, they are just ignored. + if (!arg.StartsWith("--", StringComparison.Ordinal)) + continue; + + if (arg.Length <= 2) + throw new ArgumentException($"Invalid command-line argument '{arg}': expected a parameter name after the '--' prefix."); + + // --paramname:value (colon separator; value is everything after the first colon) + var colon = arg.IndexOf(':'); + if (colon > 0) + { + var key = arg[2..colon]; + var value = arg[(colon + 1)..]; + yield return new KeyValuePair(key.ToLowerInvariant(), value); + } + else + { + // --paramname value (space separator; value is the next token) + i++; + var key = arg[2..]; + if (i >= args.Length) + { + throw new ArgumentException( + $"Missing value for command-line argument '{arg}': expected a value after the parameter name."); + } + + yield return new KeyValuePair(key.ToLowerInvariant(), args[i]); + } + } + } + } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.AppHost/HostBuilderExtensions.cs b/tools/testing-tool/TestingTool.AppHost/HostBuilderExtensions.cs new file mode 100644 index 0000000000..c132d1e1f0 --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/HostBuilderExtensions.cs @@ -0,0 +1,58 @@ +using Particular.Aspire.Hosting.ServicePlatform.Platform; + +namespace TestingTool.AppHost; + +public static class HostBuilderExtensions +{ + public static IResourceBuilder AddSqlServerPersistence(this IDistributedApplicationBuilder builder, string databaseName) + { + var password = builder.AddParameter("sql-password", "Password1!", secret: true); + var server = builder + .AddSqlServer("sqlserver", password) + .WithDataVolume("migration-sql-data"); + return server.AddDatabase("servicecontrol-sql", databaseName); + } + + public static IResourceBuilder AddPostgresPersistence( + this IDistributedApplicationBuilder builder, string databaseName) + { + var postgresPassword = builder.AddParameter("postgres-password", "Password1!", secret: true); + var postgres = builder + .AddPostgres("postgres", password: postgresPassword) + .WithPgAdmin() + .WithDataVolume("migration-postgres-data"); + return postgres.AddDatabase("servicecontrol-postgres", databaseName); + } + + public static IResourceBuilder WithPersistenceType( + this IResourceBuilder error, PersistenceType type) + { + if (type == PersistenceType.RavenDb) + { + //ravenDB is currently set up explicitly even if you aren't using it + return error; + } + + var db = type switch + { + PersistenceType.SqlServer => error.ApplicationBuilder.AddSqlServerPersistence("ServiceControl"), + PersistenceType.PostgreSql => error.ApplicationBuilder.AddPostgresPersistence("servicecontrol"), + _ => throw new ArgumentOutOfRangeException(nameof(type), type, null) + }; + + return error + .WaitFor(db) + //file storage for now + .WithEnvironment("SERVICECONTROL_MESSAGEBODY_STORAGETYPE", "FileSystem") + .WithEnvironment("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_STORAGEPATH", "/tmp/ServiceControlBodyStorage") + .WithEnvironment("SERVICECONTROL_PERSISTENCETYPE", PersistenceTypeName(type)) + .WithEnvironment("SERVICECONTROL_DATABASE_CONNECTIONSTRING", db); + } + + static string PersistenceTypeName(PersistenceType persistence) => persistence switch + { + PersistenceType.SqlServer => "SQLServer", + PersistenceType.PostgreSql => "PostgreSQL", + _ => throw new ArgumentOutOfRangeException(nameof(persistence)) + }; +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.AppHost/ObservabilityExtensions.cs b/tools/testing-tool/TestingTool.AppHost/ObservabilityExtensions.cs new file mode 100644 index 0000000000..3b0a67fcb8 --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/ObservabilityExtensions.cs @@ -0,0 +1,129 @@ +using Aspire.Hosting.ApplicationModel; + +namespace TestingTool.AppHost; + +/// +/// Adds a complete OpenTelemetry observability stack to the Aspire AppHost, giving purpose-built +/// trace analysis (Jaeger) and metric dashboards (Grafana + Prometheus) that are richer than the +/// Aspire dashboard's built-in OTel view. +/// +/// Architecture: +/// testing-tool → OTLP → OTel Collector → traces → Jaeger +/// └→ metrics → Prometheus exporter ← Prometheus ← Grafana +/// +/// All config files live under obs/ next to the AppHost project and are bind-mounted into +/// the containers at startup. +/// +public static class ObservabilityExtensions +{ + /// + /// Adds the observability stack (OTel Collector, Jaeger, Prometheus, Grafana) and returns + /// references so callers can wire the testing tool's OTLP exporter at the collector. + /// + public static ObservabilityStack AddObservabilityStack(this IDistributedApplicationBuilder builder) + { + var jaeger = AddJaeger(builder); + var collector = AddOtelCollector(builder, jaeger); + var prometheus = AddPrometheus(builder, collector); + var grafana = AddGrafana(builder, prometheus, jaeger); + + // Grafana is the user-facing entry point of the stack, so nest the backing + // OTel/trace/metric resources under it in the Aspire dashboard's resource tree. + collector.WithParentRelationship(grafana); + jaeger.WithParentRelationship(grafana); + prometheus.WithParentRelationship(grafana); + + return new ObservabilityStack(collector, grafana, jaeger, prometheus); + } + + /// + /// Overrides the image tag for every ServiceControl container in the AppHost. + /// + public static void UseServiceControlImageTag(this IDistributedApplicationBuilder builder, string? tag) + { + if (string.IsNullOrWhiteSpace(tag)) + return; + + Console.WriteLine($"Using ServiceControl image tag: {tag}"); + foreach (var c in builder.Resources.OfType()) + { + if (c.TryGetLastAnnotation(out var image) && + image.Image.StartsWith("particular/servicecontrol")) + { + builder.CreateResourceBuilder(c) + .WithImage($"ghcr.io/{image.Image}", tag); + } + } + } + + // --- Jaeger: distributed-trace UI --- + + static IResourceBuilder AddJaeger(IDistributedApplicationBuilder builder) => + builder.AddContainer("jaeger", "jaegertracing/all-in-one:1.62.0") + .WithHttpEndpoint(16686, 16686, "ui") + .WithEndpoint(4317, 4317, scheme: "http", name: "otlp-grpc") + .WithHttpEndpoint(4318, 4318, "otlp-http") + .WithUrlForEndpoint("ui", url => url.DisplayText = "Jaeger UI — Traces"); + + // --- OTel Collector: receives OTLP, fans out traces → Jaeger, metrics → Prometheus exporter --- + + static IResourceBuilder AddOtelCollector( + IDistributedApplicationBuilder builder, IResourceBuilder jaeger) => + builder.AddContainer("otel-collector", "otel/opentelemetry-collector-contrib:0.110.0") + // The contrib image reads its config from /etc/otelcol-contrib/config.yaml (the core + // image uses /etc/otelcol/config.yaml). Mounting at the wrong path silently leaves the + // image's built-in default config active — which has no Prometheus exporter, so the + // :8889 scrape target has no listener and Prometheus gets "connection refused". + .WithBindMount("obs/otel-collector-config.yaml", "/etc/otelcol-contrib/config.yaml") + .WithHttpEndpoint(8889, 8889, "metrics") // Prometheus scrape target + // The third positional arg of WithEndpoint is `scheme`, not `name` — passing "otlp-grpc" + // positionally would set UriScheme to "otlp-grpc" and yield an otlp-grpc:// URL that the + // .NET OTLP gRPC exporter (GrpcChannel) rejects, silently dropping all telemetry. Name + // the endpoint explicitly and force an `http` scheme so GetEndpoint produces an + // http:// URL the exporter can connect to (gRPC runs over HTTP/2). + .WithEndpoint(4317, 4317, scheme: "http", name: "otlp-grpc") // OTLP gRPC (testing tool sends here) + .WithHttpEndpoint(4318, 4318, "otlp-http") // OTLP HTTP (fallback) + .WaitFor(jaeger) + .WithUrlForEndpoint("metrics", url => url.DisplayText = "OTel Collector — Prometheus Metrics"); + + // --- Prometheus: scrapes the collector's metrics exporter --- + + static IResourceBuilder AddPrometheus( + IDistributedApplicationBuilder builder, IResourceBuilder collector) => + builder.AddContainer("prometheus", "prom/prometheus:v3.2.1") + .WithBindMount("obs/prometheus.yml", "/etc/prometheus/prometheus.yml") + .WithHttpEndpoint(9090, 9090, "http") + .WaitFor(collector) + .WithUrlForEndpoint("http", url => url.DisplayText = "Prometheus — Metrics"); + + // --- Grafana: dashboards with Prometheus + Jaeger data sources --- + + static IResourceBuilder AddGrafana( + IDistributedApplicationBuilder builder, + IResourceBuilder prometheus, + IResourceBuilder jaeger) + { + var grafana = builder.AddContainer("grafana", "grafana/grafana-oss:11.4.0") + .WithBindMount("obs/grafana/provisioning", "/etc/grafana/provisioning") + .WithBindMount("obs/grafana/dashboards", "/var/lib/grafana/dashboards") + .WithHttpEndpoint(3000, 3000, "http") + .WithEnvironment("GF_SECURITY_ADMIN_USER", "admin") + .WithEnvironment("GF_SECURITY_ADMIN_PASSWORD", "admin") + .WithEnvironment("GF_AUTH_ANONYMOUS_ENABLED", "true") + .WithEnvironment("GF_AUTH_ANONYMOUS_ORG_ROLE", "Viewer") + .WaitFor(prometheus) + .WaitFor(jaeger); + + grafana.WithUrlForEndpoint("http", url => url.DisplayText = "Grafana — Dashboards"); + return grafana; + } +} + +/// +/// References to the observability stack resources, returned from . +/// +public sealed record ObservabilityStack( + IResourceBuilder Collector, + IResourceBuilder Grafana, + IResourceBuilder Jaeger, + IResourceBuilder Prometheus); \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.AppHost/PersistenceType.cs b/tools/testing-tool/TestingTool.AppHost/PersistenceType.cs new file mode 100644 index 0000000000..aa10ff9f68 --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/PersistenceType.cs @@ -0,0 +1,8 @@ +namespace TestingTool.AppHost; + +public enum PersistenceType +{ + RavenDb, + SqlServer, + PostgreSql +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.AppHost/Properties/launchSettings.json b/tools/testing-tool/TestingTool.AppHost/Properties/launchSettings.json new file mode 100644 index 0000000000..7a2182ca92 --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/Properties/launchSettings.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://sc-load-tool.dev.localhost:17290;http://sc-load-tool.dev.localhost:15170", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21118", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22281" + } + }, + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://sc-load-tool.dev.localhost:15170", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19233", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20173" + } + } + } +} diff --git a/tools/testing-tool/TestingTool.AppHost/TestingTool.AppHost.csproj b/tools/testing-tool/TestingTool.AppHost/TestingTool.AppHost.csproj new file mode 100644 index 0000000000..b726bce402 --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/TestingTool.AppHost.csproj @@ -0,0 +1,22 @@ + + + net10.0 + enable + enable + Exe + false + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.AppHost/aspire.config.json b/tools/testing-tool/TestingTool.AppHost/aspire.config.json new file mode 100644 index 0000000000..1efd009745 --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/aspire.config.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://aspire.dev/reference/cli/configuration/schema.json", + "appHost": { + "path": "TestingTool.AppHost.csproj" + } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.AppHost/obs/grafana/dashboards/testing-tool.json b/tools/testing-tool/TestingTool.AppHost/obs/grafana/dashboards/testing-tool.json new file mode 100644 index 0000000000..b85ec685aa --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/obs/grafana/dashboards/testing-tool.json @@ -0,0 +1,703 @@ +{ + "title": "Testing Tool — Error Load & Observability", + "uid": "testing-tool", + "schemaVersion": 39, + "version": 4, + "timezone": "browser", + "refresh": "5s", + "tags": ["testing-tool", "servicecontrol", "otel"], + "templating": { + "list": [ + { + "name": "scenario", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(errors_sent_total, scenario)", + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" } + }, + { + "name": "sc_instance", + "label": "ServiceControl instance", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(sc_error_ingestion_message_duration_seconds_count, exported_job)", + "refresh": 2, + "includeAll": true, + "allValue": ".*", + "multi": true, + "current": { "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { + "id": 100, + "type": "row", + "title": "Load generation (testing tool)", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "panels": [] + }, + { + "id": 1, + "title": "Errors Raised / sec — handler path (by scenario)", + "description": "Errors sent to the ServiceControl error queue via the NServiceBus handler path (errors_sent_total), broken down by scenario. The bypass path is emitted separately — see the 'Bypass Errors Written / sec' panel — and combined with this in the 'Errors Raised vs Ingested' panels.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "sum by (scenario) (rate(errors_sent_total{scenario=~\"$scenario\"}[$__rate_interval]))", + "legendFormat": "{{scenario}}" + } + ] + }, + { + "id": 5, + "title": "Bypass Errors Written / sec (by scenario)", + "description": "Errors written directly to the ServiceControl error queue by the bypass path (bypass_errors_written_total), bypassing the handler. Emitted separately from errors_sent_total so handler and bypass load can be told apart; the 'Errors Raised vs Ingested' panels combine the two into the total raised.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "sum by (scenario) (rate(bypass_errors_written_total[$__rate_interval]))", + "legendFormat": "bypass: {{scenario}}" + } + ] + }, + { + "id": 3, + "title": "Errors Replayed / sec", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 9 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "rate(errors_replayed_total[$__rate_interval])", + "legendFormat": "replayed" + } + ] + }, + { + "id": 4, + "title": "Errors Archived / sec", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 9 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "rate(errors_archived_total[$__rate_interval])", + "legendFormat": "archived" + } + ] + }, + { + "id": 6, + "title": "Searches Executed / sec", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 9 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "rate(searches_executed_total[$__rate_interval])", + "legendFormat": "searches" + } + ] + }, + { + "id": 2, + "title": "Search Latency p95 (ms)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(search_latency_ms_bucket[$__rate_interval])))", + "legendFormat": "p95" + } + ] + }, + { + "id": 7, + "title": "Total Errors Raised (cumulative)", + "description": "Cumulative errors raised by the testing tool, combining the handler path (errors_sent_total) and the bypass path (bypass_errors_written_total). Compare against 'Errors Ingested' in the 'Errors Raised vs Ingested' panels to gauge ingestion backlog.", + "type": "stat", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "thresholds" }, + "thresholds": { "steps": [ { "color": "green" } ] } + } + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] } + }, + "targets": [ + { + "expr": "sum(errors_sent_total) + sum(bypass_errors_written_total)", + "legendFormat": "raised" + } + ] + }, + { + "id": 101, + "type": "row", + "title": "Raised vs ingested", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, + "panels": [] + }, + { + "id": 9, + "title": "Errors Raised vs Ingested / sec", + "description": "Compares the rate of errors the testing tool raises — combining the handler path (errors_sent_total) and the bypass path (bypass_errors_written_total) — against the rate ServiceControl successfully ingests (sc.error.ingestion.message_duration_seconds, message_category=failed-message, result=success). The raised series should lead the ingested series; a widening gap indicates an ingestion backlog or failures.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "sum(rate(errors_sent_total[$__rate_interval])) + sum(rate(bypass_errors_written_total[$__rate_interval]))", + "legendFormat": "raised (testing-tool)" + }, + { + "expr": "sum(rate(sc_error_ingestion_message_duration_seconds_count{exported_job=~\"$sc_instance\",message_category=\"failed-message\",result=\"success\"}[$__rate_interval]))", + "legendFormat": "ingested (servicecontrol)" + } + ] + }, + { + "id": 10, + "title": "Errors Raised vs Ingested (cumulative)", + "description": "Cumulative comparison: total errors raised by the testing tool (handler + bypass) vs total failed messages ServiceControl successfully ingested. A persistent delta is the ingestion backlog (still queued or failed).", + "type": "stat", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "thresholds" }, + "thresholds": { "steps": [ { "color": "green" } ] } + } + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] } + }, + "targets": [ + { + "expr": "sum(errors_sent_total) + sum(bypass_errors_written_total)", + "legendFormat": "raised" + }, + { + "expr": "sum(sc_error_ingestion_message_duration_seconds_count{exported_job=~\"$sc_instance\",message_category=\"failed-message\",result=\"success\"})", + "legendFormat": "ingested" + } + ] + }, + { + "id": 102, + "type": "row", + "title": "ServiceControl ingestion — rate & throughput", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 }, + "panels": [] + }, + { + "id": 20, + "title": "Ingestion Rate (msg/sec)", + "description": "Current successful ingestion rate across both message categories, from the count of sc.error.ingestion.message_duration_seconds with result=success. This is the headline throughput number: what ServiceControl is actually absorbing right now.", + "type": "stat", + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 35 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "decimals": 1, + "color": { "mode": "thresholds" }, + "thresholds": { "steps": [ { "color": "green" } ] } + } + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "graphMode": "area", + "colorMode": "value" + }, + "targets": [ + { + "expr": "sum(rate(sc_error_ingestion_message_duration_seconds_count{exported_job=~\"$sc_instance\",result=\"success\"}[$__rate_interval]))", + "legendFormat": "ingested" + } + ] + }, + { + "id": 21, + "title": "Peak Ingestion Rate (selected range)", + "description": "Highest 1-minute ingestion rate seen anywhere in the dashboard's time range, sampled every 15s. Use it to read off what the instance sustained at its best during a load run.", + "type": "stat", + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 35 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "decimals": 1, + "color": { "mode": "thresholds" }, + "thresholds": { "steps": [ { "color": "blue" } ] } + } + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "graphMode": "none", + "colorMode": "value" + }, + "targets": [ + { + "expr": "max_over_time((sum(rate(sc_error_ingestion_message_duration_seconds_count{exported_job=~\"$sc_instance\",result=\"success\"}[$__rate_interval])))[$__range:15s])", + "legendFormat": "peak" + } + ] + }, + { + "id": 22, + "title": "Total Messages Ingested", + "description": "Cumulative count of messages ServiceControl ingested successfully, both failed messages and retry confirmations.", + "type": "stat", + "gridPos": { "h": 6, "w": 6, "x": 12, "y": 35 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "thresholds" }, + "thresholds": { "steps": [ { "color": "green" } ] } + } + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "graphMode": "area" + }, + "targets": [ + { + "expr": "sum(sc_error_ingestion_message_duration_seconds_count{exported_job=~\"$sc_instance\",result=\"success\"})", + "legendFormat": "ingested" + } + ] + }, + { + "id": 23, + "title": "Consecutive Batch Failures", + "description": "sc.error.ingestion.consecutive_batch_failures_total — the ingestor's own circuit-breaker signal. It resets to 0 on the first batch that writes, so anything above 0 means storage is currently rejecting batches.", + "type": "stat", + "gridPos": { "h": 6, "w": 6, "x": 18, "y": 35 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 3 } + ] + } + } + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "graphMode": "area", + "colorMode": "background" + }, + "targets": [ + { + "expr": "max(sc_error_ingestion_consecutive_batch_failures_total{exported_job=~\"$sc_instance\"})", + "legendFormat": "consecutive failures" + } + ] + }, + { + "id": 24, + "title": "Ingestion Rate / sec (by result)", + "description": "Every message the ingestor completed, split by outcome: success, skipped (nothing to store) and failed. Stacked, the total is the rate messages left the ingestion pipeline at, regardless of outcome.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 25, + "stacking": { "mode": "normal", "group": "A" } + } + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "expr": "sum by (result) (rate(sc_error_ingestion_message_duration_seconds_count{exported_job=~\"$sc_instance\"}[$__rate_interval]))", + "legendFormat": "{{result}}" + } + ] + }, + { + "id": 25, + "title": "Ingestion Rate / sec (by message category and instance)", + "description": "Successful ingestion rate split by message.category — failed-message vs retry-confirmation — and by instance. Retry confirmations are much cheaper than failed messages, so a rate that climbs while latency stays flat is usually a shift in this mix, not extra capacity.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "expr": "sum by (exported_job, message_category) (rate(sc_error_ingestion_message_duration_seconds_count{exported_job=~\"$sc_instance\",result=\"success\"}[$__rate_interval]))", + "legendFormat": "{{message_category}} - {{exported_job}}" + } + ] + }, + { + "id": 26, + "title": "Batches Written / sec (by result)", + "description": "Count of sc.error.ingestion.batch_duration_seconds by result. 'full' means the batch hit the maximum size, so the ingestor was never waiting for messages: a sustained run of full batches means the load generator, not ServiceControl, is setting the pace.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 49 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "expr": "sum by (result) (rate(sc_error_ingestion_batch_duration_seconds_count{exported_job=~\"$sc_instance\"}[$__rate_interval]))", + "legendFormat": "{{result}}" + } + ] + }, + { + "id": 27, + "title": "Effective Batch Size (messages per batch)", + "description": "Message rate divided by batch rate: how many messages each batch actually carried. Sitting at the configured maximum means batching is saturated; a low number under high load points at the ingestor being starved rather than storage being slow.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 49 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 1, + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "sum(rate(sc_error_ingestion_message_duration_seconds_count{exported_job=~\"$sc_instance\"}[$__rate_interval])) / sum(rate(sc_error_ingestion_batch_duration_seconds_count{exported_job=~\"$sc_instance\"}[$__rate_interval]))", + "legendFormat": "messages / batch" + } + ] + }, + { + "id": 103, + "type": "row", + "title": "ServiceControl ingestion — latency", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 57 }, + "panels": [] + }, + { + "id": 11, + "title": "ServiceControl Ingestion Duration (s)", + "description": "ServiceControl OTel histogram sc.error.ingestion.message_duration_seconds for failed messages — p50, p95 and p99 end-to-end duration per ingested error message. Spikes here explain why 'ingested' lags 'raised'. Buckets top out at 5s, so a p99 pinned at 5 means 'at least 5s'.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 58 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le, exported_job) (rate(sc_error_ingestion_message_duration_seconds_bucket{exported_job=~\"$sc_instance\",message_category=\"failed-message\"}[$__rate_interval])))", + "legendFormat": "Message P50 - {{exported_job}}" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, exported_job) (rate(sc_error_ingestion_message_duration_seconds_bucket{exported_job=~\"$sc_instance\",message_category=\"failed-message\"}[$__rate_interval])))", + "legendFormat": "Message P95 - {{exported_job}}" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, exported_job) (rate(sc_error_ingestion_message_duration_seconds_bucket{exported_job=~\"$sc_instance\",message_category=\"failed-message\"}[$__rate_interval])))", + "legendFormat": "Message P99 - {{exported_job}}" + } + ] + }, + { + "id": 28, + "title": "Ingestion Duration Distribution (heatmap)", + "description": "The whole message_duration_seconds histogram over time rather than a single quantile. Bucket boundaries are 10ms, 50ms, 100ms, 500ms, 1s and 5s. A band drifting upward under load is the shape of the instance running out of headroom.", + "type": "heatmap", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 58 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "options": { + "calculate": false, + "cellGap": 1, + "color": { "mode": "scheme", "scheme": "Spectral", "steps": 64, "reverse": false }, + "yAxis": { "unit": "s" }, + "legend": { "show": true }, + "tooltip": { "show": true, "yHistogram": true } + }, + "targets": [ + { + "expr": "sum by (le) (increase(sc_error_ingestion_message_duration_seconds_bucket{exported_job=~\"$sc_instance\",message_category=\"failed-message\"}[$__rate_interval]))", + "format": "heatmap", + "legendFormat": "{{le}}" + } + ] + }, + { + "id": 29, + "title": "Storage Write Duration (s)", + "description": "sc.error.ingestion.storage_duration_seconds — the batch's storage write on its own, with announcing and forwarding excluded. Compare against batch duration below to see whether the persister or the rest of the pipeline is the constraint.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 66 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(sc_error_ingestion_storage_duration_seconds_bucket{exported_job=~\"$sc_instance\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(sc_error_ingestion_storage_duration_seconds_bucket{exported_job=~\"$sc_instance\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(sc_error_ingestion_storage_duration_seconds_bucket{exported_job=~\"$sc_instance\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ] + }, + { + "id": 30, + "title": "Batch Duration (s)", + "description": "sc.error.ingestion.batch_duration_seconds — the whole batch, storage write plus announcing and forwarding. Batch duration multiplied by concurrent ingestion is what caps the achievable ingestion rate.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 66 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(sc_error_ingestion_batch_duration_seconds_bucket{exported_job=~\"$sc_instance\"}[$__rate_interval])))", + "legendFormat": "p50" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(sc_error_ingestion_batch_duration_seconds_bucket{exported_job=~\"$sc_instance\"}[$__rate_interval])))", + "legendFormat": "p95" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(sc_error_ingestion_batch_duration_seconds_bucket{exported_job=~\"$sc_instance\"}[$__rate_interval])))", + "legendFormat": "p99" + } + ] + }, + { + "id": 31, + "title": "Mean Duration — message / batch / storage write (s)", + "description": "Histogram sum divided by histogram count, so these are true means rather than quantiles and are not clipped by the 5s top bucket. Mean message duration times the ingestion rate gives the concurrency the instance is sustaining.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 74 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "sum(rate(sc_error_ingestion_message_duration_seconds_sum{exported_job=~\"$sc_instance\"}[$__rate_interval])) / sum(rate(sc_error_ingestion_message_duration_seconds_count{exported_job=~\"$sc_instance\"}[$__rate_interval]))", + "legendFormat": "message" + }, + { + "expr": "sum(rate(sc_error_ingestion_batch_duration_seconds_sum{exported_job=~\"$sc_instance\"}[$__rate_interval])) / sum(rate(sc_error_ingestion_batch_duration_seconds_count{exported_job=~\"$sc_instance\"}[$__rate_interval]))", + "legendFormat": "batch" + }, + { + "expr": "sum(rate(sc_error_ingestion_storage_duration_seconds_sum{exported_job=~\"$sc_instance\"}[$__rate_interval])) / sum(rate(sc_error_ingestion_storage_duration_seconds_count{exported_job=~\"$sc_instance\"}[$__rate_interval]))", + "legendFormat": "storage write" + } + ] + }, + { + "id": 32, + "title": "Storage Share of Batch Time", + "description": "Total time spent in storage writes divided by total time spent in batches. Near 100% means the persister owns the batch and tuning belongs there; a low share means announcing and forwarding are where the batch time goes.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 74 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 20 } + } + }, + "targets": [ + { + "expr": "sum(rate(sc_error_ingestion_storage_duration_seconds_sum{exported_job=~\"$sc_instance\"}[$__rate_interval])) / sum(rate(sc_error_ingestion_batch_duration_seconds_sum{exported_job=~\"$sc_instance\"}[$__rate_interval]))", + "legendFormat": "storage / batch" + } + ] + }, + { + "id": 104, + "type": "row", + "title": "ServiceControl ingestion — failures & batch health", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 82 }, + "panels": [] + }, + { + "id": 12, + "title": "ServiceControl Ingestion Outcome / sec (by result)", + "description": "ServiceControl OTel histogram sc.error.ingestion.message_duration_seconds count, broken down by ingestion result (success / failed / skipped). 'success' is what counts as ingested; 'failed'/'skipped' explain the gap between raised and ingested.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 83 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "targets": [ + { + "expr": "sum by (result) (rate(sc_error_ingestion_message_duration_seconds_count{exported_job=~\"$sc_instance\",message_category=\"failed-message\"}[$__rate_interval]))", + "legendFormat": "{{result}}" + } + ] + }, + { + "id": 33, + "title": "Ingestion Failures / sec (by disposition)", + "description": "sc.error.ingestion.failures_total — messages the ingestor could not handle, split by what it did with them: 'retry' means the transport will redeliver, 'stored-poison' means it gave up and wrote a failed import. Rising stored-poison is permanent data loss from the load run's point of view.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 83 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10 } + } + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "expr": "sum by (result, message_category) (rate(sc_error_ingestion_failures_total{exported_job=~\"$sc_instance\"}[$__rate_interval]))", + "legendFormat": "{{result}} / {{message_category}}" + } + ] + }, + { + "id": 34, + "title": "Batch Outcome Share", + "description": "Share of batches ending full, partial or failed. A high 'full' share is a healthy saturated pipeline; any sustained 'failed' share is what drives the consecutive-batch-failure circuit breaker.", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 83 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 30, + "stacking": { "mode": "normal", "group": "A" } + } + } + }, + "targets": [ + { + "expr": "sum by (result) (rate(sc_error_ingestion_batch_duration_seconds_count{exported_job=~\"$sc_instance\"}[$__rate_interval])) / on() group_left() sum(rate(sc_error_ingestion_batch_duration_seconds_count{exported_job=~\"$sc_instance\"}[$__rate_interval]))", + "legendFormat": "{{result}}" + } + ] + } + ] +} diff --git a/tools/testing-tool/TestingTool.AppHost/obs/grafana/provisioning/dashboards/dashboards.yml b/tools/testing-tool/TestingTool.AppHost/obs/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000000..5fb03c4745 --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/obs/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,13 @@ +# Grafana dashboard provisioning — loads dashboards from the mounted dashboards directory. + +apiVersion: 1 + +providers: + - name: 'testing-tool' + orgId: 1 + folder: 'Testing Tool' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.AppHost/obs/grafana/provisioning/datasources/datasources.yml b/tools/testing-tool/TestingTool.AppHost/obs/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000000..a7b0030668 --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/obs/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,24 @@ +# Grafana data source provisioning for the testing-tool observability stack. +# Auto-configures Prometheus (metrics) and Jaeger (traces) as data sources on startup. + +apiVersion: 1 + +datasources: + # The dashboard JSON references its panels' datasource by uid "prometheus", so the provisioned + # data source MUST use that exact uid — otherwise Grafana can't resolve the panels and every + # panel shows "no data" / "datasource not found", even when Prometheus has the metrics. + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + + # Jaeger is referenced by trace panels/links via uid "jaeger". + - name: Jaeger + uid: jaeger + type: jaeger + access: proxy + url: http://jaeger:16686 + editable: true \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.AppHost/obs/otel-collector-config.yaml b/tools/testing-tool/TestingTool.AppHost/obs/otel-collector-config.yaml new file mode 100644 index 0000000000..2d23f8724d --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/obs/otel-collector-config.yaml @@ -0,0 +1,46 @@ +# OpenTelemetry Collector configuration for the testing-tool observability stack. +# +# Receives OTLP (traces + metrics) from the testing tool and fans out: +# traces → Jaeger (all-in-one OTLP receiver) +# metrics → Prometheus-compatible exporter scraped by Prometheus +# logs → dropped (console logs remain visible in the Aspire dashboard console view) +# +# Container DNS names match the Aspire resource names ("jaeger") so the collector can +# reach Jaeger on the shared container network. + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +exporters: + # Traces → Jaeger all-in-one OTLP gRPC receiver. + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + + # Metrics → Prometheus scraping endpoint (Prometheus pulls from :8889). + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp/jaeger] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus] + # No logs pipeline: logs are accepted by the OTLP receiver but dropped (no exporter). + # Console logs remain visible in the Aspire dashboard's console view. \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.AppHost/obs/prometheus.yml b/tools/testing-tool/TestingTool.AppHost/obs/prometheus.yml new file mode 100644 index 0000000000..f54c3c483b --- /dev/null +++ b/tools/testing-tool/TestingTool.AppHost/obs/prometheus.yml @@ -0,0 +1,15 @@ +# Prometheus scrape configuration for the testing-tool observability stack. +# +# Scrapes the OpenTelemetry Collector's Prometheus exporter, which exposes the testing +# tool's OTel metrics (received via OTLP) in Prometheus format at :8889. + +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: 'otel-collector' + static_configs: + - targets: ['otel-collector:8889'] + labels: + source: 'otel-collector' \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Contracts/BypassStatus.cs b/tools/testing-tool/TestingTool.Contracts/BypassStatus.cs new file mode 100644 index 0000000000..7e0df24aa9 --- /dev/null +++ b/tools/testing-tool/TestingTool.Contracts/BypassStatus.cs @@ -0,0 +1,23 @@ +namespace TestingTool.Contracts; + +/// +/// Live status of the direct error-queue bypass writer, returned by +/// GET /api/bypass/status and included in the GET /api/status snapshot. +/// +public sealed class BypassStatus +{ + /// Whether the bypass writer is currently emitting failed-message envelopes. + public bool Running { get; init; } + + /// The scenario whose failure shape is being simulated, or null if idle. + public string? Scenario { get; init; } + + /// Current target emission rate in messages/second (0 if idle). + public double Rate { get; init; } + + /// Total failed-message envelopes written directly to the error queue since process start. + public long ErrorsWritten { get; init; } + + /// When the current bypass run started (UTC ISO 8601, null if idle). + public string? StartedAt { get; init; } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Contracts/JobInfo.cs b/tools/testing-tool/TestingTool.Contracts/JobInfo.cs new file mode 100644 index 0000000000..61dc0d80f7 --- /dev/null +++ b/tools/testing-tool/TestingTool.Contracts/JobInfo.cs @@ -0,0 +1,35 @@ +namespace TestingTool.Contracts; + +/// +/// Describes a recoverability/search job that can be started and stopped from the web UI. +/// Returned by GET /api/jobs and rendered in the "Recoverability Jobs" section. +/// +public sealed class JobInfo +{ + /// The stable, url-safe job name used in API paths. + public required string Name { get; init; } + + /// A short human-readable description of what the job does each cycle. + public required string Description { get; init; } + + /// Human-readable category for grouping in the UI (e.g. "Recoverability", "Search"). + public required string Category { get; init; } + + /// Whether the job is currently running its periodic cycle. + public bool Running { get; init; } + + /// Configured cycle interval in seconds (0 if idle). + public int IntervalSeconds { get; init; } + + /// Default cycle interval in seconds. + public int DefaultIntervalSeconds { get; init; } + + /// Number of cycles completed since the job was started. + public long Cycles { get; init; } + + /// Number of items processed (groups retried/archived, searches run) since start. + public long ItemsProcessed { get; init; } + + /// When the current run started (UTC ISO 8601, null if idle). + public string? StartedAt { get; init; } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Contracts/ScenarioInfo.cs b/tools/testing-tool/TestingTool.Contracts/ScenarioInfo.cs new file mode 100644 index 0000000000..0133829b1c --- /dev/null +++ b/tools/testing-tool/TestingTool.Contracts/ScenarioInfo.cs @@ -0,0 +1,32 @@ +namespace TestingTool.Contracts; + +/// +/// Describes a scenario that can generate error load against ServiceControl. +/// Returned by GET /api/scenarios and rendered in the web UI. +/// +public sealed class ScenarioInfo +{ + /// The stable, url-safe scenario name used in API paths. + public required string Name { get; init; } + + /// A short human-readable description of the failure shape this scenario produces. + public required string Description { get; init; } + + /// Human-readable category for grouping in the UI (e.g. "Outage", "Poison", "Noise"). + public required string Category { get; init; } + + /// Whether the scenario is currently emitting load. + public bool Running { get; init; } + + /// Current target rate in messages/second (0 if idle). + public double CurrentRate { get; init; } + + /// Errors emitted by this scenario since process start. + public long ErrorsSent { get; init; } + + /// Default recommended rate in messages/second. + public double DefaultRate { get; init; } + + /// Optional cooldown duration between failure bursts (ISO 8601 duration, null = continuous). + public string? Cooldown { get; init; } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Contracts/StartBypassRequest.cs b/tools/testing-tool/TestingTool.Contracts/StartBypassRequest.cs new file mode 100644 index 0000000000..fc70cb84c7 --- /dev/null +++ b/tools/testing-tool/TestingTool.Contracts/StartBypassRequest.cs @@ -0,0 +1,18 @@ +namespace TestingTool.Contracts; + +/// +/// Request body for POST /api/bypass/start. +/// Starts the direct error-queue bypass writer, which writes failed-message envelopes +/// directly to the ServiceControl error queue without going through a handler. +/// +public sealed class StartBypassRequest +{ + /// The scenario whose failure shape to simulate (determines exception type, message, and grouping). + public string? Scenario { get; init; } + + /// Target emission rate in messages/second. Defaults to 100. + public double? Rate { get; init; } + + /// Optional auto-stop duration in seconds. Null/0 = run until explicitly stopped. + public double? DurationSeconds { get; init; } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Contracts/StartJobRequest.cs b/tools/testing-tool/TestingTool.Contracts/StartJobRequest.cs new file mode 100644 index 0000000000..b086de21e3 --- /dev/null +++ b/tools/testing-tool/TestingTool.Contracts/StartJobRequest.cs @@ -0,0 +1,11 @@ +namespace TestingTool.Contracts; + +/// +/// Request body for POST /api/jobs/{name}/start. +/// All fields are optional; defaults are taken from the job definition. +/// +public sealed class StartJobRequest +{ + /// Cycle interval in seconds. Defaults to the job's default interval. + public double? IntervalSeconds { get; init; } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Contracts/StartScenarioRequest.cs b/tools/testing-tool/TestingTool.Contracts/StartScenarioRequest.cs new file mode 100644 index 0000000000..77dd3b91a5 --- /dev/null +++ b/tools/testing-tool/TestingTool.Contracts/StartScenarioRequest.cs @@ -0,0 +1,14 @@ +namespace TestingTool.Contracts; + +/// +/// Request body for POST /api/scenarios/{name}/start. +/// All fields are optional; defaults are taken from the scenario definition. +/// +public sealed class StartScenarioRequest +{ + /// Target emission rate in messages/second. Defaults to the scenario's . + public double? Rate { get; init; } + + /// Optional auto-stop duration in seconds. Null/0 = run until explicitly stopped. + public double? DurationSeconds { get; init; } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Contracts/TestingTool.Contracts.csproj b/tools/testing-tool/TestingTool.Contracts/TestingTool.Contracts.csproj new file mode 100644 index 0000000000..7d5db420dc --- /dev/null +++ b/tools/testing-tool/TestingTool.Contracts/TestingTool.Contracts.csproj @@ -0,0 +1,5 @@ + + + + + diff --git a/tools/testing-tool/TestingTool.Contracts/TestingToolStatus.cs b/tools/testing-tool/TestingTool.Contracts/TestingToolStatus.cs new file mode 100644 index 0000000000..4e59cc7f8d --- /dev/null +++ b/tools/testing-tool/TestingTool.Contracts/TestingToolStatus.cs @@ -0,0 +1,44 @@ +namespace TestingTool.Contracts; + +/// +/// Live snapshot of testing-tool counters, returned by GET /api/status and used as the +/// container liveness/readiness probe target. +/// +public sealed class TestingToolStatus +{ + /// Whether the tool is ready to accept scenario control requests. + public bool Ready { get; init; } + + /// Total errors emitted since process start (handler + bypass paths). + public long ErrorsSent { get; init; } + + /// Total error messages replayed since process start. + public long ErrorsReplayed { get; init; } + + /// Total error messages archived since process start. + public long ErrorsArchived { get; init; } + + /// Total ServiceControl searches executed since process start. + public long SearchesExecuted { get; init; } + + /// Total errors emitted via the direct error-queue bypass writer since process start. + public long BypassErrorsWritten { get; init; } + + /// The shard id this replica owns, used for disjoint scenario slices when scaled out. + public string? ShardId { get; init; } + + /// Number of scenarios currently running. + public int ActiveScenarios { get; init; } + + /// Number of recoverability/search jobs currently running. + public int ActiveJobs { get; init; } + + /// Aggregate current emission rate across all running scenarios (msgs/sec). + public double CurrentRate { get; init; } + + /// ServiceControl API URL the tool is targeting. + public string? ServiceControlUrl { get; init; } + + /// Uptime since process start (formatted string). + public string? Uptime { get; init; } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Scenarios/DeserializationScenario.cs b/tools/testing-tool/TestingTool.Scenarios/DeserializationScenario.cs new file mode 100644 index 0000000000..c89116d9e1 --- /dev/null +++ b/tools/testing-tool/TestingTool.Scenarios/DeserializationScenario.cs @@ -0,0 +1,24 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Deserialization failure scenario: messages with malformed payloads fail during deserialization. +/// Grouped by message type, simulating a deployment that shipped an incompatible schema version. +/// All messages fail (100%) since deserialization happens before the handler runs. +/// +public sealed class DeserializationScenario(string shardId) : ScenarioBase(shardId) +{ + public override string Name => "deserialization-failure"; + public override string Description => "Messages fail during deserialization due to incompatible schema. Grouped by message type — simulates a bad deployment."; + public override string Category => "Deserialization"; + public override double DefaultRate => 20; + + public override bool ShouldFail(string messageId) => true; + + public override Exception CreateException() => + CreateException( + "NServiceBus.MessageDeserializationException", + "Unable to deserialize message: unexpected token at position 0. Expected a valid message envelope.", + "deser:SampleCommand:v2-incompatible"); +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Scenarios/IScenario.cs b/tools/testing-tool/TestingTool.Scenarios/IScenario.cs new file mode 100644 index 0000000000..33b869d40c --- /dev/null +++ b/tools/testing-tool/TestingTool.Scenarios/IScenario.cs @@ -0,0 +1,35 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Defines a real-world error scenario that produces grouped failures against ServiceControl. +/// Each implementation models a distinct failure shape (outage, poison message, timeout spike, etc.) +/// so that ServiceControl naturally groups the resulting errors. +/// +public interface IScenario +{ + /// The stable scenario name, matching . + string Name { get; } + + /// Human-readable description of the failure shape. + string Description { get; } + + /// UI grouping category (e.g. "Outage", "Poison", "Noise"). + string Category { get; } + + /// Default recommended emission rate in messages/second. + double DefaultRate { get; } + + /// Activity source used to emit OpenTelemetry traces for this scenario's work. + ActivitySource ActivitySource { get; } + + /// Determines whether a given message should fail. Must be deterministic per shard. + bool ShouldFail(string messageId); + + /// Creates the grouped, typed exception emitted when returns true. + Exception CreateException(); + + /// Burst shape: optional cooldown between failure bursts. + TimeSpan? Cooldown { get; } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Scenarios/LoadMessage.cs b/tools/testing-tool/TestingTool.Scenarios/LoadMessage.cs new file mode 100644 index 0000000000..e72db79cea --- /dev/null +++ b/tools/testing-tool/TestingTool.Scenarios/LoadMessage.cs @@ -0,0 +1,15 @@ +namespace TestingTool.Scenarios; + +/// +/// A load-generation message processed by FailingMessageHandler. The ScenarioName +/// header (set by the sender) determines which scenario's failure logic applies. The payload +/// is intentionally minimal — the testing tool generates volume, not realistic business data. +/// +public class LoadMessage +{ + /// Monotonically increasing sequence number within a generation run. + public long Sequence { get; set; } + + /// Random payload bytes to give messages some size variety. + public byte[]? Payload { get; set; } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Scenarios/PoisonMessageScenario.cs b/tools/testing-tool/TestingTool.Scenarios/PoisonMessageScenario.cs new file mode 100644 index 0000000000..ec92d89a6a --- /dev/null +++ b/tools/testing-tool/TestingTool.Scenarios/PoisonMessageScenario.cs @@ -0,0 +1,27 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Poison message scenario: a deterministic subset of messages (by hash) always fails. +/// These messages will never succeed on retry, creating a persistent error group that +/// exercises ServiceControl's retry-storm handling and message archival flows. +/// +public sealed class PoisonMessageScenario(string shardId) : ScenarioBase(shardId) +{ + public override string Name => "poison-message"; + public override string Description => "Deterministic poison messages that always fail on retry. Exercises retry-storm and archival handling."; + public override string Category => "Poison"; + public override double DefaultRate => 5; + + // ~15% of messages are poison (always fail). + private const double PoisonRatio = 0.15; + + public override bool ShouldFail(string messageId) => Hash(messageId) < PoisonRatio; + + public override Exception CreateException() => + CreateException( + "System.InvalidOperationException", + "The message payload is corrupt and cannot be processed. This message will always fail.", + "poison:invalid-payload"); +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Scenarios/RandomBackgroundNoiseScenario.cs b/tools/testing-tool/TestingTool.Scenarios/RandomBackgroundNoiseScenario.cs new file mode 100644 index 0000000000..5bb6024592 --- /dev/null +++ b/tools/testing-tool/TestingTool.Scenarios/RandomBackgroundNoiseScenario.cs @@ -0,0 +1,36 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Low baseline error rate that is always on. Produces a small, steady stream of random +/// exceptions to simulate real-world background noise. Used to keep ServiceControl's +/// ingestion pipeline warm between explicit scenario runs. +/// +public sealed class RandomBackgroundNoiseScenario(string shardId) : ScenarioBase(shardId) +{ + public override string Name => "background-noise"; + public override string Description => "Always-on low baseline error rate (≈3%). Simulates real-world background noise to keep ingestion warm."; + public override string Category => "Noise"; + public override double DefaultRate => 15; + + private const double NoiseRate = 0.03; + + public override bool ShouldFail(string messageId) => Hash(messageId) < NoiseRate; + + public override Exception CreateException() + { + // Rotate through a few exception types so we get a handful of small groups. + var types = new[] + { + ("System.NullReferenceException", "Object reference not set to an instance of an object.", "noise:nre"), + ("System.IndexOutOfRangeException", "Index was outside the bounds of the array.", "noise:oor"), + ("System.FormatException", "The input string was not in a correct format.", "noise:fmt"), + ("System.InvalidCastException", "Unable to cast object of type 'System.String' to type 'System.Int32'.", "noise:cast"), + }; + + var idx = (int)(Hash(Guid.NewGuid().ToString("N")) * types.Length) % types.Length; + var (type, msg, group) = types[idx]; + return CreateException(type, msg, group); + } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Scenarios/ScenarioBase.cs b/tools/testing-tool/TestingTool.Scenarios/ScenarioBase.cs new file mode 100644 index 0000000000..02eaf7fc9c --- /dev/null +++ b/tools/testing-tool/TestingTool.Scenarios/ScenarioBase.cs @@ -0,0 +1,48 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Base class providing common functionality for scenarios: deterministic hashing for per-shard +/// failure decisions, activity source management, and exception tagging. +/// +public abstract class ScenarioBase : IScenario +{ + private readonly string _shardId; + + protected ScenarioBase(string shardId) + { + _shardId = shardId; + ActivitySource = new ActivitySource($"testing-tool.{Name}"); + } + + public abstract string Name { get; } + public abstract string Description { get; } + public abstract string Category { get; } + public virtual double DefaultRate => 10; + public ActivitySource ActivitySource { get; } + public virtual TimeSpan? Cooldown => null; + + public abstract bool ShouldFail(string messageId); + public abstract Exception CreateException(); + + /// Deterministic hash of a message id + shard id, returning a value in [0, 1). + protected double Hash(string messageId) + { + var combined = $"{_shardId}:{messageId}"; + // Simple FNV-1a hash — no extra NuGet dependency required. + uint hash = 2166136261u; + foreach (var b in System.Text.Encoding.UTF8.GetBytes(combined)) + { + hash ^= b; + hash *= 16777619u; + } + return hash / (double)uint.MaxValue; + } + + /// Creates an exception with correlation tags that ServiceControl uses for grouping. + protected static Exception CreateException(string type, string message, string correlationGroup) + { + return new ScenarioException(type, message, correlationGroup); + } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Scenarios/ScenarioException.cs b/tools/testing-tool/TestingTool.Scenarios/ScenarioException.cs new file mode 100644 index 0000000000..00b80b15fd --- /dev/null +++ b/tools/testing-tool/TestingTool.Scenarios/ScenarioException.cs @@ -0,0 +1,17 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Exception carrying correlation metadata so ServiceControl groups errors by exception type +/// and scenario-defined correlation group rather than by individual message. +/// +public sealed class ScenarioException(string exceptionType, string message, string correlationGroup) + : Exception(message) +{ + public string ExceptionType { get; } = exceptionType; + public string CorrelationGroup { get; } = correlationGroup; + + public override string ToString() => + $"{ExceptionType}: {Message} [group: {CorrelationGroup}]"; +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Scenarios/TestingTool.Scenarios.csproj b/tools/testing-tool/TestingTool.Scenarios/TestingTool.Scenarios.csproj new file mode 100644 index 0000000000..5d8d3109b8 --- /dev/null +++ b/tools/testing-tool/TestingTool.Scenarios/TestingTool.Scenarios.csproj @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/tools/testing-tool/TestingTool.Scenarios/ThirdPartyOutageScenario.cs b/tools/testing-tool/TestingTool.Scenarios/ThirdPartyOutageScenario.cs new file mode 100644 index 0000000000..537a22c418 --- /dev/null +++ b/tools/testing-tool/TestingTool.Scenarios/ThirdPartyOutageScenario.cs @@ -0,0 +1,35 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Simulates a third-party service outage: 100% of messages fail for a burst period, then recover. +/// All failures share the same downstream host header so ServiceControl groups them as one +/// "third-party outage" error group. After the burst, messages succeed (cooldown), then the +/// cycle repeats — modelling a flaky external dependency. +/// +public sealed class ThirdPartyOutageScenario(string shardId) : ScenarioBase(shardId) +{ + public override string Name => "third-party-outage"; + public override string Description => "Simulates a third-party service outage: 100% fail during burst, then recovers. Errors grouped by downstream host."; + public override string Category => "Outage"; + public override double DefaultRate => 50; + public override TimeSpan? Cooldown => TimeSpan.FromSeconds(30); + + // Burst window: fail for 20s, then cooldown 30s, repeat. + private static readonly TimeSpan BurstWindow = TimeSpan.FromSeconds(20); + private static readonly TimeSpan CyclePeriod = BurstWindow + TimeSpan.FromSeconds(30); + + public override bool ShouldFail(string messageId) + { + var now = DateTimeOffset.UtcNow; + var phase = now.Ticks % CyclePeriod.Ticks; + return phase < BurstWindow.Ticks; + } + + public override Exception CreateException() => + CreateException( + "System.Net.Http.HttpRequestException", + "The third-party service at https://api.downstream.example.com did not respond within the timeout period.", + "downstream:api.downstream.example.com"); +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.Scenarios/TimeoutSpikeScenario.cs b/tools/testing-tool/TestingTool.Scenarios/TimeoutSpikeScenario.cs new file mode 100644 index 0000000000..a38e062db0 --- /dev/null +++ b/tools/testing-tool/TestingTool.Scenarios/TimeoutSpikeScenario.cs @@ -0,0 +1,41 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Intermittent timeout spikes: a configurable percentage of messages fail with +/// , correlated by batch id so ServiceControl groups them +/// into timeout-related error groups. The failure rate oscillates to simulate periodic spikes. +/// +public sealed class TimeoutSpikeScenario(string shardId) : ScenarioBase(shardId) +{ + public override string Name => "timeout-spike"; + public override string Description => "Intermittent timeout exceptions with correlated batch ids. Failure rate oscillates to simulate spikes."; + public override string Category => "Timeout"; + public override double DefaultRate => 30; + + // Spike every ~60s, lasting ~15s at elevated rate. + private static readonly TimeSpan SpikeWindow = TimeSpan.FromSeconds(15); + private static readonly TimeSpan SpikeCycle = TimeSpan.FromSeconds(60); + + public override bool ShouldFail(string messageId) + { + var now = DateTimeOffset.UtcNow; + var phase = now.Ticks % SpikeCycle.Ticks; + var inSpike = phase < SpikeWindow.Ticks; + + // Base 10% fail rate, spikes to ~70% during spike window. + var threshold = inSpike ? 0.70 : 0.10; + return Hash(messageId) < threshold; + } + + public override Exception CreateException() + { + // Correlate by 5-minute bucket so timeouts cluster into time-based groups. + var bucket = DateTimeOffset.UtcNow.ToUnixTimeSeconds() / 300; + return CreateException( + "System.TimeoutException", + "The operation has timed out waiting for a response from the downstream service.", + $"timeout-batch:{bucket}"); + } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.SmokeTests/SmokeTest.cs b/tools/testing-tool/TestingTool.SmokeTests/SmokeTest.cs new file mode 100644 index 0000000000..1908fa88d1 --- /dev/null +++ b/tools/testing-tool/TestingTool.SmokeTests/SmokeTest.cs @@ -0,0 +1,124 @@ +using System.Net; +using System.Net.Http.Json; +using Xunit; + +namespace TestingTool.SmokeTests; + +/// +/// Smoke test for the ServiceControl testing tool. Requires a running ServiceControl instance +/// and testing tool (e.g. via docker-compose or the Aspire AppHost). The test: +/// 1. Verifies the testing tool is accessible +/// 2. Triggers the third-party-outage scenario for 30 seconds +/// 3. Verifies error groups appear in ServiceControl +/// 4. Triggers a replay and verifies it is accepted +/// +/// Configure via environment variables: +/// TESTING_TOOL_URL (default: http://localhost:8080) +/// SERVICECONTROL_URL (default: http://localhost:33333) +/// +public class SmokeTest +{ + private static readonly string TestingToolUrl = + Environment.GetEnvironmentVariable("TESTING_TOOL_URL") ?? "http://localhost:8080"; + private static readonly string ServiceControlUrl = + Environment.GetEnvironmentVariable("SERVICECONTROL_URL") ?? "http://localhost:33333"; + + private static readonly HttpClient ToolClient = new() { BaseAddress = new Uri(TestingToolUrl), Timeout = TimeSpan.FromSeconds(30) }; + private static readonly HttpClient ScClient = new() { BaseAddress = new Uri(ServiceControlUrl), Timeout = TimeSpan.FromSeconds(30) }; + + [Fact] + public async Task TestingTool_IsAccessible_ReturnsStatus() + { + var response = await ToolClient.GetAsync("/api/status"); + response.EnsureSuccessStatusCode(); + + var status = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(status); + Assert.True(status.Ready); + } + + [Fact] + public async Task ThirdPartyOutage_GeneratesErrors_VisibleInServiceControl() + { + // 1. Stop any existing scenario first (clean slate) + await ToolClient.PostAsync("/api/scenarios/stop-all", null); + + // 2. Start the third-party-outage scenario for 30 seconds at 50 msg/s + var startResponse = await ToolClient.PostAsJsonAsync( + "/api/scenarios/third-party-outage/start", + new { rate = 50, durationSeconds = 30 }); + startResponse.EnsureSuccessStatusCode(); + + // 3. Wait for the scenario to generate errors (10s in, then check SC) + await Task.Delay(TimeSpan.FromSeconds(10)); + + // 4. Check ServiceControl for error groups + var groupsResponse = await ScClient.GetAsync("/api/recoverability/groups"); + groupsResponse.EnsureSuccessStatusCode(); + + var groups = await groupsResponse.Content.ReadFromJsonAsync>(); + Assert.NotNull(groups); + Assert.NotEmpty(groups!); + + // 5. Wait for the scenario to finish (remaining ~20s + buffer) + await Task.Delay(TimeSpan.FromSeconds(25)); + + // 6. Verify the testing tool counted errors + var statusResponse = await ToolClient.GetAsync("/api/status"); + statusResponse.EnsureSuccessStatusCode(); + var status = await statusResponse.Content.ReadFromJsonAsync(); + Assert.NotNull(status); + Assert.True(status!.ErrorsSent > 0, $"Expected errors to be sent, got {status.ErrorsSent}"); + } + + [Fact] + public async Task Replay_ErrorGroup_IsAcceptedByServiceControl() + { + // 1. Ensure there are error groups to replay + var groupsResponse = await ScClient.GetAsync("/api/recoverability/groups"); + groupsResponse.EnsureSuccessStatusCode(); + var groups = await groupsResponse.Content.ReadFromJsonAsync>(); + + if (groups is null || groups.Count == 0) + { + // Generate some errors first + await ToolClient.PostAsJsonAsync( + "/api/scenarios/third-party-outage/start", + new { rate = 50, durationSeconds = 10 }); + await Task.Delay(TimeSpan.FromSeconds(15)); + + groupsResponse = await ScClient.GetAsync("/api/recoverability/groups"); + groupsResponse.EnsureSuccessStatusCode(); + groups = await groupsResponse.Content.ReadFromJsonAsync>(); + } + + Assert.NotNull(groups); + Assert.NotEmpty(groups!); + + // 2. Trigger replay on the first group + var firstGroup = groups![0]; + var replayResponse = await ScClient.PostAsJsonAsync( + $"/api/recoverability/groups/{firstGroup.Id}/errors/retry", new { }); + + // ServiceControl should accept the replay request (202 Accepted or 200 OK) + Assert.True(replayResponse.IsSuccessStatusCode, + $"Replay request failed: {replayResponse.StatusCode} {await replayResponse.Content.ReadAsStringAsync()}"); + } + + // --- Response DTOs (minimal, matching the APIs) --- + + private sealed record StatusResponse + { + public bool Ready { get; init; } + public long ErrorsSent { get; init; } + public long ErrorsReplayed { get; init; } + public int ActiveScenarios { get; init; } + } + + private sealed record ErrorGroupResponse + { + public string Id { get; init; } = ""; + public string Title { get; init; } = ""; + public int Count { get; init; } + } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.SmokeTests/TestingTool.SmokeTests.csproj b/tools/testing-tool/TestingTool.SmokeTests/TestingTool.SmokeTests.csproj new file mode 100644 index 0000000000..5caf3219dd --- /dev/null +++ b/tools/testing-tool/TestingTool.SmokeTests/TestingTool.SmokeTests.csproj @@ -0,0 +1,24 @@ + + + + + + net10.0 + false + true + enable + enable + + + + + + + + + \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.slnx b/tools/testing-tool/TestingTool.slnx new file mode 100644 index 0000000000..63133cab9a --- /dev/null +++ b/tools/testing-tool/TestingTool.slnx @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/DirectErrorQueueWriter.cs b/tools/testing-tool/TestingTool/DirectErrorQueueWriter.cs new file mode 100644 index 0000000000..800eaae402 --- /dev/null +++ b/tools/testing-tool/TestingTool/DirectErrorQueueWriter.cs @@ -0,0 +1,195 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NServiceBus; +using TestingTool.Contracts; +using TestingTool.Scenarios; + +namespace TestingTool; + +/// +/// Bypass path: constructs failed-message envelopes with NServiceBus failure headers and writes +/// them directly to the ServiceControl error queue via the NServiceBus transport, without going +/// through the handler. This enables high-throughput error load generation that bypasses the +/// initial message creation and handler processing (requirement: "simulate high error loads, +/// bypass actually creating the initial messages"). +/// +/// Each emitted message carries the standard NServiceBus failure headers +/// (NServiceBus.ExceptionInfo.*, NServiceBus.FailedQ) so ServiceControl ingests +/// it as a genuine failed message and groups it by the scenario's exception type and correlation +/// group — exactly like handler-generated failures. +/// +public sealed class DirectErrorQueueWriter +{ + private readonly IMessageSession _session; + private readonly IScenarioRegistry _registry; + private readonly TestingToolMetrics _metrics; + private readonly Meter _meter; + private readonly TestingToolOptions _options; + private readonly ILogger _logger; + + private readonly ActivitySource _activitySource = new(TelemetrySetup.Sources.Bypass); + private readonly Counter _bypassCounter; + + private CancellationTokenSource? _cts; + private Task? _loop; + private long _errorsWritten; + private double _currentRate; + private string? _activeScenario; + private DateTimeOffset _startedAt; + + public bool IsRunning => _cts is not null; + public long ErrorsWritten => Interlocked.Read(ref _errorsWritten); + public double CurrentRate => _currentRate; + public string? ActiveScenario => _activeScenario; + + public DirectErrorQueueWriter( + IMessageSession session, + IScenarioRegistry registry, + TestingToolMetrics metrics, + Meter meter, + IOptions options, + ILogger logger) + { + _session = session; + _registry = registry; + _metrics = metrics; + _meter = meter; + _options = options.Value; + _logger = logger; + _bypassCounter = meter.CreateCounter("bypass_errors_written_total"); + } + + /// Starts writing failed-message envelopes directly to the error queue. + public bool TryStart(string scenarioName, double rate, TimeSpan? duration, out string? error) + { + var scenario = _registry.Get(scenarioName); + if (scenario is null) + { + error = $"Unknown scenario '{scenarioName}'"; + return false; + } + + if (IsRunning) + { + error = "Bypass writer is already running — stop it first"; + return false; + } + + if (rate <= 0) + { + error = "Rate must be greater than 0"; + return false; + } + + _activeScenario = scenarioName; + _currentRate = rate; + _startedAt = DateTimeOffset.UtcNow; + + var cts = duration is { } d + ? new CancellationTokenSource(d) + : new CancellationTokenSource(); + _cts = cts; + + _loop = Task.Run(() => WriteLoop(scenario, rate, cts.Token)); + + _logger.LogInformation("Started bypass writer for scenario {Scenario} at {Rate:F1} msg/s{Duration}", + scenarioName, rate, duration is null ? "" : $" for {duration.Value}"); + + error = null; + return true; + } + + /// Stops the bypass writer. + public void Stop() + { + if (_cts is null) return; + + _cts.Cancel(); + _cts.Dispose(); + _cts = null; + + _logger.LogInformation("Stopped bypass writer after {Errors} errors written", ErrorsWritten); + + _activeScenario = null; + _currentRate = 0; + } + + /// Returns the current bypass writer status for API consumers. + public BypassStatus GetStatus() => new() + { + Running = IsRunning, + Scenario = _activeScenario, + Rate = _currentRate, + ErrorsWritten = ErrorsWritten, + StartedAt = IsRunning ? _startedAt.ToString("O") : null + }; + + /// + /// The load generation loop: sends directly to the error queue + /// with failure headers at the target rate until cancelled. + /// + private async Task WriteLoop(IScenario scenario, double rate, CancellationToken ct) + { + var interval = TimeSpan.FromSeconds(1.0 / rate); + using var timer = new PeriodicTimer(interval); + long sequence = 0; + + // Pre-compute the failure metadata from the scenario so all messages in this run + // share the same exception type, message, and correlation group — ServiceControl + // will group them as one error group. + var exception = scenario.CreateException(); + var scenarioEx = exception as ScenarioException; + var exceptionType = scenarioEx?.ExceptionType ?? exception.GetType().FullName!; + var exceptionMessage = exception.Message; + var correlationGroup = scenarioEx?.CorrelationGroup ?? ""; + + try + { + while (await timer.WaitForNextTickAsync(ct)) + { + var seq = Interlocked.Increment(ref sequence); + var payload = new byte[Random.Shared.Next(64, 512)]; + Random.Shared.NextBytes(payload); + + var message = new LoadMessage { Sequence = seq, Payload = payload }; + + var sendOptions = new SendOptions(); + // Route directly to the ServiceControl error queue — bypasses the handler entirely. + sendOptions.SetDestination(_options.ErrorQueueName); + + // Set failure headers so ServiceControl recognises the message as a failed message + // and groups it by exception type + correlation group, exactly like handler failures. + sendOptions.SetHeader("TestingTool.Scenario", scenario.Name); + sendOptions.SetHeader("TestingTool.Bypass", "true"); + sendOptions.SetHeader("TestingTool.CorrelationGroup", correlationGroup); + sendOptions.SetHeader("NServiceBus.ExceptionInfo.ExceptionType", exceptionType); + sendOptions.SetHeader("NServiceBus.ExceptionInfo.Message", exceptionMessage); + sendOptions.SetHeader("NServiceBus.ExceptionInfo.Source", "TestingTool.Load"); + sendOptions.SetHeader("NServiceBus.FailedQ", "TestingTool.Load"); + + using var activity = _activitySource.StartActivity("bypass-write"); + activity?.SetTag("scenario", scenario.Name); + activity?.SetTag("sequence", seq); + activity?.SetTag("exception.type", exceptionType); + activity?.SetTag("exception.group", correlationGroup); + + try + { + await _session.Send(message, sendOptions, ct); + + Interlocked.Increment(ref _errorsWritten); + _metrics.AddErrorsSent(1); + _metrics.AddBypassErrorsWritten(1); + _bypassCounter.Add(1, new KeyValuePair("scenario", scenario.Name)); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogDebug(ex, "Bypass send failed for scenario {Scenario} seq {Seq}", scenario.Name, seq); + } + } + } + catch (OperationCanceledException) { } + } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/FailingMessageHandler.cs b/tools/testing-tool/TestingTool/FailingMessageHandler.cs new file mode 100644 index 0000000000..333fe68bd1 --- /dev/null +++ b/tools/testing-tool/TestingTool/FailingMessageHandler.cs @@ -0,0 +1,44 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using NServiceBus; +using TestingTool.Scenarios; + +namespace TestingTool; + +/// +/// Handles by delegating to the active scenario. When the scenario's +/// ShouldFail returns true, the handler throws — NServiceBus routes the failed message to +/// the configured error queue (ServiceControl). The ScenarioName header selects the scenario. +/// +public sealed class FailingMessageHandler(IScenarioRegistry registry, ILogger logger) + : IHandleMessages +{ + public Task Handle(LoadMessage message, IMessageHandlerContext context) + { + var scenarioName = context.MessageHeaders.GetValueOrDefault("TestingTool.Scenario") ?? "unknown"; + var scenario = registry.Get(scenarioName); + + if (scenario is null) + { + logger.LogDebug("No scenario '{Scenario}' registered — message {Seq} succeeds", scenarioName, message.Sequence); + return Task.CompletedTask; + } + + using var activity = scenario.ActivitySource.StartActivity("handle-load"); + activity?.SetTag("scenario", scenario.Name); + activity?.SetTag("message.sequence", message.Sequence); + activity?.SetTag("message.id", context.MessageId); + + if (scenario.ShouldFail(context.MessageId)) + { + var ex = scenario.CreateException(); + activity?.SetStatus(ActivityStatusCode.Error); + activity?.SetTag("exception.type", (ex as ScenarioException)?.ExceptionType ?? ex.GetType().Name); + activity?.SetTag("exception.group", (ex as ScenarioException)?.CorrelationGroup); + throw ex; + } + + activity?.SetTag("result", "success"); + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/IScenarioRegistry.cs b/tools/testing-tool/TestingTool/IScenarioRegistry.cs new file mode 100644 index 0000000000..ba3c52bb75 --- /dev/null +++ b/tools/testing-tool/TestingTool/IScenarioRegistry.cs @@ -0,0 +1,12 @@ +using TestingTool.Scenarios; + +namespace TestingTool; + +/// +/// Registry of all available scenarios, keyed by name. Registered at startup via DI. +/// +public interface IScenarioRegistry +{ + IScenario? Get(string name); + IReadOnlyList All { get; } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/Jobs/ArchiveJob.cs b/tools/testing-tool/TestingTool/Jobs/ArchiveJob.cs new file mode 100644 index 0000000000..67d98fe99a --- /dev/null +++ b/tools/testing-tool/TestingTool/Jobs/ArchiveJob.cs @@ -0,0 +1,54 @@ +using System.Diagnostics.Metrics; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace TestingTool.Jobs; + +/// +/// Recoverability job that fetches error groups from ServiceControl and archives each group on +/// every cycle. This exercises ServiceControl's archive pipeline and retention behaviour, and +/// complements the . Controllable from the web UI rather than running as a +/// hidden background timer. +/// +public sealed class ArchiveJob( + ServiceControlClient sc, + TestingToolMetrics metrics, + IOptions options, + Meter meter, + ILogger logger) : JobBase("testing-tool.archive") +{ + private readonly Counter _archiveCounter = meter.CreateCounter("errors_archived_total"); + + public override string Name => "archive"; + public override string Description => "Archive all error groups in ServiceControl on each cycle (moves failures to the archive)."; + public override string Category => "Recoverability"; + public override TimeSpan DefaultInterval => options.Value.ArchiveInterval; + + protected override async Task ExecuteCycleAsync(CancellationToken ct) + { + var minGroupSize = options.Value.ArchiveMinGroupSize; + var groups = await sc.GetErrorGroupsAsync(ct); + if (groups.Count == 0) + { + logger.LogDebug("No error groups to archive"); + return; + } + + foreach (var group in groups) + { + ct.ThrowIfCancellationRequested(); + if (group.Count < minGroupSize) + continue; + + if (await sc.ArchiveGroupAsync(group.Id, ct)) + { + AddItems(group.Count); + metrics.AddErrorsArchived(group.Count); + _archiveCounter.Add(group.Count, new KeyValuePair("group", group.Title)); + logger.LogInformation("Archived group {Title} ({Count} messages)", group.Title, group.Count); + } + } + } + + protected override void LogCycleError(Exception ex) => logger.LogWarning(ex, "Archive cycle failed"); +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/Jobs/JobBase.cs b/tools/testing-tool/TestingTool/Jobs/JobBase.cs new file mode 100644 index 0000000000..fbee4e3beb --- /dev/null +++ b/tools/testing-tool/TestingTool/Jobs/JobBase.cs @@ -0,0 +1,143 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; + +namespace TestingTool.Jobs; + +/// +/// Base class for UI-controllable periodic jobs (retry, archive, search). Each job runs a cycle +/// on a configurable interval until stopped, recording how many cycles and items it has +/// processed. Unlike the old config-gated BackgroundService timers, jobs are started and +/// stopped on demand from the web UI. +/// +public abstract class JobBase +{ + private readonly object _lock = new(); + private readonly ActivitySource _activitySource; + private CancellationTokenSource? _cts; + private Task? _loop; + private long _cycles; + private long _itemsProcessed; + private DateTimeOffset _startedAt; + private TimeSpan _interval; + + protected JobBase(string activitySourceName) + { + _activitySource = new ActivitySource(activitySourceName); + } + + /// The stable, url-safe job name used in API paths. + public abstract string Name { get; } + + /// Human-readable description of what the job does each cycle. + public abstract string Description { get; } + + /// UI grouping category (e.g. "Recoverability", "Search"). + public abstract string Category { get; } + + /// Default cycle interval when none is supplied on start. + public abstract TimeSpan DefaultInterval { get; } + + /// Whether the job is currently running. + public bool IsRunning => _cts is not null; + + /// Cycles completed since the current run started. + public long Cycles => Interlocked.Read(ref _cycles); + + /// Items processed since the current run started. + public long ItemsProcessed => Interlocked.Read(ref _itemsProcessed); + + /// The interval of the current run, or the default when idle. + public TimeSpan Interval => _cts is null ? DefaultInterval : _interval; + + /// When the current run started (UTC), or null when idle. + public DateTimeOffset? StartedAt => _cts is null ? null : _startedAt; + + /// Starts the job on the given interval. The first cycle runs immediately. + public bool TryStart(TimeSpan interval, out string? error) + { + lock (_lock) + { + if (IsRunning) + { + error = $"Job '{Name}' is already running"; + return false; + } + + if (interval <= TimeSpan.Zero) + { + error = "Interval must be greater than 0"; + return false; + } + + _interval = interval; + _startedAt = DateTimeOffset.UtcNow; + _cycles = 0; + _itemsProcessed = 0; + _cts = new CancellationTokenSource(); + _loop = Task.Run(() => RunAsync(_cts.Token)); + } + + OnStarted(); + error = null; + return true; + } + + /// Stops the job. + public void Stop() + { + lock (_lock) + { + if (_cts is null) + return; + + _cts.Cancel(); + _cts.Dispose(); + _cts = null; + } + + OnStopped(); + } + + private async Task RunAsync(CancellationToken ct) + { + using var timer = new PeriodicTimer(_interval); + try + { + // Run the first cycle immediately so a Start click has instant effect. + await RunCycle(ct); + + while (await timer.WaitForNextTickAsync(ct)) + { + await RunCycle(ct); + } + } + catch (OperationCanceledException) { } + } + + private async Task RunCycle(CancellationToken ct) + { + using var activity = _activitySource.StartActivity($"{Name}-cycle"); + try + { + await ExecuteCycleAsync(ct); + Interlocked.Increment(ref _cycles); + } + catch (OperationCanceledException) { throw; } + catch (Exception ex) + { + LogCycleError(ex); + } + } + + /// Performs one cycle of work. Increment for each processed item. + protected abstract Task ExecuteCycleAsync(CancellationToken ct); + + /// Logs a cycle failure. + protected abstract void LogCycleError(Exception ex); + + /// Adds to the item-processed counter for the current run. + protected void AddItems(long count) => Interlocked.Add(ref _itemsProcessed, count); + + protected virtual void OnStarted() { } + protected virtual void OnStopped() { } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/Jobs/JobRunner.cs b/tools/testing-tool/TestingTool/Jobs/JobRunner.cs new file mode 100644 index 0000000000..a2025c71f9 --- /dev/null +++ b/tools/testing-tool/TestingTool/Jobs/JobRunner.cs @@ -0,0 +1,87 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using TestingTool.Contracts; + +namespace TestingTool.Jobs; + +/// +/// Manages the lifecycle of all UI-controllable jobs (retry, archive, search): start, stop, and +/// status snapshots. This is the job counterpart of — it exposes the +/// recoverability/search jobs through the /api/jobs endpoints so they can be controlled +/// from the web UI instead of running as hidden config-gated background timers. +/// +public sealed class JobRunner +{ + private readonly ConcurrentDictionary _jobs; + private readonly ILogger _logger; + + public JobRunner(IEnumerable jobs, ILogger logger) + { + _logger = logger; + _jobs = new ConcurrentDictionary( + jobs.ToDictionary(j => j.Name, StringComparer.OrdinalIgnoreCase), + StringComparer.OrdinalIgnoreCase); + } + + /// All registered jobs. + public IReadOnlyList All => _jobs.Values.ToArray(); + + /// Starts a job, optionally overriding its cycle interval. + public bool TryStart(string name, TimeSpan? interval, out string? error) + { + if (!_jobs.TryGetValue(name, out var job)) + { + error = $"Unknown job '{name}'"; + return false; + } + + var effectiveInterval = interval ?? job.DefaultInterval; + if (!job.TryStart(effectiveInterval, out error)) + return false; + + _logger.LogInformation("Started job {Job} — interval {Interval}", name, effectiveInterval); + return true; + } + + /// Stops a running job. + public bool TryStop(string name) + { + if (!_jobs.TryGetValue(name, out var job)) + return false; + + if (!job.IsRunning) + return false; + + job.Stop(); + _logger.LogInformation("Stopped job {Job} after {Cycles} cycles ({Items} items processed)", + name, job.Cycles, job.ItemsProcessed); + return true; + } + + /// Stops all running jobs (used on shutdown). + public void StopAll() + { + foreach (var job in _jobs.Values) + { + if (job.IsRunning) + job.Stop(); + } + } + + /// Returns a snapshot of every job's state for the API/UI. + public List GetSnapshot() => + _jobs.Values.OrderBy(j => j.Category).ThenBy(j => j.Name) + .Select(j => new JobInfo + { + Name = j.Name, + Description = j.Description, + Category = j.Category, + Running = j.IsRunning, + IntervalSeconds = j.IsRunning ? (int)j.Interval.TotalSeconds : 0, + DefaultIntervalSeconds = (int)j.DefaultInterval.TotalSeconds, + Cycles = j.Cycles, + ItemsProcessed = j.ItemsProcessed, + StartedAt = j.StartedAt?.ToString("O") + }) + .ToList(); +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/Jobs/RetryJob.cs b/tools/testing-tool/TestingTool/Jobs/RetryJob.cs new file mode 100644 index 0000000000..9283113ba4 --- /dev/null +++ b/tools/testing-tool/TestingTool/Jobs/RetryJob.cs @@ -0,0 +1,54 @@ +using System.Diagnostics.Metrics; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace TestingTool.Jobs; + +/// +/// Recoverability job that fetches error groups from ServiceControl and triggers a retry for +/// each group on every cycle. Replayed messages should then succeed (simulating a fix being +/// applied), which exercises ServiceControl's retry pipeline. This is the UI-controllable +/// successor to the old hidden ReplayService background timer. +/// +public sealed class RetryJob( + ServiceControlClient sc, + TestingToolMetrics metrics, + IOptions options, + Meter meter, + ILogger logger) : JobBase("testing-tool.retry") +{ + private readonly Counter _replayCounter = meter.CreateCounter("errors_replayed_total"); + + public override string Name => "retry"; + public override string Description => "Retry all error groups in ServiceControl on each cycle (replayed messages then succeed)."; + public override string Category => "Recoverability"; + public override TimeSpan DefaultInterval => options.Value.ReplayInterval; + + protected override async Task ExecuteCycleAsync(CancellationToken ct) + { + var minGroupSize = options.Value.ReplayMinGroupSize; + var groups = await sc.GetErrorGroupsAsync(ct); + if (groups.Count == 0) + { + logger.LogDebug("No error groups to retry"); + return; + } + + foreach (var group in groups) + { + ct.ThrowIfCancellationRequested(); + if (group.Count < minGroupSize) + continue; + + if (await sc.RetryGroupAsync(group.Id, ct)) + { + AddItems(group.Count); + metrics.AddErrorsReplayed(group.Count); + _replayCounter.Add(group.Count, new KeyValuePair("group", group.Title)); + logger.LogInformation("Retried group {Title} ({Count} messages)", group.Title, group.Count); + } + } + } + + protected override void LogCycleError(Exception ex) => logger.LogWarning(ex, "Retry cycle failed"); +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/Jobs/SearchJob.cs b/tools/testing-tool/TestingTool/Jobs/SearchJob.cs new file mode 100644 index 0000000000..90fe33fc18 --- /dev/null +++ b/tools/testing-tool/TestingTool/Jobs/SearchJob.cs @@ -0,0 +1,66 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace TestingTool.Jobs; + +/// +/// Search job that runs canned full-text-search queries against ServiceControl on a cycle. +/// Exercises the ServiceControl search index under concurrent load and records latency metrics. +/// This is the UI-controllable successor to the old hidden SearchService background timer. +/// +public sealed class SearchJob( + ServiceControlClient sc, + TestingToolMetrics metrics, + IOptions options, + Meter meter, + ILogger logger) : JobBase("testing-tool.search") +{ + private readonly Counter _searchCounter = meter.CreateCounter("searches_executed_total"); + private readonly Histogram _searchLatency = meter.CreateHistogram("search_latency_ms", "ms"); + + // Canned queries that exercise different search index paths. + private static readonly string[] CannedQueries = + [ + "exception", + "timeout", + "NullReferenceException", + "downstream", + "deserialization", + "poison", + "503", + "retry" + ]; + + public override string Name => "search"; + public override string Description => "Run canned full-text-search queries against ServiceControl to exercise the search index."; + public override string Category => "Search"; + public override TimeSpan DefaultInterval => options.Value.SearchInterval; + + protected override async Task ExecuteCycleAsync(CancellationToken ct) + { + // Run a few random queries per cycle to exercise the search index. + var queries = CannedQueries.OrderBy(_ => Random.Shared.Next()).Take(3).ToList(); + foreach (var query in queries) + { + ct.ThrowIfCancellationRequested(); + + var sw = Stopwatch.StartNew(); + var result = await sc.SearchAsync(query, ct); + sw.Stop(); + + _searchLatency.Record(sw.Elapsed.TotalMilliseconds, + new KeyValuePair("query", query)); + + AddItems(1); + metrics.AddSearches(1); + _searchCounter.Add(1, new KeyValuePair("query", query)); + + logger.LogDebug("Search '{Query}' → {Count} results in {Ms:F1}ms", + query, result?.MessageCount, sw.Elapsed.TotalMilliseconds); + } + } + + protected override void LogCycleError(Exception ex) => logger.LogWarning(ex, "Search cycle failed"); +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/NServiceBusSetup.cs b/tools/testing-tool/TestingTool/NServiceBusSetup.cs new file mode 100644 index 0000000000..1f1a50399a --- /dev/null +++ b/tools/testing-tool/TestingTool/NServiceBusSetup.cs @@ -0,0 +1,53 @@ +using Microsoft.Extensions.Options; +using NServiceBus; + +namespace TestingTool; + +/// +/// Extension method to configure the NServiceBus endpoint using the NServiceBus 10 +/// AddNServiceBusEndpoint DI-integrated approach. The endpoint lifecycle is managed by +/// the ASP.NET Core host — no manual Endpoint.Start/Endpoint.Stop needed. +/// +public static class NServiceBusEndpointExtensions +{ + /// + /// Registers the NServiceBus load-generation endpoint. When a RabbitMQ connection string is + /// present (injected by the Aspire AppHost as ConnectionStrings__rabbitmq, matching the + /// ServiceControl platform transport), the endpoint uses RabbitMQ with quorum/conventional + /// routing. Otherwise it falls back to the Learning transport for local standalone runs. + /// Failed messages are routed to the ServiceControl error queue. + /// + public static IServiceCollection AddTestingToolEndpoint(this IServiceCollection services, TestingToolOptions options, IConfiguration configuration) + { + var config = new EndpointConfiguration("TestingTool.Load"); + + var rabbitConnectionString = configuration.GetConnectionString("transport"); + if (!string.IsNullOrWhiteSpace(rabbitConnectionString)) + { + var transport = config.UseTransport(); + transport.UseConventionalRoutingTopology(QueueType.Quorum); + transport.ConnectionString(rabbitConnectionString); + } + else + { + config.UseTransport(); + } + + // Route failures to the ServiceControl error queue. + config.SendFailedMessagesTo(options.ErrorQueueName); + + // Simplified serializer; the testing tool generates volume, not complex payloads. + config.UseSerialization(); + + // Disable immediate retries to make error groups cleaner; deferred retries are handled + // by ServiceControl's retry mechanism. + var recoverability = config.Recoverability(); + recoverability.Immediate(im => im.NumberOfRetries(0)); + recoverability.Delayed(d => d.NumberOfRetries(0)); + + config.EnableInstallers(); + + services.AddNServiceBusEndpoint(config); + return services; + } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/Program.cs b/tools/testing-tool/TestingTool/Program.cs new file mode 100644 index 0000000000..89ee8d483d --- /dev/null +++ b/tools/testing-tool/TestingTool/Program.cs @@ -0,0 +1,239 @@ +using System.Diagnostics.Metrics; +using TestingTool; +using TestingTool.Contracts; +using TestingTool.Jobs; +using TestingTool.Scenarios; + +// --- Configuration --- + +var builder = WebApplication.CreateBuilder(args); + +builder.Services.Configure(builder.Configuration.GetSection("TestingTool")); + +var options = builder.Configuration.GetSection("TestingTool").Get() ?? new TestingToolOptions(); +var shardId = ShardIdResolver.Resolve(); + +// --- OpenTelemetry (Phase 1) --- + +var meter = TelemetrySetup.CreateMeter(); +builder.Services.AddTestingToolTelemetry(meter); +builder.Services.AddSingleton(meter); + +// --- NServiceBus endpoint (Phase 2) --- + +builder.Services.AddTestingToolEndpoint(options, builder.Configuration); + +// --- Scenarios (Phase 3) --- + +builder.Services.AddSingleton(_ => new ThirdPartyOutageScenario(shardId)); +builder.Services.AddSingleton(_ => new TimeoutSpikeScenario(shardId)); +builder.Services.AddSingleton(_ => new PoisonMessageScenario(shardId)); +builder.Services.AddSingleton(_ => new DeserializationScenario(shardId)); +builder.Services.AddSingleton(_ => new RandomBackgroundNoiseScenario(shardId)); + +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +// --- Recoverability/search jobs (Phase 4) --- +// The retry, archive and search jobs used to be hidden config-gated background timers. They are +// now UI-controllable jobs managed by JobRunner and exposed through /api/jobs — start/stop them +// from the web UI. Intervals/minimums still come from configuration as defaults. + +builder.Services.AddHttpClient((sp, client) => +{ + var opts = sp.GetRequiredService>().Value; + client.BaseAddress = new Uri(opts.ServiceControlApiUrl); + client.Timeout = TimeSpan.FromSeconds(30); +}); + +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +// --- Application pipeline --- + +var app = builder.Build(); +app.UseStaticFiles(); +app.UseOpenTelemetryPrometheusScrapingEndpoint("/metrics"); + +var metrics = app.Services.GetRequiredService(); +var scClient = app.Services.GetRequiredService(); +var jobRunner = app.Services.GetRequiredService(); +var startedAt = DateTimeOffset.UtcNow; + +// Auto-start background noise after the endpoint is ready. +app.Lifetime.ApplicationStarted.Register(() => +{ + if (options.AutoStartBackgroundNoise) + { + var runner = app.Services.GetRequiredService(); + runner.TryStart("background-noise", null, null, out _); + } +}); + +// Graceful shutdown: stop all scenarios, jobs and bypass writer. +app.Lifetime.ApplicationStopping.Register(() => +{ + var runner = app.Services.GetRequiredService(); + runner.StopAll(); + jobRunner.StopAll(); + app.Services.GetRequiredService().Stop(); +}); + +// --- Health endpoints (Phase 6) --- +// Lightweight probe targets for Kubernetes liveness/readiness and docker-compose health checks. +// The chiseled runtime image has no shell/curl, so HTTP probes are used instead of exec probes. + +app.MapGet("/health/live", () => Results.Ok(new { status = "alive", shardId })); + +app.MapGet("/health/ready", () => Results.Ok(new { status = "ready", shardId })); + +// --- API endpoints (Phase 5) --- + +app.MapGet("/api/scenarios", (ScenarioRunner runner) => Results.Ok(runner.GetSnapshot())); + +app.MapGet("/api/status", () => Results.Ok(new TestingToolStatus +{ + Ready = true, + ErrorsSent = metrics.TotalErrorsSent, + ErrorsReplayed = metrics.TotalErrorsReplayed, + ErrorsArchived = metrics.TotalErrorsArchived, + SearchesExecuted = metrics.TotalSearches, + BypassErrorsWritten = metrics.TotalBypassErrorsWritten, + ShardId = shardId, + ActiveScenarios = metrics.ActiveScenarios, + ActiveJobs = jobRunner.GetSnapshot().Count(j => j.Running), + CurrentRate = Math.Round(metrics.CurrentRate, 1), + ServiceControlUrl = scClient.BaseUrl, + Uptime = (DateTimeOffset.UtcNow - startedAt).ToString(@"h\h\ m\m\ s\s") +})); + +app.MapPost("/api/scenarios/{name}/start", (string name, StartScenarioRequest? request, ScenarioRunner runner) => +{ + var duration = request?.DurationSeconds is { } secs and > 0 + ? TimeSpan.FromSeconds(secs) + : (TimeSpan?)null; + + if (!runner.TryStart(name, request?.Rate, duration, out var error)) + return Results.BadRequest(new { error }); + + var snapshot = runner.GetSnapshot() + .First(s => s.Name.Equals(name, StringComparison.OrdinalIgnoreCase)); + return Results.Ok(snapshot); +}); + +app.MapPost("/api/scenarios/{name}/stop", (string name, ScenarioRunner runner) => +{ + if (!runner.TryStop(name)) + return Results.BadRequest(new { error = $"Scenario '{name}' is not running" }); + + return Results.Ok(new { stopped = name }); +}); + +app.MapPost("/api/scenarios/stop-all", (ScenarioRunner runner) => +{ + runner.StopAll(); + return Results.Ok(new { stopped = "all" }); +}); + +// --- Job endpoints (recoverability + search) --- +// These replace the hidden background timers. Jobs are started/stopped on demand from the UI. + +app.MapGet("/api/jobs", () => Results.Ok(jobRunner.GetSnapshot())); + +app.MapPost("/api/jobs/{name}/start", (string name, StartJobRequest? request) => +{ + var interval = request?.IntervalSeconds is { } secs and > 0 + ? TimeSpan.FromSeconds(secs) + : (TimeSpan?)null; + + if (!jobRunner.TryStart(name, interval, out var error)) + return Results.BadRequest(new { error }); + + var snapshot = jobRunner.GetSnapshot() + .First(j => j.Name.Equals(name, StringComparison.OrdinalIgnoreCase)); + return Results.Ok(snapshot); +}); + +app.MapPost("/api/jobs/{name}/stop", (string name) => +{ + if (!jobRunner.TryStop(name)) + return Results.BadRequest(new { error = $"Job '{name}' is not running" }); + + return Results.Ok(new { stopped = name }); +}); + +app.MapPost("/api/jobs/stop-all", () => +{ + jobRunner.StopAll(); + return Results.Ok(new { stopped = "all" }); +}); + +// --- Bypass endpoints (Phase 2: direct error-queue writer) --- +// These endpoints control the bypass path that writes failed-message envelopes directly to the +// ServiceControl error queue, bypassing the handler for high-throughput error load. + +app.MapGet("/api/bypass/status", (DirectErrorQueueWriter writer) => Results.Ok(writer.GetStatus())); + +app.MapPost("/api/bypass/start", (StartBypassRequest? request, DirectErrorQueueWriter writer, IScenarioRegistry registry) => +{ + var scenarioName = request?.Scenario; + if (string.IsNullOrWhiteSpace(scenarioName)) + { + // Default to the first scenario if none specified. + scenarioName = registry.All[0].Name; + } + + var rate = request?.Rate ?? 100; + var duration = request?.DurationSeconds is { } secs and > 0 + ? TimeSpan.FromSeconds(secs) + : (TimeSpan?)null; + + if (!writer.TryStart(scenarioName, rate, duration, out var error)) + return Results.BadRequest(new { error }); + + return Results.Ok(writer.GetStatus()); +}); + +app.MapPost("/api/bypass/stop", (DirectErrorQueueWriter writer) => +{ + writer.Stop(); + return Results.Ok(writer.GetStatus()); +}); + +// --- Release-test scenario endpoints (Phase 5: release-test presets) --- +// Lists release-test presets that map to testing-tool scenarios, and allows kicking them off +// by release-test name. This satisfies the optional requirement to consider release-test +// scenarios for manual kickoff. + +app.MapGet("/api/release-tests", () => Results.Ok(ReleaseTestScenarios.Presets.Select(p => new +{ + name = p.Name, + scenario = p.ScenarioName, + description = p.Description, + rate = p.Rate, + durationSeconds = p.DurationSeconds +}))); + +app.MapPost("/api/release-tests/{name}/start", (string name, ScenarioRunner runner) => +{ + var preset = ReleaseTestScenarios.Find(name); + if (preset is null) + return Results.BadRequest(new { error = $"Unknown release-test scenario '{name}'" }); + + var duration = preset.DurationSeconds is { } secs and > 0 + ? TimeSpan.FromSeconds(secs) + : (TimeSpan?)null; + + if (!runner.TryStart(preset.ScenarioName, preset.Rate, duration, out var error)) + return Results.BadRequest(new { error }); + + return Results.Ok(new { started = preset.Name, scenario = preset.ScenarioName, rate = preset.Rate }); +}); + +app.MapFallbackToFile("index.html"); + +app.Run(); \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/Properties/launchSettings.json b/tools/testing-tool/TestingTool/Properties/launchSettings.json new file mode 100644 index 0000000000..842b5f5920 --- /dev/null +++ b/tools/testing-tool/TestingTool/Properties/launchSettings.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:5290", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:7175;http://localhost:5290", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/tools/testing-tool/TestingTool/ReleaseTestScenarios.cs b/tools/testing-tool/TestingTool/ReleaseTestScenarios.cs new file mode 100644 index 0000000000..edd2aabd25 --- /dev/null +++ b/tools/testing-tool/TestingTool/ReleaseTestScenarios.cs @@ -0,0 +1,66 @@ +using TestingTool.Contracts; +using TestingTool.Scenarios; + +namespace TestingTool; + +/// +/// Maps release-test scenario names (from docs/testing-scenarios.md) to testing-tool +/// scenarios so they can be kicked off manually from the web UI or API. This satisfies the +/// optional requirement: "any scenarios from the release tests should be considered to kick off +/// manually." +/// +public static class ReleaseTestScenarios +{ + /// + /// Release-test scenario presets. Each preset maps a release-test checklist item to a + /// testing-tool scenario with a recommended rate and duration. + /// + public static readonly IReadOnlyList Presets = + [ + new("retry-single-message", "third-party-outage", + "Recoverability — Retry single message: generate a small batch of grouped errors for single-message retry testing.", + 5, 10), + + new("retry-message-group", "third-party-outage", + "Recoverability — Retry message group: generate a larger batch of grouped errors for group retry testing.", + 50, 20), + + new("ingestion-load", "background-noise", + "Ingestion — High message load: continuous background errors to test ingestion throughput.", + 100, null), + + new("chaos-testing", "background-noise", + "Chaos testing — Continuous low-level errors while stopping/killing ServiceControl processes.", + 10, null), + + new("performance-clean-db", "third-party-outage", + "Performance — Clean database: burst of errors to test ingestion with an empty RavenDB.", + 200, 60), + + new("poison-retry-storm", "poison-message", + "Recoverability — Poison message retry storm: deterministic always-fail messages.", + 20, 30), + + new("timeout-batch", "timeout-spike", + "Recoverability — Timeout spike: oscillating timeout failures grouped by batch bucket.", + 30, 60), + + new("deserialization-bad-deploy", "deserialization-failure", + "Recoverability — Bad deployment: 100% deserialization failures grouped by message type.", + 50, 15), + ]; + + /// Find a preset by its release-test name (case-insensitive). + public static ReleaseTestPreset? Find(string name) => + Presets.FirstOrDefault(p => p.Name.Equals(name, StringComparison.OrdinalIgnoreCase)); +} + +/// +/// A named release-test preset that maps to a testing-tool scenario with recommended parameters. +/// +public sealed record ReleaseTestPreset( + string Name, + string ScenarioName, + string Description, + double Rate, + int? DurationSeconds); \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/ScenarioRegistry.cs b/tools/testing-tool/TestingTool/ScenarioRegistry.cs new file mode 100644 index 0000000000..7080bd4fc1 --- /dev/null +++ b/tools/testing-tool/TestingTool/ScenarioRegistry.cs @@ -0,0 +1,17 @@ +using TestingTool.Scenarios; + +namespace TestingTool; + +/// +/// Default scenario registry backed by a dictionary built from DI-registered instances. +/// +public sealed class ScenarioRegistry(IEnumerable scenarios) : IScenarioRegistry +{ + private readonly Dictionary _byName = + scenarios.ToDictionary(s => s.Name, StringComparer.OrdinalIgnoreCase); + + public IScenario? Get(string name) => + _byName.TryGetValue(name, out var s) ? s : null; + + public IReadOnlyList All { get; } = scenarios.ToArray().AsReadOnly(); +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/ScenarioRunner.cs b/tools/testing-tool/TestingTool/ScenarioRunner.cs new file mode 100644 index 0000000000..b0e613b953 --- /dev/null +++ b/tools/testing-tool/TestingTool/ScenarioRunner.cs @@ -0,0 +1,228 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using Microsoft.Extensions.Logging; +using NServiceBus; +using TestingTool.Contracts; +using TestingTool.Scenarios; + +namespace TestingTool; + +/// +/// Manages the lifecycle of all scenarios: start, stop, rate control, auto-stop on duration, +/// and per-scenario error counting. Load is generated by sending to the +/// NServiceBus endpoint at a controlled rate; the throws based +/// on the scenario's failure logic, routing failures to the ServiceControl error queue. +/// +public sealed class ScenarioRunner +{ + private readonly IMessageSession _session; + private readonly IScenarioRegistry _registry; + private readonly TestingToolMetrics _metrics; + private readonly Meter _meter; + private readonly ILogger _logger; + + // Per-scenario runtime state. + private readonly ConcurrentDictionary _running = new(StringComparer.OrdinalIgnoreCase); + + public ScenarioRunner(IMessageSession session, IScenarioRegistry registry, TestingToolMetrics metrics, Meter meter, ILogger logger) + { + _session = session; + _registry = registry; + _metrics = metrics; + _meter = meter; + _logger = logger; + } + + /// Starts a scenario with an optional rate override and auto-stop duration. + public bool TryStart(string scenarioName, double? rate, TimeSpan? duration, out string? error) + { + var scenario = _registry.Get(scenarioName); + if (scenario is null) + { + error = $"Unknown scenario '{scenarioName}'"; + return false; + } + + if (_running.ContainsKey(scenarioName)) + { + error = $"Scenario '{scenarioName}' is already running"; + return false; + } + + var targetRate = rate ?? scenario.DefaultRate; + if (targetRate <= 0) + { + error = "Rate must be greater than 0"; + return false; + } + + var runtime = new ScenarioRuntime(scenario, targetRate, duration, _meter, _logger); + if (_running.TryAdd(scenarioName, runtime)) + { + runtime.Start(GenerateLoadAsync); + UpdateAggregateMetrics(); + _logger.LogInformation("Started scenario {Scenario} at {Rate:F1} msg/s{Duration}", + scenarioName, targetRate, duration is null ? "" : $" for {duration.Value}"); + error = null; + return true; + } + + error = $"Scenario '{scenarioName}' is already running"; + return false; + } + + /// Stops a running scenario. + public bool TryStop(string scenarioName) + { + if (_running.TryRemove(scenarioName, out var runtime)) + { + runtime.Stop(); + UpdateAggregateMetrics(); + _logger.LogInformation("Stopped scenario {Scenario} after {Errors} errors", + scenarioName, runtime.ErrorsSent); + return true; + } + return false; + } + + /// Stops all running scenarios (used on shutdown). + public void StopAll() + { + foreach (var name in _running.Keys) + TryStop(name); + } + + public List GetSnapshot() + { + var list = new List(); + foreach (var scenario in _registry.All) + { + _running.TryGetValue(scenario.Name, out var runtime); + list.Add(new ScenarioInfo + { + Name = scenario.Name, + Description = scenario.Description, + Category = scenario.Category, + Running = runtime is not null, + CurrentRate = runtime?.TargetRate ?? 0, + ErrorsSent = runtime?.ErrorsSent ?? 0, + DefaultRate = scenario.DefaultRate, + Cooldown = scenario.Cooldown?.ToString() + }); + } + return list; + } + + private void UpdateAggregateMetrics() + { + _metrics.SetActiveScenarios(_running.Count); + _metrics.SetCurrentRate(_running.Values.Sum(r => r.TargetRate)); + } + + /// The load generation loop: sends LoadMessages at the target rate until cancelled. + private async Task GenerateLoadAsync(ScenarioRuntime runtime, CancellationToken ct) + { + var scenario = runtime.Scenario; + var interval = TimeSpan.FromSeconds(1.0 / runtime.TargetRate); + using var timer = new PeriodicTimer(interval); + long sequence = 0; + + try + { + while (await timer.WaitForNextTickAsync(ct)) + { + var seq = Interlocked.Increment(ref sequence); + var payload = new byte[Random.Shared.Next(64, 512)]; + Random.Shared.NextBytes(payload); + + // Deterministic message id shared with the handler: FailingMessageHandler calls + // scenario.ShouldFail(context.MessageId), so setting the id here ensures the + // failure decision counted below matches the decision the handler actually makes. + var messageId = $"{scenario.Name}-{seq}-{runtime.StartedAt.Ticks}"; + + var sendOptions = new SendOptions(); + sendOptions.RouteToThisEndpoint(); + sendOptions.SetMessageId(messageId); + sendOptions.SetHeader("TestingTool.Scenario", scenario.Name); + + try + { + await _session.Send(new LoadMessage { Sequence = seq, Payload = payload }, sendOptions, ct); + + // If the scenario fails for this message id (same decision as the handler), + // count it as an error sent. + if (scenario.ShouldFail(messageId)) + { + runtime.IncrementErrors(); + _metrics.AddErrorsSent(1); + runtime.ErrorsCounter.Add(1, new KeyValuePair("scenario", scenario.Name)); + } + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Send failed for scenario {Scenario} seq {Seq}", scenario.Name, seq); + } + } + } + catch (OperationCanceledException) { } + } +} + +/// Per-scenario runtime state: rate, counters, cancellation, and auto-stop timer. +internal sealed class ScenarioRuntime +{ + private readonly Meter _meter; + private readonly ILogger _logger; + private readonly Counter _errorsCounter; + private long _errorsSent; + private CancellationTokenSource? _cts; + private Task? _loop; + + public ScenarioRuntime(IScenario scenario, double rate, TimeSpan? duration, Meter meter, ILogger logger) + { + Scenario = scenario; + TargetRate = rate; + Duration = duration; + StartedAt = DateTimeOffset.UtcNow; + _meter = meter; + _logger = logger; + _errorsCounter = meter.CreateCounter("errors_sent_total"); + } + + public IScenario Scenario { get; } + public double TargetRate { get; } + public TimeSpan? Duration { get; } + public DateTimeOffset StartedAt { get; } + public long ErrorsSent => Interlocked.Read(ref _errorsSent); + public Counter ErrorsCounter => _errorsCounter; + + public long IncrementErrors() => Interlocked.Increment(ref _errorsSent); + + public void Start(Func generate) + { + _cts = Duration is { } d + ? new CancellationTokenSource(d) + : new CancellationTokenSource(); + + _loop = Task.Run(async () => + { + try + { + await generate(this, _cts.Token); + } + catch (OperationCanceledException) { } + catch (Exception ex) + { + _logger.LogError(ex, "Load generation loop for {Scenario} crashed", Scenario.Name); + } + }, _cts.Token); + } + + public void Stop() + { + _cts?.Cancel(); + _cts?.Dispose(); + _cts = null; + } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/ServiceControlClient.cs b/tools/testing-tool/TestingTool/ServiceControlClient.cs new file mode 100644 index 0000000000..4dfd41979a --- /dev/null +++ b/tools/testing-tool/TestingTool/ServiceControlClient.cs @@ -0,0 +1,94 @@ +using System.Net.Http.Json; +using System.Text.Json.Serialization; + +namespace TestingTool; + +/// +/// Thin HTTP client for the ServiceControl REST API. Used by the recoverability jobs (retry, +/// archive) and the search job to interact with the test ServiceControl instance. +/// +public sealed class ServiceControlClient(HttpClient http, ILogger logger) +{ + public string BaseUrl => http.BaseAddress?.ToString() ?? "(not configured)"; + + /// + /// Fetches the active error (failure) groups from ServiceControl. These are the recoverability + /// groups that the retry and archive jobs operate on. + /// + public async Task> GetErrorGroupsAsync(CancellationToken ct = default) + { + try + { + // ServiceControl exposes failure groups under /api/recoverability/groups. The response + // entries are GroupOperation objects (id, title, count, type, ...). + var groups = await http.GetFromJsonAsync>("/api/recoverability/groups", ct); + return groups ?? []; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to fetch error groups from ServiceControl"); + return []; + } + } + + /// Triggers a retry of all messages in an error group (async, 202 Accepted on success). + public async Task RetryGroupAsync(string groupId, CancellationToken ct = default) + { + try + { + var response = await http.PostAsJsonAsync($"/api/recoverability/groups/{groupId}/errors/retry", new { }, ct); + return response.IsSuccessStatusCode; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to retry error group {GroupId}", groupId); + return false; + } + } + + /// Triggers an archive of all messages in an error group (async, 202 Accepted on success). + public async Task ArchiveGroupAsync(string groupId, CancellationToken ct = default) + { + try + { + var response = await http.PostAsJsonAsync($"/api/recoverability/groups/{groupId}/errors/archive", new { }, ct); + return response.IsSuccessStatusCode; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to archive error group {GroupId}", groupId); + return false; + } + } + + /// Executes a full-text search query against ServiceControl. + public async Task SearchAsync(string query, CancellationToken ct = default) + { + try + { + var response = await http.GetAsync($"/api/errors/search?q={Uri.EscapeDataString(query)}", ct); + if (!response.IsSuccessStatusCode) + return null; + + var body = await response.Content.ReadFromJsonAsync(ct); + return new SearchResult(body?.MessageCount ?? 0); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Search '{Query}' failed", query); + return null; + } + } + + public sealed record ErrorGroup( + [property: JsonPropertyName("id")] string Id, + [property: JsonPropertyName("title")] string Title, + [property: JsonPropertyName("count")] int Count, + [property: JsonPropertyName("type")] string? Type, + [property: JsonPropertyName("first")] string? First, + [property: JsonPropertyName("last")] string? Last); + + public sealed record SearchResponse([property: JsonPropertyName("messageCount")] int MessageCount); + + public sealed record SearchResult(int MessageCount); +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/ShardIdResolver.cs b/tools/testing-tool/TestingTool/ShardIdResolver.cs new file mode 100644 index 0000000000..1c52d8b1de --- /dev/null +++ b/tools/testing-tool/TestingTool/ShardIdResolver.cs @@ -0,0 +1,38 @@ +using System.Text.RegularExpressions; + +namespace TestingTool; + +/// +/// Resolves the shard id for this replica. When scaled horizontally, each replica must own a +/// disjoint slice of the scenario space so that deterministic failure decisions (see +/// ) don't overlap across pods. +/// +/// Resolution order: +/// 1. SHARD_ID environment variable (explicit override — used by docker-compose and +/// manual runs). +/// 2. Ordinal extracted from a Kubernetes StatefulSet hostname (e.g. testing-tool-2 +/// → 2). StatefulSets give stable, ordered pod names so shards are deterministic +/// across restarts. +/// 3. — unique per pod for Deployments, deterministic +/// per host for bare-metal/VM runs. +/// +public static class ShardIdResolver +{ + // Matches a trailing - at the end of a hostname (StatefulSet pod naming convention). + private static readonly Regex StatefulSetOrdinal = new(@"-(\d+)$", RegexOptions.Compiled); + + /// Resolves the shard id for this replica. + public static string Resolve() + { + var explicitId = Environment.GetEnvironmentVariable("SHARD_ID"); + if (!string.IsNullOrWhiteSpace(explicitId)) + return explicitId; + + var hostname = Environment.MachineName; + var match = StatefulSetOrdinal.Match(hostname); + if (match.Success) + return match.Groups[1].Value; + + return hostname; + } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/TelemetrySetup.cs b/tools/testing-tool/TestingTool/TelemetrySetup.cs new file mode 100644 index 0000000000..292a84c522 --- /dev/null +++ b/tools/testing-tool/TestingTool/TelemetrySetup.cs @@ -0,0 +1,55 @@ +using System.Diagnostics.Metrics; +using OpenTelemetry; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; + +namespace TestingTool; + +/// +/// Configures OpenTelemetry traces, metrics, and logs for the testing tool. All telemetry is +/// exported via OTLP to the endpoint configured by OTEL_EXPORTER_OTLP_ENDPOINT (set in the +/// docker-compose environment). A Prometheus scraping endpoint is also exposed at /metrics. +/// +public static class TelemetrySetup +{ + public const string ServiceName = "testing-tool"; + public const string MeterName = "testing-tool"; + + // Shared activity source names used across the tool. + public static class Sources + { + public const string Load = "testing-tool.load"; + public const string Replay = "testing-tool.replay"; + public const string Search = "testing-tool.search"; + public const string Bypass = "testing-tool.bypass"; + } + + public static Meter CreateMeter() => new(MeterName, "1.0.0"); + + public static OpenTelemetryBuilder AddTestingToolTelemetry(this IServiceCollection services, Meter meter) + { + return services.AddOpenTelemetry() + .ConfigureResource(r => r.AddService(ServiceName, + serviceInstanceId: Environment.MachineName)) + .WithTracing(t => t + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddSource(Sources.Load) + .AddSource(Sources.Replay) + .AddSource(Sources.Search) + .AddSource(Sources.Bypass) + // Also pick up per-scenario activity sources dynamically. + .AddSource("testing-tool.*") + .AddOtlpExporter()) + .WithMetrics(m => m + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddMeter(MeterName) + .AddPrometheusExporter() + .AddOtlpExporter()) + // Phase 1: Route structured logs through the OpenTelemetry logs API to OTLP. + .WithLogging(l => l.AddOtlpExporter()); + } +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/TestingTool.csproj b/tools/testing-tool/TestingTool/TestingTool.csproj new file mode 100644 index 0000000000..168df6941e --- /dev/null +++ b/tools/testing-tool/TestingTool/TestingTool.csproj @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/TestingToolMetrics.cs b/tools/testing-tool/TestingTool/TestingToolMetrics.cs new file mode 100644 index 0000000000..b129217bb6 --- /dev/null +++ b/tools/testing-tool/TestingTool/TestingToolMetrics.cs @@ -0,0 +1,33 @@ +namespace TestingTool; + +/// +/// Shared singleton holding live counters from all subsystems (scenario runner, replay, search). +/// Provides a single read point for the GET /api/status endpoint without needing to resolve +/// individual hosted service instances. +/// +public sealed class TestingToolMetrics +{ + private long _totalErrorsSent; + private long _totalErrorsReplayed; + private long _totalErrorsArchived; + private long _totalSearches; + private long _totalBypassErrorsWritten; + private long _activeScenarios; + private double _currentRate; + + public long TotalErrorsSent => Interlocked.Read(ref _totalErrorsSent); + public long TotalErrorsReplayed => Interlocked.Read(ref _totalErrorsReplayed); + public long TotalErrorsArchived => Interlocked.Read(ref _totalErrorsArchived); + public long TotalSearches => Interlocked.Read(ref _totalSearches); + public long TotalBypassErrorsWritten => Interlocked.Read(ref _totalBypassErrorsWritten); + public int ActiveScenarios => (int)Interlocked.Read(ref _activeScenarios); + public double CurrentRate => _currentRate; + + public void AddErrorsSent(long count) => Interlocked.Add(ref _totalErrorsSent, count); + public void AddErrorsReplayed(long count) => Interlocked.Add(ref _totalErrorsReplayed, count); + public void AddErrorsArchived(long count) => Interlocked.Add(ref _totalErrorsArchived, count); + public void AddSearches(long count) => Interlocked.Add(ref _totalSearches, count); + public void AddBypassErrorsWritten(long count) => Interlocked.Add(ref _totalBypassErrorsWritten, count); + public void SetActiveScenarios(int count) => Interlocked.Exchange(ref _activeScenarios, count); + public void SetCurrentRate(double rate) => _currentRate = rate; +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/TestingToolOptions.cs b/tools/testing-tool/TestingTool/TestingToolOptions.cs new file mode 100644 index 0000000000..9b93de0b6b --- /dev/null +++ b/tools/testing-tool/TestingTool/TestingToolOptions.cs @@ -0,0 +1,32 @@ +namespace TestingTool; + +/// +/// Configuration options for the testing tool, bound from the "TestingTool" config section +/// and/or environment variables. +/// +public sealed class TestingToolOptions +{ + /// Base URL of the ServiceControl instance under test (e.g. http://servicecontrol:33333). + public string ServiceControlApiUrl { get; set; } = "http://localhost:33333"; + + /// Interval between retry cycles. + public TimeSpan ReplayInterval { get; set; } = TimeSpan.FromMinutes(2); + + /// Minimum number of messages in a group before it is retried. + public int ReplayMinGroupSize { get; set; } = 1; + + /// Interval between search cycles. + public TimeSpan SearchInterval { get; set; } = TimeSpan.FromMinutes(1); + + /// Interval between archive cycles for the recoverability archive job. + public TimeSpan ArchiveInterval { get; set; } = TimeSpan.FromMinutes(2); + + /// Minimum number of messages in a group before it is archived. + public int ArchiveMinGroupSize { get; set; } = 1; + + /// NServiceBus error queue name that ServiceControl monitors. + public string ErrorQueueName { get; set; } = "error"; + + /// Whether to start the background-noise scenario automatically on startup. + public bool AutoStartBackgroundNoise { get; set; } = false; +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/appsettings.Development.json b/tools/testing-tool/TestingTool/appsettings.Development.json new file mode 100644 index 0000000000..0c208ae918 --- /dev/null +++ b/tools/testing-tool/TestingTool/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/tools/testing-tool/TestingTool/appsettings.json b/tools/testing-tool/TestingTool/appsettings.json new file mode 100644 index 0000000000..5459b7371e --- /dev/null +++ b/tools/testing-tool/TestingTool/appsettings.json @@ -0,0 +1,20 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "TestingTool": { + "ServiceControlApiUrl": "http://localhost:33333", + "ReplayInterval": "00:02:00", + "ReplayMinGroupSize": 1, + "SearchInterval": "00:01:00", + "ArchiveInterval": "00:02:00", + "ArchiveMinGroupSize": 1, + "ErrorQueueName": "error", + "AutoStartBackgroundNoise": false + }, + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:4317" +} \ No newline at end of file diff --git a/tools/testing-tool/TestingTool/wwwroot/index.html b/tools/testing-tool/TestingTool/wwwroot/index.html new file mode 100644 index 0000000000..269e0339a3 --- /dev/null +++ b/tools/testing-tool/TestingTool/wwwroot/index.html @@ -0,0 +1,669 @@ + + + + + + + ServiceControl Testing Tool + + + + +
+
+

🎯 ServiceControl Testing Tool

+ +
+ + +
+
+
Errors Sent
+
—
+
—
+
+
+
Replayed
+
—
+
archived —
+
+
+
Searches
+
—
+
+
+
Active Scenarios
+
—
+
— jobs active
+
+
+ + +
+ Loading… +
+ + +
+ + + +
+ + +
+ +
+

Scenarios

+ +
+
+
Loading scenarios…
+
+ +
+ + +
+ +
+

Recoverability Jobs

+ +
+
+
Loading jobs…
+
+ +
+ + +
+ +
+

Direct Error-Queue Bypass

+
+
+
+ Writes failed-message envelopes directly to the ServiceControl error queue, bypassing the handler for high-throughput error load. Messages carry NServiceBus failure headers so ServiceControl ingests them as genuine failed messages. +
+
+ Scenario: + + Rate (msg/s): + + Duration (sec, optional): + + + +
+
+ Status: idle + Errors written: 0 +
+
+ +
+ + +
+ +
+ + + + + \ No newline at end of file diff --git a/tools/testing-tool/global.json b/tools/testing-tool/global.json new file mode 100644 index 0000000000..85017886e5 --- /dev/null +++ b/tools/testing-tool/global.json @@ -0,0 +1,10 @@ +{ + "sdk": { + "version": "10.0.100", + "allowPrerelease": false, + "rollForward": "latestFeature" + }, + "msbuild-sdks": { + "Aspire.AppHost.Sdk": "13.5.3" + } +} \ No newline at end of file diff --git a/tools/testing-tool/requirements-test-tool-plan.md b/tools/testing-tool/requirements-test-tool-plan.md new file mode 100644 index 0000000000..b468d6e11a --- /dev/null +++ b/tools/testing-tool/requirements-test-tool-plan.md @@ -0,0 +1,259 @@ +# Plan: Testing Tool (requirements-test-tool.md) + +A load + real-world-scenario testing tool deployed alongside a test instance of ServiceControl to validate error ingestion performance. + +> Source requirements: [requirements-test-tool.md](./requirements-test-tool.md) + +--- + +## 1. Architecture Overview + +The tool is a **stateless, horizontally-scalable .NET service** hosted in a container that generates error load against a test ServiceControl instance. It exposes a minimal web UI for manual scenario triggers and emits OpenTelemetry (otel) traces/metrics/logs for everything. + +``` + ┌────────────────────────────────────────────┐ + │ Testing Tool Pod(s) │ + │ │ + │ ┌──────────┐ ┌────────────────────────┐ │ + │ │ Web UI │ │ Scenario Hosted Service│ │ + │ │ (manual │ │ - Background replay │ │ + │ │ trigger)│ │ - Background search │ │ + │ └────┬─────┘ │ - Load generators │ │ + │ │ └───────────┬────────────┘ │ + │ │ │ │ + │ ▼ ▼ │ + │ ┌──────────────────────────────────────┐ │ + │ │ OTel SDK (traces/metrics/logs) │ │ + │ └──────────────────┬───────────────────┘ │ + └─────────────────────┼──────────────────────┘ + │ + ┌──────────────▼──────────────┐ + │ OTel Collector / Jaeger │ + └─────────────────────────────┘ + │ + ┌──────────────▼──────────────┐ + │ ServiceControl (test inst) │ + │ - Error ingestion queue │ + │ - FTS index (RavenDB) │ + └─────────────────────────────┘ +``` + +### Key design decisions + +| Concern | Decision | Rationale | +|---|---|---| +| Framework | ASP.NET Core 8 minimal API + `IHostedService` | Stateless, container-friendly, first-class otel + DI | +| Error transport | NServiceBus endpoint sending failed messages to ServiceControl error queue | Matches real handler path (requirement) | +| Direct injection | Optional raw `IMessagingDispatcher`/queue writer to bypass handler for high load | Requirement: bypass initial message creation | +| State | In-memory only, no DB | Stateless + horizontal scale | +| Scaling | Run N replicas; each owns disjoint scenario slices via env-configured shard id | Stateless requirement | +| UI | Single static HTML page + JSON endpoints | "simple web ui" | + +--- + +## 2. Work Breakdown (with progress tracking) + +### Phase 0 — Project bootstrap + +- [x] Create solution `TestingTool.slnx` with projects: + - `TestingTool` (web + hosted services) + - `TestingTool.Scenarios` (scenario definitions) + - `TestingTool.Contracts` (shared DTOs for UI API) +- [x] Add `Dockerfile` (multi-stage, `chiseled` base) +- [x] ~~Add `docker-compose.yml` with ServiceControl test instance + Jaeger (OTLP) collector~~ — **removed**; the local stack is now brought up via the Aspire AppHost (Phase 7) +- [x] Add `.github/workflows/testing-tool-ci.yml` (build + container image build; test step deferred — no tests exist in Phase 0) + +### Phase 1 — OTel foundation + +> Requirement: *"Everything should expose otel"* + +- [x] Add NuGet refs: `OpenTelemetry.Extensions.Hosting`, `OpenTelemetry.Instrumentation.AspNetCore`, `OpenTelemetry.Instrumentation.Http`, `OpenTelemetry.Exporter.OpenTelemetryProtocol`, `OpenTelemetry.Exporter.Prometheus.AspNetCore` +- [x] Configure OTLP exporter via env `OTEL_EXPORTER_OTLP_ENDPOINT` +- [x] Define `ActivitySource` instances per scenario category (`testing-tool.load`, `testing-tool.replay`, `testing-tool.search` + per-scenario sources) +- [x] Add metrics: `errors_sent_total{scenario}`, `errors_replayed_total{group}`, `searches_executed_total{query}`, `search_latency_ms{query}` (histogram) +- [x] Prometheus scraping endpoint at `/metrics` +- [x] Add structured logs routed through OTel logs API (Phase 1 complete) + +```csharp +// Program.cs — OTel wiring +var builder = WebApplication.CreateBuilder(args); + +builder.Services.AddOpenTelemetry() + .ConfigureResource(r => r.AddService("testing-tool", + serviceInstanceId: Environment.MachineName)) + .WithTracing(t => t + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddSource("testing-tool.load") + .AddSource("testing-tool.replay") + .AddSource("testing-tool.search") + .AddOtlpExporter()) + .WithMetrics(m => m + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddMeter("testing-tool") + .AddPrometheusExporter() // optional /metrics for ad-hoc + .AddOtlpExporter()) + .WithLogging(l => l.AddOtlpExporter()); + +builder.Services.AddOpenTelemetryPrometheusScrapingEndpoint(); +``` + +### Phase 2 — NServiceBus endpoint + error path + +> Requirements: *"generate errors via a real message handler"*, *"simulate high error loads … bypass actually creating the initial messages"* + +- [x] Configure NServiceBus endpoint `TestingTool.Load` with error queue pointed at ServiceControl (using NServiceBus 10 `AddNServiceBusEndpoint` DI integration + Learning transport) +- [x] Implement `FailingMessageHandler` that throws based on injected `IScenario` (via `IScenarioRegistry`) +- [x] Implement `DirectErrorQueueWriter` that constructs `MessageFailed` / transport message envelopes and writes directly to the ServiceControl error queue (bypass path) — see ref [ServiceControlFeeder](https://github.com/dvdstelt/ServiceControlFeeder) +- [x] Add a load-rate controller (token bucket) configurable per scenario (`PeriodicTimer`-based rate controller in `ScenarioRunner`) + +```csharp +// FailingMessageHandler.cs +public class FailingMessageHandler(IScenario scenario, ILogger log) + : IHandleMessages +{ + public Task Handle(SampleCommand message, IMessageHandlerContext context) + { + using var activity = scenario.ActivitySource.StartActivity("handler"); + activity?.SetTag("scenario", scenario.Name); + + if (scenario.ShouldFail(context.MessageId)) + throw scenario.CreateException(); // routed to ServiceControl error queue + + return Task.CompletedTask; + } +} +``` + +### Phase 3 — Scenarios + +> Requirement: *"error generation should be based on some real scenarios, so that we still have nice groups of errors, example a third party outage"* + +- [x] Define `IScenario` interface (with `Name`, `Description`, `Category`, `DefaultRate`, `ActivitySource`, `ShouldFail`, `CreateException`, `Cooldown`) +- [x] Implement scenarios: + - [x] `ThirdPartyOutageScenario` — 100% fail for 20s burst, then 30s cooldown, repeat (groups by downstream host) + - [x] `TimeoutSpikeScenario` — intermittent `TimeoutException` with correlated batch ids, oscillating rate + - [x] `PoisonMessageScenario` — deterministic 15% fail on FNV-1a hash of message id (stays failed = retry storm) + - [x] `DeserializationScenario` — 100% fail grouped by message type (simulates bad deployment) + - [x] `RandomBackgroundNoiseScenario` — ~3% baseline error rate (always on), rotates through exception types +- [x] Tag each emitted exception with `ExceptionType`, `CorrelationGroup` (via `ScenarioException`) so ServiceControl groups them naturally + +### Phase 4 — Background jobs + +> Requirements: background replay job (errors should then pass), background search job (exercises FTS) + +- [x] `ReplayService : BackgroundService` + - Every configurable interval, fetches recent error groups from ServiceControl REST API and triggers replay + - Emits otel activity + `errors_replayed_total` counter + - Gated by `TestingTool:ReplayEnabled` config flag + ```csharp + protected override async Task ExecuteAsync(CancellationToken ct) + { + using var timer = new PeriodicTimer(_options.ReplayInterval); + while (await timer.WaitForNextTickAsync(ct)) + { + using var activity = _replaySource.StartActivity("replay-cycle"); + var groups = await _scClient.GetErrorGroupsAsync(ct); + foreach (var g in groups.Where(ShouldReplay)) + { + await _scClient.ReplayGroupAsync(g.Id, ct); + _meter.CreateCounter("errors_replayed_total") + .Add(g.Count, new("scenario", g.Classifier)); + } + } + } + ``` +- [x] `SearchService : BackgroundService` + - Runs canned full-text-search queries against ServiceControl `/search` endpoint on a timer + - Records latency histogram `search_latency_ms{query}` + - Exercises FTS index under concurrent load +- [x] Both jobs are gated by config flags (`ReplayEnabled`, `SearchEnabled`) so replicas can opt-in/opt-out + +### Phase 5 — Web UI + +> Requirement: *"simple web ui for kicking off manual scenarios"* + +- [x] Endpoints: + - `GET /` → static `index.html` + - `GET /api/scenarios` → list available scenarios + status (with category, rate, error counts) + - `POST /api/scenarios/{name}/start` → start scenario (rate, durationSeconds) + - `POST /api/scenarios/{name}/stop` + - `POST /api/scenarios/stop-all` → stop all running scenarios + - `GET /api/status` → live counters snapshot (errors, replays, searches, active scenarios, rate, uptime, SC url) + - `GET /metrics` → Prometheus scraping endpoint +- [x] `wwwroot/index.html` — vanilla JS SPA, no build step; dark/light theme, status dashboard, category-grouped scenario cards with Start/Stop + rate/duration controls, live 2s polling, toast notifications +- [x] Optional: wire release-test scenario names so they can be manually kicked off (requirement: *"any scenarios from the release tests should be considered to kick off manually"*) + +```html + +
+ +``` + +### Phase 6 — Containerization & scaling + +> Requirements: *"hosted in a container"*, *"stateless"*, *"can be scaled horizontally"* + +- [x] Multi-stage `Dockerfile` (chiseled composite base, .NET 10, multi-arch build) +- [x] No local file/db state; all config via env vars (`TestingTool__*` section binding + `SHARD_ID` + `OTEL_*`) +- [x] Shard id derived from pod ordinal/hostname → disjoint scenario slices across replicas (`ShardIdResolver`: env var → StatefulSet ordinal → MachineName) +- [x] ~~`docker-compose` (single replica, local dev) + k8s `StatefulSet` (3 replicas, configurable)~~ — **removed**; the tool runs via `dotnet run` or the Aspire AppHost. Horizontal scaling is still supported through the `SHARD_ID` env var / hostname-ordinal resolver (`ShardIdResolver`: env var → StatefulSet-style ordinal → `MachineName`); bring-your-own orchestration for multi-replica. HTTP liveness/readiness probes remain on `/health/live` and `/health/ready` +- [x] Health endpoints: `GET /health/live` (liveness) + `GET /health/ready` (readiness) +- [x] Document horizontal scale: *N replicas each emit 1/N of target rate* (README § Horizontal scaling + Configuration + Health checks) + +### Phase 7 — Observability dashboard & verification + +- [x] Ship a prebuilt Grafana dashboard + full observability stack (OTel Collector → Jaeger for traces, Prometheus + Grafana for metrics dashboards). The testing tool sends OTLP telemetry to the collector, which fans out traces to Jaeger and metrics to Prometheus. Grafana is auto-provisioned with both data sources and a prebuilt "Testing Tool" dashboard (errors/sec by scenario, search p95, replay/archive/bypass rates, cumulative error count). The stack is wired through the `AddObservabilityStack()` extension method in `ObservabilityExtensions.cs` to keep `AppHost.cs` clean. Config files live under `obs/` next to the AppHost. +- [x] Create an Aspire AppHost that orchestrates the testing tool together with the full Particular platform (ServiceControl + transport + persistence), so a single `aspire run` brings up the whole system locally. Include tag/channel selection for the platform images so a specific ServiceControl version (or `latest`) can be pinned via the Aspire app model — see the [Aspire docs on container image tag selection](https://learn.microsoft.com/dotnet/aspire/fundamentals/containers) for the `WithImageTag` / `WithImage(...)` resource customization pattern. +- [x] Add a smoke test that: starts tool → triggers `ThirdPartyOutageScenario` for 30s → verifies errors appear in ServiceControl → verifies replay passes +- [x] Write README with run instructions + env var reference + +--- + +## 3. References + +- **Source requirements:** [requirements-test-tool.md](./requirements-test-tool.md) +- **ServiceControlFeeder** — direct error-queue feeding reference: https://github.com/dvdestelt/ServiceControlFeeder + - Pattern to reuse: raw transport-message construction written to the ServiceControl `error` queue to bypass the handler path (Phase 2 `DirectErrorQueueWriter`). +- **FakeMessageGen** — high-throughput fake message generator reference: https://github.com/ramonsmits/FakeMessageGen + - Pattern to reuse: rate-controlled message generation loop and token-bucket shaping (Phase 2 load controller). +- **NServiceBus** — `SendFailedMessagesTo("error")` routes failed messages to ServiceControl. Docs: https://docs.particular.net/nservicebus/recoverability +- **ServiceControl REST API** — `/search`, error group listing, and replay endpoints used by the background jobs. Docs: https://docs.particular.net/servicecontrol/ +- **OpenTelemetry .NET** — `AddOpenTelemetry()` host integration. Docs: https://opentelemetry.io/docs/instrumentation/net/ +- **Out of scope:** Audit testing (per requirements). + +--- + +## 4. Open questions + +- [ ] Which ServiceControl version(s) are the test target? (affects REST API shape) +- [ ] Direct error-queue writer: transport = MSMQ / SQL / ASB / ASQ / RabbitMQ / SQS? Changes envelope format. +- [ ] Release-test scenarios: is there an existing manifest file to import, or define new ones here? +- [ ] Target error throughput ceiling (helps size replicas + token bucket defaults)? +- [ ] (Future) Where should the prebuilt Grafana dashboard live — this repo or a shared infra repo? + +--- + +## 5. Milestone summary + +| Milestone | Deliverable | Phase | +|---|---|---| +| M1 | OTel-instrumented endpoint emitting grouped errors (handler + bypass paths) | Phases 0–3 ✅ | +| M2 | Background replay + search jobs running on timers | Phase 4 ✅ | +| M3 | Web UI for manual scenario control | Phase 5 ✅ | +| M4 | Containerized, horizontally-scalable deploy + Aspire AppHost + smoke test | Phases 6–7 ✅ | \ No newline at end of file diff --git a/tools/testing-tool/requirements-test-tool.md b/tools/testing-tool/requirements-test-tool.md new file mode 100644 index 0000000000..6e3ccdbfa3 --- /dev/null +++ b/tools/testing-tool/requirements-test-tool.md @@ -0,0 +1,35 @@ +# Testing tool + +## Goal + +The goal is to create a testing tool to simulate load and real world scenarios to be deployed alongside a test instance of service control to be able to test the error ingestion performance + +## Background + +Some tools have been made to assist, use these for reference + +* https://github.com/dvdstelt/ServiceControlFeeder +* https://github.com/ramonsmits/FakeMessageGen + +## Requirements + +*Functional* + +* should be able to simulate high error loads, this will have to bypass actually creating the initial messages +* should be able to generate errors via a real message handler isntead of simulated load +* the error generation should be based on some real scenarios, so that we still have nice groups of errors, example a third party outage +* Everything should explose otel +* A background job that every so often replays error groups (and these should then pass) +* A background job that every so often does a search (hopefully exercising FTS) +* (optional) any scenarios from the release tests should be bonsidered to kick off manually + +*nonfunctional* + +* hosted in a container +* simple web ui for kicking off manual scenarios +* stateless +* can be scaled horizontally + +## out of scope + +* Audit testing \ No newline at end of file