From 204cba96ab3db0a54e0f0d1a9518301a774611f4 Mon Sep 17 00:00:00 2001 From: Rhys Bevilaqua Date: Tue, 25 Aug 2026 13:34:03 +0800 Subject: [PATCH 01/13] wip: load testing tool --- .github/workflows/testing-tool-ci.yml | 49 ++ tools/testing-tool/Directory.Build.props | 24 + tools/testing-tool/Directory.Packages.props | 24 + tools/testing-tool/Dockerfile | 38 ++ tools/testing-tool/README.md | 136 ++++++ tools/testing-tool/TestingTool.slnx | 7 + tools/testing-tool/docker-compose.yml | 57 +++ tools/testing-tool/k8s/testing-tool.yaml | 99 ++++ .../requirements-test-tool-plan.md | 258 +++++++++++ tools/testing-tool/requirements-test-tool.md | 35 ++ .../src/TestingTool.Contracts/ScenarioInfo.cs | 32 ++ .../StartScenarioRequest.cs | 14 + .../TestingTool.Contracts.csproj | 5 + .../TestingToolStatus.cs | 41 ++ .../DeserializationScenario.cs | 24 + .../src/TestingTool.Scenarios/IScenario.cs | 35 ++ .../src/TestingTool.Scenarios/LoadMessage.cs | 15 + .../PoisonMessageScenario.cs | 27 ++ .../RandomBackgroundNoiseScenario.cs | 36 ++ .../src/TestingTool.Scenarios/ScenarioBase.cs | 48 ++ .../ScenarioException.cs | 17 + .../TestingTool.Scenarios.csproj | 9 + .../ThirdPartyOutageScenario.cs | 35 ++ .../TimeoutSpikeScenario.cs | 41 ++ .../src/TestingTool/FailingMessageHandler.cs | 44 ++ .../src/TestingTool/IScenarioRegistry.cs | 12 + .../src/TestingTool/NServiceBusSetup.cs | 45 ++ tools/testing-tool/src/TestingTool/Program.cs | 132 ++++++ .../Properties/launchSettings.json | 23 + .../src/TestingTool/ReplayService.cs | 71 +++ .../src/TestingTool/ScenarioRegistry.cs | 17 + .../src/TestingTool/ScenarioRunner.cs | 222 +++++++++ .../src/TestingTool/SearchService.cs | 85 ++++ .../src/TestingTool/ServiceControlClient.cs | 74 +++ .../src/TestingTool/ShardIdResolver.cs | 38 ++ .../src/TestingTool/TelemetrySetup.cs | 50 ++ .../src/TestingTool/TestingTool.csproj | 26 ++ .../src/TestingTool/TestingToolMetrics.cs | 27 ++ .../src/TestingTool/TestingToolOptions.cs | 32 ++ .../TestingTool/appsettings.Development.json | 8 + .../src/TestingTool/appsettings.json | 20 + .../src/TestingTool/wwwroot/index.html | 428 ++++++++++++++++++ 42 files changed, 2460 insertions(+) create mode 100644 .github/workflows/testing-tool-ci.yml create mode 100644 tools/testing-tool/Directory.Build.props create mode 100644 tools/testing-tool/Directory.Packages.props create mode 100644 tools/testing-tool/Dockerfile create mode 100644 tools/testing-tool/README.md create mode 100644 tools/testing-tool/TestingTool.slnx create mode 100644 tools/testing-tool/docker-compose.yml create mode 100644 tools/testing-tool/k8s/testing-tool.yaml create mode 100644 tools/testing-tool/requirements-test-tool-plan.md create mode 100644 tools/testing-tool/requirements-test-tool.md create mode 100644 tools/testing-tool/src/TestingTool.Contracts/ScenarioInfo.cs create mode 100644 tools/testing-tool/src/TestingTool.Contracts/StartScenarioRequest.cs create mode 100644 tools/testing-tool/src/TestingTool.Contracts/TestingTool.Contracts.csproj create mode 100644 tools/testing-tool/src/TestingTool.Contracts/TestingToolStatus.cs create mode 100644 tools/testing-tool/src/TestingTool.Scenarios/DeserializationScenario.cs create mode 100644 tools/testing-tool/src/TestingTool.Scenarios/IScenario.cs create mode 100644 tools/testing-tool/src/TestingTool.Scenarios/LoadMessage.cs create mode 100644 tools/testing-tool/src/TestingTool.Scenarios/PoisonMessageScenario.cs create mode 100644 tools/testing-tool/src/TestingTool.Scenarios/RandomBackgroundNoiseScenario.cs create mode 100644 tools/testing-tool/src/TestingTool.Scenarios/ScenarioBase.cs create mode 100644 tools/testing-tool/src/TestingTool.Scenarios/ScenarioException.cs create mode 100644 tools/testing-tool/src/TestingTool.Scenarios/TestingTool.Scenarios.csproj create mode 100644 tools/testing-tool/src/TestingTool.Scenarios/ThirdPartyOutageScenario.cs create mode 100644 tools/testing-tool/src/TestingTool.Scenarios/TimeoutSpikeScenario.cs create mode 100644 tools/testing-tool/src/TestingTool/FailingMessageHandler.cs create mode 100644 tools/testing-tool/src/TestingTool/IScenarioRegistry.cs create mode 100644 tools/testing-tool/src/TestingTool/NServiceBusSetup.cs create mode 100644 tools/testing-tool/src/TestingTool/Program.cs create mode 100644 tools/testing-tool/src/TestingTool/Properties/launchSettings.json create mode 100644 tools/testing-tool/src/TestingTool/ReplayService.cs create mode 100644 tools/testing-tool/src/TestingTool/ScenarioRegistry.cs create mode 100644 tools/testing-tool/src/TestingTool/ScenarioRunner.cs create mode 100644 tools/testing-tool/src/TestingTool/SearchService.cs create mode 100644 tools/testing-tool/src/TestingTool/ServiceControlClient.cs create mode 100644 tools/testing-tool/src/TestingTool/ShardIdResolver.cs create mode 100644 tools/testing-tool/src/TestingTool/TelemetrySetup.cs create mode 100644 tools/testing-tool/src/TestingTool/TestingTool.csproj create mode 100644 tools/testing-tool/src/TestingTool/TestingToolMetrics.cs create mode 100644 tools/testing-tool/src/TestingTool/TestingToolOptions.cs create mode 100644 tools/testing-tool/src/TestingTool/appsettings.Development.json create mode 100644 tools/testing-tool/src/TestingTool/appsettings.json create mode 100644 tools/testing-tool/src/TestingTool/wwwroot/index.html diff --git a/.github/workflows/testing-tool-ci.yml b/.github/workflows/testing-tool-ci.yml new file mode 100644 index 0000000000..045f6a6454 --- /dev/null +++ b/.github/workflows/testing-tool-ci.yml @@ -0,0 +1,49 @@ +name: Testing Tool CI + +on: + push: + branches: + - master + - release-* + paths: + - 'tools/testing-tool/**' + - '.github/workflows/testing-tool-ci.yml' + pull_request: + paths: + - 'tools/testing-tool/**' + - '.github/workflows/testing-tool-ci.yml' + workflow_dispatch: + +env: + DOTNET_NOLOGO: true + +defaults: + run: + shell: pwsh + +jobs: + build: + name: Build solution + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v6.0.0 + with: + global-json-file: global.json + + - name: Build + run: dotnet build tools/testing-tool/TestingTool.slnx --configuration Release + + - name: Build container image + uses: docker/build-push-action@v7.2.0 + with: + context: . + file: tools/testing-tool/Dockerfile + push: false + load: true + tags: particular/testing-tool:ci diff --git a/tools/testing-tool/Directory.Build.props b/tools/testing-tool/Directory.Build.props new file mode 100644 index 0000000000..a8f0c98b2f --- /dev/null +++ b/tools/testing-tool/Directory.Build.props @@ -0,0 +1,24 @@ + + + + + net10.0 + enable + enable + true + true + true + low + all + + + + true + + + diff --git a/tools/testing-tool/Directory.Packages.props b/tools/testing-tool/Directory.Packages.props new file mode 100644 index 0000000000..3c87c42989 --- /dev/null +++ b/tools/testing-tool/Directory.Packages.props @@ -0,0 +1,24 @@ + + + + + true + true + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tools/testing-tool/Dockerfile b/tools/testing-tool/Dockerfile new file mode 100644 index 0000000000..b6e1edcc42 --- /dev/null +++ b/tools/testing-tool/Dockerfile @@ -0,0 +1,38 @@ +# Multi-stage build for the ServiceControl testing tool. +# Build context is the repository root so global.json + nuget.config are available. +ARG RUNTIMEVERSION=10.0 + +# --- build stage --- +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:10.0 AS build +ARG TARGETARCH +WORKDIR /src +ENV CI=true + +# global.json pins the SDK; nuget.config adds the Particular feed for later NServiceBus deps. +COPY global.json nuget.config ./ +COPY tools/testing-tool/ ./tools/testing-tool/ + +# Publish the runnable host; its ProjectReferences pull in Contracts + Scenarios. +RUN dotnet publish tools/testing-tool/src/TestingTool/TestingTool.csproj \ + --configuration Release \ + --arch $TARGETARCH \ + --output /app \ + /p:UseAppHost=false + +# --- runtime stage --- +# Uses the same chiseled composite base as the rest of the ServiceControl images so that +# globalization/ICU dependencies are present. +FROM mcr.microsoft.com/dotnet/aspnet:$RUNTIMEVERSION-noble-chiseled-composite-extra +WORKDIR /app + +COPY --from=build /app ./ + +ENV ASPNETCORE_URLS=http://+:8080 +EXPOSE 8080 + +# Health probes are served by the /health/live and /health/ready HTTP endpoints (added in +# Phase 6). Kubernetes uses HTTP GET probes directly; the chiseled base ships no shell/curl so +# exec-based probes are not supported. + +USER $APP_UID +ENTRYPOINT ["dotnet", "TestingTool.dll"] diff --git a/tools/testing-tool/README.md b/tools/testing-tool/README.md new file mode 100644 index 0000000000..0ae7f0f6ef --- /dev/null +++ b/tools/testing-tool/README.md @@ -0,0 +1,136 @@ +# ServiceControl Testing Tool + +A stateless, horizontally-scalable .NET 10 service that generates error load and real-world failure +scenarios against a test ServiceControl instance, with OpenTelemetry observability and a simple web +UI for manual scenario control. + +See [requirements-test-tool-plan.md](./requirements-test-tool-plan.md) for the full plan. + +## Status + +Phases 0–6 complete (project bootstrap, OTel foundation, NServiceBus error path, scenarios, +background jobs, web UI, containerization & scaling). Remaining: direct error-queue bypass writer +(Phase 2 optional item), Grafana dashboard + smoke test (Phase 7). + +## What it does + +The tool runs an NServiceBus endpoint (`TestingTool.Load`) that sends messages through a handler +which fails based on the active scenario. Failed messages are routed to the `error` queue for +ServiceControl to ingest. Five scenarios are built in, each producing naturally-grouped errors: + +| Scenario | Category | Failure shape | +|---|---|---| +| `third-party-outage` | Outage | 100% fail for 20s bursts, 30s cooldown — grouped by downstream host | +| `timeout-spike` | Timeout | Oscillating 10–70% fail rate — grouped by 5-min batch bucket | +| `poison-message` | Poison | 15% deterministic always-fail messages — retry storm | +| `deserialization-failure` | Deserialization | 100% fail — grouped by message type (bad deployment) | +| `background-noise` | Noise | ~3% always-on baseline — rotates through exception types | + +Two background jobs (gated by config) run on timers: +- **Replay** — fetches error groups from ServiceControl and triggers retry +- **Search** — runs canned FTS queries to exercise the RavenDB search index + +All telemetry is exported via OTLP (traces + metrics) and a Prometheus `/metrics` endpoint. + +## Layout + +``` +tools/testing-tool/ + TestingTool.slnx + Directory.Build.props # repo-style conventions (warnings-as-errors, nullable, analyzers) + Directory.Packages.props # central package management (OTel + NServiceBus pinned) + Dockerfile + docker-compose.yml + k8s/testing-tool.yaml # Kubernetes StatefulSet + Service + ConfigMap + src/ + TestingTool/ # ASP.NET Core host: Program.cs, web UI, services + wwwroot/index.html # single-page web UI (vanilla JS, no build step) + Program.cs # OTel wiring, NServiceBus endpoint, DI, API endpoints + ScenarioRunner.cs # start/stop, rate control, per-scenario error counting + FailingMessageHandler.cs # NServiceBus handler that throws per scenario logic + ServiceControlClient.cs # REST API client (error groups, replay, search) + ReplayService.cs # background replay job + SearchService.cs # background search job + TelemetrySetup.cs # OTel traces + metrics + OTLP/Prometheus exporters + NServiceBusSetup.cs # endpoint config (Learning transport, error queue routing) + TestingToolOptions.cs # config (SC URL, replay/search intervals, error queue name) + TestingToolMetrics.cs # shared live counters for /api/status + ShardIdResolver.cs # shard id from env var, StatefulSet ordinal, or hostname + TestingTool.Scenarios/ # IScenario contract + 5 scenario implementations + TestingTool.Contracts/ # shared DTOs (ScenarioInfo, TestingToolStatus, StartScenarioRequest) +``` + +## Run locally + +```bash +dotnet build tools/testing-tool/TestingTool.slnx --configuration Release +dotnet run --project tools/testing-tool/src/TestingTool --configuration Release +``` + +Open http://localhost:5290 (or the port shown in the console). + +## Run the stack + +```bash +docker compose -f tools/testing-tool/docker-compose.yml up --build +``` + +This starts ServiceControl + the testing tool + Jaeger (OTLP). Open: +- Testing tool UI: http://localhost:8080 +- ServiceControl: http://localhost:33333 +- Jaeger UI: http://localhost:16686 +- Prometheus metrics: http://localhost:8080/metrics + +## Deploy on Kubernetes + +```bash +kubectl apply -f tools/testing-tool/k8s/ +``` + +The StatefulSet runs 3 replicas by default. Each pod derives its shard id from its StatefulSet +ordinal (`testing-tool-0` → shard `0`, `testing-tool-1` → shard `1`, …) so replicas own disjoint +scenario slices automatically. Scale by changing `spec.replicas` in the manifest. + +## Horizontal scaling + +The tool is **stateless** — all state is in-memory per replica. When scaled to N replicas, each +replica emits its own share of load. Shard ids ensure deterministic failure decisions don't +overlap across pods: + +| Replicas | Shard id source | Scenario slice | +|---|---|---| +| 1 (docker-compose) | `SHARD_ID=0` env var | All scenarios | +| N (k8s StatefulSet) | Pod ordinal from hostname | 1/N of each scenario's messages | + +To achieve a target aggregate rate of R msg/s across N replicas, set each replica's scenario rate +to R/N. The web UI and `/api/status` endpoint report per-replica counters; aggregate across +replicas via Prometheus queries or the OTLP backend. + +## Configuration + +All configuration is via environment variables (no files, no database). Settings are in +`appsettings.json` under the `TestingTool` section, overridable by environment variables using +`__` as the section separator (e.g. `TestingTool__ServiceControlApiUrl`): + +| Setting | Default | Description | +|---|---|---| +| `TestingTool__ServiceControlApiUrl` | `http://localhost:33333` | ServiceControl REST API base URL | +| `TestingTool__ReplayEnabled` | `false` | Enable the background replay job | +| `TestingTool__ReplayInterval` | `00:02:00` | Interval between replay cycles | +| `TestingTool__ReplayMinGroupSize` | `1` | Min messages in a group before replaying | +| `TestingTool__SearchEnabled` | `false` | Enable the background search job | +| `TestingTool__SearchInterval` | `00:01:00` | Interval between search cycles | +| `TestingTool__ErrorQueueName` | `error` | NServiceBus error queue (ServiceControl monitors this) | +| `TestingTool__AutoStartBackgroundNoise` | `false` | Auto-start the background-noise scenario on startup | +| `SHARD_ID` (env) | *(auto: pod ordinal or hostname)* | Shard id for disjoint scenario slices when scaled | +| `OTEL_EXPORTER_OTLP_ENDPOINT` (env) | `http://localhost:4317` | OTLP collector endpoint | +| `OTEL_SERVICE_NAME` (env) | `testing-tool` | OTel service name | + +## Health checks + +| Endpoint | Purpose | +|---|---| +| `GET /health/live` | Liveness — process is alive | +| `GET /health/ready` | Readiness — app is ready to serve requests | +| `GET /api/status` | Full status snapshot (counters, shard, uptime) | +| `GET /metrics` | Prometheus scraping endpoint | \ No newline at end of file diff --git a/tools/testing-tool/TestingTool.slnx b/tools/testing-tool/TestingTool.slnx new file mode 100644 index 0000000000..a988c98201 --- /dev/null +++ b/tools/testing-tool/TestingTool.slnx @@ -0,0 +1,7 @@ + + + + + + + diff --git a/tools/testing-tool/docker-compose.yml b/tools/testing-tool/docker-compose.yml new file mode 100644 index 0000000000..e26e3ff781 --- /dev/null +++ b/tools/testing-tool/docker-compose.yml @@ -0,0 +1,57 @@ +# Test instance of ServiceControl + the testing tool + a Jaeger (OTLP) backend. +# +# Usage: +# docker compose -f tools/testing-tool/docker-compose.yml up --build +# +# Then open: +# * Testing tool UI: http://localhost:8080 +# * ServiceControl: http://localhost:33333 +# * Jaeger UI: http://localhost:16686 +# +# This compose runs a single replica for local development. For horizontal scaling with +# disjoint shard ids, use the Kubernetes manifests in tools/testing-tool/k8s/ instead. +# +# Health probes: the chiseled runtime image has no shell/curl, so in-container healthchecks +# are not possible in docker-compose. The /health/live and /health/ready HTTP endpoints are +# available for Kubernetes HTTP probes and manual verification: +# curl http://localhost:8080/health/live +# curl http://localhost:8080/health/ready + +services: + + servicecontrol: + # Primary ServiceControl instance under test. Uses the embedded RavenDB persistence and the + # learning transport so the stack runs with no external broker/database dependencies. + image: ghcr.io/particular/servicecontrol:latest + environment: + TRANSPORTTYPE: Learning + RAVENDB_TARGETTYPENAME: Embedded + ports: + - "33333:33333" + # The official image is chiseled (no shell/curl), so no in-container healthcheck is defined + # here. `depends_on` only orders startup; Phase 6 adds proper readiness gating. + + jaeger: + # all-in-one Jaeger accepts OTLP (gRPC :4317 / HTTP :4318) directly, so no separate collector is + # needed for the test deployment. + image: jaegertracing/all-in-one:1.62 + ports: + - "16686:16686" # Jaeger UI + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + + testing-tool: + build: + context: ../.. + dockerfile: tools/testing-tool/Dockerfile + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4317 + OTEL_SERVICE_NAME: testing-tool + SHARD_ID: "0" + TestingTool__ServiceControlApiUrl: http://servicecontrol:33333 + TestingTool__AutoStartBackgroundNoise: "true" + ports: + - "8080:8080" + depends_on: + - servicecontrol + - jaeger diff --git a/tools/testing-tool/k8s/testing-tool.yaml b/tools/testing-tool/k8s/testing-tool.yaml new file mode 100644 index 0000000000..934ed36a6b --- /dev/null +++ b/tools/testing-tool/k8s/testing-tool.yaml @@ -0,0 +1,99 @@ +# Kubernetes manifests for the ServiceControl Testing Tool. +# +# Deploy with: +# kubectl apply -f tools/testing-tool/k8s/ +# +# This deploys: +# - A StatefulSet (stable pod names → deterministic shard ids via ShardIdResolver) +# - A ClusterIP Service +# - A ConfigMap for non-secret configuration +# - HTTP liveness/readiness probes on /health/live and /health/ready +# +# Scale by changing spec.replicas. Each pod derives its shard id from its StatefulSet ordinal +# (testing-tool-0 → shard "0", testing-tool-1 → shard "1", …) so replicas own disjoint scenario +# slices automatically. See README.md § Horizontal scaling for details. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: testing-tool-config + labels: + app: testing-tool +data: + # All config is environment-based — the tool is stateless. + TestingTool__ServiceControlApiUrl: "http://servicecontrol:33333" + TestingTool__ReplayEnabled: "false" + TestingTool__SearchEnabled: "false" + TestingTool__AutoStartBackgroundNoise: "true" + OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317" + OTEL_SERVICE_NAME: "testing-tool" + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: testing-tool + labels: + app: testing-tool +spec: + serviceName: testing-tool + replicas: 3 + selector: + matchLabels: + app: testing-tool + template: + metadata: + labels: + app: testing-tool + spec: + containers: + - name: testing-tool + image: ghcr.io/particular/testing-tool:latest + ports: + - containerPort: 8080 + name: http + envFrom: + - configMapRef: + name: testing-tool-config + # SHARD_ID is not set — ShardIdResolver extracts the ordinal from the StatefulSet + # pod hostname (e.g. testing-tool-0 → "0"). + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 1000m + memory: 512Mi + # HTTP probes — the chiseled base has no shell, so exec probes are not possible. + livenessProbe: + httpGet: + path: /health/live + port: http + initialDelaySeconds: 10 + periodSeconds: 15 + timeoutSeconds: 3 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health/ready + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + +--- +apiVersion: v1 +kind: Service +metadata: + name: testing-tool + labels: + app: testing-tool +spec: + type: ClusterIP + selector: + app: testing-tool + ports: + - port: 8080 + targetPort: http + name: http \ No newline at end of file diff --git a/tools/testing-tool/requirements-test-tool-plan.md b/tools/testing-tool/requirements-test-tool-plan.md new file mode 100644 index 0000000000..9e8f962a48 --- /dev/null +++ b/tools/testing-tool/requirements-test-tool-plan.md @@ -0,0 +1,258 @@ +# Plan: Testing Tool (requirements-test-tool.md) + +A load + real-world-scenario testing tool deployed alongside a test instance of ServiceControl to validate error ingestion performance. + +> Source requirements: [requirements-test-tool.md](./requirements-test-tool.md) + +--- + +## 1. Architecture Overview + +The tool is a **stateless, horizontally-scalable .NET service** hosted in a container that generates error load against a test ServiceControl instance. It exposes a minimal web UI for manual scenario triggers and emits OpenTelemetry (otel) traces/metrics/logs for everything. + +``` + ┌────────────────────────────────────────────┐ + │ Testing Tool Pod(s) │ + │ │ + │ ┌──────────┐ ┌────────────────────────┐ │ + │ │ Web UI │ │ Scenario Hosted Service│ │ + │ │ (manual │ │ - Background replay │ │ + │ │ trigger)│ │ - Background search │ │ + │ └────┬─────┘ │ - Load generators │ │ + │ │ └───────────┬────────────┘ │ + │ │ │ │ + │ ▼ ▼ │ + │ ┌──────────────────────────────────────┐ │ + │ │ OTel SDK (traces/metrics/logs) │ │ + │ └──────────────────┬───────────────────┘ │ + └─────────────────────┼──────────────────────┘ + │ + ┌──────────────▼──────────────┐ + │ OTel Collector / Jaeger │ + └─────────────────────────────┘ + │ + ┌──────────────▼──────────────┐ + │ ServiceControl (test inst) │ + │ - Error ingestion queue │ + │ - FTS index (RavenDB) │ + └─────────────────────────────┘ +``` + +### Key design decisions + +| Concern | Decision | Rationale | +|---|---|---| +| Framework | ASP.NET Core 8 minimal API + `IHostedService` | Stateless, container-friendly, first-class otel + DI | +| Error transport | NServiceBus endpoint sending failed messages to ServiceControl error queue | Matches real handler path (requirement) | +| Direct injection | Optional raw `IMessagingDispatcher`/queue writer to bypass handler for high load | Requirement: bypass initial message creation | +| State | In-memory only, no DB | Stateless + horizontal scale | +| Scaling | Run N replicas; each owns disjoint scenario slices via env-configured shard id | Stateless requirement | +| UI | Single static HTML page + JSON endpoints | "simple web ui" | + +--- + +## 2. Work Breakdown (with progress tracking) + +### Phase 0 — Project bootstrap + +- [x] Create solution `TestingTool.slnx` with projects: + - `TestingTool` (web + hosted services) + - `TestingTool.Scenarios` (scenario definitions) + - `TestingTool.Contracts` (shared DTOs for UI API) +- [x] Add `Dockerfile` (multi-stage, `chiseled` base) +- [x] Add `docker-compose.yml` with ServiceControl test instance + Jaeger (OTLP) collector +- [x] Add `.github/workflows/testing-tool-ci.yml` (build + container image build; test step deferred — no tests exist in Phase 0) + +### Phase 1 — OTel foundation + +> Requirement: *"Everything should expose otel"* + +- [x] Add NuGet refs: `OpenTelemetry.Extensions.Hosting`, `OpenTelemetry.Instrumentation.AspNetCore`, `OpenTelemetry.Instrumentation.Http`, `OpenTelemetry.Exporter.OpenTelemetryProtocol`, `OpenTelemetry.Exporter.Prometheus.AspNetCore` +- [x] Configure OTLP exporter via env `OTEL_EXPORTER_OTLP_ENDPOINT` +- [x] Define `ActivitySource` instances per scenario category (`testing-tool.load`, `testing-tool.replay`, `testing-tool.search` + per-scenario sources) +- [x] Add metrics: `errors_sent_total{scenario}`, `errors_replayed_total{group}`, `searches_executed_total{query}`, `search_latency_ms{query}` (histogram) +- [x] Prometheus scraping endpoint at `/metrics` +- [ ] Add structured logs routed through OTel logs API (deferred — console logging is active) + +```csharp +// Program.cs — OTel wiring +var builder = WebApplication.CreateBuilder(args); + +builder.Services.AddOpenTelemetry() + .ConfigureResource(r => r.AddService("testing-tool", + serviceInstanceId: Environment.MachineName)) + .WithTracing(t => t + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddSource("testing-tool.load") + .AddSource("testing-tool.replay") + .AddSource("testing-tool.search") + .AddOtlpExporter()) + .WithMetrics(m => m + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddMeter("testing-tool") + .AddPrometheusExporter() // optional /metrics for ad-hoc + .AddOtlpExporter()) + .WithLogging(l => l.AddOtlpExporter()); + +builder.Services.AddOpenTelemetryPrometheusScrapingEndpoint(); +``` + +### Phase 2 — NServiceBus endpoint + error path + +> Requirements: *"generate errors via a real message handler"*, *"simulate high error loads … bypass actually creating the initial messages"* + +- [x] Configure NServiceBus endpoint `TestingTool.Load` with error queue pointed at ServiceControl (using NServiceBus 10 `AddNServiceBusEndpoint` DI integration + Learning transport) +- [x] Implement `FailingMessageHandler` that throws based on injected `IScenario` (via `IScenarioRegistry`) +- [ ] Implement `DirectErrorQueueWriter` that constructs `MessageFailed` / transport message envelopes and writes directly to the ServiceControl error queue (bypass path) — see ref [ServiceControlFeeder](https://github.com/dvdstelt/ServiceControlFeeder) +- [x] Add a load-rate controller (token bucket) configurable per scenario (`PeriodicTimer`-based rate controller in `ScenarioRunner`) + +```csharp +// FailingMessageHandler.cs +public class FailingMessageHandler(IScenario scenario, ILogger log) + : IHandleMessages +{ + public Task Handle(SampleCommand message, IMessageHandlerContext context) + { + using var activity = scenario.ActivitySource.StartActivity("handler"); + activity?.SetTag("scenario", scenario.Name); + + if (scenario.ShouldFail(context.MessageId)) + throw scenario.CreateException(); // routed to ServiceControl error queue + + return Task.CompletedTask; + } +} +``` + +### Phase 3 — Scenarios + +> Requirement: *"error generation should be based on some real scenarios, so that we still have nice groups of errors, example a third party outage"* + +- [x] Define `IScenario` interface (with `Name`, `Description`, `Category`, `DefaultRate`, `ActivitySource`, `ShouldFail`, `CreateException`, `Cooldown`) +- [x] Implement scenarios: + - [x] `ThirdPartyOutageScenario` — 100% fail for 20s burst, then 30s cooldown, repeat (groups by downstream host) + - [x] `TimeoutSpikeScenario` — intermittent `TimeoutException` with correlated batch ids, oscillating rate + - [x] `PoisonMessageScenario` — deterministic 15% fail on FNV-1a hash of message id (stays failed = retry storm) + - [x] `DeserializationScenario` — 100% fail grouped by message type (simulates bad deployment) + - [x] `RandomBackgroundNoiseScenario` — ~3% baseline error rate (always on), rotates through exception types +- [x] Tag each emitted exception with `ExceptionType`, `CorrelationGroup` (via `ScenarioException`) so ServiceControl groups them naturally + +### Phase 4 — Background jobs + +> Requirements: background replay job (errors should then pass), background search job (exercises FTS) + +- [x] `ReplayService : BackgroundService` + - Every configurable interval, fetches recent error groups from ServiceControl REST API and triggers replay + - Emits otel activity + `errors_replayed_total` counter + - Gated by `TestingTool:ReplayEnabled` config flag + ```csharp + protected override async Task ExecuteAsync(CancellationToken ct) + { + using var timer = new PeriodicTimer(_options.ReplayInterval); + while (await timer.WaitForNextTickAsync(ct)) + { + using var activity = _replaySource.StartActivity("replay-cycle"); + var groups = await _scClient.GetErrorGroupsAsync(ct); + foreach (var g in groups.Where(ShouldReplay)) + { + await _scClient.ReplayGroupAsync(g.Id, ct); + _meter.CreateCounter("errors_replayed_total") + .Add(g.Count, new("scenario", g.Classifier)); + } + } + } + ``` +- [x] `SearchService : BackgroundService` + - Runs canned full-text-search queries against ServiceControl `/search` endpoint on a timer + - Records latency histogram `search_latency_ms{query}` + - Exercises FTS index under concurrent load +- [x] Both jobs are gated by config flags (`ReplayEnabled`, `SearchEnabled`) so replicas can opt-in/opt-out + +### Phase 5 — Web UI + +> Requirement: *"simple web ui for kicking off manual scenarios"* + +- [x] Endpoints: + - `GET /` → static `index.html` + - `GET /api/scenarios` → list available scenarios + status (with category, rate, error counts) + - `POST /api/scenarios/{name}/start` → start scenario (rate, durationSeconds) + - `POST /api/scenarios/{name}/stop` + - `POST /api/scenarios/stop-all` → stop all running scenarios + - `GET /api/status` → live counters snapshot (errors, replays, searches, active scenarios, rate, uptime, SC url) + - `GET /metrics` → Prometheus scraping endpoint +- [x] `wwwroot/index.html` — vanilla JS SPA, no build step; dark/light theme, status dashboard, category-grouped scenario cards with Start/Stop + rate/duration controls, live 2s polling, toast notifications +- [ ] Optional: wire release-test scenario names so they can be manually kicked off (requirement: *"any scenarios from the release tests should be considered to kick off manually"*) + +```html + +
+ +``` + +### Phase 6 — Containerization & scaling + +> Requirements: *"hosted in a container"*, *"stateless"*, *"can be scaled horizontally"* + +- [x] Multi-stage `Dockerfile` (chiseled composite base, .NET 10, multi-arch build) +- [x] No local file/db state; all config via env vars (`TestingTool__*` section binding + `SHARD_ID` + `OTEL_*`) +- [x] Shard id derived from pod ordinal/hostname → disjoint scenario slices across replicas (`ShardIdResolver`: env var → StatefulSet ordinal → MachineName) +- [x] `docker-compose` (single replica, local dev) + k8s `StatefulSet` (3 replicas, configurable) with HTTP liveness/readiness probes on `/health/live` and `/health/ready` +- [x] Health endpoints: `GET /health/live` (liveness) + `GET /health/ready` (readiness) +- [x] Document horizontal scale: *N replicas each emit 1/N of target rate* (README § Horizontal scaling + Configuration + Health checks) + +### Phase 7 — Observability dashboard & verification + +- [ ] Ship a prebuilt Grafana dashboard JSON (errors/sec, ingestion lag, search p95, replay success) +- [ ] Add a smoke test that: starts tool → triggers `ThirdPartyOutageScenario` for 30s → verifies errors appear in ServiceControl → verifies replay passes +- [ ] Write README with run instructions + env var reference + +--- + +## 3. References + +- **Source requirements:** [requirements-test-tool.md](./requirements-test-tool.md) +- **ServiceControlFeeder** — direct error-queue feeding reference: https://github.com/dvdestelt/ServiceControlFeeder + - Pattern to reuse: raw transport-message construction written to the ServiceControl `error` queue to bypass the handler path (Phase 2 `DirectErrorQueueWriter`). +- **FakeMessageGen** — high-throughput fake message generator reference: https://github.com/ramonsmits/FakeMessageGen + - Pattern to reuse: rate-controlled message generation loop and token-bucket shaping (Phase 2 load controller). +- **NServiceBus** — `SendFailedMessagesTo("error")` routes failed messages to ServiceControl. Docs: https://docs.particular.net/nservicebus/recoverability +- **ServiceControl REST API** — `/search`, error group listing, and replay endpoints used by the background jobs. Docs: https://docs.particular.net/servicecontrol/ +- **OpenTelemetry .NET** — `AddOpenTelemetry()` host integration. Docs: https://opentelemetry.io/docs/instrumentation/net/ +- **Out of scope:** Audit testing (per requirements). + +--- + +## 4. Open questions + +- [ ] Which ServiceControl version(s) are the test target? (affects REST API shape) +- [ ] Direct error-queue writer: transport = MSMQ / SQL / ASB / ASQ / RabbitMQ / SQS? Changes envelope format. +- [ ] Release-test scenarios: is there an existing manifest file to import, or define new ones here? +- [ ] Target error throughput ceiling (helps size replicas + token bucket defaults)? +- [ ] Where should the Grafana dashboard + compose live — this repo or a shared infra repo? + +--- + +## 5. Milestone summary + +| Milestone | Deliverable | Phase | +|---|---|---| +| M1 | OTel-instrumented endpoint emitting grouped errors (handler + bypass paths) | Phases 0–3 | +| M2 | Background replay + search jobs running on timers | Phase 4 | +| M3 | Web UI for manual scenario control | Phase 5 | +| M4 | Containerized, horizontally-scalable deploy (done) + dashboard + smoke test (Phase 7) | Phases 6–7 | \ No newline at end of file diff --git a/tools/testing-tool/requirements-test-tool.md b/tools/testing-tool/requirements-test-tool.md new file mode 100644 index 0000000000..6e3ccdbfa3 --- /dev/null +++ b/tools/testing-tool/requirements-test-tool.md @@ -0,0 +1,35 @@ +# Testing tool + +## Goal + +The goal is to create a testing tool to simulate load and real world scenarios to be deployed alongside a test instance of service control to be able to test the error ingestion performance + +## Background + +Some tools have been made to assist, use these for reference + +* https://github.com/dvdstelt/ServiceControlFeeder +* https://github.com/ramonsmits/FakeMessageGen + +## Requirements + +*Functional* + +* should be able to simulate high error loads, this will have to bypass actually creating the initial messages +* should be able to generate errors via a real message handler isntead of simulated load +* the error generation should be based on some real scenarios, so that we still have nice groups of errors, example a third party outage +* Everything should explose otel +* A background job that every so often replays error groups (and these should then pass) +* A background job that every so often does a search (hopefully exercising FTS) +* (optional) any scenarios from the release tests should be bonsidered to kick off manually + +*nonfunctional* + +* hosted in a container +* simple web ui for kicking off manual scenarios +* stateless +* can be scaled horizontally + +## out of scope + +* Audit testing \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Contracts/ScenarioInfo.cs b/tools/testing-tool/src/TestingTool.Contracts/ScenarioInfo.cs new file mode 100644 index 0000000000..0133829b1c --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Contracts/ScenarioInfo.cs @@ -0,0 +1,32 @@ +namespace TestingTool.Contracts; + +/// +/// Describes a scenario that can generate error load against ServiceControl. +/// Returned by GET /api/scenarios and rendered in the web UI. +/// +public sealed class ScenarioInfo +{ + /// The stable, url-safe scenario name used in API paths. + public required string Name { get; init; } + + /// A short human-readable description of the failure shape this scenario produces. + public required string Description { get; init; } + + /// Human-readable category for grouping in the UI (e.g. "Outage", "Poison", "Noise"). + public required string Category { get; init; } + + /// Whether the scenario is currently emitting load. + public bool Running { get; init; } + + /// Current target rate in messages/second (0 if idle). + public double CurrentRate { get; init; } + + /// Errors emitted by this scenario since process start. + public long ErrorsSent { get; init; } + + /// Default recommended rate in messages/second. + public double DefaultRate { get; init; } + + /// Optional cooldown duration between failure bursts (ISO 8601 duration, null = continuous). + public string? Cooldown { get; init; } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Contracts/StartScenarioRequest.cs b/tools/testing-tool/src/TestingTool.Contracts/StartScenarioRequest.cs new file mode 100644 index 0000000000..77dd3b91a5 --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Contracts/StartScenarioRequest.cs @@ -0,0 +1,14 @@ +namespace TestingTool.Contracts; + +/// +/// Request body for POST /api/scenarios/{name}/start. +/// All fields are optional; defaults are taken from the scenario definition. +/// +public sealed class StartScenarioRequest +{ + /// Target emission rate in messages/second. Defaults to the scenario's . + public double? Rate { get; init; } + + /// Optional auto-stop duration in seconds. Null/0 = run until explicitly stopped. + public double? DurationSeconds { get; init; } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Contracts/TestingTool.Contracts.csproj b/tools/testing-tool/src/TestingTool.Contracts/TestingTool.Contracts.csproj new file mode 100644 index 0000000000..7d5db420dc --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Contracts/TestingTool.Contracts.csproj @@ -0,0 +1,5 @@ + + + + + diff --git a/tools/testing-tool/src/TestingTool.Contracts/TestingToolStatus.cs b/tools/testing-tool/src/TestingTool.Contracts/TestingToolStatus.cs new file mode 100644 index 0000000000..6d940baaca --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Contracts/TestingToolStatus.cs @@ -0,0 +1,41 @@ +namespace TestingTool.Contracts; + +/// +/// Live snapshot of testing-tool counters, returned by GET /api/status and used as the +/// container liveness/readiness probe target. +/// +public sealed class TestingToolStatus +{ + /// Whether the tool is ready to accept scenario control requests. + public bool Ready { get; init; } + + /// Total errors emitted since process start (handler + bypass paths). + public long ErrorsSent { get; init; } + + /// Total error groups replayed since process start. + public long ErrorsReplayed { get; init; } + + /// Total ServiceControl searches executed since process start. + public long SearchesExecuted { get; init; } + + /// The shard id this replica owns, used for disjoint scenario slices when scaled out. + public string? ShardId { get; init; } + + /// Number of scenarios currently running. + public int ActiveScenarios { get; init; } + + /// Aggregate current emission rate across all running scenarios (msgs/sec). + public double CurrentRate { get; init; } + + /// Whether the background replay job is enabled. + public bool ReplayEnabled { get; init; } + + /// Whether the background search job is enabled. + public bool SearchEnabled { get; init; } + + /// ServiceControl API URL the tool is targeting. + public string? ServiceControlUrl { get; init; } + + /// Uptime since process start (formatted string). + public string? Uptime { get; init; } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Scenarios/DeserializationScenario.cs b/tools/testing-tool/src/TestingTool.Scenarios/DeserializationScenario.cs new file mode 100644 index 0000000000..c89116d9e1 --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Scenarios/DeserializationScenario.cs @@ -0,0 +1,24 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Deserialization failure scenario: messages with malformed payloads fail during deserialization. +/// Grouped by message type, simulating a deployment that shipped an incompatible schema version. +/// All messages fail (100%) since deserialization happens before the handler runs. +/// +public sealed class DeserializationScenario(string shardId) : ScenarioBase(shardId) +{ + public override string Name => "deserialization-failure"; + public override string Description => "Messages fail during deserialization due to incompatible schema. Grouped by message type — simulates a bad deployment."; + public override string Category => "Deserialization"; + public override double DefaultRate => 20; + + public override bool ShouldFail(string messageId) => true; + + public override Exception CreateException() => + CreateException( + "NServiceBus.MessageDeserializationException", + "Unable to deserialize message: unexpected token at position 0. Expected a valid message envelope.", + "deser:SampleCommand:v2-incompatible"); +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Scenarios/IScenario.cs b/tools/testing-tool/src/TestingTool.Scenarios/IScenario.cs new file mode 100644 index 0000000000..33b869d40c --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Scenarios/IScenario.cs @@ -0,0 +1,35 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Defines a real-world error scenario that produces grouped failures against ServiceControl. +/// Each implementation models a distinct failure shape (outage, poison message, timeout spike, etc.) +/// so that ServiceControl naturally groups the resulting errors. +/// +public interface IScenario +{ + /// The stable scenario name, matching . + string Name { get; } + + /// Human-readable description of the failure shape. + string Description { get; } + + /// UI grouping category (e.g. "Outage", "Poison", "Noise"). + string Category { get; } + + /// Default recommended emission rate in messages/second. + double DefaultRate { get; } + + /// Activity source used to emit OpenTelemetry traces for this scenario's work. + ActivitySource ActivitySource { get; } + + /// Determines whether a given message should fail. Must be deterministic per shard. + bool ShouldFail(string messageId); + + /// Creates the grouped, typed exception emitted when returns true. + Exception CreateException(); + + /// Burst shape: optional cooldown between failure bursts. + TimeSpan? Cooldown { get; } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Scenarios/LoadMessage.cs b/tools/testing-tool/src/TestingTool.Scenarios/LoadMessage.cs new file mode 100644 index 0000000000..e72db79cea --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Scenarios/LoadMessage.cs @@ -0,0 +1,15 @@ +namespace TestingTool.Scenarios; + +/// +/// A load-generation message processed by FailingMessageHandler. The ScenarioName +/// header (set by the sender) determines which scenario's failure logic applies. The payload +/// is intentionally minimal — the testing tool generates volume, not realistic business data. +/// +public class LoadMessage +{ + /// Monotonically increasing sequence number within a generation run. + public long Sequence { get; set; } + + /// Random payload bytes to give messages some size variety. + public byte[]? Payload { get; set; } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Scenarios/PoisonMessageScenario.cs b/tools/testing-tool/src/TestingTool.Scenarios/PoisonMessageScenario.cs new file mode 100644 index 0000000000..ec92d89a6a --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Scenarios/PoisonMessageScenario.cs @@ -0,0 +1,27 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Poison message scenario: a deterministic subset of messages (by hash) always fails. +/// These messages will never succeed on retry, creating a persistent error group that +/// exercises ServiceControl's retry-storm handling and message archival flows. +/// +public sealed class PoisonMessageScenario(string shardId) : ScenarioBase(shardId) +{ + public override string Name => "poison-message"; + public override string Description => "Deterministic poison messages that always fail on retry. Exercises retry-storm and archival handling."; + public override string Category => "Poison"; + public override double DefaultRate => 5; + + // ~15% of messages are poison (always fail). + private const double PoisonRatio = 0.15; + + public override bool ShouldFail(string messageId) => Hash(messageId) < PoisonRatio; + + public override Exception CreateException() => + CreateException( + "System.InvalidOperationException", + "The message payload is corrupt and cannot be processed. This message will always fail.", + "poison:invalid-payload"); +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Scenarios/RandomBackgroundNoiseScenario.cs b/tools/testing-tool/src/TestingTool.Scenarios/RandomBackgroundNoiseScenario.cs new file mode 100644 index 0000000000..5bb6024592 --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Scenarios/RandomBackgroundNoiseScenario.cs @@ -0,0 +1,36 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Low baseline error rate that is always on. Produces a small, steady stream of random +/// exceptions to simulate real-world background noise. Used to keep ServiceControl's +/// ingestion pipeline warm between explicit scenario runs. +/// +public sealed class RandomBackgroundNoiseScenario(string shardId) : ScenarioBase(shardId) +{ + public override string Name => "background-noise"; + public override string Description => "Always-on low baseline error rate (≈3%). Simulates real-world background noise to keep ingestion warm."; + public override string Category => "Noise"; + public override double DefaultRate => 15; + + private const double NoiseRate = 0.03; + + public override bool ShouldFail(string messageId) => Hash(messageId) < NoiseRate; + + public override Exception CreateException() + { + // Rotate through a few exception types so we get a handful of small groups. + var types = new[] + { + ("System.NullReferenceException", "Object reference not set to an instance of an object.", "noise:nre"), + ("System.IndexOutOfRangeException", "Index was outside the bounds of the array.", "noise:oor"), + ("System.FormatException", "The input string was not in a correct format.", "noise:fmt"), + ("System.InvalidCastException", "Unable to cast object of type 'System.String' to type 'System.Int32'.", "noise:cast"), + }; + + var idx = (int)(Hash(Guid.NewGuid().ToString("N")) * types.Length) % types.Length; + var (type, msg, group) = types[idx]; + return CreateException(type, msg, group); + } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Scenarios/ScenarioBase.cs b/tools/testing-tool/src/TestingTool.Scenarios/ScenarioBase.cs new file mode 100644 index 0000000000..02eaf7fc9c --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Scenarios/ScenarioBase.cs @@ -0,0 +1,48 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Base class providing common functionality for scenarios: deterministic hashing for per-shard +/// failure decisions, activity source management, and exception tagging. +/// +public abstract class ScenarioBase : IScenario +{ + private readonly string _shardId; + + protected ScenarioBase(string shardId) + { + _shardId = shardId; + ActivitySource = new ActivitySource($"testing-tool.{Name}"); + } + + public abstract string Name { get; } + public abstract string Description { get; } + public abstract string Category { get; } + public virtual double DefaultRate => 10; + public ActivitySource ActivitySource { get; } + public virtual TimeSpan? Cooldown => null; + + public abstract bool ShouldFail(string messageId); + public abstract Exception CreateException(); + + /// Deterministic hash of a message id + shard id, returning a value in [0, 1). + protected double Hash(string messageId) + { + var combined = $"{_shardId}:{messageId}"; + // Simple FNV-1a hash — no extra NuGet dependency required. + uint hash = 2166136261u; + foreach (var b in System.Text.Encoding.UTF8.GetBytes(combined)) + { + hash ^= b; + hash *= 16777619u; + } + return hash / (double)uint.MaxValue; + } + + /// Creates an exception with correlation tags that ServiceControl uses for grouping. + protected static Exception CreateException(string type, string message, string correlationGroup) + { + return new ScenarioException(type, message, correlationGroup); + } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Scenarios/ScenarioException.cs b/tools/testing-tool/src/TestingTool.Scenarios/ScenarioException.cs new file mode 100644 index 0000000000..00b80b15fd --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Scenarios/ScenarioException.cs @@ -0,0 +1,17 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Exception carrying correlation metadata so ServiceControl groups errors by exception type +/// and scenario-defined correlation group rather than by individual message. +/// +public sealed class ScenarioException(string exceptionType, string message, string correlationGroup) + : Exception(message) +{ + public string ExceptionType { get; } = exceptionType; + public string CorrelationGroup { get; } = correlationGroup; + + public override string ToString() => + $"{ExceptionType}: {Message} [group: {CorrelationGroup}]"; +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Scenarios/TestingTool.Scenarios.csproj b/tools/testing-tool/src/TestingTool.Scenarios/TestingTool.Scenarios.csproj new file mode 100644 index 0000000000..5d8d3109b8 --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Scenarios/TestingTool.Scenarios.csproj @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/tools/testing-tool/src/TestingTool.Scenarios/ThirdPartyOutageScenario.cs b/tools/testing-tool/src/TestingTool.Scenarios/ThirdPartyOutageScenario.cs new file mode 100644 index 0000000000..537a22c418 --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Scenarios/ThirdPartyOutageScenario.cs @@ -0,0 +1,35 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Simulates a third-party service outage: 100% of messages fail for a burst period, then recover. +/// All failures share the same downstream host header so ServiceControl groups them as one +/// "third-party outage" error group. After the burst, messages succeed (cooldown), then the +/// cycle repeats — modelling a flaky external dependency. +/// +public sealed class ThirdPartyOutageScenario(string shardId) : ScenarioBase(shardId) +{ + public override string Name => "third-party-outage"; + public override string Description => "Simulates a third-party service outage: 100% fail during burst, then recovers. Errors grouped by downstream host."; + public override string Category => "Outage"; + public override double DefaultRate => 50; + public override TimeSpan? Cooldown => TimeSpan.FromSeconds(30); + + // Burst window: fail for 20s, then cooldown 30s, repeat. + private static readonly TimeSpan BurstWindow = TimeSpan.FromSeconds(20); + private static readonly TimeSpan CyclePeriod = BurstWindow + TimeSpan.FromSeconds(30); + + public override bool ShouldFail(string messageId) + { + var now = DateTimeOffset.UtcNow; + var phase = now.Ticks % CyclePeriod.Ticks; + return phase < BurstWindow.Ticks; + } + + public override Exception CreateException() => + CreateException( + "System.Net.Http.HttpRequestException", + "The third-party service at https://api.downstream.example.com did not respond within the timeout period.", + "downstream:api.downstream.example.com"); +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Scenarios/TimeoutSpikeScenario.cs b/tools/testing-tool/src/TestingTool.Scenarios/TimeoutSpikeScenario.cs new file mode 100644 index 0000000000..a38e062db0 --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Scenarios/TimeoutSpikeScenario.cs @@ -0,0 +1,41 @@ +using System.Diagnostics; + +namespace TestingTool.Scenarios; + +/// +/// Intermittent timeout spikes: a configurable percentage of messages fail with +/// , correlated by batch id so ServiceControl groups them +/// into timeout-related error groups. The failure rate oscillates to simulate periodic spikes. +/// +public sealed class TimeoutSpikeScenario(string shardId) : ScenarioBase(shardId) +{ + public override string Name => "timeout-spike"; + public override string Description => "Intermittent timeout exceptions with correlated batch ids. Failure rate oscillates to simulate spikes."; + public override string Category => "Timeout"; + public override double DefaultRate => 30; + + // Spike every ~60s, lasting ~15s at elevated rate. + private static readonly TimeSpan SpikeWindow = TimeSpan.FromSeconds(15); + private static readonly TimeSpan SpikeCycle = TimeSpan.FromSeconds(60); + + public override bool ShouldFail(string messageId) + { + var now = DateTimeOffset.UtcNow; + var phase = now.Ticks % SpikeCycle.Ticks; + var inSpike = phase < SpikeWindow.Ticks; + + // Base 10% fail rate, spikes to ~70% during spike window. + var threshold = inSpike ? 0.70 : 0.10; + return Hash(messageId) < threshold; + } + + public override Exception CreateException() + { + // Correlate by 5-minute bucket so timeouts cluster into time-based groups. + var bucket = DateTimeOffset.UtcNow.ToUnixTimeSeconds() / 300; + return CreateException( + "System.TimeoutException", + "The operation has timed out waiting for a response from the downstream service.", + $"timeout-batch:{bucket}"); + } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/FailingMessageHandler.cs b/tools/testing-tool/src/TestingTool/FailingMessageHandler.cs new file mode 100644 index 0000000000..333fe68bd1 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/FailingMessageHandler.cs @@ -0,0 +1,44 @@ +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using NServiceBus; +using TestingTool.Scenarios; + +namespace TestingTool; + +/// +/// Handles by delegating to the active scenario. When the scenario's +/// ShouldFail returns true, the handler throws — NServiceBus routes the failed message to +/// the configured error queue (ServiceControl). The ScenarioName header selects the scenario. +/// +public sealed class FailingMessageHandler(IScenarioRegistry registry, ILogger logger) + : IHandleMessages +{ + public Task Handle(LoadMessage message, IMessageHandlerContext context) + { + var scenarioName = context.MessageHeaders.GetValueOrDefault("TestingTool.Scenario") ?? "unknown"; + var scenario = registry.Get(scenarioName); + + if (scenario is null) + { + logger.LogDebug("No scenario '{Scenario}' registered — message {Seq} succeeds", scenarioName, message.Sequence); + return Task.CompletedTask; + } + + using var activity = scenario.ActivitySource.StartActivity("handle-load"); + activity?.SetTag("scenario", scenario.Name); + activity?.SetTag("message.sequence", message.Sequence); + activity?.SetTag("message.id", context.MessageId); + + if (scenario.ShouldFail(context.MessageId)) + { + var ex = scenario.CreateException(); + activity?.SetStatus(ActivityStatusCode.Error); + activity?.SetTag("exception.type", (ex as ScenarioException)?.ExceptionType ?? ex.GetType().Name); + activity?.SetTag("exception.group", (ex as ScenarioException)?.CorrelationGroup); + throw ex; + } + + activity?.SetTag("result", "success"); + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/IScenarioRegistry.cs b/tools/testing-tool/src/TestingTool/IScenarioRegistry.cs new file mode 100644 index 0000000000..ba3c52bb75 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/IScenarioRegistry.cs @@ -0,0 +1,12 @@ +using TestingTool.Scenarios; + +namespace TestingTool; + +/// +/// Registry of all available scenarios, keyed by name. Registered at startup via DI. +/// +public interface IScenarioRegistry +{ + IScenario? Get(string name); + IReadOnlyList All { get; } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/NServiceBusSetup.cs b/tools/testing-tool/src/TestingTool/NServiceBusSetup.cs new file mode 100644 index 0000000000..af584f70ca --- /dev/null +++ b/tools/testing-tool/src/TestingTool/NServiceBusSetup.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Options; +using NServiceBus; + +namespace TestingTool; + +/// +/// Extension method to configure the NServiceBus endpoint using the NServiceBus 10 +/// AddNServiceBusEndpoint DI-integrated approach. The endpoint lifecycle is managed by +/// the ASP.NET Core host — no manual Endpoint.Start/Endpoint.Stop needed. +/// +public static class NServiceBusEndpointExtensions +{ + /// + /// Registers the NServiceBus load-generation endpoint. Uses the Learning transport by default + /// (sufficient for local/single-host testing); swap for a real transport when targeting a + /// distributed ServiceControl deployment. Failed messages are routed to the ServiceControl + /// error queue. + /// + public static IServiceCollection AddTestingToolEndpoint(this IServiceCollection services, TestingToolOptions options) + { + var config = new EndpointConfiguration("TestingTool.Load"); + + // Learning transport — zero-config, single-host. For multi-container deployments, + // replace with a real transport matching the ServiceControl instance under test. + // Learning transport — routing to this endpoint is handled by SendOptions.RouteToThisEndpoint(). + config.UseTransport(); + + // Route failures to the ServiceControl error queue. + config.SendFailedMessagesTo(options.ErrorQueueName); + + // Simplified serializer; the testing tool generates volume, not complex payloads. + config.UseSerialization(); + + // Disable immediate retries to make error groups cleaner; deferred retries are handled + // by ServiceControl's retry mechanism. + var recoverability = config.Recoverability(); + recoverability.Immediate(im => im.NumberOfRetries(0)); + recoverability.Delayed(d => d.NumberOfRetries(0)); + + config.EnableInstallers(); + + services.AddNServiceBusEndpoint(config); + return services; + } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/Program.cs b/tools/testing-tool/src/TestingTool/Program.cs new file mode 100644 index 0000000000..86243b02e2 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/Program.cs @@ -0,0 +1,132 @@ +using System.Diagnostics.Metrics; +using TestingTool; +using TestingTool.Contracts; +using TestingTool.Scenarios; + +// --- Configuration --- + +var builder = WebApplication.CreateBuilder(args); + +builder.Services.Configure(builder.Configuration.GetSection("TestingTool")); + +var options = builder.Configuration.GetSection("TestingTool").Get() ?? new TestingToolOptions(); +var shardId = ShardIdResolver.Resolve(); + +// --- OpenTelemetry (Phase 1) --- + +var meter = TelemetrySetup.CreateMeter(); +builder.Services.AddTestingToolTelemetry(meter); +builder.Services.AddSingleton(meter); + +// --- NServiceBus endpoint (Phase 2) --- + +builder.Services.AddTestingToolEndpoint(options); + +// --- Scenarios (Phase 3) --- + +builder.Services.AddSingleton(_ => new ThirdPartyOutageScenario(shardId)); +builder.Services.AddSingleton(_ => new TimeoutSpikeScenario(shardId)); +builder.Services.AddSingleton(_ => new PoisonMessageScenario(shardId)); +builder.Services.AddSingleton(_ => new DeserializationScenario(shardId)); +builder.Services.AddSingleton(_ => new RandomBackgroundNoiseScenario(shardId)); + +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +// --- Background jobs (Phase 4) --- + +builder.Services.AddHttpClient((sp, client) => +{ + var opts = sp.GetRequiredService>().Value; + client.BaseAddress = new Uri(opts.ServiceControlApiUrl); + client.Timeout = TimeSpan.FromSeconds(30); +}); +builder.Services.AddHostedService(); +builder.Services.AddHostedService(); + +// --- Application pipeline --- + +var app = builder.Build(); +app.UseStaticFiles(); +app.UseOpenTelemetryPrometheusScrapingEndpoint("/metrics"); + +var metrics = app.Services.GetRequiredService(); +var scClient = app.Services.GetRequiredService(); +var startedAt = DateTimeOffset.UtcNow; + +// Auto-start background noise after the endpoint is ready. +app.Lifetime.ApplicationStarted.Register(() => +{ + if (options.AutoStartBackgroundNoise) + { + var runner = app.Services.GetRequiredService(); + runner.TryStart("background-noise", null, null, out _); + } +}); + +// Graceful shutdown: stop all scenarios. +app.Lifetime.ApplicationStopping.Register(() => +{ + var runner = app.Services.GetRequiredService(); + runner.StopAll(); +}); + +// --- Health endpoints (Phase 6) --- +// Lightweight probe targets for Kubernetes liveness/readiness and docker-compose health checks. +// The chiseled runtime image has no shell/curl, so HTTP probes are used instead of exec probes. + +app.MapGet("/health/live", () => Results.Ok(new { status = "alive", shardId })); + +app.MapGet("/health/ready", () => Results.Ok(new { status = "ready", shardId })); + +// --- API endpoints (Phase 5) --- + +app.MapGet("/api/scenarios", (ScenarioRunner runner) => Results.Ok(runner.GetSnapshot())); + +app.MapGet("/api/status", () => Results.Ok(new TestingToolStatus +{ + Ready = true, + ErrorsSent = metrics.TotalErrorsSent, + ErrorsReplayed = metrics.TotalErrorsReplayed, + SearchesExecuted = metrics.TotalSearches, + ShardId = shardId, + ActiveScenarios = metrics.ActiveScenarios, + CurrentRate = Math.Round(metrics.CurrentRate, 1), + ReplayEnabled = options.ReplayEnabled, + SearchEnabled = options.SearchEnabled, + ServiceControlUrl = scClient.BaseUrl, + Uptime = (DateTimeOffset.UtcNow - startedAt).ToString(@"h\h\ m\m\ s\s") +})); + +app.MapPost("/api/scenarios/{name}/start", (string name, StartScenarioRequest? request, ScenarioRunner runner) => +{ + var duration = request?.DurationSeconds is { } secs and > 0 + ? TimeSpan.FromSeconds(secs) + : (TimeSpan?)null; + + if (!runner.TryStart(name, request?.Rate, duration, out var error)) + return Results.BadRequest(new { error }); + + var snapshot = runner.GetSnapshot() + .First(s => s.Name.Equals(name, StringComparison.OrdinalIgnoreCase)); + return Results.Ok(snapshot); +}); + +app.MapPost("/api/scenarios/{name}/stop", (string name, ScenarioRunner runner) => +{ + if (!runner.TryStop(name)) + return Results.BadRequest(new { error = $"Scenario '{name}' is not running" }); + + return Results.Ok(new { stopped = name }); +}); + +app.MapPost("/api/scenarios/stop-all", (ScenarioRunner runner) => +{ + runner.StopAll(); + return Results.Ok(new { stopped = "all" }); +}); + +app.MapFallbackToFile("index.html"); + +app.Run(); \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/Properties/launchSettings.json b/tools/testing-tool/src/TestingTool/Properties/launchSettings.json new file mode 100644 index 0000000000..842b5f5920 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/Properties/launchSettings.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:5290", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:7175;http://localhost:5290", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/tools/testing-tool/src/TestingTool/ReplayService.cs b/tools/testing-tool/src/TestingTool/ReplayService.cs new file mode 100644 index 0000000000..acb5080a50 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/ReplayService.cs @@ -0,0 +1,71 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using Microsoft.Extensions.Options; + +namespace TestingTool; + +/// +/// Background service that periodically fetches error groups from ServiceControl and triggers +/// retry/replay. Replayed messages should then succeed (simulating a fix being applied), which +/// exercises ServiceControl's retry pipeline. Gated by configuration so replicas can opt in/out. +/// +public sealed class ReplayService( + ServiceControlClient sc, + TestingToolMetrics metrics, + IOptions options, + Meter meter, + ILogger logger) : BackgroundService +{ + private readonly Counter _replayCounter = meter.CreateCounter("errors_replayed_total"); + private readonly ActivitySource _activitySource = new("testing-tool.replay"); + private long _totalReplayed; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (!options.Value.ReplayEnabled) + { + logger.LogInformation("Replay job disabled by configuration"); + return; + } + + var interval = options.Value.ReplayInterval; + logger.LogInformation("Replay job started — interval {Interval}", interval); + + using var timer = new PeriodicTimer(interval); + while (await timer.WaitForNextTickAsync(stoppingToken)) + { + try + { + using var activity = _activitySource.StartActivity("replay-cycle"); + var groups = await sc.GetErrorGroupsAsync(stoppingToken); + + if (groups.Count == 0) + { + logger.LogDebug("No error groups to replay"); + continue; + } + + foreach (var group in groups) + { + if (group.Count < options.Value.ReplayMinGroupSize) + continue; + + var success = await sc.ReplayGroupAsync(group.Id, stoppingToken); + if (success) + { + Interlocked.Add(ref _totalReplayed, group.Count); + metrics.AddErrorsReplayed(group.Count); + _replayCounter.Add(group.Count, new KeyValuePair("group", group.Title)); + logger.LogInformation("Replayed group {Title} ({Count} messages)", group.Title, group.Count); + } + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Replay cycle failed"); + } + } + } + + public long TotalReplayed => Interlocked.Read(ref _totalReplayed); +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/ScenarioRegistry.cs b/tools/testing-tool/src/TestingTool/ScenarioRegistry.cs new file mode 100644 index 0000000000..7080bd4fc1 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/ScenarioRegistry.cs @@ -0,0 +1,17 @@ +using TestingTool.Scenarios; + +namespace TestingTool; + +/// +/// Default scenario registry backed by a dictionary built from DI-registered instances. +/// +public sealed class ScenarioRegistry(IEnumerable scenarios) : IScenarioRegistry +{ + private readonly Dictionary _byName = + scenarios.ToDictionary(s => s.Name, StringComparer.OrdinalIgnoreCase); + + public IScenario? Get(string name) => + _byName.TryGetValue(name, out var s) ? s : null; + + public IReadOnlyList All { get; } = scenarios.ToArray().AsReadOnly(); +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/ScenarioRunner.cs b/tools/testing-tool/src/TestingTool/ScenarioRunner.cs new file mode 100644 index 0000000000..9d37e13d31 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/ScenarioRunner.cs @@ -0,0 +1,222 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using Microsoft.Extensions.Logging; +using NServiceBus; +using TestingTool.Contracts; +using TestingTool.Scenarios; + +namespace TestingTool; + +/// +/// Manages the lifecycle of all scenarios: start, stop, rate control, auto-stop on duration, +/// and per-scenario error counting. Load is generated by sending to the +/// NServiceBus endpoint at a controlled rate; the throws based +/// on the scenario's failure logic, routing failures to the ServiceControl error queue. +/// +public sealed class ScenarioRunner +{ + private readonly IMessageSession _session; + private readonly IScenarioRegistry _registry; + private readonly TestingToolMetrics _metrics; + private readonly Meter _meter; + private readonly ILogger _logger; + + // Per-scenario runtime state. + private readonly ConcurrentDictionary _running = new(StringComparer.OrdinalIgnoreCase); + + public ScenarioRunner(IMessageSession session, IScenarioRegistry registry, TestingToolMetrics metrics, Meter meter, ILogger logger) + { + _session = session; + _registry = registry; + _metrics = metrics; + _meter = meter; + _logger = logger; + } + + /// Starts a scenario with an optional rate override and auto-stop duration. + public bool TryStart(string scenarioName, double? rate, TimeSpan? duration, out string? error) + { + var scenario = _registry.Get(scenarioName); + if (scenario is null) + { + error = $"Unknown scenario '{scenarioName}'"; + return false; + } + + if (_running.ContainsKey(scenarioName)) + { + error = $"Scenario '{scenarioName}' is already running"; + return false; + } + + var targetRate = rate ?? scenario.DefaultRate; + if (targetRate <= 0) + { + error = "Rate must be greater than 0"; + return false; + } + + var runtime = new ScenarioRuntime(scenario, targetRate, duration, _meter, _logger); + if (_running.TryAdd(scenarioName, runtime)) + { + runtime.Start(GenerateLoadAsync); + UpdateAggregateMetrics(); + _logger.LogInformation("Started scenario {Scenario} at {Rate:F1} msg/s{Duration}", + scenarioName, targetRate, duration is null ? "" : $" for {duration.Value}"); + error = null; + return true; + } + + error = $"Scenario '{scenarioName}' is already running"; + return false; + } + + /// Stops a running scenario. + public bool TryStop(string scenarioName) + { + if (_running.TryRemove(scenarioName, out var runtime)) + { + runtime.Stop(); + UpdateAggregateMetrics(); + _logger.LogInformation("Stopped scenario {Scenario} after {Errors} errors", + scenarioName, runtime.ErrorsSent); + return true; + } + return false; + } + + /// Stops all running scenarios (used on shutdown). + public void StopAll() + { + foreach (var name in _running.Keys) + TryStop(name); + } + + public List GetSnapshot() + { + var list = new List(); + foreach (var scenario in _registry.All) + { + _running.TryGetValue(scenario.Name, out var runtime); + list.Add(new ScenarioInfo + { + Name = scenario.Name, + Description = scenario.Description, + Category = scenario.Category, + Running = runtime is not null, + CurrentRate = runtime?.TargetRate ?? 0, + ErrorsSent = runtime?.ErrorsSent ?? 0, + DefaultRate = scenario.DefaultRate, + Cooldown = scenario.Cooldown?.ToString() + }); + } + return list; + } + + private void UpdateAggregateMetrics() + { + _metrics.SetActiveScenarios(_running.Count); + _metrics.SetCurrentRate(_running.Values.Sum(r => r.TargetRate)); + } + + /// The load generation loop: sends LoadMessages at the target rate until cancelled. + private async Task GenerateLoadAsync(ScenarioRuntime runtime, CancellationToken ct) + { + var scenario = runtime.Scenario; + var interval = TimeSpan.FromSeconds(1.0 / runtime.TargetRate); + using var timer = new PeriodicTimer(interval); + long sequence = 0; + + try + { + while (await timer.WaitForNextTickAsync(ct)) + { + var seq = Interlocked.Increment(ref sequence); + var payload = new byte[Random.Shared.Next(64, 512)]; + Random.Shared.NextBytes(payload); + + var sendOptions = new SendOptions(); + sendOptions.RouteToThisEndpoint(); + sendOptions.SetHeader("TestingTool.Scenario", scenario.Name); + + try + { + await _session.Send(new LoadMessage { Sequence = seq, Payload = payload }, sendOptions, ct); + + // If the scenario would fail for this message id, count it as an error sent. + var messageId = $"{scenario.Name}-{seq}-{runtime.StartedAt.Ticks}"; + if (scenario.ShouldFail(messageId)) + { + runtime.IncrementErrors(); + _metrics.AddErrorsSent(1); + runtime.ErrorsCounter.Add(1, new KeyValuePair("scenario", scenario.Name)); + } + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Send failed for scenario {Scenario} seq {Seq}", scenario.Name, seq); + } + } + } + catch (OperationCanceledException) { } + } +} + +/// Per-scenario runtime state: rate, counters, cancellation, and auto-stop timer. +internal sealed class ScenarioRuntime +{ + private readonly Meter _meter; + private readonly ILogger _logger; + private readonly Counter _errorsCounter; + private long _errorsSent; + private CancellationTokenSource? _cts; + private Task? _loop; + + public ScenarioRuntime(IScenario scenario, double rate, TimeSpan? duration, Meter meter, ILogger logger) + { + Scenario = scenario; + TargetRate = rate; + Duration = duration; + StartedAt = DateTimeOffset.UtcNow; + _meter = meter; + _logger = logger; + _errorsCounter = meter.CreateCounter("errors_sent_total"); + } + + public IScenario Scenario { get; } + public double TargetRate { get; } + public TimeSpan? Duration { get; } + public DateTimeOffset StartedAt { get; } + public long ErrorsSent => Interlocked.Read(ref _errorsSent); + public Counter ErrorsCounter => _errorsCounter; + + public long IncrementErrors() => Interlocked.Increment(ref _errorsSent); + + public void Start(Func generate) + { + _cts = Duration is { } d + ? new CancellationTokenSource(d) + : new CancellationTokenSource(); + + _loop = Task.Run(async () => + { + try + { + await generate(this, _cts.Token); + } + catch (OperationCanceledException) { } + catch (Exception ex) + { + _logger.LogError(ex, "Load generation loop for {Scenario} crashed", Scenario.Name); + } + }, _cts.Token); + } + + public void Stop() + { + _cts?.Cancel(); + _cts?.Dispose(); + _cts = null; + } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/SearchService.cs b/tools/testing-tool/src/TestingTool/SearchService.cs new file mode 100644 index 0000000000..8a8c681f4c --- /dev/null +++ b/tools/testing-tool/src/TestingTool/SearchService.cs @@ -0,0 +1,85 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using Microsoft.Extensions.Options; + +namespace TestingTool; + +/// +/// Background service that runs canned full-text-search queries against ServiceControl on a timer. +/// Exercises the RavenDB FTS index under concurrent load and records latency metrics. Gated by +/// configuration so replicas can opt in/out. +/// +public sealed class SearchService( + ServiceControlClient sc, + TestingToolMetrics metrics, + IOptions options, + Meter meter, + ILogger logger) : BackgroundService +{ + private readonly Counter _searchCounter = meter.CreateCounter("searches_executed_total"); + private readonly Histogram _searchLatency = meter.CreateHistogram("search_latency_ms", "ms"); + private readonly ActivitySource _activitySource = new("testing-tool.search"); + private long _totalSearches; + + // Canned queries that exercise different FTS index paths. + private static readonly string[] CannedQueries = + [ + "exception", + "timeout", + "NullReferenceException", + "downstream", + "deserialization", + "poison", + "503", + "retry" + ]; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (!options.Value.SearchEnabled) + { + logger.LogInformation("Search job disabled by configuration"); + return; + } + + var interval = options.Value.SearchInterval; + logger.LogInformation("Search job started — interval {Interval}", interval); + + using var timer = new PeriodicTimer(interval); + while (await timer.WaitForNextTickAsync(stoppingToken)) + { + try + { + using var activity = _activitySource.StartActivity("search-cycle"); + + // Run a few random queries per tick to exercise the FTS index. + var queries = CannedQueries.OrderBy(_ => Random.Shared.Next()).Take(3).ToList(); + foreach (var query in queries) + { + var sw = Stopwatch.StartNew(); + var result = await sc.SearchAsync(query, stoppingToken); + sw.Stop(); + + _searchLatency.Record(sw.Elapsed.TotalMilliseconds, + new KeyValuePair("query", query)); + + Interlocked.Increment(ref _totalSearches); + metrics.AddSearches(1); + _searchCounter.Add(1, new KeyValuePair("query", query)); + + activity?.SetTag($"search.{query}.count", result?.MessageCount); + activity?.SetTag($"search.{query}.latency_ms", sw.Elapsed.TotalMilliseconds); + + logger.LogDebug("Search '{Query}' → {Count} results in {Ms:F1}ms", + query, result?.MessageCount, sw.Elapsed.TotalMilliseconds); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Search cycle failed"); + } + } + } + + public long TotalSearches => Interlocked.Read(ref _totalSearches); +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/ServiceControlClient.cs b/tools/testing-tool/src/TestingTool/ServiceControlClient.cs new file mode 100644 index 0000000000..b221e5e23d --- /dev/null +++ b/tools/testing-tool/src/TestingTool/ServiceControlClient.cs @@ -0,0 +1,74 @@ +using System.Net.Http.Json; +using System.Text.Json.Serialization; + +namespace TestingTool; + +/// +/// Thin HTTP client for the ServiceControl REST API. Used by the background replay and search +/// jobs to interact with the test ServiceControl instance. +/// +public sealed class ServiceControlClient(HttpClient http, ILogger logger) +{ + public string BaseUrl => http.BaseAddress?.ToString() ?? "(not configured)"; + + /// Fetches all error groups from ServiceControl. + public async Task> GetErrorGroupsAsync(CancellationToken ct = default) + { + try + { + var groups = await http.GetFromJsonAsync>("/api/errors/groups", ct); + return groups ?? []; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to fetch error groups from ServiceControl"); + return []; + } + } + + /// Triggers a retry/replay of all messages in an error group. + public async Task ReplayGroupAsync(string groupId, CancellationToken ct = default) + { + try + { + var response = await http.PostAsJsonAsync($"/api/errors/groups/{groupId}/retry", new { }, ct); + return response.IsSuccessStatusCode; + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to replay error group {GroupId}", groupId); + return false; + } + } + + /// Executes a full-text search query against ServiceControl. + public async Task SearchAsync(string query, CancellationToken ct = default) + { + try + { + var response = await http.GetAsync($"/api/errors/search?q={Uri.EscapeDataString(query)}", ct); + if (!response.IsSuccessStatusCode) + return null; + + var body = await response.Content.ReadFromJsonAsync(ct); + return new SearchResult(body?.MessageCount ?? 0); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Search '{Query}' failed", query); + return null; + } + } + + public sealed record ErrorGroup( + [property: JsonPropertyName("id")] string Id, + [property: JsonPropertyName("title")] string Title, + [property: JsonPropertyName("count")] int Count, + [property: JsonPropertyName("exceptionType")] string? ExceptionType, + [property: JsonPropertyName("firstTime")] string? FirstTime, + [property: JsonPropertyName("lastTime")] string? LastTime); + + public sealed record SearchResponse([property: JsonPropertyName("messageCount")] int MessageCount); + + public sealed record SearchResult(int MessageCount); +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/ShardIdResolver.cs b/tools/testing-tool/src/TestingTool/ShardIdResolver.cs new file mode 100644 index 0000000000..1c52d8b1de --- /dev/null +++ b/tools/testing-tool/src/TestingTool/ShardIdResolver.cs @@ -0,0 +1,38 @@ +using System.Text.RegularExpressions; + +namespace TestingTool; + +/// +/// Resolves the shard id for this replica. When scaled horizontally, each replica must own a +/// disjoint slice of the scenario space so that deterministic failure decisions (see +/// ) don't overlap across pods. +/// +/// Resolution order: +/// 1. SHARD_ID environment variable (explicit override — used by docker-compose and +/// manual runs). +/// 2. Ordinal extracted from a Kubernetes StatefulSet hostname (e.g. testing-tool-2 +/// → 2). StatefulSets give stable, ordered pod names so shards are deterministic +/// across restarts. +/// 3. — unique per pod for Deployments, deterministic +/// per host for bare-metal/VM runs. +/// +public static class ShardIdResolver +{ + // Matches a trailing - at the end of a hostname (StatefulSet pod naming convention). + private static readonly Regex StatefulSetOrdinal = new(@"-(\d+)$", RegexOptions.Compiled); + + /// Resolves the shard id for this replica. + public static string Resolve() + { + var explicitId = Environment.GetEnvironmentVariable("SHARD_ID"); + if (!string.IsNullOrWhiteSpace(explicitId)) + return explicitId; + + var hostname = Environment.MachineName; + var match = StatefulSetOrdinal.Match(hostname); + if (match.Success) + return match.Groups[1].Value; + + return hostname; + } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/TelemetrySetup.cs b/tools/testing-tool/src/TestingTool/TelemetrySetup.cs new file mode 100644 index 0000000000..824913c7a6 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/TelemetrySetup.cs @@ -0,0 +1,50 @@ +using System.Diagnostics.Metrics; +using OpenTelemetry; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; + +namespace TestingTool; + +/// +/// Configures OpenTelemetry traces, metrics, and logs for the testing tool. All telemetry is +/// exported via OTLP to the endpoint configured by OTEL_EXPORTER_OTLP_ENDPOINT (set in the +/// docker-compose environment). A Prometheus scraping endpoint is also exposed at /metrics. +/// +public static class TelemetrySetup +{ + public const string ServiceName = "testing-tool"; + public const string MeterName = "testing-tool"; + + // Shared activity source names used across the tool. + public static class Sources + { + public const string Load = "testing-tool.load"; + public const string Replay = "testing-tool.replay"; + public const string Search = "testing-tool.search"; + } + + public static Meter CreateMeter() => new(MeterName, "1.0.0"); + + public static OpenTelemetryBuilder AddTestingToolTelemetry(this IServiceCollection services, Meter meter) + { + return services.AddOpenTelemetry() + .ConfigureResource(r => r.AddService(ServiceName, + serviceInstanceId: Environment.MachineName)) + .WithTracing(t => t + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddSource(Sources.Load) + .AddSource(Sources.Replay) + .AddSource(Sources.Search) + // Also pick up per-scenario activity sources dynamically. + .AddSource("testing-tool.*") + .AddOtlpExporter()) + .WithMetrics(m => m + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddMeter(MeterName) + .AddPrometheusExporter() + .AddOtlpExporter()); + } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/TestingTool.csproj b/tools/testing-tool/src/TestingTool/TestingTool.csproj new file mode 100644 index 0000000000..4d9474bf40 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/TestingTool.csproj @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/TestingToolMetrics.cs b/tools/testing-tool/src/TestingTool/TestingToolMetrics.cs new file mode 100644 index 0000000000..2419865713 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/TestingToolMetrics.cs @@ -0,0 +1,27 @@ +namespace TestingTool; + +/// +/// Shared singleton holding live counters from all subsystems (scenario runner, replay, search). +/// Provides a single read point for the GET /api/status endpoint without needing to resolve +/// individual hosted service instances. +/// +public sealed class TestingToolMetrics +{ + private long _totalErrorsSent; + private long _totalErrorsReplayed; + private long _totalSearches; + private long _activeScenarios; + private double _currentRate; + + public long TotalErrorsSent => Interlocked.Read(ref _totalErrorsSent); + public long TotalErrorsReplayed => Interlocked.Read(ref _totalErrorsReplayed); + public long TotalSearches => Interlocked.Read(ref _totalSearches); + public int ActiveScenarios => (int)Interlocked.Read(ref _activeScenarios); + public double CurrentRate => _currentRate; + + public void AddErrorsSent(long count) => Interlocked.Add(ref _totalErrorsSent, count); + public void AddErrorsReplayed(long count) => Interlocked.Add(ref _totalErrorsReplayed, count); + public void AddSearches(long count) => Interlocked.Add(ref _totalSearches, count); + public void SetActiveScenarios(int count) => Interlocked.Exchange(ref _activeScenarios, count); + public void SetCurrentRate(double rate) => _currentRate = rate; +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/TestingToolOptions.cs b/tools/testing-tool/src/TestingTool/TestingToolOptions.cs new file mode 100644 index 0000000000..f1b25564a2 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/TestingToolOptions.cs @@ -0,0 +1,32 @@ +namespace TestingTool; + +/// +/// Configuration options for the testing tool, bound from the "TestingTool" config section +/// and/or environment variables. +/// +public sealed class TestingToolOptions +{ + /// Base URL of the ServiceControl instance under test (e.g. http://servicecontrol:33333). + public string ServiceControlApiUrl { get; set; } = "http://localhost:33333"; + + /// Whether the background replay job is enabled. + public bool ReplayEnabled { get; set; } = false; + + /// Interval between replay cycles. + public TimeSpan ReplayInterval { get; set; } = TimeSpan.FromMinutes(2); + + /// Minimum number of messages in a group before it is replayed. + public int ReplayMinGroupSize { get; set; } = 1; + + /// Whether the background search job is enabled. + public bool SearchEnabled { get; set; } = false; + + /// Interval between search cycles. + public TimeSpan SearchInterval { get; set; } = TimeSpan.FromMinutes(1); + + /// NServiceBus error queue name that ServiceControl monitors. + public string ErrorQueueName { get; set; } = "error"; + + /// Whether to start the background-noise scenario automatically on startup. + public bool AutoStartBackgroundNoise { get; set; } = false; +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/appsettings.Development.json b/tools/testing-tool/src/TestingTool/appsettings.Development.json new file mode 100644 index 0000000000..0c208ae918 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/tools/testing-tool/src/TestingTool/appsettings.json b/tools/testing-tool/src/TestingTool/appsettings.json new file mode 100644 index 0000000000..58f95a9e7e --- /dev/null +++ b/tools/testing-tool/src/TestingTool/appsettings.json @@ -0,0 +1,20 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*", + "TestingTool": { + "ServiceControlApiUrl": "http://localhost:33333", + "ReplayEnabled": false, + "ReplayInterval": "00:02:00", + "ReplayMinGroupSize": 1, + "SearchEnabled": false, + "SearchInterval": "00:01:00", + "ErrorQueueName": "error", + "AutoStartBackgroundNoise": false + }, + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:4317" +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/wwwroot/index.html b/tools/testing-tool/src/TestingTool/wwwroot/index.html new file mode 100644 index 0000000000..2d62433a10 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/wwwroot/index.html @@ -0,0 +1,428 @@ + + + + + + + ServiceControl Testing Tool + + + + +
+
+

🎯 ServiceControl Testing Tool

+ +
+ + +
+
+
Errors Sent
+
—
+
—
+
+
+
Errors Replayed
+
—
+
+
+
Searches
+
—
+
+
+
Active Scenarios
+
—
+
+
+
Uptime
+
—
+
+
+ + +
+ Loading… +
+ + +
+

Scenarios

+ +
+
+
Loading scenarios…
+
+ + +
+ +
+ + + + + \ No newline at end of file From 85cf7e19514058116245306ed890e346fb6c17df Mon Sep 17 00:00:00 2001 From: Rhys Bevilaqua Date: Tue, 25 Aug 2026 14:18:00 +0800 Subject: [PATCH 02/13] add aspire apphost to load testing tool --- .github/workflows/testing-tool-ci.yml | 3 + tools/testing-tool/Directory.Packages.props | 5 + tools/testing-tool/README.md | 68 +++++- tools/testing-tool/TestingTool.slnx | 1 + tools/testing-tool/aspire/AppHost.cs | 68 ++++++ .../testing-tool/aspire/Directory.Build.props | 15 ++ .../aspire/Directory.Packages.props | 13 ++ tools/testing-tool/aspire/aspire.config.json | 5 + .../requirements-test-tool-plan.md | 21 +- .../src/TestingTool.Contracts/BypassStatus.cs | 23 +++ .../StartBypassRequest.cs | 18 ++ .../TestingToolStatus.cs | 3 + .../src/TestingTool.SmokeTests/SmokeTest.cs | 124 +++++++++++ .../TestingTool.SmokeTests.csproj | 24 +++ .../src/TestingTool/DirectErrorQueueWriter.cs | 195 ++++++++++++++++++ tools/testing-tool/src/TestingTool/Program.cs | 67 +++++- .../src/TestingTool/ReleaseTestScenarios.cs | 66 ++++++ .../src/TestingTool/TelemetrySetup.cs | 7 +- .../src/TestingTool/TestingToolMetrics.cs | 3 + .../src/TestingTool/wwwroot/index.html | 87 +++++++- 20 files changed, 797 insertions(+), 19 deletions(-) create mode 100644 tools/testing-tool/aspire/AppHost.cs create mode 100644 tools/testing-tool/aspire/Directory.Build.props create mode 100644 tools/testing-tool/aspire/Directory.Packages.props create mode 100644 tools/testing-tool/aspire/aspire.config.json create mode 100644 tools/testing-tool/src/TestingTool.Contracts/BypassStatus.cs create mode 100644 tools/testing-tool/src/TestingTool.Contracts/StartBypassRequest.cs create mode 100644 tools/testing-tool/src/TestingTool.SmokeTests/SmokeTest.cs create mode 100644 tools/testing-tool/src/TestingTool.SmokeTests/TestingTool.SmokeTests.csproj create mode 100644 tools/testing-tool/src/TestingTool/DirectErrorQueueWriter.cs create mode 100644 tools/testing-tool/src/TestingTool/ReleaseTestScenarios.cs diff --git a/.github/workflows/testing-tool-ci.yml b/.github/workflows/testing-tool-ci.yml index 045f6a6454..3e8f1af917 100644 --- a/.github/workflows/testing-tool-ci.yml +++ b/.github/workflows/testing-tool-ci.yml @@ -39,6 +39,9 @@ jobs: - name: Build run: dotnet build tools/testing-tool/TestingTool.slnx --configuration Release + - name: Build Aspire AppHost + run: dotnet build tools/testing-tool/aspire/AppHost.cs --configuration Release + - name: Build container image uses: docker/build-push-action@v7.2.0 with: diff --git a/tools/testing-tool/Directory.Packages.props b/tools/testing-tool/Directory.Packages.props index 3c87c42989..94aab87214 100644 --- a/tools/testing-tool/Directory.Packages.props +++ b/tools/testing-tool/Directory.Packages.props @@ -19,6 +19,11 @@ + + + + + \ No newline at end of file diff --git a/tools/testing-tool/README.md b/tools/testing-tool/README.md index 0ae7f0f6ef..44535db75a 100644 --- a/tools/testing-tool/README.md +++ b/tools/testing-tool/README.md @@ -8,9 +8,10 @@ See [requirements-test-tool-plan.md](./requirements-test-tool-plan.md) for the f ## Status -Phases 0–6 complete (project bootstrap, OTel foundation, NServiceBus error path, scenarios, -background jobs, web UI, containerization & scaling). Remaining: direct error-queue bypass writer -(Phase 2 optional item), Grafana dashboard + smoke test (Phase 7). +All phases complete (0–7): project bootstrap, OTel foundation (traces + metrics + logs), +NServiceBus error path (handler + direct error-queue bypass writer), scenarios, background jobs, +web UI, containerization & scaling, Aspire AppHost, and smoke tests. The only unplanned item is +a prebuilt Grafana dashboard (otel traces/metrics already flow to the collector). ## What it does @@ -30,7 +31,24 @@ Two background jobs (gated by config) run on timers: - **Replay** — fetches error groups from ServiceControl and triggers retry - **Search** — runs canned FTS queries to exercise the RavenDB search index -All telemetry is exported via OTLP (traces + metrics) and a Prometheus `/metrics` endpoint. +All telemetry is exported via OTLP (traces + metrics + logs) and a Prometheus `/metrics` endpoint. + +### Direct error-queue bypass writer + +In addition to the handler path, the tool can write failed-message envelopes directly to the +ServiceControl error queue, bypassing the handler entirely for high-throughput error load. +Each message carries standard NServiceBus failure headers (`NServiceBus.ExceptionInfo.*`, +`NServiceBus.FailedQ`) so ServiceControl ingests it as a genuine failed message. Control via: +- `POST /api/bypass/start` — `{ "scenario": "third-party-outage", "rate": 100, "durationSeconds": 60 }` +- `POST /api/bypass/stop` +- `GET /api/bypass/status` + +### Release-test scenario presets + +The tool ships with presets mapped from `docs/testing-scenarios.md` so release-test scenarios can +be kicked off manually by name: +- `GET /api/release-tests` — list all presets +- `POST /api/release-tests/{name}/start` — start a preset (e.g. `retry-message-group`, `ingestion-load`) ## Layout @@ -47,17 +65,21 @@ tools/testing-tool/ wwwroot/index.html # single-page web UI (vanilla JS, no build step) Program.cs # OTel wiring, NServiceBus endpoint, DI, API endpoints ScenarioRunner.cs # start/stop, rate control, per-scenario error counting + DirectErrorQueueWriter.cs # bypass path: writes failed-message envelopes directly to error queue FailingMessageHandler.cs # NServiceBus handler that throws per scenario logic + ReleaseTestScenarios.cs # release-test preset mappings (Phase 5) ServiceControlClient.cs # REST API client (error groups, replay, search) ReplayService.cs # background replay job SearchService.cs # background search job - TelemetrySetup.cs # OTel traces + metrics + OTLP/Prometheus exporters + TelemetrySetup.cs # OTel traces + metrics + logs + OTLP/Prometheus exporters NServiceBusSetup.cs # endpoint config (Learning transport, error queue routing) TestingToolOptions.cs # config (SC URL, replay/search intervals, error queue name) TestingToolMetrics.cs # shared live counters for /api/status ShardIdResolver.cs # shard id from env var, StatefulSet ordinal, or hostname TestingTool.Scenarios/ # IScenario contract + 5 scenario implementations - TestingTool.Contracts/ # shared DTOs (ScenarioInfo, TestingToolStatus, StartScenarioRequest) + TestingTool.Contracts/ # shared DTOs (ScenarioInfo, TestingToolStatus, BypassStatus, etc.) + TestingTool.SmokeTests/ # xunit smoke tests (requires running SC + tool) + aspire/ # file-based Aspire AppHost (platform + tool + Jaeger) ``` ## Run locally @@ -81,6 +103,40 @@ This starts ServiceControl + the testing tool + Jaeger (OTLP). Open: - Jaeger UI: http://localhost:16686 - Prometheus metrics: http://localhost:8080/metrics +## Run with Aspire + +The Aspire AppHost orchestrates the testing tool together with the full Particular platform +(ServiceControl + Learning transport + RavenDB + ServicePulse) and Jaeger, so a single command +brings up the whole system locally: + +```bash +aspire run tools/testing-tool/aspire/AppHost.cs +``` + +To test a specific ServiceControl image tag (e.g. a PR-based prerelease tag): + +```bash +aspire run tools/testing-tool/aspire/AppHost.cs -- pr-1234 +``` + +The Aspire dashboard provides allocated ports for each service. The testing tool automatically +connects to ServiceControl via the platform's transport and REST API URL. + +## Run smoke tests + +The smoke tests require a running ServiceControl + testing tool (via docker-compose or Aspire): + +```bash +# Start the stack first (see above) +dotnet test tools/testing-tool/src/TestingTool.SmokeTests +``` + +Configure the test URLs via environment variables if not using defaults: +```bash +TESTING_TOOL_URL=http://localhost:8080 SERVICECONTROL_URL=http://localhost:33333 \ + dotnet test tools/testing-tool/src/TestingTool.SmokeTests +``` + ## Deploy on Kubernetes ```bash diff --git a/tools/testing-tool/TestingTool.slnx b/tools/testing-tool/TestingTool.slnx index a988c98201..9c4fa8fda6 100644 --- a/tools/testing-tool/TestingTool.slnx +++ b/tools/testing-tool/TestingTool.slnx @@ -3,5 +3,6 @@ + diff --git a/tools/testing-tool/aspire/AppHost.cs b/tools/testing-tool/aspire/AppHost.cs new file mode 100644 index 0000000000..49724ecfe2 --- /dev/null +++ b/tools/testing-tool/aspire/AppHost.cs @@ -0,0 +1,68 @@ +#:sdk Aspire.AppHost.Sdk@13.4.5 +#:package Particular.Aspire.Hosting.ServicePlatform@1.* +#:project ../src/TestingTool/TestingTool.csproj + +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; +using Particular.Aspire.Hosting.ServicePlatform.Platform; + +var builder = DistributedApplication.CreateBuilder(args); + +// --- Particular Platform (ServiceControl + Learning transport + RavenDB persistence) --- +// AddDefaultComponents wires up the Learning transport, RavenDB, ServiceControl error/audit/monitoring +// instances, and ServicePulse — a complete local platform in one call. + +var platform = builder + .AddParticularPlatform("particular") + .AddDefaultComponents(); + +// Find the ServiceControl error instance to wire its REST API URL into the testing tool. +var errorInstance = builder.Resources.OfType().First(); +var errorInstanceBuilder = builder.CreateResourceBuilder(errorInstance); + +// --- Jaeger (OTLP collector + UI) --- +// All-in-one Jaeger accepts OTLP (gRPC :4317) directly and serves the Jaeger UI on :16686. + +var jaeger = builder.AddContainer("jaeger", "jaegertracing/all-in-one", "1.62") + .WithHttpEndpoint(targetPort: 16686, name: "ui") + .WithEndpoint(targetPort: 4317, name: "otlp-grpc") + .WithUrlForEndpoint("ui", url => url.DisplayText = "Jaeger UI"); + +// --- Testing Tool --- +// Added as a .NET project so it can be debugged locally. WithParticularPlatform wires the +// transport connection string and license. The ServiceControl REST API URL and OTLP endpoint +// are injected as environment variables so the testing tool can drive replay/search jobs and +// export telemetry. + +builder.AddProject("testing-tool") + .WithParticularPlatform(platform) + .WithEnvironment("TestingTool__ServiceControlApiUrl", errorInstanceBuilder.GetEndpoint("http")) + .WithEnvironment("OTEL_EXPORTER_OTLP_ENDPOINT", + ReferenceExpression.Create($"http://{jaeger.GetEndpoint("otlp-grpc")}")) + .WithEnvironment("TestingTool__AutoStartBackgroundNoise", "true") + .WithEnvironment("TestingTool__ReplayEnabled", "true") + .WithEnvironment("TestingTool__SearchEnabled", "true") + .WaitFor(errorInstanceBuilder); + +// --- Optional: override ServiceControl image tag for prerelease testing --- +// Pass a tag as the first argument: `aspire run AppHost.cs -- pr-1234` +// Defaults to the 'latest' tag configured by AddDefaultComponents. + +if (args.Length > 0) +{ + var tag = args[0]; + Console.WriteLine($"Using ServiceControl image tag: {tag}"); + foreach (var c in builder.Resources.OfType()) + { + if (c.TryGetLastAnnotation(out var image) && + (image.Image.StartsWith("particular/servicecontrol") || + image.Image.StartsWith("particular/servicepulse"))) + { + builder + .CreateResourceBuilder(c) + .WithImageTag(tag); + } + } +} + +builder.Build().Run(); \ No newline at end of file diff --git a/tools/testing-tool/aspire/Directory.Build.props b/tools/testing-tool/aspire/Directory.Build.props new file mode 100644 index 0000000000..040f259069 --- /dev/null +++ b/tools/testing-tool/aspire/Directory.Build.props @@ -0,0 +1,15 @@ + + + + + net10.0 + enable + enable + + + \ No newline at end of file diff --git a/tools/testing-tool/aspire/Directory.Packages.props b/tools/testing-tool/aspire/Directory.Packages.props new file mode 100644 index 0000000000..f83f49b387 --- /dev/null +++ b/tools/testing-tool/aspire/Directory.Packages.props @@ -0,0 +1,13 @@ + + + + + false + + + \ No newline at end of file diff --git a/tools/testing-tool/aspire/aspire.config.json b/tools/testing-tool/aspire/aspire.config.json new file mode 100644 index 0000000000..09075c45d9 --- /dev/null +++ b/tools/testing-tool/aspire/aspire.config.json @@ -0,0 +1,5 @@ +{ + "appHost": { + "path": "AppHost.cs" + } +} \ No newline at end of file diff --git a/tools/testing-tool/requirements-test-tool-plan.md b/tools/testing-tool/requirements-test-tool-plan.md index 9e8f962a48..416ac779e7 100644 --- a/tools/testing-tool/requirements-test-tool-plan.md +++ b/tools/testing-tool/requirements-test-tool-plan.md @@ -72,7 +72,7 @@ The tool is a **stateless, horizontally-scalable .NET service** hosted in a cont - [x] Define `ActivitySource` instances per scenario category (`testing-tool.load`, `testing-tool.replay`, `testing-tool.search` + per-scenario sources) - [x] Add metrics: `errors_sent_total{scenario}`, `errors_replayed_total{group}`, `searches_executed_total{query}`, `search_latency_ms{query}` (histogram) - [x] Prometheus scraping endpoint at `/metrics` -- [ ] Add structured logs routed through OTel logs API (deferred — console logging is active) +- [x] Add structured logs routed through OTel logs API (Phase 1 complete) ```csharp // Program.cs — OTel wiring @@ -105,7 +105,7 @@ builder.Services.AddOpenTelemetryPrometheusScrapingEndpoint(); - [x] Configure NServiceBus endpoint `TestingTool.Load` with error queue pointed at ServiceControl (using NServiceBus 10 `AddNServiceBusEndpoint` DI integration + Learning transport) - [x] Implement `FailingMessageHandler` that throws based on injected `IScenario` (via `IScenarioRegistry`) -- [ ] Implement `DirectErrorQueueWriter` that constructs `MessageFailed` / transport message envelopes and writes directly to the ServiceControl error queue (bypass path) — see ref [ServiceControlFeeder](https://github.com/dvdstelt/ServiceControlFeeder) +- [x] Implement `DirectErrorQueueWriter` that constructs `MessageFailed` / transport message envelopes and writes directly to the ServiceControl error queue (bypass path) — see ref [ServiceControlFeeder](https://github.com/dvdstelt/ServiceControlFeeder) - [x] Add a load-rate controller (token bucket) configurable per scenario (`PeriodicTimer`-based rate controller in `ScenarioRunner`) ```csharp @@ -183,7 +183,7 @@ public class FailingMessageHandler(IScenario scenario, ILogger @@ -218,9 +218,10 @@ public class FailingMessageHandler(IScenario scenario, ILogger +/// Live status of the direct error-queue bypass writer, returned by +/// GET /api/bypass/status and included in the GET /api/status snapshot. +/// +public sealed class BypassStatus +{ + /// Whether the bypass writer is currently emitting failed-message envelopes. + public bool Running { get; init; } + + /// The scenario whose failure shape is being simulated, or null if idle. + public string? Scenario { get; init; } + + /// Current target emission rate in messages/second (0 if idle). + public double Rate { get; init; } + + /// Total failed-message envelopes written directly to the error queue since process start. + public long ErrorsWritten { get; init; } + + /// When the current bypass run started (UTC ISO 8601, null if idle). + public string? StartedAt { get; init; } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Contracts/StartBypassRequest.cs b/tools/testing-tool/src/TestingTool.Contracts/StartBypassRequest.cs new file mode 100644 index 0000000000..fc70cb84c7 --- /dev/null +++ b/tools/testing-tool/src/TestingTool.Contracts/StartBypassRequest.cs @@ -0,0 +1,18 @@ +namespace TestingTool.Contracts; + +/// +/// Request body for POST /api/bypass/start. +/// Starts the direct error-queue bypass writer, which writes failed-message envelopes +/// directly to the ServiceControl error queue without going through a handler. +/// +public sealed class StartBypassRequest +{ + /// The scenario whose failure shape to simulate (determines exception type, message, and grouping). + public string? Scenario { get; init; } + + /// Target emission rate in messages/second. Defaults to 100. + public double? Rate { get; init; } + + /// Optional auto-stop duration in seconds. Null/0 = run until explicitly stopped. + public double? DurationSeconds { get; init; } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.Contracts/TestingToolStatus.cs b/tools/testing-tool/src/TestingTool.Contracts/TestingToolStatus.cs index 6d940baaca..19cefda119 100644 --- a/tools/testing-tool/src/TestingTool.Contracts/TestingToolStatus.cs +++ b/tools/testing-tool/src/TestingTool.Contracts/TestingToolStatus.cs @@ -18,6 +18,9 @@ public sealed class TestingToolStatus /// Total ServiceControl searches executed since process start. public long SearchesExecuted { get; init; } + /// Total errors emitted via the direct error-queue bypass writer since process start. + public long BypassErrorsWritten { get; init; } + /// The shard id this replica owns, used for disjoint scenario slices when scaled out. public string? ShardId { get; init; } diff --git a/tools/testing-tool/src/TestingTool.SmokeTests/SmokeTest.cs b/tools/testing-tool/src/TestingTool.SmokeTests/SmokeTest.cs new file mode 100644 index 0000000000..8a5e84ef90 --- /dev/null +++ b/tools/testing-tool/src/TestingTool.SmokeTests/SmokeTest.cs @@ -0,0 +1,124 @@ +using System.Net; +using System.Net.Http.Json; +using Xunit; + +namespace TestingTool.SmokeTests; + +/// +/// Smoke test for the ServiceControl testing tool. Requires a running ServiceControl instance +/// and testing tool (e.g. via docker-compose or the Aspire AppHost). The test: +/// 1. Verifies the testing tool is accessible +/// 2. Triggers the third-party-outage scenario for 30 seconds +/// 3. Verifies error groups appear in ServiceControl +/// 4. Triggers a replay and verifies it is accepted +/// +/// Configure via environment variables: +/// TESTING_TOOL_URL (default: http://localhost:8080) +/// SERVICECONTROL_URL (default: http://localhost:33333) +/// +public class SmokeTest +{ + private static readonly string TestingToolUrl = + Environment.GetEnvironmentVariable("TESTING_TOOL_URL") ?? "http://localhost:8080"; + private static readonly string ServiceControlUrl = + Environment.GetEnvironmentVariable("SERVICECONTROL_URL") ?? "http://localhost:33333"; + + private static readonly HttpClient ToolClient = new() { BaseAddress = new Uri(TestingToolUrl), Timeout = TimeSpan.FromSeconds(30) }; + private static readonly HttpClient ScClient = new() { BaseAddress = new Uri(ServiceControlUrl), Timeout = TimeSpan.FromSeconds(30) }; + + [Fact] + public async Task TestingTool_IsAccessible_ReturnsStatus() + { + var response = await ToolClient.GetAsync("/api/status"); + response.EnsureSuccessStatusCode(); + + var status = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(status); + Assert.True(status.Ready); + } + + [Fact] + public async Task ThirdPartyOutage_GeneratesErrors_VisibleInServiceControl() + { + // 1. Stop any existing scenario first (clean slate) + await ToolClient.PostAsync("/api/scenarios/stop-all", null); + + // 2. Start the third-party-outage scenario for 30 seconds at 50 msg/s + var startResponse = await ToolClient.PostAsJsonAsync( + "/api/scenarios/third-party-outage/start", + new { rate = 50, durationSeconds = 30 }); + startResponse.EnsureSuccessStatusCode(); + + // 3. Wait for the scenario to generate errors (10s in, then check SC) + await Task.Delay(TimeSpan.FromSeconds(10)); + + // 4. Check ServiceControl for error groups + var groupsResponse = await ScClient.GetAsync("/api/errors/groups"); + groupsResponse.EnsureSuccessStatusCode(); + + var groups = await groupsResponse.Content.ReadFromJsonAsync>(); + Assert.NotNull(groups); + Assert.NotEmpty(groups!); + + // 5. Wait for the scenario to finish (remaining ~20s + buffer) + await Task.Delay(TimeSpan.FromSeconds(25)); + + // 6. Verify the testing tool counted errors + var statusResponse = await ToolClient.GetAsync("/api/status"); + statusResponse.EnsureSuccessStatusCode(); + var status = await statusResponse.Content.ReadFromJsonAsync(); + Assert.NotNull(status); + Assert.True(status!.ErrorsSent > 0, $"Expected errors to be sent, got {status.ErrorsSent}"); + } + + [Fact] + public async Task Replay_ErrorGroup_IsAcceptedByServiceControl() + { + // 1. Ensure there are error groups to replay + var groupsResponse = await ScClient.GetAsync("/api/errors/groups"); + groupsResponse.EnsureSuccessStatusCode(); + var groups = await groupsResponse.Content.ReadFromJsonAsync>(); + + if (groups is null || groups.Count == 0) + { + // Generate some errors first + await ToolClient.PostAsJsonAsync( + "/api/scenarios/third-party-outage/start", + new { rate = 50, durationSeconds = 10 }); + await Task.Delay(TimeSpan.FromSeconds(15)); + + groupsResponse = await ScClient.GetAsync("/api/errors/groups"); + groupsResponse.EnsureSuccessStatusCode(); + groups = await groupsResponse.Content.ReadFromJsonAsync>(); + } + + Assert.NotNull(groups); + Assert.NotEmpty(groups!); + + // 2. Trigger replay on the first group + var firstGroup = groups![0]; + var replayResponse = await ScClient.PostAsJsonAsync( + $"/api/errors/groups/{firstGroup.Id}/retry", new { }); + + // ServiceControl should accept the replay request (202 Accepted or 200 OK) + Assert.True(replayResponse.IsSuccessStatusCode, + $"Replay request failed: {replayResponse.StatusCode} {await replayResponse.Content.ReadAsStringAsync()}"); + } + + // --- Response DTOs (minimal, matching the APIs) --- + + private sealed record StatusResponse + { + public bool Ready { get; init; } + public long ErrorsSent { get; init; } + public long ErrorsReplayed { get; init; } + public int ActiveScenarios { get; init; } + } + + private sealed record ErrorGroupResponse + { + public string Id { get; init; } = ""; + public string Title { get; init; } = ""; + public int Count { get; init; } + } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool.SmokeTests/TestingTool.SmokeTests.csproj b/tools/testing-tool/src/TestingTool.SmokeTests/TestingTool.SmokeTests.csproj new file mode 100644 index 0000000000..5caf3219dd --- /dev/null +++ b/tools/testing-tool/src/TestingTool.SmokeTests/TestingTool.SmokeTests.csproj @@ -0,0 +1,24 @@ + + + + + + net10.0 + false + true + enable + enable + + + + + + + + + \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/DirectErrorQueueWriter.cs b/tools/testing-tool/src/TestingTool/DirectErrorQueueWriter.cs new file mode 100644 index 0000000000..800eaae402 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/DirectErrorQueueWriter.cs @@ -0,0 +1,195 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NServiceBus; +using TestingTool.Contracts; +using TestingTool.Scenarios; + +namespace TestingTool; + +/// +/// Bypass path: constructs failed-message envelopes with NServiceBus failure headers and writes +/// them directly to the ServiceControl error queue via the NServiceBus transport, without going +/// through the handler. This enables high-throughput error load generation that bypasses the +/// initial message creation and handler processing (requirement: "simulate high error loads, +/// bypass actually creating the initial messages"). +/// +/// Each emitted message carries the standard NServiceBus failure headers +/// (NServiceBus.ExceptionInfo.*, NServiceBus.FailedQ) so ServiceControl ingests +/// it as a genuine failed message and groups it by the scenario's exception type and correlation +/// group — exactly like handler-generated failures. +/// +public sealed class DirectErrorQueueWriter +{ + private readonly IMessageSession _session; + private readonly IScenarioRegistry _registry; + private readonly TestingToolMetrics _metrics; + private readonly Meter _meter; + private readonly TestingToolOptions _options; + private readonly ILogger _logger; + + private readonly ActivitySource _activitySource = new(TelemetrySetup.Sources.Bypass); + private readonly Counter _bypassCounter; + + private CancellationTokenSource? _cts; + private Task? _loop; + private long _errorsWritten; + private double _currentRate; + private string? _activeScenario; + private DateTimeOffset _startedAt; + + public bool IsRunning => _cts is not null; + public long ErrorsWritten => Interlocked.Read(ref _errorsWritten); + public double CurrentRate => _currentRate; + public string? ActiveScenario => _activeScenario; + + public DirectErrorQueueWriter( + IMessageSession session, + IScenarioRegistry registry, + TestingToolMetrics metrics, + Meter meter, + IOptions options, + ILogger logger) + { + _session = session; + _registry = registry; + _metrics = metrics; + _meter = meter; + _options = options.Value; + _logger = logger; + _bypassCounter = meter.CreateCounter("bypass_errors_written_total"); + } + + /// Starts writing failed-message envelopes directly to the error queue. + public bool TryStart(string scenarioName, double rate, TimeSpan? duration, out string? error) + { + var scenario = _registry.Get(scenarioName); + if (scenario is null) + { + error = $"Unknown scenario '{scenarioName}'"; + return false; + } + + if (IsRunning) + { + error = "Bypass writer is already running — stop it first"; + return false; + } + + if (rate <= 0) + { + error = "Rate must be greater than 0"; + return false; + } + + _activeScenario = scenarioName; + _currentRate = rate; + _startedAt = DateTimeOffset.UtcNow; + + var cts = duration is { } d + ? new CancellationTokenSource(d) + : new CancellationTokenSource(); + _cts = cts; + + _loop = Task.Run(() => WriteLoop(scenario, rate, cts.Token)); + + _logger.LogInformation("Started bypass writer for scenario {Scenario} at {Rate:F1} msg/s{Duration}", + scenarioName, rate, duration is null ? "" : $" for {duration.Value}"); + + error = null; + return true; + } + + /// Stops the bypass writer. + public void Stop() + { + if (_cts is null) return; + + _cts.Cancel(); + _cts.Dispose(); + _cts = null; + + _logger.LogInformation("Stopped bypass writer after {Errors} errors written", ErrorsWritten); + + _activeScenario = null; + _currentRate = 0; + } + + /// Returns the current bypass writer status for API consumers. + public BypassStatus GetStatus() => new() + { + Running = IsRunning, + Scenario = _activeScenario, + Rate = _currentRate, + ErrorsWritten = ErrorsWritten, + StartedAt = IsRunning ? _startedAt.ToString("O") : null + }; + + /// + /// The load generation loop: sends directly to the error queue + /// with failure headers at the target rate until cancelled. + /// + private async Task WriteLoop(IScenario scenario, double rate, CancellationToken ct) + { + var interval = TimeSpan.FromSeconds(1.0 / rate); + using var timer = new PeriodicTimer(interval); + long sequence = 0; + + // Pre-compute the failure metadata from the scenario so all messages in this run + // share the same exception type, message, and correlation group — ServiceControl + // will group them as one error group. + var exception = scenario.CreateException(); + var scenarioEx = exception as ScenarioException; + var exceptionType = scenarioEx?.ExceptionType ?? exception.GetType().FullName!; + var exceptionMessage = exception.Message; + var correlationGroup = scenarioEx?.CorrelationGroup ?? ""; + + try + { + while (await timer.WaitForNextTickAsync(ct)) + { + var seq = Interlocked.Increment(ref sequence); + var payload = new byte[Random.Shared.Next(64, 512)]; + Random.Shared.NextBytes(payload); + + var message = new LoadMessage { Sequence = seq, Payload = payload }; + + var sendOptions = new SendOptions(); + // Route directly to the ServiceControl error queue — bypasses the handler entirely. + sendOptions.SetDestination(_options.ErrorQueueName); + + // Set failure headers so ServiceControl recognises the message as a failed message + // and groups it by exception type + correlation group, exactly like handler failures. + sendOptions.SetHeader("TestingTool.Scenario", scenario.Name); + sendOptions.SetHeader("TestingTool.Bypass", "true"); + sendOptions.SetHeader("TestingTool.CorrelationGroup", correlationGroup); + sendOptions.SetHeader("NServiceBus.ExceptionInfo.ExceptionType", exceptionType); + sendOptions.SetHeader("NServiceBus.ExceptionInfo.Message", exceptionMessage); + sendOptions.SetHeader("NServiceBus.ExceptionInfo.Source", "TestingTool.Load"); + sendOptions.SetHeader("NServiceBus.FailedQ", "TestingTool.Load"); + + using var activity = _activitySource.StartActivity("bypass-write"); + activity?.SetTag("scenario", scenario.Name); + activity?.SetTag("sequence", seq); + activity?.SetTag("exception.type", exceptionType); + activity?.SetTag("exception.group", correlationGroup); + + try + { + await _session.Send(message, sendOptions, ct); + + Interlocked.Increment(ref _errorsWritten); + _metrics.AddErrorsSent(1); + _metrics.AddBypassErrorsWritten(1); + _bypassCounter.Add(1, new KeyValuePair("scenario", scenario.Name)); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogDebug(ex, "Bypass send failed for scenario {Scenario} seq {Seq}", scenario.Name, seq); + } + } + } + catch (OperationCanceledException) { } + } +} \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/Program.cs b/tools/testing-tool/src/TestingTool/Program.cs index 86243b02e2..503b370db4 100644 --- a/tools/testing-tool/src/TestingTool/Program.cs +++ b/tools/testing-tool/src/TestingTool/Program.cs @@ -32,6 +32,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(); // --- Background jobs (Phase 4) --- @@ -65,11 +66,12 @@ } }); -// Graceful shutdown: stop all scenarios. +// Graceful shutdown: stop all scenarios and bypass writer. app.Lifetime.ApplicationStopping.Register(() => { var runner = app.Services.GetRequiredService(); runner.StopAll(); + app.Services.GetRequiredService().Stop(); }); // --- Health endpoints (Phase 6) --- @@ -90,6 +92,7 @@ ErrorsSent = metrics.TotalErrorsSent, ErrorsReplayed = metrics.TotalErrorsReplayed, SearchesExecuted = metrics.TotalSearches, + BypassErrorsWritten = metrics.TotalBypassErrorsWritten, ShardId = shardId, ActiveScenarios = metrics.ActiveScenarios, CurrentRate = Math.Round(metrics.CurrentRate, 1), @@ -127,6 +130,68 @@ return Results.Ok(new { stopped = "all" }); }); +// --- Bypass endpoints (Phase 2: direct error-queue writer) --- +// These endpoints control the bypass path that writes failed-message envelopes directly to the +// ServiceControl error queue, bypassing the handler for high-throughput error load. + +app.MapGet("/api/bypass/status", (DirectErrorQueueWriter writer) => Results.Ok(writer.GetStatus())); + +app.MapPost("/api/bypass/start", (StartBypassRequest? request, DirectErrorQueueWriter writer, IScenarioRegistry registry) => +{ + var scenarioName = request?.Scenario; + if (string.IsNullOrWhiteSpace(scenarioName)) + { + // Default to the first scenario if none specified. + scenarioName = registry.All[0].Name; + } + + var rate = request?.Rate ?? 100; + var duration = request?.DurationSeconds is { } secs and > 0 + ? TimeSpan.FromSeconds(secs) + : (TimeSpan?)null; + + if (!writer.TryStart(scenarioName, rate, duration, out var error)) + return Results.BadRequest(new { error }); + + return Results.Ok(writer.GetStatus()); +}); + +app.MapPost("/api/bypass/stop", (DirectErrorQueueWriter writer) => +{ + writer.Stop(); + return Results.Ok(writer.GetStatus()); +}); + +// --- Release-test scenario endpoints (Phase 5: release-test presets) --- +// Lists release-test presets that map to testing-tool scenarios, and allows kicking them off +// by release-test name. This satisfies the optional requirement to consider release-test +// scenarios for manual kickoff. + +app.MapGet("/api/release-tests", () => Results.Ok(ReleaseTestScenarios.Presets.Select(p => new +{ + name = p.Name, + scenario = p.ScenarioName, + description = p.Description, + rate = p.Rate, + durationSeconds = p.DurationSeconds +}))); + +app.MapPost("/api/release-tests/{name}/start", (string name, ScenarioRunner runner) => +{ + var preset = ReleaseTestScenarios.Find(name); + if (preset is null) + return Results.BadRequest(new { error = $"Unknown release-test scenario '{name}'" }); + + var duration = preset.DurationSeconds is { } secs and > 0 + ? TimeSpan.FromSeconds(secs) + : (TimeSpan?)null; + + if (!runner.TryStart(preset.ScenarioName, preset.Rate, duration, out var error)) + return Results.BadRequest(new { error }); + + return Results.Ok(new { started = preset.Name, scenario = preset.ScenarioName, rate = preset.Rate }); +}); + app.MapFallbackToFile("index.html"); app.Run(); \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/ReleaseTestScenarios.cs b/tools/testing-tool/src/TestingTool/ReleaseTestScenarios.cs new file mode 100644 index 0000000000..edd2aabd25 --- /dev/null +++ b/tools/testing-tool/src/TestingTool/ReleaseTestScenarios.cs @@ -0,0 +1,66 @@ +using TestingTool.Contracts; +using TestingTool.Scenarios; + +namespace TestingTool; + +/// +/// Maps release-test scenario names (from docs/testing-scenarios.md) to testing-tool +/// scenarios so they can be kicked off manually from the web UI or API. This satisfies the +/// optional requirement: "any scenarios from the release tests should be considered to kick off +/// manually." +/// +public static class ReleaseTestScenarios +{ + /// + /// Release-test scenario presets. Each preset maps a release-test checklist item to a + /// testing-tool scenario with a recommended rate and duration. + /// + public static readonly IReadOnlyList Presets = + [ + new("retry-single-message", "third-party-outage", + "Recoverability — Retry single message: generate a small batch of grouped errors for single-message retry testing.", + 5, 10), + + new("retry-message-group", "third-party-outage", + "Recoverability — Retry message group: generate a larger batch of grouped errors for group retry testing.", + 50, 20), + + new("ingestion-load", "background-noise", + "Ingestion — High message load: continuous background errors to test ingestion throughput.", + 100, null), + + new("chaos-testing", "background-noise", + "Chaos testing — Continuous low-level errors while stopping/killing ServiceControl processes.", + 10, null), + + new("performance-clean-db", "third-party-outage", + "Performance — Clean database: burst of errors to test ingestion with an empty RavenDB.", + 200, 60), + + new("poison-retry-storm", "poison-message", + "Recoverability — Poison message retry storm: deterministic always-fail messages.", + 20, 30), + + new("timeout-batch", "timeout-spike", + "Recoverability — Timeout spike: oscillating timeout failures grouped by batch bucket.", + 30, 60), + + new("deserialization-bad-deploy", "deserialization-failure", + "Recoverability — Bad deployment: 100% deserialization failures grouped by message type.", + 50, 15), + ]; + + /// Find a preset by its release-test name (case-insensitive). + public static ReleaseTestPreset? Find(string name) => + Presets.FirstOrDefault(p => p.Name.Equals(name, StringComparison.OrdinalIgnoreCase)); +} + +/// +/// A named release-test preset that maps to a testing-tool scenario with recommended parameters. +/// +public sealed record ReleaseTestPreset( + string Name, + string ScenarioName, + string Description, + double Rate, + int? DurationSeconds); \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/TelemetrySetup.cs b/tools/testing-tool/src/TestingTool/TelemetrySetup.cs index 824913c7a6..292a84c522 100644 --- a/tools/testing-tool/src/TestingTool/TelemetrySetup.cs +++ b/tools/testing-tool/src/TestingTool/TelemetrySetup.cs @@ -1,5 +1,6 @@ using System.Diagnostics.Metrics; using OpenTelemetry; +using OpenTelemetry.Logs; using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; @@ -22,6 +23,7 @@ public static class Sources public const string Load = "testing-tool.load"; public const string Replay = "testing-tool.replay"; public const string Search = "testing-tool.search"; + public const string Bypass = "testing-tool.bypass"; } public static Meter CreateMeter() => new(MeterName, "1.0.0"); @@ -37,6 +39,7 @@ public static OpenTelemetryBuilder AddTestingToolTelemetry(this IServiceCollecti .AddSource(Sources.Load) .AddSource(Sources.Replay) .AddSource(Sources.Search) + .AddSource(Sources.Bypass) // Also pick up per-scenario activity sources dynamically. .AddSource("testing-tool.*") .AddOtlpExporter()) @@ -45,6 +48,8 @@ public static OpenTelemetryBuilder AddTestingToolTelemetry(this IServiceCollecti .AddHttpClientInstrumentation() .AddMeter(MeterName) .AddPrometheusExporter() - .AddOtlpExporter()); + .AddOtlpExporter()) + // Phase 1: Route structured logs through the OpenTelemetry logs API to OTLP. + .WithLogging(l => l.AddOtlpExporter()); } } \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/TestingToolMetrics.cs b/tools/testing-tool/src/TestingTool/TestingToolMetrics.cs index 2419865713..d7d89f3ac4 100644 --- a/tools/testing-tool/src/TestingTool/TestingToolMetrics.cs +++ b/tools/testing-tool/src/TestingTool/TestingToolMetrics.cs @@ -10,18 +10,21 @@ public sealed class TestingToolMetrics private long _totalErrorsSent; private long _totalErrorsReplayed; private long _totalSearches; + private long _totalBypassErrorsWritten; private long _activeScenarios; private double _currentRate; public long TotalErrorsSent => Interlocked.Read(ref _totalErrorsSent); public long TotalErrorsReplayed => Interlocked.Read(ref _totalErrorsReplayed); public long TotalSearches => Interlocked.Read(ref _totalSearches); + public long TotalBypassErrorsWritten => Interlocked.Read(ref _totalBypassErrorsWritten); public int ActiveScenarios => (int)Interlocked.Read(ref _activeScenarios); public double CurrentRate => _currentRate; public void AddErrorsSent(long count) => Interlocked.Add(ref _totalErrorsSent, count); public void AddErrorsReplayed(long count) => Interlocked.Add(ref _totalErrorsReplayed, count); public void AddSearches(long count) => Interlocked.Add(ref _totalSearches, count); + public void AddBypassErrorsWritten(long count) => Interlocked.Add(ref _totalBypassErrorsWritten, count); public void SetActiveScenarios(int count) => Interlocked.Exchange(ref _activeScenarios, count); public void SetCurrentRate(double rate) => _currentRate = rate; } \ No newline at end of file diff --git a/tools/testing-tool/src/TestingTool/wwwroot/index.html b/tools/testing-tool/src/TestingTool/wwwroot/index.html index 2d62433a10..a09431aa24 100644 --- a/tools/testing-tool/src/TestingTool/wwwroot/index.html +++ b/tools/testing-tool/src/TestingTool/wwwroot/index.html @@ -171,6 +171,10 @@

🎯 ServiceControl Testing Tool

Uptime
—
+
+
Bypass Errors
+
—
+
@@ -187,6 +191,32 @@

Scenarios

Loading scenarios…
+ +
+

Direct Error-Queue Bypass

+
+
+
+ Writes failed-message envelopes directly to the ServiceControl error queue, bypassing the handler for high-throughput error load. Messages carry NServiceBus failure headers so ServiceControl ingests them as genuine failed messages. +
+
+ Scenario: + + Rate (msg/s): + + Duration (sec, optional): + + + +
+
+ Status: idle + Errors written: 0 +
+
+