From 500569762f5895642cfd81539f2dacc1002efd3a Mon Sep 17 00:00:00 2001 From: John Simons Date: Tue, 25 Aug 2026 10:18:46 +1000 Subject: [PATCH 1/2] Switch to standard OpenTelemetry environment variables for OTLP configuration Removes the custom OtlpEndpointUrl setting in favor of standard environment variables such as OTEL_EXPORTER_OTLP_ENDPOINT and OTEL_EXPORTER_OTLP_METRICS_ENDPOINT. This ensures compatibility with native OpenTelemetry SDK configuration, supports both gRPC and HTTP protocols, and simplifies instance configuration. --- docs/telemetry.md | 16 +++---- ...rovals.PlatformSampleSettings.approved.txt | 3 +- src/ServiceControl.Audit/App.config | 10 +++-- .../HostApplicationBuilderExtensions.cs | 45 +++++++++---------- .../Infrastructure/Settings/Settings.cs | 1 - .../OtlpEndpoint.cs | 15 +++++++ ...sTests.PlatformSampleSettings.approved.txt | 2 +- src/ServiceControl.Monitoring/App.config | 10 +++-- src/ServiceControl.Monitoring/Settings.cs | 2 +- ...rovals.PlatformSampleSettings.approved.txt | 3 +- src/ServiceControl/App.config | 4 +- .../HostApplicationBuilderExtensions.cs | 13 +++--- .../Infrastructure/Settings/Settings.cs | 1 - 13 files changed, 69 insertions(+), 56 deletions(-) create mode 100644 src/ServiceControl.Infrastructure/OtlpEndpoint.cs diff --git a/docs/telemetry.md b/docs/telemetry.md index 5dbad512d9..1eec152d0a 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -2,17 +2,17 @@ Instances can be configured to emit telemetry to aid in performance testing or troubleshooting performance-related issues. -Both the error and the audit instance report their ingestion the same way. Set `OtlpEndpointUrl` -in that instance's settings root to a valid [OTLP endpoint url](https://opentelemetry.io/docs/specs/otel/protocol/exporter/#configuration-options). -Only GRPC endpoints are supported at this stage. +Both the error and the audit instance report their ingestion the same way. Exporting is configured with the standard [OpenTelemetry environment variables](https://opentelemetry.io/docs/specs/otel/protocol/exporter/#configuration-options), not with instance settings, so the same variables that configure any other OpenTelemetry process apply here. Setting `OTEL_EXPORTER_OTLP_ENDPOINT` is enough to turn metrics on, and `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` overrides it for metrics alone. Both gRPC and HTTP endpoints are supported, and `OTEL_EXPORTER_OTLP_PROTOCOL` selects between them. -The instruments differ only in their prefix and in the categories a message can fall into, so the -same dashboard works for both with the prefix swapped. What the batches being measured actually are -is covered in [ingestion-pipeline.md](ingestion-pipeline.md). +These variables have to be set in the instance's environment. They cannot be set in the instance's app.config, because the OpenTelemetry SDK reads them itself rather than going through the ServiceControl settings reader. + +Logs are exported separately. Add `Otlp` to the instance's `LoggingProviders` setting, which is what turns the OTLP log exporter on, and it then reads the same environment variables for its endpoint. + +The instruments differ only in their prefix and in the categories a message can fall into, so the same dashboard works for both with the prefix swapped. What the batches being measured actually are is covered in [ingestion-pipeline.md](ingestion-pipeline.md). ## Error -Meter `Particular.ServiceControl`, configured with `ServiceControl/OtlpEndpointUrl`. +Meter `Particular.ServiceControl`. - `sc.error.ingestion.batch_duration_seconds` - Message batch processing duration in seconds - `result` - Whether the batch was written at its configured size: `full`, `partial` or `failed` @@ -29,7 +29,7 @@ Meter `Particular.ServiceControl`, configured with `ServiceControl/OtlpEndpointU ## Audit -Meter `Particular.ServiceControl.Audit`, configured with `ServiceControl.Audit/OtlpEndpointUrl`. +Meter `Particular.ServiceControl.Audit`. - `sc.audit.ingestion.batch_duration_seconds` - Message batch processing duration in seconds - `result` - Whether the batch was written at its configured size: `full`, `partial` or `failed` diff --git a/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt index a1901c1f2c..c1066673e7 100644 --- a/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt @@ -1,4 +1,4 @@ -{ +{ "LoggingSettings": { "LogLevel": "Information", "LogPath": "C:\\Logs" @@ -47,7 +47,6 @@ "ApiUrl": "http://localhost:8888/api", "Port": 8888, "PrintMetrics": false, - "OtlpEndpointUrl": null, "Hostname": "localhost", "VirtualDirectory": "", "TransportType": "LearningTransport", diff --git a/src/ServiceControl.Audit/App.config b/src/ServiceControl.Audit/App.config index a3f5781c51..6e4cc74869 100644 --- a/src/ServiceControl.Audit/App.config +++ b/src/ServiceControl.Audit/App.config @@ -1,4 +1,4 @@ - + - - - + + + + + diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index 151d7b9315..6add21574b 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -1,4 +1,4 @@ -namespace ServiceControl.Audit; +namespace ServiceControl.Audit; using System; using System.Diagnostics; @@ -97,31 +97,30 @@ public static void AddMetrics(this IHostApplicationBuilder builder, Settings set { builder.Services.AddSingleton(); - if (!string.IsNullOrEmpty(settings.OtlpEndpointUrl)) + var otlpEndpoint = OtlpEndpoint.MetricsEndpointFromEnvironment(); + + if (otlpEndpoint is null) { - if (!Uri.TryCreate(settings.OtlpEndpointUrl, UriKind.Absolute, out var otelMetricsUri)) + return; + } + + builder.Services.AddOpenTelemetry() + .ConfigureResource(b => b.AddService( + serviceName: settings.InstanceName, + serviceVersion: InstanceVersion, + autoGenerateServiceInstanceId: true)) + .WithMetrics(b => { - throw new UriFormatException($"Invalid OtlpEndpointUrl: {settings.OtlpEndpointUrl}"); - } - - builder.Services.AddOpenTelemetry() - .ConfigureResource(b => b.AddService( - serviceName: settings.InstanceName, - serviceVersion: InstanceVersion, - autoGenerateServiceInstanceId: true)) - .WithMetrics(b => + b.AddIngestionMetrics(); + b.AddOtlpExporter(); + if (Debugger.IsAttached) { - b.AddIngestionMetrics(); - b.AddOtlpExporter(e => e.Endpoint = otelMetricsUri); - if (Debugger.IsAttached) - { - b.AddConsoleExporter(); - } - }); - - var logger = LoggerUtil.CreateStaticLogger(typeof(HostApplicationBuilderExtensions), settings.LoggingSettings.LogLevel); - logger.LogInformation("OpenTelemetry metrics exporter enabled: {OtlpEndpointUrl}", settings.OtlpEndpointUrl); - } + b.AddConsoleExporter(); + } + }); + + var logger = LoggerUtil.CreateStaticLogger(typeof(HostApplicationBuilderExtensions), settings.LoggingSettings.LogLevel); + logger.LogInformation("OpenTelemetry metrics exporter enabled: {OtlpEndpoint}", otlpEndpoint); } static void RecordStartup(Settings settings, EndpointConfiguration endpointConfiguration, IPersistenceConfiguration persistenceConfiguration) diff --git a/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs b/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs index e60961c560..7a6b8d978f 100644 --- a/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs @@ -148,7 +148,6 @@ public string RootUrl public int Port { get; set; } public bool PrintMetrics => SettingsReader.Read(SettingsRootNamespace, "PrintMetrics"); - public string OtlpEndpointUrl { get; set; } = SettingsReader.Read(SettingsRootNamespace, nameof(OtlpEndpointUrl)); public string Hostname { get; private set; } public string VirtualDirectory => SettingsReader.Read(SettingsRootNamespace, "VirtualDirectory", string.Empty); diff --git a/src/ServiceControl.Infrastructure/OtlpEndpoint.cs b/src/ServiceControl.Infrastructure/OtlpEndpoint.cs new file mode 100644 index 0000000000..5f6a754724 --- /dev/null +++ b/src/ServiceControl.Infrastructure/OtlpEndpoint.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Infrastructure; + +using System; + +public static class OtlpEndpoint +{ + public static string MetricsEndpointFromEnvironment() => + Read("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT") ?? Read("OTEL_EXPORTER_OTLP_ENDPOINT"); + + static string Read(string variable) + { + var value = Environment.GetEnvironmentVariable(variable); + return string.IsNullOrWhiteSpace(value) ? null : value; + } +} diff --git a/src/ServiceControl.Monitoring.UnitTests/ApprovalFiles/SettingsTests.PlatformSampleSettings.approved.txt b/src/ServiceControl.Monitoring.UnitTests/ApprovalFiles/SettingsTests.PlatformSampleSettings.approved.txt index b8f0237609..ca16ad95a4 100644 --- a/src/ServiceControl.Monitoring.UnitTests/ApprovalFiles/SettingsTests.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.Monitoring.UnitTests/ApprovalFiles/SettingsTests.PlatformSampleSettings.approved.txt @@ -1,4 +1,4 @@ -{ +{ "LoggingSettings": { "LogLevel": "Information", "LogPath": "C:\\Logs" diff --git a/src/ServiceControl.Monitoring/App.config b/src/ServiceControl.Monitoring/App.config index ab620acab4..be1b700c36 100644 --- a/src/ServiceControl.Monitoring/App.config +++ b/src/ServiceControl.Monitoring/App.config @@ -1,4 +1,4 @@ - + - - - + + + + + diff --git a/src/ServiceControl.Monitoring/Settings.cs b/src/ServiceControl.Monitoring/Settings.cs index e6f6b84f02..a1b11750b8 100644 --- a/src/ServiceControl.Monitoring/Settings.cs +++ b/src/ServiceControl.Monitoring/Settings.cs @@ -1,4 +1,4 @@ -namespace ServiceControl.Monitoring +namespace ServiceControl.Monitoring { using System; using System.Collections.Generic; diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt index a638e3df4f..abd4c98313 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt @@ -1,4 +1,4 @@ -{ +{ "LoggingSettings": { "LogLevel": "Information", "LogPath": "C:\\Logs" @@ -59,7 +59,6 @@ "Port": 8888, "PersisterSpecificSettings": null, "PrintMetrics": false, - "OtlpEndpointUrl": null, "Hostname": "localhost", "VirtualDirectory": "", "HeartbeatGracePeriod": "00:00:40", diff --git a/src/ServiceControl/App.config b/src/ServiceControl/App.config index 4a77515a3b..98d8fc8433 100644 --- a/src/ServiceControl/App.config +++ b/src/ServiceControl/App.config @@ -1,4 +1,4 @@ - + + + diff --git a/src/ServiceControl/HostApplicationBuilderExtensions.cs b/src/ServiceControl/HostApplicationBuilderExtensions.cs index cbef15f93b..7ae3370059 100644 --- a/src/ServiceControl/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl/HostApplicationBuilderExtensions.cs @@ -152,14 +152,11 @@ public static void AddIngestionMetrics(this IHostApplicationBuilder hostBuilder, { hostBuilder.Services.AddSingleton(); - if (string.IsNullOrEmpty(settings.OtlpEndpointUrl)) - { - return; - } + var otlpEndpoint = OtlpEndpoint.MetricsEndpointFromEnvironment(); - if (!Uri.TryCreate(settings.OtlpEndpointUrl, UriKind.Absolute, out var otlpEndpoint)) + if (otlpEndpoint is null) { - throw new UriFormatException($"Invalid OtlpEndpointUrl: {settings.OtlpEndpointUrl}"); + return; } hostBuilder.Services.AddOpenTelemetry() @@ -170,7 +167,7 @@ public static void AddIngestionMetrics(this IHostApplicationBuilder hostBuilder, .WithMetrics(metrics => { metrics.AddIngestionMetrics(); - metrics.AddOtlpExporter(exporter => exporter.Endpoint = otlpEndpoint); + metrics.AddOtlpExporter(); if (Debugger.IsAttached) { @@ -179,7 +176,7 @@ public static void AddIngestionMetrics(this IHostApplicationBuilder hostBuilder, }); LoggerUtil.CreateStaticLogger(typeof(HostApplicationBuilderExtensions), settings.LoggingSettings.LogLevel) - .LogInformation("OpenTelemetry metrics exporter enabled: {OtlpEndpointUrl}", settings.OtlpEndpointUrl); + .LogInformation("OpenTelemetry metrics exporter enabled: {OtlpEndpoint}", otlpEndpoint); } static void RecordStartup(Settings settings, EndpointConfiguration endpointConfiguration) diff --git a/src/ServiceControl/Infrastructure/Settings/Settings.cs b/src/ServiceControl/Infrastructure/Settings/Settings.cs index 8be72ac55e..af52f4bf88 100644 --- a/src/ServiceControl/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl/Infrastructure/Settings/Settings.cs @@ -178,7 +178,6 @@ public string InstanceId public bool PrintMetrics => SettingsReader.Read(SettingsRootNamespace, "PrintMetrics"); - public string OtlpEndpointUrl { get; set; } = SettingsReader.Read(SettingsRootNamespace, nameof(OtlpEndpointUrl)); public string Hostname { get; private set; } public string VirtualDirectory => SettingsReader.Read(SettingsRootNamespace, "VirtualDirectory", string.Empty); From 8116477219272d4288b969b7b10d710cbbe0f1da Mon Sep 17 00:00:00 2001 From: John Simons Date: Tue, 25 Aug 2026 16:54:45 +1000 Subject: [PATCH 2/2] Only use OTEL_EXPORTER_OTLP_ENDPOINT for OTLP metrics configuration The OpenTelemetry SDK only honors signal-specific environment variables, such as OTEL_EXPORTER_OTLP_METRICS_ENDPOINT, when using UseOtlpExporter. Since instances use AddOtlpExporter to restrict OTLP to metrics, these specific variables have no effect and have been removed to avoid confusion. This also switches the endpoint detection to use IConfiguration and returns a Uri to ensure the configured value is valid. --- docs/telemetry.md | 4 ++-- .../HostApplicationBuilderExtensions.cs | 2 +- .../OtlpEndpoint.cs | 23 +++++++++++++++---- .../HostApplicationBuilderExtensions.cs | 2 +- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/docs/telemetry.md b/docs/telemetry.md index 1eec152d0a..94b3c1ca10 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -2,9 +2,9 @@ Instances can be configured to emit telemetry to aid in performance testing or troubleshooting performance-related issues. -Both the error and the audit instance report their ingestion the same way. Exporting is configured with the standard [OpenTelemetry environment variables](https://opentelemetry.io/docs/specs/otel/protocol/exporter/#configuration-options), not with instance settings, so the same variables that configure any other OpenTelemetry process apply here. Setting `OTEL_EXPORTER_OTLP_ENDPOINT` is enough to turn metrics on, and `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` overrides it for metrics alone. Both gRPC and HTTP endpoints are supported, and `OTEL_EXPORTER_OTLP_PROTOCOL` selects between them. +Both the error and the audit instance report their ingestion the same way. Exporting is configured with the standard [OpenTelemetry environment variables](https://opentelemetry.io/docs/specs/otel/protocol/exporter/#configuration-options), not with instance settings, so the same variables that configure any other OpenTelemetry process apply here. Setting `OTEL_EXPORTER_OTLP_ENDPOINT` is enough to turn metrics on. Both gRPC and HTTP endpoints are supported, and `OTEL_EXPORTER_OTLP_PROTOCOL` selects between them. -These variables have to be set in the instance's environment. They cannot be set in the instance's app.config, because the OpenTelemetry SDK reads them itself rather than going through the ServiceControl settings reader. +The signal-specific variables, `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` and its siblings, have no effect. The SDK only honours those under `UseOtlpExporter`, and instances use `AddOtlpExporter` so that OTLP applies to metrics without also being turned on for every other signal. Logs are exported separately. Add `Otlp` to the instance's `LoggingProviders` setting, which is what turns the OTLP log exporter on, and it then reads the same environment variables for its endpoint. diff --git a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs index 6add21574b..ccd2c635cc 100644 --- a/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl.Audit/HostApplicationBuilderExtensions.cs @@ -97,7 +97,7 @@ public static void AddMetrics(this IHostApplicationBuilder builder, Settings set { builder.Services.AddSingleton(); - var otlpEndpoint = OtlpEndpoint.MetricsEndpointFromEnvironment(); + var otlpEndpoint = OtlpEndpoint.Read(builder.Configuration); if (otlpEndpoint is null) { diff --git a/src/ServiceControl.Infrastructure/OtlpEndpoint.cs b/src/ServiceControl.Infrastructure/OtlpEndpoint.cs index 5f6a754724..eee1f7cbf6 100644 --- a/src/ServiceControl.Infrastructure/OtlpEndpoint.cs +++ b/src/ServiceControl.Infrastructure/OtlpEndpoint.cs @@ -1,15 +1,28 @@ namespace ServiceControl.Infrastructure; using System; +using Microsoft.Extensions.Configuration; public static class OtlpEndpoint { - public static string MetricsEndpointFromEnvironment() => - Read("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT") ?? Read("OTEL_EXPORTER_OTLP_ENDPOINT"); + // This is the default environment variable name used by the OpenTelemetry .NET SDK to configure the OTLP exporter endpoint + // as specified in https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md. + const string EndpointKey = "OTEL_EXPORTER_OTLP_ENDPOINT"; - static string Read(string variable) + public static Uri Read(IConfiguration configuration) { - var value = Environment.GetEnvironmentVariable(variable); - return string.IsNullOrWhiteSpace(value) ? null : value; + var configuredEndpoint = configuration[EndpointKey]; + + if (string.IsNullOrWhiteSpace(configuredEndpoint)) + { + return null; + } + + if (!Uri.TryCreate(configuredEndpoint, UriKind.Absolute, out var endpoint)) + { + throw new UriFormatException($"Invalid {EndpointKey}: {configuredEndpoint}"); + } + + return endpoint; } } diff --git a/src/ServiceControl/HostApplicationBuilderExtensions.cs b/src/ServiceControl/HostApplicationBuilderExtensions.cs index 7ae3370059..6f132742bc 100644 --- a/src/ServiceControl/HostApplicationBuilderExtensions.cs +++ b/src/ServiceControl/HostApplicationBuilderExtensions.cs @@ -152,7 +152,7 @@ public static void AddIngestionMetrics(this IHostApplicationBuilder hostBuilder, { hostBuilder.Services.AddSingleton(); - var otlpEndpoint = OtlpEndpoint.MetricsEndpointFromEnvironment(); + var otlpEndpoint = OtlpEndpoint.Read(hostBuilder.Configuration); if (otlpEndpoint is null) {