From 10af15966b6d00f2466b9d77a80709033b3cf1d5 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Sun, 6 Sep 2026 12:43:01 +0200 Subject: [PATCH 01/14] fix(opcua): recover the OPC UA session after a bad start Two failures share the same shape: the plugin decides something once, at startup, and can never revise it while it runs. Re-scan while no session is up. Config-less discovery (#544, #509) ran a single scan a couple of seconds after start. A gateway that boots alongside its PLC scans while the PLC is still coming up, finds nothing, falls back to opc.tcp://localhost:4840 and retries that endpoint for as long as it runs. Only a restart found the PLC. The poller's reconnect arm now asks the plugin for a fresh scan, rate limited by discovery.interval_s (default 30 s), and adopts a newly found server for its next connect attempt, resetting the backoff so the new endpoint is tried at once rather than after the dead one's accumulated wait. The rules that made discovery safe are unchanged: an explicitly configured endpoint_url still wins and is never rescanned, the scan stays a bounded read-only TCP sweep plus GetEndpoints, and nothing is scanned while a session is up. interval_s stops being an accepted-but-ignored knob. Clear PLC_COMMS_LOST on every successful connect. The fault raised for a sustained outage (#496) was cleared only when the running process still remembered raising it. The fault manager keys faults by fault code and persists them, so a fault raised before a gateway restart is standing in the store with nothing in memory to remember it, and the arm that would clear it is never entered when the first connect succeeds. The fault then stayed CONFIRMED for good. Both connect paths, the initial one and every reconnect, now send the clear regardless of what this process raised. The clear is fire and forget, so a clear for a fault that is not there costs nothing. The debounce that governs raising is untouched. Tests: the discovery pass and the endpoint adoption rule are exercised with injected probes, including the positive control that a configured endpoint is refused the very server an unconfigured one accepts. The comms-lost heal runs against the live test server, because only a connect that actually succeeds reaches that arm. --- .../ros2_medkit_opcua/README.md | 33 +++- .../ros2_medkit_opcua/network_discovery.hpp | 10 +- .../ros2_medkit_opcua/opcua_plugin.hpp | 58 +++++++ .../ros2_medkit_opcua/opcua_poller.hpp | 25 +++ .../ros2_medkit_opcua/src/opcua_plugin.cpp | 148 +++++++++++++++--- .../ros2_medkit_opcua/src/opcua_poller.cpp | 44 +++++- .../test/test_opcua_identity.cpp | 82 +++++++++- .../test/test_opcua_plugin.cpp | 141 +++++++++++++++++ 8 files changed, 499 insertions(+), 42 deletions(-) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md index 0381ef765..17cc1115e 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md @@ -548,7 +548,7 @@ ros2_medkit_gateway: | `subscription_interval_ms` | `500` | Publishing interval for OPC-UA subscriptions when `prefer_subscriptions: true` | | `condition_replay_strategy` | `auto` | Active-condition replay on reconnect: `method`, `read`, `auto`, `off` (see below) | | `require_confirm_for_clear` | `true` | Require both Acknowledge AND Confirm before a native alarm auto-clears. Set `false` for Confirm-less servers (e.g. Siemens S7-1500) so alarms clear on Acknowledge alone (see below) | -| `comms_lost_fault_enabled` | `true` | Raise a component-scoped `PLC_COMMS_LOST` fault when the connection stays down (issue #496) | +| `comms_lost_fault_enabled` | `true` | Raise a component-scoped `PLC_COMMS_LOST` fault when the connection stays down, and clear it on every successful connect (issue #496) | | `comms_lost_debounce_ms` | `5000` | Continuous down time before `PLC_COMMS_LOST` is raised (debounces reconnect blips; clamped to [0, 3600000] ms) | | `comms_lost_severity` | `ERROR` | SOVD severity bucket for the `PLC_COMMS_LOST` fault | | `discovery.enabled` | `false` | Opt-in read-only PLC network discovery (auto endpoint). See below | @@ -604,7 +604,7 @@ plugins.opcua.discovery: connect_timeout_ms: 600 # per-port TCP connect timeout scan_concurrency: 100 # bounded, polite concurrent connect count identify_timeout_ms: 6000 # per GetEndpoints identify - interval_s: 0 # 0 = one-shot at startup (periodic re-scan: TODO) + interval_s: 0 # re-scan cadence while disconnected (0 = default 30 s) anonymous_none_only: true # only auto-connect None/Anonymous servers ``` @@ -625,6 +625,15 @@ How it works: auto-selects the best None/Anonymous data server (deterministic, lowest ip:port) and connects to the **scanned ip:port** - not the advertised EndpointUrl, which a server may report as a non-resolvable hostname. +5. While no session is established, the reconnect loop scans again every + `interval_s` (default 30 s) and adopts a newly found server for its next + connect attempt, logging the swap at INFO. This is what covers the common + race where the gateway and the PLC boot together: the startup scan finds + nothing because the PLC is still coming up, and without a re-scan the plugin + would retry the fallback endpoint until someone restarted it. + +Re-scanning stops as soon as a session is up, and never starts at all when an +`endpoint_url` is configured. Safety / OT posture: - Everything is read-only: TCP connect + `GetEndpoints` only. No writes, no @@ -640,9 +649,9 @@ Safety / OT posture: Note on passive discovery: a stock Siemens S7-1500 neither multicast-announces (mDNS `_opcua-tcp._tcp`) nor registers with an OPC-UA LDS, so passive sources find nothing there; the active scan is what discovers it. Passive mDNS / LDS -`FindServers` sources (useful on Kepware / Prosys / GDS estates) and periodic -re-scan + multi-endpoint registration are planned follow-ups; this iteration -delivers the active-scan core and single "auto endpoint" mode. +`FindServers` sources (useful on Kepware / Prosys / GDS estates) and +multi-endpoint registration are planned follow-ups. This iteration delivers the +active-scan core and a single "auto endpoint" mode. ### Active-condition replay on reconnect (issue #389/#478) @@ -694,6 +703,20 @@ so the alarm clears on `Acknowledge` alone. The default (`true`) is unchanged and spec-strict; the relaxed path still requires acknowledgement and needs real-S7-1500 validation. +### Connection loss and `PLC_COMMS_LOST` (issue #496) + +When the OPC-UA connection stays down for `comms_lost_debounce_ms` continuously, +the plugin raises one component-scoped `PLC_COMMS_LOST` fault (a shorter blip +during a normal reconnect does not flap it). + +The fault is cleared on **every** successful connect, both the initial one and +every later reconnect, whether or not this process was the one that raised it. +The fault manager keys faults by fault code and persists them, so a fault raised +before a gateway restart is still standing while the new process has no memory +of it. Clearing only what the running process remembered left exactly that fault +CONFIRMED for good. The clear is fire-and-forget, so a clear for a fault that is +not there is harmless. + Node map entries also support an optional `ros2_topic` field to override the auto-generated ROS 2 topic name for the PLC value bridge: ```yaml diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/network_discovery.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/network_discovery.hpp index ed68d33e5..56c31a8ae 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/network_discovery.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/network_discovery.hpp @@ -105,9 +105,13 @@ struct OpcuaDiscoveryConfig { int scan_concurrency{100}; ///< bounded, polite concurrent connect count int identify_timeout_ms{6000}; - /// Re-scan cadence. 0 = one-shot at startup (the only mode implemented in - /// this iteration); a positive value is accepted and validated but periodic - /// re-scan is a documented follow-up. + /// Re-scan cadence, in seconds, while no OPC-UA session is established. 0 + /// selects the built-in default (see OpcuaPlugin::effective_rescan_interval_s). + /// The startup scan always runs once. The cadence only governs how often the + /// disconnected reconnect loop scans again, so a gateway that started before + /// its PLC finished booting adopts the PLC when it appears instead of retrying + /// the fallback endpoint forever. Never used once an endpoint is configured + /// explicitly, and never while a session is up. int interval_s{0}; /// Only auto-register endpoints that expose a None + Anonymous endpoint (what diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp index e77a3a519..95d2415e3 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp @@ -31,10 +31,12 @@ #include #include +#include #include #include #include #include +#include #include #include #include @@ -141,6 +143,42 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, static void apply_auto_alarms_param(const nlohmann::json & value, AutoAlarmsConfig & cfg, const std::function & warn); + // Run one read-only discovery pass and return the endpoint URL to adopt. + // + // Returns nullopt - meaning "keep the endpoint you have" - when discovery is + // disabled, when an endpoint was configured explicitly, when no subnet could + // be resolved, or when the pass found no auto-connectable None/Anonymous data + // server. Both the startup scan and the reconnect rescan go through here, so + // the two cannot drift apart. Static with injected probes and log sinks so + // both are unit-testable without a network. + // + // @param config discovery configuration (subnets, ports, timeouts, ...) + // @param endpoint_configured true when the operator pinned endpoint_url, so + // discovery then selects nothing and can neither override the + // operator's target nor open a second session on an already polled PLC + // @param scan injected TCP port probe + // @param identify injected OPC-UA GetEndpoints identify + // @param log_info operator-visible info sink + // @param log_warn operator-visible warning sink + static std::optional discover_endpoint(const OpcuaDiscoveryConfig & config, bool endpoint_configured, + const PortScanFn & scan, const IdentifyFn & identify, + const std::function & log_info, + const std::function & log_warn); + + // Seconds between reconnect rescans, or 0 when the reconnect loop must never + // rescan (discovery disabled, or an endpoint configured explicitly). A + // configured ``interval_s`` wins. interval_s = 0 means "discovery is on but no + // cadence was stated" and takes the built-in default rather than never + // rescanning: a config-less deployment is the one that cannot name a cadence + // and the one that most needs its PLC adopted once it finishes booting. + static int effective_rescan_interval_s(const OpcuaDiscoveryConfig & config, bool endpoint_configured); + + // Default reconnect rescan cadence, in seconds, when discovery is enabled with + // no explicit ``interval_s``. Long enough that a bounded subnet sweep stays a + // background cost on the poll thread, short enough that a PLC finishing its + // boot is picked up in well under a minute. + static constexpr int kDefaultRescanIntervalS = 30; + private: // Route handlers void handle_plc_data(const ros2_medkit_gateway::PluginRequest & req, ros2_medkit_gateway::PluginResponse & res); @@ -160,6 +198,14 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, const std::string & severity_str, const std::string & message); void send_clear_fault(const std::string & fault_code); + // Clear PLC_COMMS_LOST after the initial connect in set_context() succeeded. + // Unconditional on purpose: the fault manager keys faults by fault_code and + // persists them, so a comms-lost fault raised before a gateway restart is + // still standing in the store while this process has no memory of raising it. + // The poller's own reconnect clear can never reach that case, because a + // successful first connect means the reconnect arm is never entered. + void clear_comms_lost_on_connect(); + // Dispatch now if the fault_manager service is matched, else buffer the // dispatch (bounded, order-preserving) to be flushed once it appears. void send_or_buffer(std::function dispatch); @@ -206,6 +252,14 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // an endpoint is already configured. void run_startup_discovery(); + // Poll-thread hook bound into PollerConfig::rediscover_endpoint whenever + // discovery runs without a configured endpoint. Called from the poller's + // reconnect arm, so only while no session is up, and rate-limited to one scan + // per effective_rescan_interval_s(). Returns the newly selected endpoint when + // a rescan found a different server (and logs the swap at INFO), nullopt when + // the rescan is not due yet or changed nothing. + std::optional rescan_endpoint_for_reconnect(); + // Build JSON response for data endpoint nlohmann::json build_data_response(const std::string & entity_id) const; @@ -231,6 +285,10 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // a network. Set in configure(), consumed in run_startup_discovery(). PortScanFn discovery_scan_fn_; IdentifyFn discovery_identify_fn_; + // When the last discovery pass ran, so the reconnect rescan honours the + // cadence instead of sweeping the subnet on every reconnect attempt. Stamped + // by the startup scan, then only ever read/written on the poll thread. + std::chrono::steady_clock::time_point last_discovery_scan_{}; std::unique_ptr client_; NodeMap node_map_; diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp index 711c3ec47..73f670f72 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -133,8 +134,21 @@ struct PollerConfig { /// fire-and-forget report is never dropped-and-forgotten while the sink is /// unmatched - it retries on the next poll instead. Empty => assume ready. std::function report_sink_ready; + /// Optional endpoint rediscovery, bound by a plugin running config-less + /// network discovery with no endpoint configured. Called from the reconnect + /// arm - so only while no session is up - and expected to rate-limit itself. + /// Returns a new endpoint URL to reconnect against, nullopt to keep the + /// current one. Without it the reconnect loop retries the same endpoint + /// forever, which strands a gateway that scanned before its PLC had booted. + std::function()> rediscover_endpoint; }; +/// Fault code of the component-scoped OPC-UA connection fault the poller raises +/// on a sustained outage and clears on the next successful connect (issue #496). +/// Named here so the plugin can clear the same code from its own connect path +/// without keeping a second copy of the literal. +inline constexpr const char * kCommsLostFaultCode = "PLC_COMMS_LOST"; + /// Manages OPC-UA data collection via subscriptions (preferred) or polling class OpcuaPoller { public: @@ -237,6 +251,17 @@ class OpcuaPoller { std::chrono::steady_clock::time_point down_since, std::chrono::steady_clock::time_point now, std::chrono::milliseconds debounce); + /// Endpoint the next reconnect attempt should target. Asks + /// ``rediscover_endpoint`` (when bound) for a freshly discovered server and + /// returns it only when it names a DIFFERENT endpoint than ``current``. + /// nullopt means "keep the current one", which is also the answer when no + /// callback is bound, when the callback declines, or when it hands back an + /// empty string. Pure and static (the callback is injected) so the adoption + /// rule is unit-testable without a network. + static std::optional + adopt_rediscovered_endpoint(const std::string & current, + const std::function()> & rediscover); + /// Zero-config native A&C (``auto_alarms``): the alarm sources that should /// actually be subscribed / replayed, i.e. every explicit ``event_alarms`` /// entry plus (when ``auto_cfg.enabled`` and no explicit entry already diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp index aebb164f9..884d599ae 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp @@ -585,6 +585,7 @@ void OpcuaPlugin::set_context(PluginContext & context) { const bool connected = client_->connect(client_config_); if (connected) { log_info("Connected to OPC-UA server: " + client_config_.endpoint_url); + clear_comms_lost_on_connect(); } else { log_warn("Failed to connect to OPC-UA server: " + client_config_.endpoint_url); } @@ -660,6 +661,16 @@ void OpcuaPlugin::set_context(PluginContext & context) { poller_config_.report_sink_ready = [this]() { return fault_clients_->report && fault_clients_->report->service_is_ready(); }; + // Config-less discovery with no configured endpoint: let the poller's + // reconnect arm ask for a fresh scan while it is down. Without this the + // startup scan is the only one that ever runs, so a gateway that scanned + // while its PLC was still booting retries the fallback endpoint forever and + // only a restart finds the PLC. + if (effective_rescan_interval_s(discovery_config_, endpoint_configured_) > 0) { + poller_config_.rediscover_endpoint = [this]() { + return rescan_endpoint_for_reconnect(); + }; + } poller_->start(poller_config_); log_info("OPC-UA poller started (mode: " + std::string(poller_->using_subscriptions() ? "subscription" : "poll") + ")"); @@ -733,7 +744,11 @@ IntrospectionResult OpcuaPlugin::introspect(const IntrospectionInput & /*input*/ // Fault scope grants bare-id ownership only to external entities; the poller // reports PLC_COMMS_LOST under this component's own id. comp.external = true; - comp.description = "PLC runtime connected at " + client_config_.endpoint_url; + // Read the endpoint off the client, not off client_config_: a reconnect + // rescan can adopt a different server after startup, and the client is the + // one that holds the endpoint actually being connected to. + const std::string live_endpoint = client_ ? client_->endpoint_url() : client_config_.endpoint_url; + comp.description = "PLC runtime connected at " + live_endpoint; // INV2: fill the asset-identity nameplate from the live server's device-info // (ServerStatus/BuildInfo + optional OPC-UA DI nameplate). Read once per @@ -744,7 +759,7 @@ IntrospectionResult OpcuaPlugin::introspect(const IntrospectionInput & /*input*/ if (client_ && client_->is_connected()) { const uint64_t session_generation = client_->connection_generation(); if (session_generation != device_identity_generation_) { - device_identity_ = opcua_device_info_to_identity(client_->read_device_info(), client_config_.endpoint_url); + device_identity_ = opcua_device_info_to_identity(client_->read_device_info(), live_endpoint); device_identity_generation_ = session_generation; if (!device_identity_.empty()) { log_info("Populated asset identity from OPC-UA device-info (manufacturer='" + device_identity_.manufacturer + @@ -1253,6 +1268,17 @@ void OpcuaPlugin::send_clear_fault(const std::string & fault_code) { }); } +void OpcuaPlugin::clear_comms_lost_on_connect() { + if (!poller_config_.comms_lost_fault_enabled) { + return; + } + // ClearFault is idempotent from this side: send_clear_fault is + // fire-and-forget, so a "Fault not found" answer for a code that was never + // raised costs nothing here and is the normal case on a healthy start. + log_info(std::string("OPC-UA connection established; clearing any standing ") + kCommsLostFaultCode); + send_clear_fault(kCommsLostFaultCode); +} + void OpcuaPlugin::send_or_buffer(std::function dispatch) { // Bound the buffer so a deployment with no fault_manager cannot grow it // without limit; drop the oldest (least relevant) pending dispatch. @@ -1466,35 +1492,39 @@ void OpcuaPlugin::log_security_profile() const { } } -void OpcuaPlugin::run_startup_discovery() { - if (!discovery_config_.enabled) { - return; +int OpcuaPlugin::effective_rescan_interval_s(const OpcuaDiscoveryConfig & config, bool endpoint_configured) { + if (!config.enabled || endpoint_configured) { + return 0; + } + return config.interval_s > 0 ? config.interval_s : kDefaultRescanIntervalS; +} + +std::optional OpcuaPlugin::discover_endpoint(const OpcuaDiscoveryConfig & config, bool endpoint_configured, + const PortScanFn & scan, const IdentifyFn & identify, + const std::function & log_info, + const std::function & log_warn) { + if (!config.enabled) { + return std::nullopt; } // Never override an explicitly configured endpoint: discovery must not open a // second session on a PLC the operator already targets (and already polls). - if (endpoint_configured_) { - log_info("OPC-UA discovery enabled but endpoint_url is explicitly configured (" + client_config_.endpoint_url + - "); skipping auto-discovery to avoid a second session."); - return; - } - if (discovery_config_.interval_s > 0) { - log_warn("OPC-UA discovery interval_s=" + std::to_string(discovery_config_.interval_s) + - " set, but periodic re-scan is not implemented yet; running a one-shot scan at startup."); + if (endpoint_configured) { + return std::nullopt; } - NetworkDiscovery discovery(discovery_config_, discovery_scan_fn_, discovery_identify_fn_); + NetworkDiscovery discovery(config, scan, identify); const auto subnets = discovery.resolve_subnets(); if (subnets.empty()) { log_warn("OPC-UA discovery: no subnet configured and could not derive a local /24; nothing to scan."); - return; + return std::nullopt; } std::string subnet_list; for (const auto & s : subnets) { subnet_list += (subnet_list.empty() ? "" : ", ") + s; } log_info("OPC-UA discovery: read-only active scan of [" + subnet_list + "] on " + - std::to_string(discovery_config_.ports.size()) + " port(s)..."); + std::to_string(config.ports.size()) + " port(s)..."); const std::vector found = discovery.run(); @@ -1528,18 +1558,92 @@ void OpcuaPlugin::run_startup_discovery() { std::to_string(discovery_servers) + " discovery server(s)/LDS, " + std::to_string(secured_only) + " secured-only (need credentials), " + std::to_string(leads) + " non-OPC-UA/unidentified lead(s)."); - const DiscoveredEndpoint * chosen = - NetworkDiscovery::select_auto_endpoint(found, discovery_config_.anonymous_none_only); + const DiscoveredEndpoint * chosen = NetworkDiscovery::select_auto_endpoint(found, config.anonymous_none_only); if (chosen == nullptr) { log_warn( - "OPC-UA discovery: no auto-connectable None/Anonymous data server found; leaving endpoint at default. " + "OPC-UA discovery: no auto-connectable None/Anonymous data server found; leaving the endpoint unchanged. " "Secured-only servers require operator credentials."); + return std::nullopt; + } + + log_info("OPC-UA discovery: selected endpoint " + chosen->endpoint_url + " (uri='" + chosen->application_uri + "')"); + return chosen->endpoint_url; +} + +void OpcuaPlugin::run_startup_discovery() { + if (!discovery_config_.enabled) { return; } + if (endpoint_configured_) { + log_info("OPC-UA discovery enabled but endpoint_url is explicitly configured (" + client_config_.endpoint_url + + "); skipping auto-discovery to avoid a second session."); + return; + } + + // Stamp the scan before running it: the rescan cadence measures the gap + // between the START of two sweeps, so a slow sweep does not immediately earn + // another one. + last_discovery_scan_ = std::chrono::steady_clock::now(); + const auto chosen = discover_endpoint( + discovery_config_, endpoint_configured_, discovery_scan_fn_, discovery_identify_fn_, + [this](const std::string & m) { + log_info(m); + }, + [this](const std::string & m) { + log_warn(m); + }); + + if (!chosen) { + // The startup scan can legitimately find nothing - a gateway that boots + // alongside its PLC routinely scans while the PLC is still coming up. The + // endpoint stays at its default and the poller's reconnect arm rescans on + // the cadence below, so this is a delay rather than a dead end. + log_info("OPC-UA discovery: startup scan selected no endpoint; the reconnect loop rescans every " + + std::to_string(effective_rescan_interval_s(discovery_config_, endpoint_configured_)) + "s while down."); + return; + } + + client_config_.endpoint_url = *chosen; + log_info("OPC-UA discovery: auto-selected endpoint " + *chosen + " - handing to the connect + introspect path."); +} + +std::optional OpcuaPlugin::rescan_endpoint_for_reconnect() { + // A sweep is a bounded but multi-second blocking call on the poll thread, and + // stop() has to wait for whatever it is in the middle of. Do not start one the + // shutdown is going to throw away. + if (shutdown_requested_.load()) { + return std::nullopt; + } + const int interval_s = effective_rescan_interval_s(discovery_config_, endpoint_configured_); + if (interval_s <= 0) { + return std::nullopt; + } + + const auto now = std::chrono::steady_clock::now(); + if (now - last_discovery_scan_ < std::chrono::seconds(interval_s)) { + return std::nullopt; + } + last_discovery_scan_ = now; + + const auto chosen = discover_endpoint( + discovery_config_, endpoint_configured_, discovery_scan_fn_, discovery_identify_fn_, + [this](const std::string & m) { + log_info(m); + }, + [this](const std::string & m) { + log_warn(m); + }); + // The live client config, not client_config_: this runs on the poll thread + // and client_config_ is read by the refresh thread in introspect(). The + // client owns the endpoint once connect() has been called with it, and its + // accessors are mutex-guarded. + const std::string current = client_ ? client_->endpoint_url() : client_config_.endpoint_url; + if (!chosen || *chosen == current) { + return std::nullopt; + } - client_config_.endpoint_url = chosen->endpoint_url; - log_info("OPC-UA discovery: auto-selected endpoint " + chosen->endpoint_url + " (uri='" + chosen->application_uri + - "') - handing to the connect + introspect path."); + log_info("OPC-UA discovery: rescan while disconnected adopted endpoint " + *chosen + " (was " + current + ")"); + return chosen; } nlohmann::json OpcuaPlugin::build_data_response(const std::string & entity_id) const { diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_poller.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_poller.cpp index c24f9ca0b..d972a3e05 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_poller.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_poller.cpp @@ -1142,9 +1142,22 @@ bool OpcuaPoller::comms_lost_should_raise(bool enabled, bool already_raised, return (now - down_since) >= debounce; } +std::optional +OpcuaPoller::adopt_rediscovered_endpoint(const std::string & current, + const std::function()> & rediscover) { + if (!rediscover) { + return std::nullopt; + } + const std::optional found = rediscover(); + if (!found || found->empty() || *found == current) { + return std::nullopt; + } + return found; +} + void OpcuaPoller::emit_comms_lost(bool active) { ros2_medkit::fault_detection::FaultSignal signal; - signal.fault_code = "PLC_COMMS_LOST"; + signal.fault_code = kCommsLostFaultCode; signal.severity = config_.comms_lost_severity; signal.message = active ? ("OPC-UA connection lost to " + client_.endpoint_url()) : ("OPC-UA connection restored to " + client_.endpoint_url()); @@ -1173,15 +1186,32 @@ void OpcuaPoller::poll_loop() { comms_down_since_ = std::chrono::steady_clock::now(); } - // Attempt reconnect with original config (preserves timeout, etc.) - if (client_.connect(client_.current_config())) { + // Reconnect with the original config (preserves timeout, security, ...). + // The endpoint is the one exception: when a rediscovery callback is bound + // and offers a different server, adopt it for this attempt. connect() + // stores the config it is given, so current_config() carries the adopted + // endpoint from here on and every later retry targets the new server. + OpcuaClientConfig reconnect_config = client_.current_config(); + if (auto adopted = adopt_rediscovered_endpoint(reconnect_config.endpoint_url, config_.rediscover_endpoint)) { + reconnect_config.endpoint_url = *adopted; + // A freshly discovered server deserves a prompt attempt: without this + // reset the backoff (up to 60 s) would keep the newly found PLC waiting + // for as long as the old dead endpoint had earned. + reconnect_wait = config_.reconnect_interval; + } + + if (client_.connect(reconnect_config)) { reconnect_wait = config_.reconnect_interval; // reset on success - // Issue #496: connection restored - clear the comms-lost fault if it - // was raised, then reset the debounce timer. - if (comms_lost_raised_) { + // Issue #496: connection restored - clear the comms-lost fault. Sent on + // EVERY successful reconnect, not only when this process raised it: the + // fault manager keys faults by fault_code and persists them, so a fault + // raised before a restart is standing in the store with nothing in + // memory to remember it. The clear is fire-and-forget and the store + // answers "not found" harmlessly when there is nothing to clear. + if (config_.comms_lost_fault_enabled) { emit_comms_lost(/*active=*/false); - comms_lost_raised_ = false; } + comms_lost_raised_ = false; comms_down_since_.reset(); if (config_.prefer_subscriptions) { setup_subscriptions(); diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp index 48294503f..93b84ee1f 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp @@ -12,15 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -// INV2 end-to-end (no HW): boot the test_alarm_server OPC-UA fixture, connect, -// and prove the asset-identity nameplate is filled from the server's device-info -// (ServerStatus/BuildInfo + the OPC-UA DI DeviceSet nameplate) with no manual -// entry. Exercises both the raw OpcuaClient::read_device_info read and the full -// OpcuaPlugin::introspect() path that lands identity on the SOVD Component. +// End-to-end against a live OPC-UA server (no HW): boot the test_alarm_server +// fixture and exercise the paths that only a real session can reach. +// +// INV2 identity: prove the asset-identity nameplate is filled from the server's +// device-info (ServerStatus/BuildInfo + the OPC-UA DI DeviceSet nameplate) with +// no manual entry, through both the raw OpcuaClient::read_device_info read and +// the full OpcuaPlugin::introspect() path that lands identity on the SOVD +// Component. +// +// Connection lifecycle: prove a successful connect clears the standing +// PLC_COMMS_LOST fault, which needs a connect that actually succeeds. #include "ros2_medkit_opcua/device_identity.hpp" #include "ros2_medkit_opcua/opcua_client.hpp" #include "ros2_medkit_opcua/opcua_plugin.hpp" +#include "ros2_medkit_opcua/opcua_poller.hpp" #include @@ -33,13 +40,16 @@ #include #include +#include #include #include #include #include +#include #include #include #include +#include #include #include @@ -559,4 +569,66 @@ TEST_F(OpcuaIdentityE2ETest, DiNameplateReadFollowsBrowseContinuationPoints) { client.disconnect(); } +// A gateway that restarts after a comms outage never raised PLC_COMMS_LOST in +// THIS process, yet the fault manager keys faults by fault_code alone and +// persists them, so the fault raised before the restart is still standing. +// The reconnect arm used to clear only when its own in-memory +// ``comms_lost_raised_`` flag was set, which no restart can satisfy, so the +// fault stayed CONFIRMED for good. The clear now goes out on every successful +// connect. Driven against the live fixture because the arm can only be reached +// by a connect that actually succeeds. +TEST_F(OpcuaIdentityE2ETest, SuccessfulConnectClearsCommsLostNeverRaisedHere) { + OpcuaClient client; + OpcuaClientConfig config; + config.endpoint_url = endpoint_; + config.connect_timeout = std::chrono::milliseconds(5000); + // Connect once to seed the client's stored config (what the poller reconnects + // with), then drop the session so the poll loop starts in its reconnect arm - + // the state a freshly started gateway is in while the PLC is already up. + ASSERT_TRUE(client.connect(config)); + client.disconnect(); + ASSERT_FALSE(client.is_connected()); + + NodeMap node_map; // config-less: no entries, nothing to poll + OpcuaPoller poller(client, node_map); + + std::mutex signals_mutex; + std::vector> signals; // (fault_code, active) + poller.set_alarm_callback( + [&signals_mutex, &signals](const std::string &, const ros2_medkit::fault_detection::FaultSignal & signal) { + std::lock_guard lock(signals_mutex); + signals.emplace_back(signal.fault_code, signal.active); + }); + + PollerConfig poller_config; + poller_config.poll_interval = std::chrono::milliseconds(100); + poller_config.reconnect_interval = std::chrono::milliseconds(100); + poller_config.comms_lost_fault_enabled = true; + poller.start(poller_config); + + bool cleared = false; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(15); + while (!cleared && std::chrono::steady_clock::now() < deadline) { + { + std::lock_guard lock(signals_mutex); + cleared = std::find(signals.begin(), signals.end(), std::make_pair(std::string(kCommsLostFaultCode), false)) != + signals.end(); + } + if (!cleared) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + } + poller.stop(); + + EXPECT_TRUE(cleared) << "a successful connect must clear PLC_COMMS_LOST even when this process never raised it"; + + // Absence control on the same harness: the connect succeeded, so nothing may + // have RAISED the fault. Without this a clear-everything-always regression + // would still pass the assertion above. + std::lock_guard lock(signals_mutex); + EXPECT_EQ(std::find(signals.begin(), signals.end(), std::make_pair(std::string(kCommsLostFaultCode), true)), + signals.end()) + << "comms-lost must not be raised while the connection is up"; +} + } // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp index 4bfdf4bc6..f47108005 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp @@ -27,7 +27,11 @@ #include #include #include +#include +#include #include +#include +#include #include #include #include @@ -638,6 +642,143 @@ TEST(CommsLostShouldRaise, IdempotentAndDisabled) { EXPECT_FALSE(OpcuaPoller::comms_lost_should_raise(/*enabled=*/false, false, t0, late, debounce)); } +// --------------------------------------------------------------------------- +// Endpoint rediscovery while disconnected (config-less discovery) +// --------------------------------------------------------------------------- + +namespace { + +// Fake port scanner backed by a set of open "ip:port" hosts. +PortScanFn fake_scan(std::set open) { + return [open = std::move(open)](const std::string & ip, uint16_t port, int) { + return open.count(ip + ":" + std::to_string(port)) > 0; + }; +} + +// Fake GetEndpoints identify keyed by connect URL. Anything else is unreachable. +IdentifyFn fake_identify(std::map table) { + return [table = std::move(table)](const std::string & url, int) -> IdentifyResult { + const auto it = table.find(url); + if (it != table.end()) { + return it->second; + } + IdentifyResult r; + r.error = "unreachable"; + return r; + }; +} + +IdentifyResult plc_identity() { + IdentifyResult r; + r.ok = true; + r.advertised_url = "opc.tcp://192.168.1.10:4840"; + r.application_uri = "urn:SIMATIC.S7-1500.OPC-UA.Application:Software PLC_1"; + r.product_uri = "https://www.siemens.com/s7-1500"; + r.application_name = "SIMATIC.S7-1500"; + r.application_type = 0; // Server + r.security_policies = {{"None", 1}}; + r.anonymous_none_available = true; + return r; +} + +OpcuaDiscoveryConfig rescan_cfg() { + OpcuaDiscoveryConfig cfg; + cfg.enabled = true; + cfg.subnets = {"192.168.1.0/24"}; // explicit, so no local-interface derivation + cfg.ports = {4840}; + return cfg; +} + +// Discards log output. The tests assert on the selected endpoint, not the text. +const std::function kSilent = [](const std::string &) {}; + +} // namespace + +TEST(DiscoverEndpoint, ScanBeforeThePlcIsUpSelectsNothingAndALaterRescanAdoptsIt) { + // The field race: the gateway scans 2 s after start while the PLC is still + // booting. Nothing answers, so nothing is selected and the caller keeps the + // default endpoint. + const auto empty_pass = OpcuaPlugin::discover_endpoint(rescan_cfg(), /*endpoint_configured=*/false, fake_scan({}), + fake_identify({}), kSilent, kSilent); + EXPECT_FALSE(empty_pass.has_value()); + + // The PLC finishes booting. The same call with the same config now finds it, + // which is what the reconnect arm applies to the next connect attempt. + const auto later_pass = OpcuaPlugin::discover_endpoint( + rescan_cfg(), /*endpoint_configured=*/false, fake_scan({"192.168.1.10:4840"}), + fake_identify({{"opc.tcp://192.168.1.10:4840", plc_identity()}}), kSilent, kSilent); + ASSERT_TRUE(later_pass.has_value()); + EXPECT_EQ(*later_pass, "opc.tcp://192.168.1.10:4840"); +} + +TEST(DiscoverEndpoint, AnExplicitEndpointIsNeverRescanned) { + // Positive control: the very scan that DOES find a server above finds the + // same server here, and is still refused because the operator pinned an + // endpoint. Discovery must not open a second session on a polled PLC. + const auto chosen = OpcuaPlugin::discover_endpoint( + rescan_cfg(), /*endpoint_configured=*/true, fake_scan({"192.168.1.10:4840"}), + fake_identify({{"opc.tcp://192.168.1.10:4840", plc_identity()}}), kSilent, kSilent); + EXPECT_FALSE(chosen.has_value()); +} + +TEST(DiscoverEndpoint, DisabledDiscoveryScansNothing) { + OpcuaDiscoveryConfig cfg = rescan_cfg(); + cfg.enabled = false; + bool scanned = false; + auto counting_scan = [&scanned](const std::string &, uint16_t, int) { + scanned = true; + return true; + }; + const auto chosen = OpcuaPlugin::discover_endpoint(cfg, /*endpoint_configured=*/false, counting_scan, + fake_identify({}), kSilent, kSilent); + EXPECT_FALSE(chosen.has_value()); + EXPECT_FALSE(scanned) << "a disabled discovery must not touch the network"; +} + +TEST(EffectiveRescanInterval, DefaultsWhenDiscoveryIsOnWithNoCadenceAndIsOffOtherwise) { + OpcuaDiscoveryConfig cfg = rescan_cfg(); + // Config-less: discovery on, no interval stated -> the built-in cadence, not + // "never rescan". This is the deployment that most needs the rescan. + EXPECT_EQ(OpcuaPlugin::effective_rescan_interval_s(cfg, /*endpoint_configured=*/false), + OpcuaPlugin::kDefaultRescanIntervalS); + // An operator-stated cadence wins. + cfg.interval_s = 120; + EXPECT_EQ(OpcuaPlugin::effective_rescan_interval_s(cfg, false), 120); + // An explicit endpoint, or discovery off, means no rescan at all. + EXPECT_EQ(OpcuaPlugin::effective_rescan_interval_s(cfg, /*endpoint_configured=*/true), 0); + cfg.enabled = false; + EXPECT_EQ(OpcuaPlugin::effective_rescan_interval_s(cfg, false), 0); +} + +TEST(AdoptRediscoveredEndpoint, AdoptsOnlyADifferentNonEmptyUrl) { + const std::string current = "opc.tcp://localhost:4840"; + + // No callback bound (an explicit endpoint, or discovery off) -> keep current. + EXPECT_FALSE(OpcuaPoller::adopt_rediscovered_endpoint(current, nullptr).has_value()); + + // Rescan not due, or found nothing -> keep current. + EXPECT_FALSE(OpcuaPoller::adopt_rediscovered_endpoint(current, [] { + return std::optional{}; + }).has_value()); + + // Same server as before -> nothing to adopt, so no needless reconnect churn. + EXPECT_FALSE(OpcuaPoller::adopt_rediscovered_endpoint(current, [¤t] { + return std::optional{current}; + }).has_value()); + + // An empty URL is not an endpoint. + EXPECT_FALSE(OpcuaPoller::adopt_rediscovered_endpoint(current, [] { + return std::optional{""}; + }).has_value()); + + // A different server -> adopt it for the next connect attempt. + const auto adopted = OpcuaPoller::adopt_rediscovered_endpoint(current, [] { + return std::optional{"opc.tcp://192.168.1.10:4840"}; + }); + ASSERT_TRUE(adopted.has_value()); + EXPECT_EQ(*adopted, "opc.tcp://192.168.1.10:4840"); +} + // Issue #478 safety-gate: an empty scan from a source that has NEVER yielded a // condition instance node (EventNotifier-only server, e.g. S7-1500) must NOT // clear the still-active tracked fault. This is the single most important From 67b072a7619143072ba5d472f7d537db73c82ded Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Sun, 6 Sep 2026 12:43:01 +0200 Subject: [PATCH 02/14] fix(gateway): say where a plugin entity's freeze-frame values came from A freeze-frame captured for a plugin-backed entity (#564) reaches a client with an empty topic and an empty message_type. That is correct, the values are the plugin's live entity data and not a ROS message, but it leaves the snapshot with no field at all naming its origin. Two of these frames from different bridges are indistinguishable, and a frame is indistinguishable from a topic capture whose metadata went missing. Frame now carries the capture path that read it, and it is served as x-medkit.source: plugin_data_provider for a read through the owning plugin's DataProvider, plugin_x_plc_data_route for the in-process dispatch of the plugin's own x-plc-data route (bridges that export no DataProvider). topic and message_type are left empty rather than overloaded, since neither names a ROS topic here. The field is omitted, not emptied, when the capture named no path, so a fault-manager freeze-frame taken from a real topic is unaffected and carries its topic and message_type as before. --- docs/tutorials/snapshots.rst | 21 ++++++++ .../entity_freeze_frame_capture.hpp | 19 +++++++- .../src/entity_freeze_frame_capture.cpp | 7 +-- .../src/http/handlers/fault_handlers.cpp | 13 +++++ .../test/test_entity_freeze_frame_capture.cpp | 31 ++++++++++++ .../test/test_fault_handlers.cpp | 48 +++++++++++++++++++ 6 files changed, 134 insertions(+), 5 deletions(-) diff --git a/docs/tutorials/snapshots.rst b/docs/tutorials/snapshots.rst index f17d55d7e..c23e845ef 100644 --- a/docs/tutorials/snapshots.rst +++ b/docs/tutorials/snapshots.rst @@ -216,6 +216,26 @@ with: ros2 run ros2_medkit_gateway gateway_node --ros-args \ -p entity_freeze_frame.enabled:=false +A plugin entity's values are not a ROS message, so ``topic`` and +``message_type`` are empty on these frames. ``x-medkit.source`` names the +capture path instead, so a consumer can still tell where the values came +from: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - ``x-medkit.source`` + - Meaning + * - ``plugin_data_provider`` + - Read through the owning plugin's ``DataProvider::list_data``. + * - ``plugin_x_plc_data_route`` + - Read by dispatching the owning plugin's own ``x-plc-data`` route + in-process (plugins that export no ``DataProvider``). + +The field is absent on freeze-frames captured by the fault manager from a ROS +topic. Those carry a real ``topic`` and ``message_type`` instead. + Example plugin-entity freeze-frame in the fault response: .. code-block:: json @@ -227,6 +247,7 @@ Example plugin-entity freeze-frame in the fault response: "x-medkit": { "topic": "", "message_type": "", + "source": "plugin_x_plc_data_route", "full_data": {"tank_level": 87.5, "pump_running": true}, "captured_at": "2026-07-14T12:00:00.000Z" } diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp index 7dc081dc0..d3c041ee4 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp @@ -60,6 +60,13 @@ namespace ros2_medkit_gateway { */ class EntityFreezeFrameCapture { public: + /// Capture-path identifiers stored in Frame::source and served as + /// ``x-medkit.source``. The plugin's own DataProvider, and the in-process + /// dispatch of the plugin's `x-plc-data` route for plugins that export no + /// DataProvider. + static constexpr const char * kSourceDataProvider = "plugin_data_provider"; + static constexpr const char * kSourceXPlcDataRoute = "plugin_x_plc_data_route"; + /// One captured frame: the entity's data values at fault-confirm time. /// captured_at_ns dates the capture, not the values - a disconnected entity /// serves its last known values, whose age is bounded only by the outage. @@ -72,6 +79,13 @@ class EntityFreezeFrameCapture { bool startup_catchup{false}; std::optional connected; ///< payload's top-level link flag, when reported nlohmann::json source_timestamp; ///< payload's own "timestamp" field verbatim (null when absent) + /// Which capture path read the values (kSourceDataProvider / + /// kSourceXPlcDataRoute), served as ``x-medkit.source``. These values are + /// entity data, not a ROS message, so ``topic`` and ``message_type`` are + /// empty on the wire and would otherwise leave a consumer with nothing at + /// all saying where the numbers came from. Empty when the caller named no + /// path. + std::string source; }; /// Resolves an entity id to its owning plugin's DataProvider (nullptr when @@ -187,9 +201,10 @@ class EntityFreezeFrameCapture { bool capture_for_event(const ros2_medkit_msgs::msg::FaultEvent & event, bool startup_catchup = false); /// Build a frame from list-data-shaped content, enforcing the shared - /// no-row-of-nulls invariant on both capture paths. + /// no-row-of-nulls invariant on both capture paths. @p source names the path + /// that read the content and is stored verbatim in Frame::source. std::optional frame_from_content(const std::string & entity_id, const std::string & fault_code, - const nlohmann::json & content); + const nlohmann::json & content, const std::string & source); /// Capture via the plugin's own x-plc-data route (no DataProvider exported). /// Returns nullopt when the route yields nothing usable. diff --git a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp index fe060826c..01bde9674 100644 --- a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp @@ -188,13 +188,14 @@ EntityFreezeFrameCapture::standing_faults_from_list_reply(const nlohmann::json & std::optional EntityFreezeFrameCapture::frame_from_content(const std::string & entity_id, const std::string & fault_code, - const nlohmann::json & content) { + const nlohmann::json & content, const std::string & source) { if (!content_has_live_data(content)) { log_fallback_failure_once(fault_code, "entity '" + entity_id + "' returned no data items"); return std::nullopt; } Frame frame; frame.entity_id = entity_id; + frame.source = source; frame.values = values_from_list_content(content); if (!values_have_data(frame.values)) { // Items present but nothing usable in them (all-null values, or no usable @@ -228,7 +229,7 @@ EntityFreezeFrameCapture::capture_via_route(const std::string & entity_id, const if (!content) { return std::nullopt; // not plugin-owned, no x-plc-data route, or handler error } - return frame_from_content(entity_id, fault_code, *content); + return frame_from_content(entity_id, fault_code, *content, kSourceXPlcDataRoute); } void EntityFreezeFrameCapture::log_fallback_failure_once(const std::string & fault_code, const std::string & message) { @@ -410,7 +411,7 @@ bool EntityFreezeFrameCapture::capture_for_event(const ros2_medkit_msgs::msg::Fa log_fallback_failure_once(fault_code, "list_data('" + source + "') failed: " + result.error().message); continue; } - if (auto frame = frame_from_content(source, fault_code, result->content)) { + if (auto frame = frame_from_content(source, fault_code, result->content, kSourceDataProvider)) { frames.push_back(std::move(*frame)); } } catch (const std::exception & e) { diff --git a/src/ros2_medkit_gateway/src/http/handlers/fault_handlers.cpp b/src/ros2_medkit_gateway/src/http/handlers/fault_handlers.cpp index b235acc53..c7282fd7b 100644 --- a/src/ros2_medkit_gateway/src/http/handlers/fault_handlers.cpp +++ b/src/ros2_medkit_gateway/src/http/handlers/fault_handlers.cpp @@ -263,6 +263,13 @@ json FaultHandlers::merge_entity_freeze_frames(json env_data, snap["topic"] = ""; // entity data values, not a ROS topic snap["message_type"] = ""; snap["captured_at_ns"] = frame.captured_at_ns; + // Capture provenance. topic/message_type stay empty because these values + // are not a ROS message, which leaves "source" as the only field naming + // where the numbers came from - so carry it whenever the capture named a + // path. + if (!frame.source.empty()) { + snap["source"] = frame.source; + } if (frame.startup_catchup) { // Values were read at gateway start, not when the fault confirmed; // absent marker = captured on the confirm edge. @@ -344,6 +351,12 @@ dto::FaultDetail FaultHandlers::build_sovd_fault_response(const json & fault_jso snap["x-medkit"]["capture_origin"] = s["capture_origin"]; } // Entity-frame provenance (merge_entity_freeze_frames), only when known. + // "source" names the capture path (a plugin DataProvider or the + // plugin's x-plc-data route). A consumer reads it instead of the + // empty topic/message_type an entity frame necessarily carries. + if (s.contains("source") && s["source"].is_string()) { + snap["x-medkit"]["source"] = s["source"]; + } if (s.contains("connected") && s["connected"].is_boolean()) { snap["x-medkit"]["connected"] = s["connected"]; } diff --git a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp index b9a44b2e0..152297ba0 100644 --- a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp @@ -787,6 +787,37 @@ TEST(MergeEntityFreezeFrames, AppendsWhenNoConfiguredFreezeFrame) { EXPECT_FALSE(snap.contains("capture_origin")); // confirm-edge frames carry no marker } +TEST(MergeEntityFreezeFrames, CarriesCapturePathAsSource) { + // An entity frame has no ROS topic, so topic/message_type are necessarily + // empty, so "source" is the only field left saying where they came from. + json env_data = {{"snapshots", json::array()}}; + EntityFreezeFrameCapture::Frame frame; + frame.entity_id = "plc_app"; + frame.values = {{"temperature", 42.5}}; + frame.captured_at_ns = 1234; + frame.source = EntityFreezeFrameCapture::kSourceXPlcDataRoute; + + auto merged = FaultHandlers::merge_entity_freeze_frames(env_data, {frame}); + ASSERT_EQ(merged["snapshots"].size(), 1u); + const auto & snap = merged["snapshots"][0]; + EXPECT_EQ(snap["source"], EntityFreezeFrameCapture::kSourceXPlcDataRoute); + EXPECT_EQ(snap["topic"], ""); + EXPECT_EQ(snap["message_type"], ""); +} + +TEST(MergeEntityFreezeFrames, OmitsSourceWhenTheCaptureNamedNoPath) { + // Absence control for the test above, on the same harness: a frame whose + // capture path is unknown must not have one invented for it. + json env_data = {{"snapshots", json::array()}}; + EntityFreezeFrameCapture::Frame frame; + frame.entity_id = "plc_app"; + frame.values = {{"temperature", 42.5}}; + + auto merged = FaultHandlers::merge_entity_freeze_frames(env_data, {frame}); + ASSERT_EQ(merged["snapshots"].size(), 1u); + EXPECT_FALSE(merged["snapshots"][0].contains("source")); +} + TEST(MergeEntityFreezeFrames, StartupCatchUpFrameCarriesCaptureOrigin) { json env_data = {{"snapshots", json::array()}}; EntityFreezeFrameCapture::Frame frame; diff --git a/src/ros2_medkit_gateway/test/test_fault_handlers.cpp b/src/ros2_medkit_gateway/test/test_fault_handlers.cpp index 9c7f08a91..29b53ef22 100644 --- a/src/ros2_medkit_gateway/test/test_fault_handlers.cpp +++ b/src/ros2_medkit_gateway/test/test_fault_handlers.cpp @@ -142,6 +142,54 @@ TEST_F(FaultHandlersTest, BuildSovdFaultResponsePropagatesCaptureOrigin) { EXPECT_EQ(snap["x-medkit"]["capture_origin"], "startup"); } +TEST_F(FaultHandlersTest, BuildSovdFaultResponseServesEntityFrameSource) { + // A plugin-captured entity frame reaches the wire with an empty topic and + // message_type (the values are not a ROS message) plus x-medkit.source + // naming the capture path that read them. + ros2_medkit_msgs::msg::Fault fault; + fault.fault_code = "PLC_ALARM"; + + json env_data = {{"snapshots", json::array({{{"type", "freeze_frame"}, + {"snapshot_type", "freeze_frame"}, + {"name", "plc_app"}, + {"data", R"({"tank_level": 87.5})"}, + {"topic", ""}, + {"message_type", ""}, + {"captured_at_ns", 1234}, + {"source", "plugin_x_plc_data_route"}}})}}; + + auto response = to_json(FaultHandlers::build_sovd_fault_response(fault_json(fault), env_data, "/apps/plc_app")); + + auto & snap = response["environment_data"]["snapshots"][0]; + EXPECT_EQ(snap["x-medkit"]["source"], "plugin_x_plc_data_route"); + EXPECT_EQ(snap["x-medkit"]["topic"], ""); + EXPECT_EQ(snap["x-medkit"]["message_type"], ""); +} + +TEST_F(FaultHandlersTest, BuildSovdFaultResponseOmitsSourceWhenTheSnapshotHasNone) { + // Absence control for the test above, on the same harness: a topic-captured + // freeze frame (the fault_manager's own) carries no source, and none is + // invented for it. + ros2_medkit_msgs::msg::Fault fault; + fault.fault_code = "TEMP_FAULT"; + + ros2_medkit_msgs::msg::EnvironmentData env_data; + ros2_medkit_msgs::msg::Snapshot freeze_frame; + freeze_frame.type = "freeze_frame"; + freeze_frame.name = "temperature"; + freeze_frame.data = R"({"temperature": 85.5})"; + freeze_frame.topic = "/motor/temperature"; + freeze_frame.message_type = "sensor_msgs/msg/Temperature"; + env_data.snapshots.push_back(freeze_frame); + + auto response = + to_json(FaultHandlers::build_sovd_fault_response(fault_json(fault), env_json(env_data), "/apps/motor")); + + auto & snap = response["environment_data"]["snapshots"][0]; + EXPECT_FALSE(snap["x-medkit"].contains("source")); + EXPECT_EQ(snap["x-medkit"]["topic"], "/motor/temperature"); +} + // Conversion layer must emit an explicit "snapshot_type" discriminator so // downstream consumers (handler, SSE, MCP) can dispatch on a single key // regardless of which optional payload fields are present. From f9e6aa55dc601543dd7be1a41538ebe763b93b2d Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Sun, 6 Sep 2026 12:43:01 +0200 Subject: [PATCH 03/14] fix(gateway): stop listing the gateway's own nodes as apps The gateway runs four nodes inside its own process, all named after itself: the gateway node, "_sub" for the subscription executor, "_fault_clients" for the fault-service transport, and "_lifecycle_state_reader" for the lifecycle reader. None of them starts with an underscore, so the ROS 2 hidden-node convention does not cover them, and runtime introspection returned all four as ordinary Apps. The gateway advertised its own plumbing as diagnosable entities, and an operator browsing /api/v1/apps saw four entries that answer nothing useful. There were two half-answers to the same question. count_peer_nodes knew the gateway's own FQN plus "_sub" and "_fault_clients" but not the lifecycle reader, and the app filter knew only the underscore rule, so it dropped none of the four. Both now go through one predicate, is_own_gateway_node, so a fifth helper is declared in one place instead of two. The match is exact per suffix, never a prefix test: a genuine peer named "_monitor" or "2" must stay visible, and hiding a real node is the worse error. A fault_manager sharing the process is not ours either and stays listed. Remote entities are left alone: a peer's helper nodes carry the same fully qualified names and are the peer's own filter's business. --- .../ros2_medkit_gateway/gateway_node.hpp | 30 ++++++- src/ros2_medkit_gateway/src/gateway_node.cpp | 47 ++++++++-- .../test/test_handler_context.cpp | 90 +++++++++++++++++-- 3 files changed, 150 insertions(+), 17 deletions(-) diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp index a46a11bf5..d4585d99a 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp @@ -497,6 +497,27 @@ class GatewayNode : public rclcpp::Node { std::unique_ptr server_thread_; }; +/** + * @brief Is this node FQN the gateway's own, rather than a diagnosable peer? + * + * True for the gateway node itself and for the helper nodes it creates inside + * its own process: the subscription executor's `_sub`, the fault-service + * transport's `_fault_clients`, and the lifecycle reader's + * `_lifecycle_state_reader`. None of these begins with '_', so the ROS 2 + * hidden-node convention does not cover them and the gateway would otherwise + * count them as peers and list them as diagnosable Apps - reporting on itself. + * + * A fault_manager node sharing the process is NOT ours: it is a separate, + * diagnosable component and stays visible. + * + * Exact matches only. A prefix test would also claim a genuine peer named + * `_monitor` or `2`, and dropping a real node is the worse error. + * + * @param node_fqn Fully qualified node name to test ("/ns/node") + * @param self_fqn The gateway node's own FQN; an empty value matches nothing + */ +bool is_own_gateway_node(const std::string & node_fqn, const std::string & self_fqn); + /** * @brief Filter ROS 2 internal nodes from an app list * @@ -505,11 +526,18 @@ class GatewayNode : public rclcpp::Node { * before checking for the underscore prefix, using the routing table for precise * prefix detection. * + * Also removes local apps bound to one of the gateway's own nodes + * (is_own_gateway_node), which the underscore rule cannot see. The test is on + * the bound node FQN, and only for apps with no routing-table entry: a peer's + * helper nodes are the peer's business and are left to the peer's own filter. + * * @param apps App vector to filter in place * @param peer_routing_table Maps entity_id -> peer_name for remote entities + * @param self_fqn The gateway node's own FQN; empty disables the self check * @return Number of apps removed */ size_t filter_internal_node_apps(std::vector & apps, - const std::unordered_map & peer_routing_table); + const std::unordered_map & peer_routing_table, + const std::string & self_fqn); } // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_gateway/src/gateway_node.cpp b/src/ros2_medkit_gateway/src/gateway_node.cpp index a8f32986e..0d4e8c5ff 100644 --- a/src/ros2_medkit_gateway/src/gateway_node.cpp +++ b/src/ros2_medkit_gateway/src/gateway_node.cpp @@ -15,6 +15,7 @@ #include "ros2_medkit_gateway/gateway_node.hpp" #include +#include #include #include #include @@ -1596,6 +1597,27 @@ GatewayNode::GatewayNode(const rclcpp::NodeOptions & options) : Node("ros2_medki }); } +bool is_own_gateway_node(const std::string & node_fqn, const std::string & self_fqn) { + if (self_fqn.empty() || node_fqn.empty()) { + return false; + } + if (node_fqn == self_fqn) { + return true; + } + // The helper nodes the gateway creates inside its own process, each named + // after this node plus a fixed suffix. Where each one is set: + // "_sub" Ros2SubscriptionExecutor::Config + // (subscription_node_name_suffix) + // "_fault_clients" Ros2FaultServiceTransport + // "_lifecycle_state_reader" Ros2LifecycleStateReader + // Exact matches only: a prefix test would also claim a genuine peer named + // "_monitor" or "2", and hiding a real node is the worse error. + static constexpr std::array kHelperSuffixes{"_sub", "_fault_clients", "_lifecycle_state_reader"}; + return std::any_of(kHelperSuffixes.begin(), kHelperSuffixes.end(), [&](const char * suffix) { + return node_fqn == self_fqn + suffix; + }); +} + size_t GatewayNode::count_peer_nodes(const std::vector> & nodes_and_namespaces, const std::string & self_fqn) { size_t count = 0; @@ -1608,10 +1630,7 @@ size_t GatewayNode::count_peer_nodes(const std::vector_monitor" or "2"). - if (fqn == self_fqn || fqn == self_fqn + "_sub" || fqn == self_fqn + "_fault_clients") { + if (is_own_gateway_node(fqn, self_fqn)) { continue; } ++count; @@ -2457,14 +2476,15 @@ void GatewayNode::refresh_cache() { } } - // Filter ROS 2 internal nodes (underscore prefix convention). + // Filter ROS 2 internal nodes (underscore prefix convention) and this + // gateway's own helper nodes. // Controlled by discovery.runtime.filter_internal_nodes parameter (default: true). // Covers local heuristic apps (which bypass the merge pipeline orphan filter // in runtime_only mode) and any peer apps that slipped through fetch_entities. if (filter_internal_nodes_) { - auto removed = filter_internal_node_apps(apps, peer_routing_table); + auto removed = filter_internal_node_apps(apps, peer_routing_table, get_fully_qualified_name()); if (removed > 0) { - RCLCPP_DEBUG(get_logger(), "Filtered %zu internal node apps (_ prefix)", removed); + RCLCPP_DEBUG(get_logger(), "Filtered %zu internal node apps (_ prefix or own helper node)", removed); } } @@ -2561,9 +2581,10 @@ void GatewayNode::stop_rest_server() { } size_t filter_internal_node_apps(std::vector & apps, - const std::unordered_map & peer_routing_table) { + const std::unordered_map & peer_routing_table, + const std::string & self_fqn) { auto before = apps.size(); - auto end = std::remove_if(apps.begin(), apps.end(), [&peer_routing_table](const App & app) { + auto end = std::remove_if(apps.begin(), apps.end(), [&peer_routing_table, &self_fqn](const App & app) { std::string original_id = app.id; auto rt_it = peer_routing_table.find(app.id); if (rt_it != peer_routing_table.end()) { @@ -2573,6 +2594,14 @@ size_t filter_internal_node_apps(std::vector & apps, if (original_id.size() > prefix.size() && original_id.compare(0, prefix.size(), prefix) == 0) { original_id = original_id.substr(prefix.size()); } + } else if (is_own_gateway_node(app.effective_fqn(), self_fqn)) { + // A local app bound to one of this gateway's own nodes. Those names do + // not start with '_' ("_sub", "_fault_clients", ...), + // so only the FQN test catches them, and without it the gateway + // advertises its own plumbing as diagnosable apps. Remote entities are + // skipped deliberately: a peer's helper nodes carry the same FQNs and are + // the peer's own filter's business. + return true; } // ROS 2 internal nodes use _ prefix convention return !original_id.empty() && original_id[0] == '_'; diff --git a/src/ros2_medkit_gateway/test/test_handler_context.cpp b/src/ros2_medkit_gateway/test/test_handler_context.cpp index 968f29469..573c356c0 100644 --- a/src/ros2_medkit_gateway/test/test_handler_context.cpp +++ b/src/ros2_medkit_gateway/test/test_handler_context.cpp @@ -874,7 +874,7 @@ TEST(FilterInternalNodeAppsTest, FiltersLocalInternalNodes) { apps.push_back(another_internal); std::unordered_map routing; - auto removed = filter_internal_node_apps(apps, routing); + auto removed = filter_internal_node_apps(apps, routing, "/ros2_medkit_gateway"); EXPECT_EQ(removed, 2u); ASSERT_EQ(apps.size(), 1u); @@ -896,7 +896,7 @@ TEST(FilterInternalNodeAppsTest, PreservesAllNormalNodes) { apps.push_back(a3); std::unordered_map routing; - auto removed = filter_internal_node_apps(apps, routing); + auto removed = filter_internal_node_apps(apps, routing, "/ros2_medkit_gateway"); EXPECT_EQ(removed, 0u); EXPECT_EQ(apps.size(), 3u); @@ -921,7 +921,7 @@ TEST(FilterInternalNodeAppsTest, FiltersPeerPrefixedInternalNodes) { routing["peer_subsystem___ros2cli_daemon"] = "peer_subsystem"; routing["peer_subsystem__lidar_driver"] = "peer_subsystem"; - auto removed = filter_internal_node_apps(apps, routing); + auto removed = filter_internal_node_apps(apps, routing, "/ros2_medkit_gateway"); EXPECT_EQ(removed, 1u); ASSERT_EQ(apps.size(), 1u); @@ -939,7 +939,7 @@ TEST(FilterInternalNodeAppsTest, DoesNotStripPrefixWithoutRoutingEntry) { apps.push_back(ambiguous); std::unordered_map routing; - auto removed = filter_internal_node_apps(apps, routing); + auto removed = filter_internal_node_apps(apps, routing, "/ros2_medkit_gateway"); EXPECT_EQ(removed, 0u); ASSERT_EQ(apps.size(), 1u); @@ -950,7 +950,7 @@ TEST(FilterInternalNodeAppsTest, HandlesEmptyAppList) { std::vector apps; std::unordered_map routing; - auto removed = filter_internal_node_apps(apps, routing); + auto removed = filter_internal_node_apps(apps, routing, "/ros2_medkit_gateway"); EXPECT_EQ(removed, 0u); EXPECT_TRUE(apps.empty()); @@ -981,7 +981,7 @@ TEST(FilterInternalNodeAppsTest, MixedLocalAndRemoteInternalNodes) { routing["sub_b__actuator"] = "sub_b"; routing["sub_b___parameter_bridge"] = "sub_b"; - auto removed = filter_internal_node_apps(apps, routing); + auto removed = filter_internal_node_apps(apps, routing, "/ros2_medkit_gateway"); EXPECT_EQ(removed, 2u); ASSERT_EQ(apps.size(), 2u); @@ -1006,12 +1006,88 @@ TEST(FilterInternalNodeAppsTest, PeerPrefixMatchMustBeExact) { std::unordered_map routing; routing["my_peer__sensor"] = "my_peer"; - auto removed = filter_internal_node_apps(apps, routing); + auto removed = filter_internal_node_apps(apps, routing, "/ros2_medkit_gateway"); EXPECT_EQ(removed, 0u); ASSERT_EQ(apps.size(), 1u); } +namespace { + +App bound_app(const std::string & id, const std::string & fqn) { + App app; + app.id = id; + app.name = id; + app.bound_fqn = fqn; + return app; +} + +} // namespace + +TEST(FilterInternalNodeAppsTest, DropsTheGatewaysOwnHelperNodes) { + // The gateway creates "_sub", "_fault_clients" and + // "_lifecycle_state_reader" in its own process. None starts with '_', + // so runtime introspection returns them as ordinary apps and the gateway ends + // up listing its own plumbing as diagnosable. + const std::string self_fqn = "/ros2_medkit_gateway"; + std::vector apps{ + bound_app("ros2_medkit_gateway", self_fqn), + bound_app("ros2_medkit_gateway_sub", self_fqn + "_sub"), + bound_app("ros2_medkit_gateway_fault_clients", self_fqn + "_fault_clients"), + bound_app("ros2_medkit_gateway_lifecycle_state_reader", self_fqn + "_lifecycle_state_reader"), + // Positive controls on the same harness: a similarly suffixed FOREIGN + // node, a node whose name merely extends the gateway's, and the + // fault_manager, which is a separate diagnosable component even when it + // shares the process. + bound_app("other_gateway_sub", "/other_gateway_sub"), + bound_app("ros2_medkit_gateway_monitor", self_fqn + "_monitor"), + bound_app("fault_manager", "/fault_manager"), + }; + + std::unordered_map routing; + auto removed = filter_internal_node_apps(apps, routing, self_fqn); + + EXPECT_EQ(removed, 4u); + std::set remaining; + for (const auto & app : apps) { + remaining.insert(app.id); + } + EXPECT_EQ(remaining, (std::set{"other_gateway_sub", "ros2_medkit_gateway_monitor", "fault_manager"})); +} + +TEST(FilterInternalNodeAppsTest, LeavesPeerHelperNodesToThePeer) { + // A remote entity carrying the same FQN belongs to the peer that reported + // it, so this gateway must not reach across and filter it. + const std::string self_fqn = "/ros2_medkit_gateway"; + std::vector apps{bound_app("sub_b__ros2_medkit_gateway_sub", self_fqn + "_sub")}; + + std::unordered_map routing; + routing["sub_b__ros2_medkit_gateway_sub"] = "sub_b"; + + auto removed = filter_internal_node_apps(apps, routing, self_fqn); + + EXPECT_EQ(removed, 0u); + ASSERT_EQ(apps.size(), 1u); +} + +TEST(IsOwnGatewayNodeTest, MatchesSelfAndHelpersExactlyAndNothingElse) { + const std::string self_fqn = "/ros2_medkit_gateway"; + EXPECT_TRUE(is_own_gateway_node(self_fqn, self_fqn)); + EXPECT_TRUE(is_own_gateway_node(self_fqn + "_sub", self_fqn)); + EXPECT_TRUE(is_own_gateway_node(self_fqn + "_fault_clients", self_fqn)); + EXPECT_TRUE(is_own_gateway_node(self_fqn + "_lifecycle_state_reader", self_fqn)); + + // Prefix neighbours are genuine peers, not ours. + EXPECT_FALSE(is_own_gateway_node(self_fqn + "_monitor", self_fqn)); + EXPECT_FALSE(is_own_gateway_node(self_fqn + "2", self_fqn)); + EXPECT_FALSE(is_own_gateway_node("/other" + self_fqn + "_sub", self_fqn)); + EXPECT_FALSE(is_own_gateway_node("/fault_manager", self_fqn)); + + // An unknown self FQN must claim nothing rather than everything. + EXPECT_FALSE(is_own_gateway_node(self_fqn, "")); + EXPECT_FALSE(is_own_gateway_node("", self_fqn)); +} + // ============================================================================= // Area fault/log aggregation handler tests (via REST API) // ============================================================================= From 7b11924ef82b2c95e174929e72a1f78abc2574df Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Sun, 6 Sep 2026 13:25:05 +0200 Subject: [PATCH 04/14] test(opcua): cover the config-less discovery start-up race in docker Every existing opcua docker scenario pins OPCUA_ENDPOINT_URL, which short-circuits discovery, so none of them can reach the failure this covers: the gateway and the PLC power on together, the start-up scan runs while the PLC is still booting, and the plugin is left retrying its fallback endpoint. The scenario starts the gateway first with discovery on and no endpoint configured, asserts it settled on the fallback endpoint with no session, then brings an OPC-UA server up on the same subnet and asserts the endpoint is adopted within two re-scan intervals. It also asserts the container never restarted, since a restart would satisfy the endpoint check while proving nothing: a restart is exactly what used to be needed. The network is created with an explicit /24 so the read-only sweep stays 254 hosts and finishes in seconds. --- .../ros2_medkit_opcua/README.md | 10 + .../docker/scripts/run_discovery_race_test.sh | 183 ++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100755 src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md index 17cc1115e..7f1f8e4a6 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md @@ -850,6 +850,16 @@ bash scripts/run_integration_tests.sh bash scripts/stop.sh ``` +A separate scenario covers the config-less discovery start-up race, which the +suite above cannot see because it pins `OPCUA_ENDPOINT_URL` and so +short-circuits discovery. It starts the gateway before any server, with +discovery on and no endpoint configured, then brings a server up and asserts +the gateway adopts it without a restart: + +```bash +bash src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh +``` + ### Test Coverage | Category | Tests | What it validates | diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh new file mode 100755 index 000000000..66fcad32f --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# Copyright 2026 mfaferek93 +# +# Integration test for the config-less discovery start-up race. +# +# The field failure this reproduces: the gateway and the PLC power on +# together, the gateway's start-up scan runs while the PLC is still booting +# and finds nothing, and without a re-scan the plugin retries its fallback +# endpoint for as long as it runs. Only a restart ever found the PLC. +# +# So this scenario starts the gateway FIRST, with discovery on and no +# OPCUA_ENDPOINT_URL, asserts it settled on the fallback endpoint with no +# session, then starts an OPC-UA server and asserts the gateway adopts it +# within two re-scan intervals without being restarted. +# +# Every other opcua docker scenario pins OPCUA_ENDPOINT_URL, which +# short-circuits discovery, so none of them can see this. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../../../../.." && pwd)" +NET_NAME=discovery-race-net +NET_SUBNET=172.31.77.0/24 +SERVER_NAME=discovery-race-server +GATEWAY_NAME=discovery-race-gateway +SERVER_PORT=4840 +GATEWAY_PORT=8089 +RESCAN_INTERVAL_S=5 +CONFIG_DIR=/tmp/discovery_race_config + +# The fallback the plugin keeps when a scan selects nothing (OpcuaClientConfig). +FALLBACK_ENDPOINT="opc.tcp://localhost:4840" + +cleanup() { + local rc=$? + if [[ ${rc} -ne 0 ]]; then + for c in "${SERVER_NAME}" "${GATEWAY_NAME}"; do + echo "=== ${c} logs (cleanup trap) ===" >&2 + docker logs "${c}" >&2 2>&1 || true + done + fi + docker rm -f "${SERVER_NAME}" "${GATEWAY_NAME}" >/dev/null 2>&1 || true + docker network rm "${NET_NAME}" >/dev/null 2>&1 || true + rm -rf "${CONFIG_DIR}" +} +trap cleanup EXIT + +fail() { + echo " FAIL: $*" >&2 + exit 1 +} + +status_json() { + curl -sf "http://localhost:${GATEWAY_PORT}/api/v1/components/discovery_race_runtime/x-plc-status" || echo '{}' +} + +status_field() { + status_json | python3 -c "import json,sys; print(json.load(sys.stdin).get('$1', ''))" +} + +cd "${REPO_ROOT}" + +# Idempotent teardown of anything a hard-killed earlier run left behind. +docker rm -f "${SERVER_NAME}" "${GATEWAY_NAME}" >/dev/null 2>&1 || true +docker network rm "${NET_NAME}" >/dev/null 2>&1 || true + +echo "[1/6] Build images" +docker build --network=host \ + -f src/ros2_medkit_plugins/ros2_medkit_opcua/docker/test_alarm_server/Dockerfile \ + -t ros2_medkit_alarm_test_server:dev . >/dev/null +docker build --network=host \ + -f src/ros2_medkit_plugins/ros2_medkit_opcua/docker/Dockerfile.gateway \ + -t gateway-opcua:discovery-race . >/dev/null + +# A /24 keeps the read-only sweep to 254 hosts, so a scan finishes in seconds. +# Discovery rejects anything wider than /16 outright. +docker network create --subnet "${NET_SUBNET}" "${NET_NAME}" >/dev/null + +echo "[2/6] Start the gateway BEFORE any server, discovery on, no endpoint pinned" +mkdir -p "${CONFIG_DIR}" +cat >"${CONFIG_DIR}/discovery_nodes.yaml" <<'EOF' +area_id: plc_systems +component_id: discovery_race_runtime +# One node is enough: the assertion is on the session, not on any value. The +# node id need not resolve on the server, since a failed read does not drop +# the connection. +nodes: + - node_id: "ns=2;s=StatusWord" + entity_id: tank_process + data_name: status_word + data_type: int +EOF +cat >"${CONFIG_DIR}/manifest.yaml" <<'EOF' +manifest_version: "1.0" +EOF +cp src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml \ + "${CONFIG_DIR}/gateway_params.yaml" + +docker run -d --name "${GATEWAY_NAME}" --network "${NET_NAME}" \ + -p "${GATEWAY_PORT}:8080" \ + -v "${CONFIG_DIR}:/config:ro" \ + -e ROS_DOMAIN_ID=67 \ + -e OPCUA_DISCOVERY_ENABLED=1 \ + -e OPCUA_DISCOVERY_SUBNETS="${NET_SUBNET}" \ + -e OPCUA_DISCOVERY_INTERVAL_S="${RESCAN_INTERVAL_S}" \ + -e OPCUA_NODE_MAP_PATH=/config/discovery_nodes.yaml \ + gateway-opcua:discovery-race \ + bash -c ' + set -e + mkdir -p /var/lib/ros2_medkit/rosbags + source /opt/ros/jazzy/setup.bash + source /root/ws/install/setup.bash + ros2 run ros2_medkit_fault_manager fault_manager_node \ + > /var/lib/ros2_medkit/fault_manager.log 2>&1 & + PLUGIN_PATH=$(find /root/ws/install -name "libros2_medkit_opcua_plugin.so" | head -1) + exec ros2 run ros2_medkit_gateway gateway_node \ + --ros-args --params-file /config/gateway_params.yaml \ + -p plugins.opcua.path:="${PLUGIN_PATH}" \ + -p discovery.mode:=hybrid \ + -p discovery.manifest_path:=/config/manifest.yaml \ + -p discovery.manifest_strict_validation:=false' >/dev/null + +echo "[3/6] Wait for the REST API" +for _ in $(seq 1 60); do + if curl -sf "http://localhost:${GATEWAY_PORT}/api/v1/components" >/dev/null 2>&1; then + break + fi + sleep 1 +done +curl -sf "http://localhost:${GATEWAY_PORT}/api/v1/components" >/dev/null \ + || fail "gateway REST API never came up" + +echo "[4/6] Assert the start-up scan found nothing and left the fallback endpoint" +# The scan runs during set_context(), so by the time the API answers it has +# already completed against an empty network. +endpoint="$(status_field endpoint_url)" +connected="$(status_field connected)" +[[ "${endpoint}" == "${FALLBACK_ENDPOINT}" ]] \ + || fail "expected the fallback endpoint before the server exists, got '${endpoint}'" +[[ "${connected}" == "False" ]] \ + || fail "expected no session before the server exists, got connected='${connected}'" +echo " OK no server found at start-up, endpoint left at ${FALLBACK_ENDPOINT}" + +echo "[5/6] Start the OPC-UA server (the PLC finishing its boot)" +docker run -d --name "${SERVER_NAME}" --network "${NET_NAME}" \ + ros2_medkit_alarm_test_server:dev --port "${SERVER_PORT}" >/dev/null +for _ in $(seq 1 30); do + if docker logs "${SERVER_NAME}" 2>&1 | grep -q '^READY '; then + break + fi + sleep 1 +done +docker logs "${SERVER_NAME}" 2>&1 | grep -q '^READY ' || fail "test server never became ready" +SERVER_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"${NET_NAME}\").IPAddress}}" "${SERVER_NAME}")" +echo " server up at ${SERVER_IP}:${SERVER_PORT}" + +echo "[6/6] Assert the gateway adopts it within two re-scan intervals, unrestarted" +# Budget: two intervals for the re-scan to come round, plus the sweep and +# connect themselves. Generous enough not to flake, far short of "never", +# which is what the bug did. +DEADLINE=$((SECONDS + 2 * RESCAN_INTERVAL_S + 40)) +adopted="" +while [[ ${SECONDS} -lt ${DEADLINE} ]]; do + endpoint="$(status_field endpoint_url)" + connected="$(status_field connected)" + if [[ "${endpoint}" == "opc.tcp://${SERVER_IP}:${SERVER_PORT}" && "${connected}" == "True" ]]; then + adopted="${endpoint}" + break + fi + sleep 2 +done +[[ -n "${adopted}" ]] \ + || fail "endpoint still '${endpoint}' (connected='${connected}') after $((2 * RESCAN_INTERVAL_S + 40))s" +echo " OK re-scan adopted ${adopted} without a gateway restart" + +# The gateway must have adopted the server in the process that started before +# it, not in a fresh one: a restarted container would pass the check above +# while proving nothing. +restarts="$(docker inspect -f '{{.RestartCount}}' "${GATEWAY_NAME}")" +[[ "${restarts}" == "0" ]] || fail "gateway restarted ${restarts} time(s) during the run" +echo " OK gateway never restarted" + +echo "Discovery race scenario passed." From 054e5c6559a45b92035c4753232c1fa264aa6011 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Sun, 6 Sep 2026 16:35:01 +0200 Subject: [PATCH 05/14] fix(opcua): keep config-less discovery honest while it stays disconnected A gateway that starts before its PLC runs the discovery rescan for the life of the outage, and several things it does in that state were wrong. Component identity. With no node map the SOVD component is named from the device. When the start-up connect fails there is no device to ask, so the name comes from the fallback endpoint and an empty DeviceInfo, and it was then pinned forever: after discovery adopted the real PLC the component still served opcua- while introspect() reported the adopted endpoint. The identity is now re-derived on the first poll of a new session (config-less mode only, an explicit node map still owns the name), the derived alarms entity follows the rename, and the change is logged at INFO. The docker race scenario gained a config-less pass that asserts the rename against a real server. Rescan cadence. The cadence was stamped when a sweep STARTED, so a sweep of a legal /16 (minutes at the defaults) made the next one due the moment it returned: the poll thread swept back to back and the reconnect attempt dropped to one per sweep. It is now stamped when the sweep ends. NetworkDiscovery::run() also takes a cancel predicate, bound to the shutdown flag and checked before each probe and between the sweep and identify phases, so shutdown() no longer has to wait out a sweep. Reconnect backoff. The rescan is consulted once per reconnect attempt and attempts are spaced by the exponential backoff, so the real cadence was max(interval_s, backoff) while the README, the header and the start-up log all said "every interval_s". The backoff ceiling is now capped at the rescan cadence while discovery is rescanning. interval_s. An unset interval and an explicit 0 both mapped to the 30 s default, so there was no way to keep discovery on and stop rescanning. They are now distinct: unset takes the default, an explicit 0 leaves the start-up scan one-shot, and a negative value is refused with a warning that no longer claims the kept default is one-shot. The start-up line no longer promises a loop that rescans "every 0s" either. Discovery report. A pass re-emitted its whole report every rescan, so a site with a secured-only server logged the same WARN every 30 s for the life of the process. A pass whose outcome matches the previous one now reports at DEBUG. The first pass, and every changed outcome, still reports at INFO/WARN. Connect-time clear. It is a link-state clear, not an operator resolving a root cause, so it now sets skip_correlation_auto_clear and cannot cascade-clear the symptom faults a rule attributes to PLC_COMMS_LOST. The poller's own clear on a successful reconnect is the same event and does the same. Those clears were also buffered unconditionally while the fault manager was unmatched, so a flapping link pushed real alarm reports out of the bounded buffer: the buffer now keeps at most one pending clear per fault code, evicts a clear before a report, and refuses a clear rather than dropping a report. --- .../ros2_medkit_opcua/README.md | 38 +- .../docker/scripts/run_discovery_race_test.sh | 174 +++++++- .../ros2_medkit_opcua/network_discovery.hpp | 20 +- .../ros2_medkit_opcua/opcua_plugin.hpp | 169 +++++++- .../ros2_medkit_opcua/opcua_poller.hpp | 19 +- .../src/network_discovery.cpp | 33 +- .../ros2_medkit_opcua/src/opcua_plugin.cpp | 365 +++++++++++++---- .../ros2_medkit_opcua/src/opcua_poller.cpp | 12 +- .../test/test_network_discovery.cpp | 116 +++++- .../test/test_opcua_plugin.cpp | 383 +++++++++++++++++- 10 files changed, 1194 insertions(+), 135 deletions(-) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md index 7f1f8e4a6..65c3d2225 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md @@ -604,12 +604,18 @@ plugins.opcua.discovery: connect_timeout_ms: 600 # per-port TCP connect timeout scan_concurrency: 100 # bounded, polite concurrent connect count identify_timeout_ms: 6000 # per GetEndpoints identify - interval_s: 0 # re-scan cadence while disconnected (0 = default 30 s) + # re-scan cadence while disconnected. Omit the key for the built-in 30 s; + # set it to 0 to keep discovery on but never re-scan (start-up scan only). + interval_s: 30 anonymous_none_only: true # only auto-connect None/Anonymous servers ``` Environment overrides (Docker / appliance): `OPCUA_DISCOVERY_ENABLED`, `OPCUA_DISCOVERY_SUBNETS` (comma-separated CIDRs), `OPCUA_DISCOVERY_INTERVAL_S`. +Leaving `interval_s` (and `OPCUA_DISCOVERY_INTERVAL_S`) unset means "no cadence +stated" and takes the 30 s default; an explicit `0` is honoured as written and +turns the recurring sweep off. A negative value is refused with a warning and +leaves the cadence unset. How it works: 1. Bounded concurrent TCP connect sweep of the configured ports across the @@ -626,11 +632,19 @@ How it works: ip:port) and connects to the **scanned ip:port** - not the advertised EndpointUrl, which a server may report as a non-resolvable hostname. 5. While no session is established, the reconnect loop scans again every - `interval_s` (default 30 s) and adopts a newly found server for its next - connect attempt, logging the swap at INFO. This is what covers the common - race where the gateway and the PLC boot together: the startup scan finds - nothing because the PLC is still coming up, and without a re-scan the plugin - would retry the fallback endpoint until someone restarted it. + `interval_s` (default 30 s), measured from the END of the previous sweep, and + adopts a newly found server for its next connect attempt, logging the swap at + INFO. The re-scan is consulted once per reconnect attempt, and those are + spaced by an exponential backoff, so the backoff ceiling is capped at + `interval_s` while discovery is re-scanning - otherwise the real cadence + would be `max(interval_s, backoff)` rather than the stated one. This covers + the common race where the gateway and the PLC boot together: the startup scan + finds nothing because the PLC is still coming up, and without a re-scan the + plugin would retry the fallback endpoint until someone restarted it. +6. On the first session after such an adoption, a config-less deployment (no + node map) re-derives the SOVD component identity from the device itself, so + the component stops being served under the provisional `opcua-` name it + got when nothing answered. The change is logged at INFO. Re-scanning stops as soon as a session is up, and never starts at all when an `endpoint_url` is configured. @@ -638,11 +652,19 @@ Re-scanning stops as soon as a session is up, and never starts at all when an Safety / OT posture: - Everything is read-only: TCP connect + `GetEndpoints` only. No writes, no subscriptions, no second long-lived session. +- The scan is NOT one-shot: while the plugin has no session it repeats every + `interval_s` (default 30 s) for as long as it stays disconnected. Set + `interval_s: 0` (or `OPCUA_DISCOVERY_INTERVAL_S=0`) to keep discovery on with + the start-up scan only, or `enabled: false` to switch it off entirely. +- A sweep is cancelled when the plugin shuts down, so a stop does not have to + wait out a subnet the size of a /16. - An explicitly configured `endpoint_url` (or `OPCUA_ENDPOINT_URL`) always wins; discovery then does nothing, so it never opens a second session on a PLC the plugin already polls. -- Secured-only servers (no None/Anonymous endpoint) are surfaced in the startup - log as leads requiring operator credentials - never auto-connected or probed. +- Secured-only servers (no None/Anonymous endpoint) are surfaced in the log as + leads requiring operator credentials - never auto-connected or probed. A + re-scan whose outcome has not changed reports at DEBUG instead of repeating + the whole report, so a recurring sweep does not bury the rest of the log. - The scan is bounded (short connect timeout, capped concurrency) and CIDRs wider than /16 are rejected to prevent an accidental broad sweep. diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh index 66fcad32f..dc52d3823 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh @@ -13,6 +13,12 @@ # session, then starts an OPC-UA server and asserts the gateway adopts it # within two re-scan intervals without being restarted. # +# It then repeats the race with NO node map at all (the config-less deployment). +# There the component identity is derived from the device, and a gateway that +# scanned before its PLC existed can only name it after the fallback endpoint - +# so the second pass asserts the SOVD component stops being served under that +# provisional name once the PLC is adopted. +# # Every other opcua docker scenario pins OPCUA_ENDPOINT_URL, which # short-circuits discovery, so none of them can see this. @@ -30,6 +36,12 @@ CONFIG_DIR=/tmp/discovery_race_config # The fallback the plugin keeps when a scan selects nothing (OpcuaClientConfig). FALLBACK_ENDPOINT="opc.tcp://localhost:4840" +# Config-less naming: with no node map the component id is derived from the +# device. Before any server exists that can only be the fallback endpoint's host; +# after adoption it is the test server's DI nameplate (Manufacturer "SelfPatch +# Devices" + Model "SPX-1000"), slugified. +FALLBACK_COMPONENT_ID="opcua-localhost" +DEVICE_COMPONENT_ID="selfpatch_devices_spx_1000" cleanup() { local rc=$? @@ -50,21 +62,53 @@ fail() { exit 1 } +# x-plc-status of a named component (the node-map pass pins the id; the +# config-less pass has to look it up first). +status_json_for() { + curl -sf "http://localhost:${GATEWAY_PORT}/api/v1/components/$1/x-plc-status" || echo '{}' +} + status_json() { - curl -sf "http://localhost:${GATEWAY_PORT}/api/v1/components/discovery_race_runtime/x-plc-status" || echo '{}' + status_json_for discovery_race_runtime } status_field() { status_json | python3 -c "import json,sys; print(json.load(sys.stdin).get('$1', ''))" } +status_field_for() { + status_json_for "$1" | python3 -c "import json,sys; print(json.load(sys.stdin).get('$2', ''))" +} + +component_ids() { + curl -sf "http://localhost:${GATEWAY_PORT}/api/v1/components" 2>/dev/null | + python3 -c " +import json,sys +try: + print(' '.join(c.get('id','') for c in json.load(sys.stdin).get('items', []))) +except Exception: + print('') +" +} + +wait_for_rest_api() { + for _ in $(seq 1 60); do + if curl -sf "http://localhost:${GATEWAY_PORT}/api/v1/components" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + curl -sf "http://localhost:${GATEWAY_PORT}/api/v1/components" >/dev/null \ + || fail "gateway REST API never came up" +} + cd "${REPO_ROOT}" # Idempotent teardown of anything a hard-killed earlier run left behind. docker rm -f "${SERVER_NAME}" "${GATEWAY_NAME}" >/dev/null 2>&1 || true docker network rm "${NET_NAME}" >/dev/null 2>&1 || true -echo "[1/6] Build images" +echo "[1/9] Build images" docker build --network=host \ -f src/ros2_medkit_plugins/ros2_medkit_opcua/docker/test_alarm_server/Dockerfile \ -t ros2_medkit_alarm_test_server:dev . >/dev/null @@ -76,7 +120,7 @@ docker build --network=host \ # Discovery rejects anything wider than /16 outright. docker network create --subnet "${NET_SUBNET}" "${NET_NAME}" >/dev/null -echo "[2/6] Start the gateway BEFORE any server, discovery on, no endpoint pinned" +echo "[2/9] Start the gateway BEFORE any server, discovery on, no endpoint pinned" mkdir -p "${CONFIG_DIR}" cat >"${CONFIG_DIR}/discovery_nodes.yaml" <<'EOF' area_id: plc_systems @@ -120,17 +164,10 @@ docker run -d --name "${GATEWAY_NAME}" --network "${NET_NAME}" \ -p discovery.manifest_path:=/config/manifest.yaml \ -p discovery.manifest_strict_validation:=false' >/dev/null -echo "[3/6] Wait for the REST API" -for _ in $(seq 1 60); do - if curl -sf "http://localhost:${GATEWAY_PORT}/api/v1/components" >/dev/null 2>&1; then - break - fi - sleep 1 -done -curl -sf "http://localhost:${GATEWAY_PORT}/api/v1/components" >/dev/null \ - || fail "gateway REST API never came up" +echo "[3/9] Wait for the REST API" +wait_for_rest_api -echo "[4/6] Assert the start-up scan found nothing and left the fallback endpoint" +echo "[4/9] Assert the start-up scan found nothing and left the fallback endpoint" # The scan runs during set_context(), so by the time the API answers it has # already completed against an empty network. endpoint="$(status_field endpoint_url)" @@ -141,7 +178,7 @@ connected="$(status_field connected)" || fail "expected no session before the server exists, got connected='${connected}'" echo " OK no server found at start-up, endpoint left at ${FALLBACK_ENDPOINT}" -echo "[5/6] Start the OPC-UA server (the PLC finishing its boot)" +echo "[5/9] Start the OPC-UA server (the PLC finishing its boot)" docker run -d --name "${SERVER_NAME}" --network "${NET_NAME}" \ ros2_medkit_alarm_test_server:dev --port "${SERVER_PORT}" >/dev/null for _ in $(seq 1 30); do @@ -154,10 +191,15 @@ docker logs "${SERVER_NAME}" 2>&1 | grep -q '^READY ' || fail "test server never SERVER_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"${NET_NAME}\").IPAddress}}" "${SERVER_NAME}")" echo " server up at ${SERVER_IP}:${SERVER_PORT}" -echo "[6/6] Assert the gateway adopts it within two re-scan intervals, unrestarted" -# Budget: two intervals for the re-scan to come round, plus the sweep and -# connect themselves. Generous enough not to flake, far short of "never", -# which is what the bug did. +echo "[6/9] Assert the gateway adopts it within two re-scan intervals, unrestarted" +# Budget arithmetic. The re-scan is consulted once per reconnect attempt, and +# those are spaced by the reconnect backoff, so the adoption cadence is +# max(interval_s, backoff). The backoff ceiling is capped at interval_s while +# discovery is re-scanning, which is what keeps this budget in terms of +# interval_s alone: worst case is one full interval before the sweep is due plus +# one for the attempt that follows it, and the +40 s covers the sweep of a /24, +# the connect and the REST refresh. Generous enough not to flake, far short of +# "never", which is what the bug did. DEADLINE=$((SECONDS + 2 * RESCAN_INTERVAL_S + 40)) adopted="" while [[ ${SECONDS} -lt ${DEADLINE} ]]; do @@ -180,4 +222,100 @@ restarts="$(docker inspect -f '{{.RestartCount}}' "${GATEWAY_NAME}")" [[ "${restarts}" == "0" ]] || fail "gateway restarted ${restarts} time(s) during the run" echo " OK gateway never restarted" +# --------------------------------------------------------------------------- +# Config-less variant: the same race with NO node map. +# +# Without a node map the SOVD component is named from the device itself. A +# gateway that scanned before its PLC existed has no device to ask, so it can +# only name the component after the fallback endpoint. The defect this covers is +# that identity being pinned once and never revisited: the component kept +# serving opcua-localhost for the life of the process while the plugin was +# happily polling the adopted PLC. +# --------------------------------------------------------------------------- + +echo "[7/9] Config-less pass: stop everything, start the gateway with no node map" +docker rm -f "${SERVER_NAME}" "${GATEWAY_NAME}" >/dev/null 2>&1 || true + +docker run -d --name "${GATEWAY_NAME}" --network "${NET_NAME}" \ + -p "${GATEWAY_PORT}:8080" \ + -v "${CONFIG_DIR}:/config:ro" \ + -e ROS_DOMAIN_ID=67 \ + -e OPCUA_DISCOVERY_ENABLED=1 \ + -e OPCUA_DISCOVERY_SUBNETS="${NET_SUBNET}" \ + -e OPCUA_DISCOVERY_INTERVAL_S="${RESCAN_INTERVAL_S}" \ + gateway-opcua:discovery-race \ + bash -c ' + set -e + mkdir -p /var/lib/ros2_medkit/rosbags + source /opt/ros/jazzy/setup.bash + source /root/ws/install/setup.bash + ros2 run ros2_medkit_fault_manager fault_manager_node \ + > /var/lib/ros2_medkit/fault_manager.log 2>&1 & + PLUGIN_PATH=$(find /root/ws/install -name "libros2_medkit_opcua_plugin.so" | head -1) + exec ros2 run ros2_medkit_gateway gateway_node \ + --ros-args --params-file /config/gateway_params.yaml \ + -p plugins.opcua.path:="${PLUGIN_PATH}" \ + -p discovery.mode:=hybrid \ + -p discovery.manifest_path:=/config/manifest.yaml \ + -p discovery.manifest_strict_validation:=false' >/dev/null + +wait_for_rest_api + +echo "[8/9] Assert the component is named after the fallback while nothing answers" +ids="" +DEADLINE=$((SECONDS + 30)) +while [[ ${SECONDS} -lt ${DEADLINE} ]]; do + ids="$(component_ids)" + if [[ " ${ids} " == *" ${FALLBACK_COMPONENT_ID} "* ]]; then + break + fi + sleep 2 +done +[[ " ${ids} " == *" ${FALLBACK_COMPONENT_ID} "* ]] \ + || fail "expected the provisional component '${FALLBACK_COMPONENT_ID}' before any server, got '${ids}'" +echo " OK config-less component provisionally named ${FALLBACK_COMPONENT_ID}" + +echo "[9/9] Start the server and assert the component is renamed from the device" +docker run -d --name "${SERVER_NAME}" --network "${NET_NAME}" \ + ros2_medkit_alarm_test_server:dev --port "${SERVER_PORT}" >/dev/null +for _ in $(seq 1 30); do + if docker logs "${SERVER_NAME}" 2>&1 | grep -q '^READY '; then + break + fi + sleep 1 +done +docker logs "${SERVER_NAME}" 2>&1 | grep -q '^READY ' || fail "test server never became ready" +SERVER_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"${NET_NAME}\").IPAddress}}" "${SERVER_NAME}")" +echo " server up at ${SERVER_IP}:${SERVER_PORT}" + +# Same budget as step 6, plus the discovery refresh that republishes entities. +DEADLINE=$((SECONDS + 2 * RESCAN_INTERVAL_S + 60)) +renamed="" +while [[ ${SECONDS} -lt ${DEADLINE} ]]; do + ids="$(component_ids)" + if [[ " ${ids} " == *" ${DEVICE_COMPONENT_ID} "* ]]; then + renamed="${DEVICE_COMPONENT_ID}" + break + fi + sleep 2 +done +[[ -n "${renamed}" ]] \ + || fail "component still '${ids}' after adoption - expected the device-derived '${DEVICE_COMPONENT_ID}'" + +# The renamed component is the one actually polling the adopted PLC, so the id +# the operator sees is not a second, stale entity next to a live opcua-localhost. +endpoint="$(status_field_for "${renamed}" endpoint_url)" +connected="$(status_field_for "${renamed}" connected)" +[[ "${endpoint}" == "opc.tcp://${SERVER_IP}:${SERVER_PORT}" ]] \ + || fail "renamed component reports endpoint '${endpoint}', expected the adopted server" +[[ "${connected}" == "True" ]] \ + || fail "renamed component reports connected='${connected}', expected a live session" +[[ "${renamed}" != "${FALLBACK_COMPONENT_ID}" ]] \ + || fail "component id never moved off ${FALLBACK_COMPONENT_ID}" +echo " OK component renamed to ${renamed}, connected at ${endpoint}" + +restarts="$(docker inspect -f '{{.RestartCount}}' "${GATEWAY_NAME}")" +[[ "${restarts}" == "0" ]] || fail "gateway restarted ${restarts} time(s) during the config-less run" +echo " OK gateway never restarted" + echo "Discovery race scenario passed." diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/network_discovery.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/network_discovery.hpp index 56c31a8ae..9f73d21d2 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/network_discovery.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/network_discovery.hpp @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -105,14 +106,17 @@ struct OpcuaDiscoveryConfig { int scan_concurrency{100}; ///< bounded, polite concurrent connect count int identify_timeout_ms{6000}; - /// Re-scan cadence, in seconds, while no OPC-UA session is established. 0 - /// selects the built-in default (see OpcuaPlugin::effective_rescan_interval_s). + /// Re-scan cadence, in seconds, while no OPC-UA session is established. + /// Unset (the key absent) selects the built-in default, an explicit 0 turns + /// re-scanning off and keeps the startup scan one-shot - the two are + /// deliberately distinct, so a deployment can keep discovery on and still + /// stop the recurring sweep (see OpcuaPlugin::effective_rescan_interval_s). /// The startup scan always runs once. The cadence only governs how often the /// disconnected reconnect loop scans again, so a gateway that started before /// its PLC finished booting adopts the PLC when it appears instead of retrying /// the fallback endpoint forever. Never used once an endpoint is configured /// explicitly, and never while a session is up. - int interval_s{0}; + std::optional interval_s; /// Only auto-register endpoints that expose a None + Anonymous endpoint (what /// the plugin connects with today). Secured-only servers are surfaced as @@ -158,7 +162,15 @@ class NetworkDiscovery { /// Run one full discovery pass (blocking). Read-only: TCP connect + /// GetEndpoints only. Deduplicated by ApplicationUri (fallback ip:port), /// sorted deterministically by ip:port. - std::vector run(); + /// + /// @param cancelled optional abort predicate, polled before every probe and + /// between the sweep and identify phases. A sweep of a legal /16 is + /// tens of thousands of probes and takes minutes, and the caller runs + /// it on the poll thread that a shutdown has to join, so without this + /// a ``docker stop`` grace period would expire mid-sweep. A pass still + /// cancelled at the next phase boundary returns an empty result rather + /// than a partial one, within one in-flight probe per worker. + std::vector run(const std::function & cancelled = {}); /// Resolve the subnets to scan: configured ``subnets`` if any, else the /// derived local /24. Exposed for logging / tests. diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp index 95d2415e3..a646bfccd 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp @@ -15,11 +15,14 @@ #pragma once #include "ros2_medkit_opcua/address_space_browser.hpp" +#include "ros2_medkit_opcua/device_identity.hpp" #include "ros2_medkit_opcua/network_discovery.hpp" #include "ros2_medkit_opcua/node_map.hpp" #include "ros2_medkit_opcua/opcua_client.hpp" #include "ros2_medkit_opcua/opcua_poller.hpp" +#include + #include #include #include @@ -143,6 +146,23 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, static void apply_auto_alarms_param(const nlohmann::json & value, AutoAlarmsConfig & cfg, const std::function & warn); + // Where one discovery pass reports to, plus the memory that keeps a repeated + // identical pass quiet. A rescan runs every ``interval_s`` for the life of a + // disconnected process, so re-emitting the same scan line, per-server lines, + // summary and "no auto-connectable server" WARN each time buries every other + // message in the log. ``previous_outcome`` is owned by the caller (the plugin + // keeps one across rescans): when it is non-null and the pass reaches the same + // outcome as the pass before it, the whole report goes to ``debug`` instead. + // The first pass, and every pass whose outcome changed, is always reported at + // info/warn. A null ``previous_outcome`` (the startup scan, and tests that do + // not care) reports every pass. + struct DiscoveryReporter { + std::function info; + std::function warn; + std::function debug; + std::string * previous_outcome{nullptr}; + }; + // Run one read-only discovery pass and return the endpoint URL to adopt. // // Returns nullopt - meaning "keep the endpoint you have" - when discovery is @@ -158,19 +178,21 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // operator's target nor open a second session on an already polled PLC // @param scan injected TCP port probe // @param identify injected OPC-UA GetEndpoints identify - // @param log_info operator-visible info sink - // @param log_warn operator-visible warning sink + // @param reporter operator-visible log sinks + repeat-suppression memory + // @param cancelled abort predicate handed to NetworkDiscovery::run, so a + // shutdown does not have to wait out a full sweep static std::optional discover_endpoint(const OpcuaDiscoveryConfig & config, bool endpoint_configured, const PortScanFn & scan, const IdentifyFn & identify, - const std::function & log_info, - const std::function & log_warn); + const DiscoveryReporter & reporter, + const std::function & cancelled = {}); // Seconds between reconnect rescans, or 0 when the reconnect loop must never - // rescan (discovery disabled, or an endpoint configured explicitly). A - // configured ``interval_s`` wins. interval_s = 0 means "discovery is on but no - // cadence was stated" and takes the built-in default rather than never - // rescanning: a config-less deployment is the one that cannot name a cadence - // and the one that most needs its PLC adopted once it finishes booting. + // rescan. That is the answer when discovery is disabled, when an endpoint was + // configured explicitly, and when the operator set ``interval_s: 0``, which + // means "keep discovery on but leave the startup scan one-shot". An UNSET + // interval is the config-less case - it cannot name a cadence and is the one + // that most needs its PLC adopted once it finishes booting - so it takes the + // built-in default instead of never rescanning. static int effective_rescan_interval_s(const OpcuaDiscoveryConfig & config, bool endpoint_configured); // Default reconnect rescan cadence, in seconds, when discovery is enabled with @@ -179,6 +201,93 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // boot is picked up in well under a minute. static constexpr int kDefaultRescanIntervalS = 30; + // One rate-limited rescan step: run ``sweep`` when the cadence is due, + // otherwise do nothing. + // + // The cadence is measured from the END of the previous sweep, which + // ``*last_scan_end`` stores. A sweep of a legal /16 takes minutes, so + // stamping its start would make the next one due the moment it returned: the + // poll thread would sweep back to back and the reconnect attempt would drop to + // one per sweep. ``now`` is injected so the spacing is unit-testable without + // sleeping. + // + // @return whatever ``sweep`` returned, or nullopt when it was not due yet. + static std::optional rescan_step(int interval_s, + const std::function & now, + std::chrono::steady_clock::time_point * last_scan_end, + const std::function()> & sweep); + + // Ceiling for the poller's exponential reconnect backoff. + // + // Without discovery this is ``default_ceiling`` (60 s). While the reconnect + // loop is rescanning, the rescan is only consulted once per reconnect attempt, + // so the real adoption cadence is max(interval_s, backoff) - a documented + // "every 30 s" would silently become every 60 s. Capping the backoff at the + // rescan interval makes the documented cadence the true one. Never shorter + // than ``base`` (the configured reconnect interval), so a tiny interval cannot + // turn the backoff into a hot retry loop. + static std::chrono::milliseconds effective_max_reconnect_wait(std::chrono::milliseconds base, + std::chrono::milliseconds default_ceiling, + int rescan_interval_s); + + // The component identity a config-less deployment should serve after a + // connect, or nullopt when the identity it already has still holds. + // + // A gateway that starts before its PLC connects to nothing, so the identity + // derived at start-up comes from an empty DeviceInfo and the fallback + // endpoint: the neutral ``opcua-`` placeholder. Once discovery adopts + // the real server, the device can finally name itself, and the SOVD component + // must stop serving the placeholder. Pure / static so the rule is testable + // without a server. + static std::optional rederived_component_identity(const std::string & current_id, + const OpcuaClient::DeviceInfo & info, + const std::string & endpoint_url); + + // Build the ClearFault request for one fault code. ``link_state`` marks a + // clear that only reports the OPC-UA link came back (the connect-time + // ``PLC_COMMS_LOST`` clear). Such a clear must not cascade: a correlation rule + // may name PLC_COMMS_LOST as the root cause of every symptom the outage + // produced, and the link returning is not an operator resolving those. An + // operator-driven clear (the SOVD DELETE route) leaves the flag off and keeps + // the cascade. Static so the wire field is assertable without a fault manager. + static ros2_medkit_msgs::srv::ClearFault::Request make_clear_fault_request(const std::string & fault_code, + bool link_state); + + // One entry in the bounded buffer of fault dispatches held while the + // fault_manager service is unmatched. + struct PendingFaultDispatch { + enum class Kind { Report, Clear }; + Kind kind{Kind::Report}; + std::string fault_code; ///< dedup key for a Clear; diagnostic for a Report + std::function dispatch; + }; + + // What ``enqueue_pending_dispatch`` did, so the caller can log it. + enum class PendingEnqueueOutcome { + Buffered, ///< appended, nothing lost + ReplacedClear, ///< superseded the pending clear for the same fault code + EvictedClear, ///< buffer was full: dropped a pending clear to make room + EvictedReport, ///< buffer was full of reports and a report arrived + Refused ///< buffer was full of reports and a clear arrived + }; + + // Enqueue policy for the bounded pending-dispatch buffer. + // + // Reports outrank clears. A report is a one-shot edge from the PLC that + // nothing will re-send, while a clear is re-derivable: the link state is + // re-observed on the next reconnect. So at most ONE clear per fault code is + // ever pending (a newer one moves to the back, keeping report-then-clear + // order), a full buffer gives up its oldest pending clear first, and a clear + // arriving at a buffer full of reports is refused rather than evicting one. + // Without this a flapping link enqueued one connect-time clear per reconnect + // attempt and pushed real alarm reports out of the buffer. + static PendingEnqueueOutcome enqueue_pending_dispatch(std::vector & buffer, size_t max_size, + PendingFaultDispatch entry); + + // Bound on the pending-dispatch buffer, so a deployment with no fault_manager + // cannot grow it without limit. + static constexpr size_t kMaxPendingDispatches = 256; + private: // Route handlers void handle_plc_data(const ros2_medkit_gateway::PluginRequest & req, ros2_medkit_gateway::PluginResponse & res); @@ -196,7 +305,9 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // Report/clear fault via ROS 2 service (private helpers, not the FaultProvider overrides) void send_report_fault(const std::string & entity_id, const std::string & fault_code, const std::string & severity_str, const std::string & message); - void send_clear_fault(const std::string & fault_code); + // ``link_state`` marks a clear that reports the OPC-UA link came back rather + // than an operator resolving a root cause; see make_clear_fault_request. + void send_clear_fault(const std::string & fault_code, bool link_state = false); // Clear PLC_COMMS_LOST after the initial connect in set_context() succeeded. // Unconditional on purpose: the fault manager keys faults by fault_code and @@ -208,7 +319,7 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // Dispatch now if the fault_manager service is matched, else buffer the // dispatch (bounded, order-preserving) to be flushed once it appears. - void send_or_buffer(std::function dispatch); + void send_or_buffer(PendingFaultDispatch entry); // Flush buffered fault dispatches when the fault_manager service is ready. void flush_pending_reports(); @@ -231,6 +342,16 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // node_map_mutex_. No-op (never called) when auto_browse is disabled. void run_auto_browse(); + // Poll-thread hook (from publish_values): re-derive the SOVD component + // identity from the device once a NEW session is up, in config-less mode only + // (an explicit node map owns the name). This is what stops a gateway that + // started before its PLC from serving the ``opcua-`` + // placeholder for the life of the process after discovery adopted the real + // server. Logs the change at INFO and rebuilds every derived reference (the + // ``_alarms`` entity, entity_defs) under the node-map lock. + // No-op when the identity is unchanged. + void maybe_rederive_component_identity(); + // Poll-thread hook (from publish_values): re-run auto_browse when the client // has established a new session since the last walk. Covers the field case // where the gateway starts before the PLC is reachable (initial connect @@ -252,6 +373,12 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // an endpoint is already configured. void run_startup_discovery(); + // Log sinks for a discovery pass: the plugin's operator-visible info/warn + // plus the named ``opcua.plugin`` debug logger a repeated identical pass falls + // back to. ``previous_outcome`` is the caller's repeat-suppression memory + // (null to report every pass in full). + DiscoveryReporter discovery_reporter(std::string * previous_outcome) const; + // Poll-thread hook bound into PollerConfig::rediscover_endpoint whenever // discovery runs without a configured endpoint. Called from the poller's // reconnect arm, so only while no session is up, and rate-limited to one scan @@ -285,10 +412,15 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // a network. Set in configure(), consumed in run_startup_discovery(). PortScanFn discovery_scan_fn_; IdentifyFn discovery_identify_fn_; - // When the last discovery pass ran, so the reconnect rescan honours the + // When the last discovery pass FINISHED, so the reconnect rescan honours the // cadence instead of sweeping the subnet on every reconnect attempt. Stamped - // by the startup scan, then only ever read/written on the poll thread. - std::chrono::steady_clock::time_point last_discovery_scan_{}; + // by the startup scan, then only ever read/written on the poll thread. See + // rescan_step for why the end of the sweep is the reference point. + std::chrono::steady_clock::time_point last_discovery_scan_end_{}; + // Outcome digest of the previous discovery pass, so an unchanged rescan + // reports at DEBUG instead of repeating the whole report every interval_s. + // Poll thread only (the startup scan runs before the poller exists). + std::string last_discovery_outcome_; std::unique_ptr client_; NodeMap node_map_; @@ -317,6 +449,13 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, AssetIdentity device_identity_; uint64_t device_identity_generation_{0}; + // OpcuaClient::connection_generation the config-less component identity was + // derived at (0 = derived with no session, i.e. from the fallback endpoint and + // an empty DeviceInfo). The poll thread re-derives when the live generation + // differs, mirroring device_identity_generation_. Written on the set_context + // thread (happens-before the poller starts) then only on the poll thread. + uint64_t component_identity_generation_{0}; + // ROS 2 service clients for fault reporting struct FaultClients; std::unique_ptr fault_clients_; @@ -336,7 +475,7 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // never held across the actual dispatch (async_send_request) to keep ROS I/O out // of the critical section. std::mutex pending_reports_mutex_; - std::vector> pending_reports_; + std::vector pending_reports_; // Tracks which non-numeric nodes have already been warned about (avoids log spam). // Instance member instead of static to survive plugin reload (dlclose/dlopen). diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp index 73f670f72..5e3c183fd 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp @@ -103,6 +103,13 @@ struct PollerConfig { double subscription_interval_ms{500.0}; std::chrono::milliseconds poll_interval{1000}; std::chrono::milliseconds reconnect_interval{5000}; + /// Ceiling for the exponential reconnect backoff (it doubles from + /// ``reconnect_interval`` up to this). A plugin whose reconnect arm also + /// rescans for a new endpoint lowers this to the rescan cadence: the rescan is + /// consulted once per reconnect attempt, so a backoff longer than the cadence + /// would silently stretch the documented "re-scan every interval_s" to the + /// backoff instead (see OpcuaPlugin::effective_max_reconnect_wait). + std::chrono::milliseconds max_reconnect_interval{60000}; /// Active-condition replay strategy on (re)subscribe (issue #389). /// Default Auto: ConditionRefresh with a read-based fallback so hardened /// servers that reject the method still recover their active alarms. @@ -262,6 +269,13 @@ class OpcuaPoller { adopt_rediscovered_endpoint(const std::string & current, const std::function()> & rediscover); + /// Wait before the next reconnect attempt: the current wait doubled, clamped + /// to ``max_wait`` (a wait already above the cap comes back down to it). Pure + /// and static so the backoff - and the cap that keeps a rescanning reconnect + /// loop on its documented cadence - is unit-testable. + static std::chrono::milliseconds next_reconnect_wait(std::chrono::milliseconds current, + std::chrono::milliseconds max_wait); + /// Zero-config native A&C (``auto_alarms``): the alarm sources that should /// actually be subscribed / replayed, i.e. every explicit ``event_alarms`` /// entry plus (when ``auto_cfg.enabled`` and no explicit entry already @@ -455,7 +469,10 @@ class OpcuaPoller { // Issue #496: comms-lost debounce state, touched only on the poll thread. // ``comms_down_since_`` is set the first poll iteration the connection is // observed down and cleared on reconnect; ``comms_lost_raised_`` guards the - // one-shot raise / matching clear so the fault is idempotent. + // one-shot RAISE only. The clear is deliberately not guarded by it: it is sent + // on every successful reconnect, because the fault manager persists faults by + // fault_code and a comms-lost fault raised before a restart is standing in the + // store with nothing in this process's memory to remember it. std::optional comms_down_since_; bool comms_lost_raised_{false}; diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/network_discovery.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/network_discovery.cpp index 40f7dd2e5..f3eaceea7 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/network_discovery.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/network_discovery.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -71,8 +72,12 @@ bool ipv4_less(const std::string & a, const std::string & b) { // ``max_workers`` threads (never more than ``count``) that pull indices off a // shared atomic cursor. The single primitive backs both the connect sweep and // the GetEndpoints identify so they share the same concurrency bound. +// +// ``cancelled`` is polled by every worker before it takes the next index, so an +// abort stops the fan-out after at most one more probe per worker instead of +// running the remaining tens of thousands. An empty predicate never cancels. template -void parallel_for(size_t count, int max_workers, Body && body) { +void parallel_for(size_t count, int max_workers, const std::function & cancelled, Body && body) { if (count == 0) { return; } @@ -80,6 +85,9 @@ void parallel_for(size_t count, int max_workers, Body && body) { std::atomic next{0}; const auto worker = [&]() { for (;;) { + if (cancelled && cancelled()) { + return; + } const size_t i = next.fetch_add(1); if (i >= count) { return; @@ -306,9 +314,12 @@ OpcuaDiscoveryConfig parse_discovery_config(const nlohmann::json & j, if (j.contains("interval_s") && j["interval_s"].is_number_integer()) { const int v = j["interval_s"].get(); if (v >= 0) { + // An explicit 0 is kept as an explicit 0: it means "keep discovery on but + // never re-scan", which an unset interval (the built-in cadence) cannot + // express. cfg.interval_s = v; } else { - warn_fn("discovery: interval_s must be >= 0 - keeping default (0 = one-shot)"); + warn_fn("discovery: interval_s must be >= 0 (0 disables re-scanning) - keeping the default cadence"); } } if (j.contains("anonymous_none_only") && j["anonymous_none_only"].is_boolean()) { @@ -333,7 +344,7 @@ std::vector NetworkDiscovery::resolve_subnets() const { return {local}; } -std::vector NetworkDiscovery::run() { +std::vector NetworkDiscovery::run(const std::function & cancelled) { // "passive" has no active scan implementation yet (mDNS / LDS FindServers // are a documented follow-up) - a no-op stub rather than silently running // the active scan mode wasn't asked for. parse_discovery_config() already @@ -365,7 +376,7 @@ std::vector NetworkDiscovery::run() { // only the polite fan-out lives here. std::vector open_hits; std::mutex hits_mu; - parallel_for(targets.size(), cfg_.scan_concurrency, [&](size_t i) { + parallel_for(targets.size(), cfg_.scan_concurrency, cancelled, [&](size_t i) { const Target & t = targets[i]; if (scan_(t.ip, t.port, cfg_.connect_timeout_ms)) { std::lock_guard lk(hits_mu); @@ -373,6 +384,12 @@ std::vector NetworkDiscovery::run() { } }); + // Cancelled mid-sweep: the hit list is partial, so do not spend an identify + // round-trip per hit on a pass whose caller is shutting down. + if (cancelled && cancelled()) { + return {}; + } + // Deterministic identify order (numerically lowest ip:port first). std::sort(open_hits.begin(), open_hits.end(), [](const Target & a, const Target & b) { if (a.ip != b.ip) { @@ -388,7 +405,7 @@ std::vector NetworkDiscovery::run() { // sweep, writing each result by index so the deterministic ip:port order // (open_hits was sorted above) survives regardless of completion order. std::vector built(open_hits.size()); - parallel_for(open_hits.size(), cfg_.scan_concurrency, [&](size_t i) { + parallel_for(open_hits.size(), cfg_.scan_concurrency, cancelled, [&](size_t i) { const Target & t = open_hits[i]; DiscoveredEndpoint ep; ep.ip = t.ip; @@ -415,6 +432,12 @@ std::vector NetworkDiscovery::run() { built[i] = std::move(ep); }); + // Cancelled during the identify phase: ``built`` holds default-constructed + // (empty) entries for the hits no worker reached, which is not a result set. + if (cancelled && cancelled()) { + return {}; + } + // 4. Dedup by ApplicationUri (fallback ip:port), sequentially over the // deterministic order so the lowest ip:port always wins. std::vector ordered; diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp index 884d599ae..355570fa4 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp @@ -604,6 +604,13 @@ void OpcuaPlugin::set_context(PluginContext & context) { node_map_.set_component_identity(ci.id, ci.name); log_info("Component identity derived from device: id='" + ci.id + "', name='" + ci.name + "'"); } + // Which session this identity speaks for. 0 when the connect failed: the id + // above then came from the fallback endpoint and an empty DeviceInfo, so it + // is provisional and the poll thread re-derives it on the first session + // (maybe_rederive_component_identity). Without that a gateway which started + // before its PLC would serve the placeholder for the life of the process, + // even after discovery adopted the real server. + component_identity_generation_ = connected ? client_->connection_generation() : 0; // Zero-config native A&C: with no node map and no explicit auto_alarms // block, subscribe the Server EventNotifier by default so discovered @@ -666,10 +673,16 @@ void OpcuaPlugin::set_context(PluginContext & context) { // startup scan is the only one that ever runs, so a gateway that scanned // while its PLC was still booting retries the fallback endpoint forever and // only a restart finds the PLC. - if (effective_rescan_interval_s(discovery_config_, endpoint_configured_) > 0) { + const int rescan_interval_s = effective_rescan_interval_s(discovery_config_, endpoint_configured_); + if (rescan_interval_s > 0) { poller_config_.rediscover_endpoint = [this]() { return rescan_endpoint_for_reconnect(); }; + // The rescan is consulted once per reconnect attempt, so the real adoption + // cadence is max(interval_s, backoff). Cap the backoff at the cadence so + // the documented "re-scan every interval_s while down" is the true one. + poller_config_.max_reconnect_interval = effective_max_reconnect_wait( + poller_config_.reconnect_interval, poller_config_.max_reconnect_interval, rescan_interval_s); } poller_->start(poller_config_); log_info("OPC-UA poller started (mode: " + std::string(poller_->using_subscriptions() ? "subscription" : "poll") + @@ -1035,7 +1048,11 @@ void OpcuaPlugin::on_alarm_change(const std::string & entity_id, send_report_fault(entity_id, signal.fault_code, signal.severity, signal.message); } else { log_info("Alarm cleared: " + signal.fault_code + " on " + entity_id); - send_clear_fault(signal.fault_code); + // The poller's own comms-lost clear on a successful reconnect is the same + // link-state clear as the connect-time one, so it does not cascade either. + // Every other code is a real alarm going inactive on the device and keeps + // the default correlation behaviour. + send_clear_fault(signal.fault_code, /*link_state=*/signal.fault_code == kCommsLostFaultCode); } } @@ -1249,23 +1266,36 @@ void OpcuaPlugin::send_report_fault(const std::string & entity_id, const std::st request->severity = ros2_medkit_msgs::msg::Fault::SEVERITY_INFO; } - send_or_buffer([this, request]() { - fault_clients_->report->async_send_request(request); - }); + send_or_buffer({PendingFaultDispatch::Kind::Report, fault_code, [this, request]() { + fault_clients_->report->async_send_request(request); + }}); +} + +ros2_medkit_msgs::srv::ClearFault::Request OpcuaPlugin::make_clear_fault_request(const std::string & fault_code, + bool link_state) { + ros2_medkit_msgs::srv::ClearFault::Request request; + request.fault_code = fault_code; + // A link-state clear reports that the OPC-UA session came back. It is not an + // operator resolving a root cause, so it must not trip the correlation + // engine's auto_clear_with_root cascade: a rule naming PLC_COMMS_LOST as the + // root cause would otherwise clear every symptom fault the outage produced, + // none of which this plugin has any evidence about. + request.skip_correlation_auto_clear = link_state; + return request; } -void OpcuaPlugin::send_clear_fault(const std::string & fault_code) { +void OpcuaPlugin::send_clear_fault(const std::string & fault_code, bool link_state) { if (!fault_clients_->clear) { log_warn("ClearFault service client not available"); return; } - auto request = std::make_shared(); - request->fault_code = fault_code; + auto request = + std::make_shared(make_clear_fault_request(fault_code, link_state)); - send_or_buffer([this, request]() { - fault_clients_->clear->async_send_request(request); - }); + send_or_buffer({PendingFaultDispatch::Kind::Clear, fault_code, [this, request]() { + fault_clients_->clear->async_send_request(request); + }}); } void OpcuaPlugin::clear_comms_lost_on_connect() { @@ -1276,26 +1306,69 @@ void OpcuaPlugin::clear_comms_lost_on_connect() { // fire-and-forget, so a "Fault not found" answer for a code that was never // raised costs nothing here and is the normal case on a healthy start. log_info(std::string("OPC-UA connection established; clearing any standing ") + kCommsLostFaultCode); - send_clear_fault(kCommsLostFaultCode); + send_clear_fault(kCommsLostFaultCode, /*link_state=*/true); } -void OpcuaPlugin::send_or_buffer(std::function dispatch) { +OpcuaPlugin::PendingEnqueueOutcome OpcuaPlugin::enqueue_pending_dispatch(std::vector & buffer, + size_t max_size, PendingFaultDispatch entry) { + const bool is_clear = entry.kind == PendingFaultDispatch::Kind::Clear; + + // At most one pending clear per fault code. A repeat moves to the BACK rather + // than overwriting in place, so an interleaved report-then-clear for the same + // code still flushes in the order the PLC produced it. + bool replaced = false; + if (is_clear) { + const auto same_code = std::find_if(buffer.begin(), buffer.end(), [&entry](const PendingFaultDispatch & pending) { + return pending.kind == PendingFaultDispatch::Kind::Clear && pending.fault_code == entry.fault_code; + }); + if (same_code != buffer.end()) { + buffer.erase(same_code); + replaced = true; + } + } + + PendingEnqueueOutcome outcome = replaced ? PendingEnqueueOutcome::ReplacedClear : PendingEnqueueOutcome::Buffered; + if (buffer.size() >= max_size) { + // A report is a one-shot edge from the PLC that nothing will re-send; a + // clear is re-derivable from the next reconnect. So a full buffer gives up a + // pending clear first, and refuses an incoming clear rather than evicting a + // report for it. + const auto oldest_clear = std::find_if(buffer.begin(), buffer.end(), [](const PendingFaultDispatch & pending) { + return pending.kind == PendingFaultDispatch::Kind::Clear; + }); + if (oldest_clear != buffer.end()) { + buffer.erase(oldest_clear); + outcome = PendingEnqueueOutcome::EvictedClear; + } else if (is_clear) { + return PendingEnqueueOutcome::Refused; + } else { + buffer.erase(buffer.begin()); + outcome = PendingEnqueueOutcome::EvictedReport; + } + } + + buffer.push_back(std::move(entry)); + return outcome; +} + +void OpcuaPlugin::send_or_buffer(PendingFaultDispatch entry) { // Bound the buffer so a deployment with no fault_manager cannot grow it - // without limit; drop the oldest (least relevant) pending dispatch. - // Runs on both the poll thread and the REST clear_fault thread, so the vector - // mutation is serialised by pending_reports_mutex_. - constexpr size_t kMaxPendingReports = 256; - bool dropped_oldest = false; + // without limit. Runs on both the poll thread and the REST clear_fault thread, + // so the vector mutation is serialised by pending_reports_mutex_. + PendingEnqueueOutcome outcome = PendingEnqueueOutcome::Buffered; { std::lock_guard lock(pending_reports_mutex_); - if (pending_reports_.size() >= kMaxPendingReports) { - pending_reports_.erase(pending_reports_.begin()); - dropped_oldest = true; - } - pending_reports_.push_back(std::move(dispatch)); - } - if (dropped_oldest) { - log_warn("pending fault report buffer full (" + std::to_string(kMaxPendingReports) + "), dropping oldest"); + outcome = enqueue_pending_dispatch(pending_reports_, kMaxPendingDispatches, std::move(entry)); + } + if (outcome == PendingEnqueueOutcome::EvictedReport) { + log_warn("pending fault dispatch buffer full (" + std::to_string(kMaxPendingDispatches) + + "), dropping the oldest report"); + } else if (outcome == PendingEnqueueOutcome::EvictedClear) { + log_warn("pending fault dispatch buffer full (" + std::to_string(kMaxPendingDispatches) + + "), dropping the oldest pending clear"); + } else if (outcome == PendingEnqueueOutcome::Refused) { + log_warn("pending fault dispatch buffer full of reports (" + std::to_string(kMaxPendingDispatches) + + "), dropping this clear instead of a report"); } // Drains immediately (in order) if the sink is already matched. flush_pending_reports(); @@ -1311,7 +1384,7 @@ void OpcuaPlugin::flush_pending_reports() { // the vector is never being reallocated by a concurrent send_or_buffer while it // is iterated here (the use-after-free that corrupted the heap), and so the ROS // service call never runs under the mutex. - std::vector> batch; + std::vector batch; { std::lock_guard lock(pending_reports_mutex_); if (pending_reports_.empty()) { @@ -1319,8 +1392,8 @@ void OpcuaPlugin::flush_pending_reports() { } batch.swap(pending_reports_); } - for (auto & dispatch : batch) { - dispatch(); + for (auto & entry : batch) { + entry.dispatch(); } } @@ -1380,6 +1453,56 @@ void OpcuaPlugin::run_auto_browse() { (result.depth_cap_hit ? " [depth cap reached on at least one branch]" : "")); } +std::optional OpcuaPlugin::rederived_component_identity(const std::string & current_id, + const OpcuaClient::DeviceInfo & info, + const std::string & endpoint_url) { + const ComponentIdentity ci = derive_component_identity(info, endpoint_url); + if (ci.id.empty() || ci.id == current_id) { + return std::nullopt; + } + return ci; +} + +void OpcuaPlugin::maybe_rederive_component_identity() { + // An explicit node map owns the component name; only the config-less path + // derives it from the device. + if (!node_map_path_.empty() || !client_ || !client_->is_connected()) { + return; + } + const uint64_t generation = client_->connection_generation(); + if (generation == component_identity_generation_) { + return; // identity already speaks for this session + } + + const std::string live_endpoint = client_->endpoint_url(); + const auto rederived = + rederived_component_identity(node_map_.component_id(), client_->read_device_info(), live_endpoint); + component_identity_generation_ = generation; + if (!rederived) { + return; + } + + const std::string previous_id = node_map_.component_id(); + { + // Serialize against the REST read paths, which hold references into + // node_map_ (entity_defs) while they answer. + std::unique_lock lock(node_map_mutex_); + // The auto_alarms fallback entity is derived from the component id, so a + // default-derived one has to follow the rename. An operator-chosen entity_id + // does not match the derived form and is left alone. + auto & auto_alarms = node_map_.mutable_auto_alarms(); + if (auto_alarms.entity_id == previous_id + "_alarms") { + auto_alarms.entity_id.clear(); + } + node_map_.set_component_identity(rederived->id, rederived->name); + // Re-derives the fallback entity id and rebuilds entity_defs, so every + // reference to the component id moves together. + node_map_.finalize_auto_alarms_overlay(); + } + log_info("Component identity re-derived from the adopted device at " + live_endpoint + ": id='" + rederived->id + + "', name='" + rederived->name + "' (was '" + previous_id + "')"); +} + void OpcuaPlugin::maybe_rebrowse_on_reconnect() { if (!node_map_.auto_browse_config().enabled || !client_ || !client_->is_connected()) { return; @@ -1398,6 +1521,11 @@ void OpcuaPlugin::publish_values(const PollSnapshot & snap) { // Poll-thread hook: drain any fault reports buffered before fault_manager was // discovered, so a late sink still receives them. flush_pending_reports(); + // Poll-thread hook: re-derive the config-less component identity from the + // device once a session is up, so an adopted PLC stops being served under the + // provisional endpoint-derived id. Runs BEFORE the re-walk below, which + // rebuilds entity_defs off the component id. + maybe_rederive_component_identity(); // Poll-thread hook: (re)run auto_browse after a fresh session so a PLC that // came up (or restarted) after the initial connect still gets walked. maybe_rebrowse_on_reconnect(); @@ -1496,13 +1624,44 @@ int OpcuaPlugin::effective_rescan_interval_s(const OpcuaDiscoveryConfig & config if (!config.enabled || endpoint_configured) { return 0; } - return config.interval_s > 0 ? config.interval_s : kDefaultRescanIntervalS; + // Unset means "discovery is on but no cadence was stated" - the config-less + // deployment, which takes the built-in default. An explicit 0 is an operator + // saying "do not re-scan", and is honoured as written. + return config.interval_s.value_or(kDefaultRescanIntervalS); +} + +std::chrono::milliseconds OpcuaPlugin::effective_max_reconnect_wait(std::chrono::milliseconds base, + std::chrono::milliseconds default_ceiling, + int rescan_interval_s) { + if (rescan_interval_s <= 0) { + return default_ceiling; // no rescan: the plain backoff ceiling applies + } + const auto cadence = std::chrono::milliseconds(static_cast(rescan_interval_s) * 1000); + return std::max(base, std::min(default_ceiling, cadence)); +} + +std::optional OpcuaPlugin::rescan_step(int interval_s, + const std::function & now, + std::chrono::steady_clock::time_point * last_scan_end, + const std::function()> & sweep) { + if (interval_s <= 0 || !now || last_scan_end == nullptr || !sweep) { + return std::nullopt; + } + if (now() - *last_scan_end < std::chrono::seconds(interval_s)) { + return std::nullopt; + } + const auto result = sweep(); + // Stamp the END of the sweep: a legal /16 runs for minutes, and stamping its + // start would make the next one due the moment this one returned - the poll + // thread would sweep back to back and only attempt a reconnect once a sweep. + *last_scan_end = now(); + return result; } std::optional OpcuaPlugin::discover_endpoint(const OpcuaDiscoveryConfig & config, bool endpoint_configured, const PortScanFn & scan, const IdentifyFn & identify, - const std::function & log_info, - const std::function & log_warn) { + const DiscoveryReporter & reporter, + const std::function & cancelled) { if (!config.enabled) { return std::nullopt; } @@ -1512,21 +1671,56 @@ std::optional OpcuaPlugin::discover_endpoint(const OpcuaDiscoveryCo return std::nullopt; } + // The pass reports through a buffer rather than straight to the log: whether + // the report is operator-visible or a DEBUG trace depends on the outcome, + // which is only known once the sweep is done. A rescan runs for the life of a + // disconnected process, so an unchanged outcome must not repeat its whole + // report (a secured-only site would log the same WARN every interval_s). + struct ReportLine { + bool warning; + std::string text; + }; + std::vector report; + std::string outcome; + const auto info_line = [&report, &outcome](const std::string & text) { + report.push_back({false, text}); + outcome += text; + outcome += '\n'; + }; + const auto warn_line = [&report, &outcome](const std::string & text) { + report.push_back({true, text}); + outcome += text; + outcome += '\n'; + }; + const auto emit = [&report, &outcome, &reporter]() { + const bool repeat = reporter.previous_outcome != nullptr && *reporter.previous_outcome == outcome; + if (reporter.previous_outcome != nullptr) { + *reporter.previous_outcome = outcome; + } + for (const auto & line : report) { + const auto & sink = repeat ? reporter.debug : (line.warning ? reporter.warn : reporter.info); + if (sink) { + sink(line.text); + } + } + }; + NetworkDiscovery discovery(config, scan, identify); const auto subnets = discovery.resolve_subnets(); if (subnets.empty()) { - log_warn("OPC-UA discovery: no subnet configured and could not derive a local /24; nothing to scan."); + warn_line("OPC-UA discovery: no subnet configured and could not derive a local /24; nothing to scan."); + emit(); return std::nullopt; } std::string subnet_list; for (const auto & s : subnets) { subnet_list += (subnet_list.empty() ? "" : ", ") + s; } - log_info("OPC-UA discovery: read-only active scan of [" + subnet_list + "] on " + - std::to_string(config.ports.size()) + " port(s)..."); + info_line("OPC-UA discovery: read-only active scan of [" + subnet_list + "] on " + + std::to_string(config.ports.size()) + " port(s)..."); - const std::vector found = discovery.run(); + const std::vector found = discovery.run(cancelled); // Summarize what was found and what was skipped (leads, LDS, secured-only). size_t data_servers = 0; @@ -1550,23 +1744,25 @@ std::optional OpcuaPlugin::discover_endpoint(const OpcuaDiscoveryCo if (!ep.anonymous_none_available) { ++secured_only; } - log_info("OPC-UA discovery: found data server " + ep.endpoint_url + " (uri='" + ep.application_uri + - "', product='" + ep.product_uri + "', None/Anonymous=" + (ep.anonymous_none_available ? "yes" : "no") + - ")"); + info_line("OPC-UA discovery: found data server " + ep.endpoint_url + " (uri='" + ep.application_uri + + "', product='" + ep.product_uri + "', None/Anonymous=" + (ep.anonymous_none_available ? "yes" : "no") + + ")"); } - log_info("OPC-UA discovery summary: " + std::to_string(data_servers) + " data server(s), " + - std::to_string(discovery_servers) + " discovery server(s)/LDS, " + std::to_string(secured_only) + - " secured-only (need credentials), " + std::to_string(leads) + " non-OPC-UA/unidentified lead(s)."); + info_line("OPC-UA discovery summary: " + std::to_string(data_servers) + " data server(s), " + + std::to_string(discovery_servers) + " discovery server(s)/LDS, " + std::to_string(secured_only) + + " secured-only (need credentials), " + std::to_string(leads) + " non-OPC-UA/unidentified lead(s)."); const DiscoveredEndpoint * chosen = NetworkDiscovery::select_auto_endpoint(found, config.anonymous_none_only); if (chosen == nullptr) { - log_warn( + warn_line( "OPC-UA discovery: no auto-connectable None/Anonymous data server found; leaving the endpoint unchanged. " "Secured-only servers require operator credentials."); + emit(); return std::nullopt; } - log_info("OPC-UA discovery: selected endpoint " + chosen->endpoint_url + " (uri='" + chosen->application_uri + "')"); + info_line("OPC-UA discovery: selected endpoint " + chosen->endpoint_url + " (uri='" + chosen->application_uri + "')"); + emit(); return chosen->endpoint_url; } @@ -1580,26 +1776,35 @@ void OpcuaPlugin::run_startup_discovery() { return; } - // Stamp the scan before running it: the rescan cadence measures the gap - // between the START of two sweeps, so a slow sweep does not immediately earn + // The startup scan is always reported in full (no previous outcome to compare + // against) and always cancellable, so a shutdown during set_context does not + // wait out a whole sweep. + const auto chosen = discover_endpoint(discovery_config_, endpoint_configured_, discovery_scan_fn_, + discovery_identify_fn_, discovery_reporter(&last_discovery_outcome_), [this]() { + return shutdown_requested_.load(); + }); + // Stamp when the sweep FINISHED: the rescan cadence is measured from the end + // of the previous sweep, so a long sweep is not immediately followed by // another one. - last_discovery_scan_ = std::chrono::steady_clock::now(); - const auto chosen = discover_endpoint( - discovery_config_, endpoint_configured_, discovery_scan_fn_, discovery_identify_fn_, - [this](const std::string & m) { - log_info(m); - }, - [this](const std::string & m) { - log_warn(m); - }); + last_discovery_scan_end_ = std::chrono::steady_clock::now(); if (!chosen) { // The startup scan can legitimately find nothing - a gateway that boots - // alongside its PLC routinely scans while the PLC is still coming up. The - // endpoint stays at its default and the poller's reconnect arm rescans on - // the cadence below, so this is a delay rather than a dead end. - log_info("OPC-UA discovery: startup scan selected no endpoint; the reconnect loop rescans every " + - std::to_string(effective_rescan_interval_s(discovery_config_, endpoint_configured_)) + "s while down."); + // alongside its PLC routinely scans while the PLC is still coming up. With a + // cadence the endpoint stays at its default and the reconnect arm rescans, + // so this is a delay rather than a dead end. With re-scanning switched off + // (an explicit interval_s: 0) it IS the end, and the operator has to be told + // which of the two they configured. + const int startup_interval_s = effective_rescan_interval_s(discovery_config_, endpoint_configured_); + if (startup_interval_s > 0) { + log_info("OPC-UA discovery: startup scan selected no endpoint; the reconnect loop rescans every " + + std::to_string(startup_interval_s) + "s while down."); + } else { + log_warn( + "OPC-UA discovery: startup scan selected no endpoint and re-scanning is off (interval_s: 0); the endpoint " + "stays at " + + client_config_.endpoint_url + " until the plugin is restarted."); + } return; } @@ -1607,6 +1812,21 @@ void OpcuaPlugin::run_startup_discovery() { log_info("OPC-UA discovery: auto-selected endpoint " + *chosen + " - handing to the connect + introspect path."); } +OpcuaPlugin::DiscoveryReporter OpcuaPlugin::discovery_reporter(std::string * previous_outcome) const { + DiscoveryReporter reporter; + reporter.info = [this](const std::string & m) { + log_info(m); + }; + reporter.warn = [this](const std::string & m) { + log_warn(m); + }; + reporter.debug = [](const std::string & m) { + RCLCPP_DEBUG(opcua_plugin_logger(), "%s", m.c_str()); + }; + reporter.previous_outcome = previous_outcome; + return reporter; +} + std::optional OpcuaPlugin::rescan_endpoint_for_reconnect() { // A sweep is a bounded but multi-second blocking call on the poll thread, and // stop() has to wait for whatever it is in the middle of. Do not start one the @@ -1615,23 +1835,18 @@ std::optional OpcuaPlugin::rescan_endpoint_for_reconnect() { return std::nullopt; } const int interval_s = effective_rescan_interval_s(discovery_config_, endpoint_configured_); - if (interval_s <= 0) { - return std::nullopt; - } - - const auto now = std::chrono::steady_clock::now(); - if (now - last_discovery_scan_ < std::chrono::seconds(interval_s)) { - return std::nullopt; - } - last_discovery_scan_ = now; - const auto chosen = discover_endpoint( - discovery_config_, endpoint_configured_, discovery_scan_fn_, discovery_identify_fn_, - [this](const std::string & m) { - log_info(m); + const auto chosen = rescan_step( + interval_s, + []() { + return std::chrono::steady_clock::now(); }, - [this](const std::string & m) { - log_warn(m); + &last_discovery_scan_end_, + [this]() { + return discover_endpoint(discovery_config_, endpoint_configured_, discovery_scan_fn_, discovery_identify_fn_, + discovery_reporter(&last_discovery_outcome_), [this]() { + return shutdown_requested_.load(); + }); }); // The live client config, not client_config_: this runs on the poll thread // and client_config_ is read by the refresh thread in introspect(). The diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_poller.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_poller.cpp index d972a3e05..019d0604c 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_poller.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_poller.cpp @@ -1155,6 +1155,11 @@ OpcuaPoller::adopt_rediscovered_endpoint(const std::string & current, return found; } +std::chrono::milliseconds OpcuaPoller::next_reconnect_wait(std::chrono::milliseconds current, + std::chrono::milliseconds max_wait) { + return std::min(current * 2, max_wait); +} + void OpcuaPoller::emit_comms_lost(bool active) { ros2_medkit::fault_detection::FaultSignal signal; signal.fault_code = kCommsLostFaultCode; @@ -1170,7 +1175,6 @@ void OpcuaPoller::emit_comms_lost(bool active) { void OpcuaPoller::poll_loop() { auto reconnect_wait = config_.reconnect_interval; - constexpr auto max_reconnect_wait = std::chrono::milliseconds(60000); while (running_.load()) { // Handle reconnection @@ -1244,14 +1248,16 @@ void OpcuaPoller::poll_loop() { comms_lost_raised_ = true; } } - // Exponential backoff capped at 60s. condition_variable so stop() wakes immediately. + // Exponential backoff, capped at config_.max_reconnect_interval (60 s by + // default, the rescan cadence while the reconnect arm also rescans). + // condition_variable so stop() wakes immediately. { std::unique_lock lock(stop_mutex_); stop_cv_.wait_for(lock, reconnect_wait, [this] { return !running_.load(); }); } - reconnect_wait = std::min(reconnect_wait * 2, max_reconnect_wait); + reconnect_wait = next_reconnect_wait(reconnect_wait, config_.max_reconnect_interval); continue; } } diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_network_discovery.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_network_discovery.cpp index 03643beb3..96c32e339 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_network_discovery.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_network_discovery.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -106,7 +107,32 @@ TEST(ParseDiscoveryConfig, DefaultsDisabled) { ASSERT_EQ(cfg.ports.size(), 1u); EXPECT_EQ(cfg.ports[0], 4840); EXPECT_TRUE(cfg.anonymous_none_only); - EXPECT_EQ(cfg.interval_s, 0); + // Unset, NOT 0: an absent key means "no cadence stated" (the caller takes its + // built-in default), while an explicit 0 means "never re-scan". + EXPECT_FALSE(cfg.interval_s.has_value()); +} + +TEST(ParseDiscoveryConfig, ExplicitZeroIntervalIsKeptAsAnExplicitZero) { + std::vector warnings; + const auto cfg = parse_discovery_config(nlohmann::json{{"interval_s", 0}}, [&](const std::string & m) { + warnings.push_back(m); + }); + ASSERT_TRUE(cfg.interval_s.has_value()); + EXPECT_EQ(*cfg.interval_s, 0); + EXPECT_TRUE(warnings.empty()); +} + +TEST(ParseDiscoveryConfig, NegativeIntervalWarnsAndLeavesTheCadenceUnset) { + std::vector warnings; + const auto cfg = parse_discovery_config(nlohmann::json{{"interval_s", -5}}, [&](const std::string & m) { + warnings.push_back(m); + }); + EXPECT_FALSE(cfg.interval_s.has_value()); + ASSERT_EQ(warnings.size(), 1u); + EXPECT_NE(warnings[0].find("interval_s"), std::string::npos); + // The warning must not tell the operator the kept default is one-shot: an + // unset interval re-scans on the built-in cadence, only an explicit 0 stops. + EXPECT_EQ(warnings[0].find("0 = one-shot"), std::string::npos) << warnings[0]; } TEST(ParseDiscoveryConfig, ReadsAllKnownKeys) { @@ -133,7 +159,8 @@ TEST(ParseDiscoveryConfig, ReadsAllKnownKeys) { EXPECT_EQ(cfg.connect_timeout_ms, 300); EXPECT_EQ(cfg.scan_concurrency, 64); EXPECT_EQ(cfg.identify_timeout_ms, 2000); - EXPECT_EQ(cfg.interval_s, 900); + ASSERT_TRUE(cfg.interval_s.has_value()); + EXPECT_EQ(*cfg.interval_s, 900); EXPECT_FALSE(cfg.anonymous_none_only); EXPECT_TRUE(warnings.empty()); } @@ -377,6 +404,91 @@ TEST(NetworkDiscoveryRun, IdentifyFailureRecordedAsLead) { EXPECT_EQ(NetworkDiscovery::select_auto_endpoint(eps, true), nullptr); } +// --------------------------------------------------------------------------- // +// run(cancelled): a shutdown must not wait out a whole sweep +// --------------------------------------------------------------------------- // +TEST(NetworkDiscoveryRun, CancelStopsTheSweepInsteadOfProbingEveryHost) { + // A /24 is 254 probes and a legal /16 is 65k; the caller runs them on the poll + // thread a shutdown has to join. With scan_concurrency 1 the sweep is + // sequential, so the probe count is exactly what the cancel predicate allowed. + std::atomic probes{0}; + std::atomic stop{false}; + auto scan = [&probes, &stop](const std::string &, uint16_t, int) { + if (probes.fetch_add(1) + 1 >= 5) { + stop.store(true); // the shutdown flag flipping mid-sweep + } + return false; + }; + OpcuaDiscoveryConfig cfg; + cfg.enabled = true; + cfg.subnets = {"192.168.1.0/24"}; + cfg.ports = {4840}; + cfg.scan_concurrency = 1; + + NetworkDiscovery disc(cfg, scan, make_identify({})); + const auto eps = disc.run([&stop]() { + return stop.load(); + }); + + EXPECT_TRUE(eps.empty()); + // One in-flight probe per worker may still complete after the flag flips. + EXPECT_GE(probes.load(), 5); + EXPECT_LE(probes.load(), 6) << "the sweep kept probing after it was cancelled"; +} + +TEST(NetworkDiscoveryRun, WithoutCancellationEveryHostIsStillProbed) { + // Positive control for the test above on the same harness: the identical + // sweep with no cancel predicate visits all 254 hosts, so a low probe count + // there is the cancellation and not a broken fake. + std::atomic probes{0}; + auto scan = [&probes](const std::string &, uint16_t, int) { + probes.fetch_add(1); + return false; + }; + OpcuaDiscoveryConfig cfg; + cfg.enabled = true; + cfg.subnets = {"192.168.1.0/24"}; + cfg.ports = {4840}; + cfg.scan_concurrency = 1; + + NetworkDiscovery disc(cfg, scan, make_identify({})); + const auto eps = disc.run(); + EXPECT_TRUE(eps.empty()); + EXPECT_EQ(probes.load(), 254); +} + +TEST(NetworkDiscoveryRun, CancelBetweenSweepAndIdentifySkipsTheIdentifyRoundTrips) { + // The identify phase is a separate batch, and each GetEndpoints blocks for up + // to identify_timeout_ms. A cancel that arrives once the sweep is done must + // not still pay for one round-trip per hit. + std::atomic stop{false}; + auto scan = [&stop](const std::string & ip, uint16_t, int) { + const bool hit = ip == "192.168.1.10" || ip == "192.168.1.11"; + if (ip == "192.168.1.254") { + stop.store(true); // sweep finished, shutdown requested + } + return hit; + }; + std::atomic identifies{0}; + auto identify = [&identifies](const std::string &, int) { + identifies.fetch_add(1); + return IdentifyResult{}; + }; + + OpcuaDiscoveryConfig cfg; + cfg.enabled = true; + cfg.subnets = {"192.168.1.0/24"}; + cfg.ports = {4840}; + cfg.scan_concurrency = 1; + + NetworkDiscovery disc(cfg, scan, identify); + const auto eps = disc.run([&stop]() { + return stop.load(); + }); + EXPECT_TRUE(eps.empty()); + EXPECT_EQ(identifies.load(), 0); +} + // --------------------------------------------------------------------------- // // select_auto_endpoint // --------------------------------------------------------------------------- // diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp index f47108005..9984cef85 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp @@ -692,6 +692,15 @@ OpcuaDiscoveryConfig rescan_cfg() { // Discards log output. The tests assert on the selected endpoint, not the text. const std::function kSilent = [](const std::string &) {}; +// Silent reporter: no repeat-suppression memory, so every pass reports in full +// (into the void). Tests that assert on the log build their own. +OpcuaPlugin::DiscoveryReporter silent_reporter() { + OpcuaPlugin::DiscoveryReporter reporter; + reporter.info = kSilent; + reporter.warn = kSilent; + return reporter; +} + } // namespace TEST(DiscoverEndpoint, ScanBeforeThePlcIsUpSelectsNothingAndALaterRescanAdoptsIt) { @@ -699,14 +708,14 @@ TEST(DiscoverEndpoint, ScanBeforeThePlcIsUpSelectsNothingAndALaterRescanAdoptsIt // booting. Nothing answers, so nothing is selected and the caller keeps the // default endpoint. const auto empty_pass = OpcuaPlugin::discover_endpoint(rescan_cfg(), /*endpoint_configured=*/false, fake_scan({}), - fake_identify({}), kSilent, kSilent); + fake_identify({}), silent_reporter()); EXPECT_FALSE(empty_pass.has_value()); // The PLC finishes booting. The same call with the same config now finds it, // which is what the reconnect arm applies to the next connect attempt. const auto later_pass = OpcuaPlugin::discover_endpoint( rescan_cfg(), /*endpoint_configured=*/false, fake_scan({"192.168.1.10:4840"}), - fake_identify({{"opc.tcp://192.168.1.10:4840", plc_identity()}}), kSilent, kSilent); + fake_identify({{"opc.tcp://192.168.1.10:4840", plc_identity()}}), silent_reporter()); ASSERT_TRUE(later_pass.has_value()); EXPECT_EQ(*later_pass, "opc.tcp://192.168.1.10:4840"); } @@ -717,7 +726,7 @@ TEST(DiscoverEndpoint, AnExplicitEndpointIsNeverRescanned) { // endpoint. Discovery must not open a second session on a polled PLC. const auto chosen = OpcuaPlugin::discover_endpoint( rescan_cfg(), /*endpoint_configured=*/true, fake_scan({"192.168.1.10:4840"}), - fake_identify({{"opc.tcp://192.168.1.10:4840", plc_identity()}}), kSilent, kSilent); + fake_identify({{"opc.tcp://192.168.1.10:4840", plc_identity()}}), silent_reporter()); EXPECT_FALSE(chosen.has_value()); } @@ -730,7 +739,7 @@ TEST(DiscoverEndpoint, DisabledDiscoveryScansNothing) { return true; }; const auto chosen = OpcuaPlugin::discover_endpoint(cfg, /*endpoint_configured=*/false, counting_scan, - fake_identify({}), kSilent, kSilent); + fake_identify({}), silent_reporter()); EXPECT_FALSE(chosen.has_value()); EXPECT_FALSE(scanned) << "a disabled discovery must not touch the network"; } @@ -750,6 +759,372 @@ TEST(EffectiveRescanInterval, DefaultsWhenDiscoveryIsOnWithNoCadenceAndIsOffOthe EXPECT_EQ(OpcuaPlugin::effective_rescan_interval_s(cfg, false), 0); } +TEST(EffectiveRescanInterval, ExplicitZeroKeepsDiscoveryOnAndStopsRescanning) { + // The three states an operator can be in, all with discovery enabled and no + // endpoint pinned. + OpcuaDiscoveryConfig unset = rescan_cfg(); // (1) unset -> the built-in cadence + EXPECT_FALSE(unset.interval_s.has_value()); + EXPECT_EQ(OpcuaPlugin::effective_rescan_interval_s(unset, false), OpcuaPlugin::kDefaultRescanIntervalS); + + OpcuaDiscoveryConfig explicit_zero = rescan_cfg(); // (2) explicit 0 -> one-shot + explicit_zero.interval_s = 0; + EXPECT_EQ(OpcuaPlugin::effective_rescan_interval_s(explicit_zero, false), 0) + << "an explicit interval_s: 0 must stop the rescan, not fall back to the default"; + + // (3) A negative value never reaches here: the parse warns and leaves the + // cadence unset, so what arrives is case (1). + std::vector warnings; + const auto parsed = + parse_discovery_config(nlohmann::json{{"enabled", true}, {"interval_s", -1}}, [&warnings](const std::string & m) { + warnings.push_back(m); + }); + EXPECT_EQ(warnings.size(), 1u); + EXPECT_EQ(OpcuaPlugin::effective_rescan_interval_s(parsed, false), OpcuaPlugin::kDefaultRescanIntervalS); +} + +// --------------------------------------------------------------------------- +// Rescan cadence: measured from the END of the previous sweep +// --------------------------------------------------------------------------- + +TEST(RescanStep, SpacesSweepsFromTheEndOfThePreviousOne) { + // A legal /16 sweep runs for minutes. With the cadence stamped at the START, + // the next sweep is due the instant the current one returns, so the poll + // thread sweeps back to back and the reconnect attempt drops to one a sweep. + const auto t0 = std::chrono::steady_clock::time_point{}; + const auto sweep_duration = std::chrono::seconds(390); // a /16 at the defaults + auto clock_now = t0; + const auto now = [&clock_now]() { + return clock_now; + }; + + int sweeps = 0; + const auto sweep = [&sweeps, &clock_now, sweep_duration]() -> std::optional { + ++sweeps; + clock_now += sweep_duration; // the sweep blocks for its whole duration + return std::nullopt; + }; + + auto last_end = t0; + clock_now = t0 + std::chrono::seconds(30); + OpcuaPlugin::rescan_step(30, now, &last_end, sweep); + ASSERT_EQ(sweeps, 1); + EXPECT_EQ(last_end, clock_now) << "the cadence must be stamped when the sweep finished"; + + // One second after the sweep returned: not due, even though it STARTED 391 s + // ago. + clock_now += std::chrono::seconds(1); + OpcuaPlugin::rescan_step(30, now, &last_end, sweep); + EXPECT_EQ(sweeps, 1) << "a rescan ran less than one interval after the previous sweep ended"; + + // A full interval after the end: due again. + clock_now += std::chrono::seconds(29); + OpcuaPlugin::rescan_step(30, now, &last_end, sweep); + EXPECT_EQ(sweeps, 2); +} + +TEST(RescanStep, DoesNothingWithoutACadence) { + const auto t0 = std::chrono::steady_clock::time_point{}; + auto last_end = t0; + int sweeps = 0; + const auto now = [t0]() { + return t0 + std::chrono::hours(1); + }; + const auto sweep = [&sweeps]() -> std::optional { + ++sweeps; + return std::string("opc.tcp://192.168.1.10:4840"); + }; + // 0 is the operator's "do not re-scan" (and also discovery off / endpoint + // pinned, both of which effective_rescan_interval_s maps to 0). + EXPECT_FALSE(OpcuaPlugin::rescan_step(0, now, &last_end, sweep).has_value()); + EXPECT_EQ(sweeps, 0); + // Positive control on the same harness: with a cadence the very same call + // sweeps and hands the endpoint back. + const auto adopted = OpcuaPlugin::rescan_step(30, now, &last_end, sweep); + ASSERT_TRUE(adopted.has_value()); + EXPECT_EQ(*adopted, "opc.tcp://192.168.1.10:4840"); + EXPECT_EQ(sweeps, 1); +} + +// --------------------------------------------------------------------------- +// Reconnect backoff ceiling while the reconnect arm also rescans +// --------------------------------------------------------------------------- + +TEST(EffectiveMaxReconnectWait, CapsTheBackoffAtTheRescanCadence) { + using namespace std::chrono_literals; + // No rescan: the plain 60 s ceiling. + EXPECT_EQ(OpcuaPlugin::effective_max_reconnect_wait(5000ms, 60000ms, /*rescan_interval_s=*/0), 60000ms); + // Rescanning every 30 s: an uncapped backoff would make the real adoption + // cadence max(30 s, 60 s), not the documented 30 s. + EXPECT_EQ(OpcuaPlugin::effective_max_reconnect_wait(5000ms, 60000ms, 30), 30000ms); + // A cadence longer than the ceiling does not raise the ceiling. + EXPECT_EQ(OpcuaPlugin::effective_max_reconnect_wait(5000ms, 60000ms, 900), 60000ms); + // A cadence shorter than the configured reconnect interval does not turn the + // backoff into a hot retry loop. + EXPECT_EQ(OpcuaPlugin::effective_max_reconnect_wait(5000ms, 60000ms, 1), 5000ms); +} + +TEST(NextReconnectWait, DoublesUpToTheCeiling) { + using namespace std::chrono_literals; + EXPECT_EQ(OpcuaPoller::next_reconnect_wait(5000ms, 60000ms), 10000ms); + EXPECT_EQ(OpcuaPoller::next_reconnect_wait(40000ms, 60000ms), 60000ms); + EXPECT_EQ(OpcuaPoller::next_reconnect_wait(60000ms, 60000ms), 60000ms); + // Capped at a 30 s rescan cadence: the wait never exceeds it, so the rescan is + // consulted every cadence instead of every max(cadence, backoff). + EXPECT_EQ(OpcuaPoller::next_reconnect_wait(20000ms, 30000ms), 30000ms); + EXPECT_EQ(OpcuaPoller::next_reconnect_wait(30000ms, 30000ms), 30000ms); +} + +// --------------------------------------------------------------------------- +// Discovery report: quiet while the outcome does not change +// --------------------------------------------------------------------------- + +TEST(DiscoverEndpoint, AnUnchangedRescanReportsAtDebugInsteadOfRepeatingItself) { + // A secured-only site rescans for the life of the process and would otherwise + // log the whole report - scan line, per-server line, summary and the + // "no auto-connectable server" WARN - every interval_s. + IdentifyResult secured = plc_identity(); + secured.anonymous_none_available = false; + + std::vector info; + std::vector warn; + std::vector debug; + std::string outcome; + OpcuaPlugin::DiscoveryReporter reporter; + reporter.info = [&info](const std::string & m) { + info.push_back(m); + }; + reporter.warn = [&warn](const std::string & m) { + warn.push_back(m); + }; + reporter.debug = [&debug](const std::string & m) { + debug.push_back(m); + }; + reporter.previous_outcome = &outcome; + + const auto pass = [&]() { + return OpcuaPlugin::discover_endpoint(rescan_cfg(), /*endpoint_configured=*/false, fake_scan({"192.168.1.10:4840"}), + fake_identify({{"opc.tcp://192.168.1.10:4840", secured}}), reporter); + }; + + EXPECT_FALSE(pass().has_value()); + const size_t first_info = info.size(); + const size_t first_warn = warn.size(); + EXPECT_GT(first_info, 0u); + EXPECT_EQ(first_warn, 1u) << "the first pass always reports the secured-only outcome"; + EXPECT_TRUE(debug.empty()); + + // Same network, same outcome: nothing new at INFO/WARN, the report goes to + // the debug logger instead. + EXPECT_FALSE(pass().has_value()); + EXPECT_EQ(info.size(), first_info) << "an unchanged rescan repeated its report at INFO"; + EXPECT_EQ(warn.size(), first_warn) << "an unchanged rescan repeated its WARN"; + EXPECT_EQ(debug.size(), first_info + first_warn) << "the repeated report must still be traceable at DEBUG"; + + // The server opens up an anonymous endpoint: the outcome changed, so the + // operator hears about it at INFO again. + const auto chosen = + OpcuaPlugin::discover_endpoint(rescan_cfg(), /*endpoint_configured=*/false, fake_scan({"192.168.1.10:4840"}), + fake_identify({{"opc.tcp://192.168.1.10:4840", plc_identity()}}), reporter); + ASSERT_TRUE(chosen.has_value()); + EXPECT_GT(info.size(), first_info) << "a changed outcome must be reported at INFO"; +} + +TEST(DiscoverEndpoint, WithNoRepeatMemoryEveryPassIsReported) { + // Positive control for the test above: the same two identical passes with no + // previous_outcome (the startup scan's own reporter) report in full twice, so + // the silence above is the suppression and not a dead sink. + std::vector info; + OpcuaPlugin::DiscoveryReporter reporter; + reporter.info = [&info](const std::string & m) { + info.push_back(m); + }; + reporter.warn = kSilent; + + const auto pass = [&]() { + return OpcuaPlugin::discover_endpoint(rescan_cfg(), /*endpoint_configured=*/false, fake_scan({"192.168.1.10:4840"}), + fake_identify({{"opc.tcp://192.168.1.10:4840", plc_identity()}}), reporter); + }; + EXPECT_TRUE(pass().has_value()); + const size_t first = info.size(); + EXPECT_GT(first, 0u); + EXPECT_TRUE(pass().has_value()); + EXPECT_EQ(info.size(), 2 * first); +} + +// --------------------------------------------------------------------------- +// Config-less component identity across an adoption +// --------------------------------------------------------------------------- + +TEST(RederivedComponentIdentity, AdoptionReplacesTheProvisionalEndpointDerivedId) { + // The config-less race, end to end over the derivation path: the gateway + // starts before the PLC, its start-up scan finds nothing, and the identity is + // derived from the fallback endpoint plus an empty DeviceInfo. + const auto startup_pass = OpcuaPlugin::discover_endpoint(rescan_cfg(), /*endpoint_configured=*/false, fake_scan({}), + fake_identify({}), silent_reporter()); + ASSERT_FALSE(startup_pass.has_value()); + const std::string fallback_endpoint = "opc.tcp://localhost:4840"; // OpcuaClientConfig's default + const ComponentIdentity provisional = derive_component_identity(OpcuaClient::DeviceInfo{}, fallback_endpoint); + EXPECT_EQ(provisional.id, "opcua-localhost"); + + // The PLC finishes booting and the rescan adopts it. + const auto adopted = OpcuaPlugin::discover_endpoint( + rescan_cfg(), /*endpoint_configured=*/false, fake_scan({"192.168.1.10:4840"}), + fake_identify({{"opc.tcp://192.168.1.10:4840", plc_identity()}}), silent_reporter()); + ASSERT_TRUE(adopted.has_value()); + + // The session is up, so the device can finally name itself: the component + // must stop being served under the placeholder. + OpcuaClient::DeviceInfo info; + info.di_manufacturer = "Siemens AG"; + info.di_model = "CPU 1505SP F"; + const auto rederived = OpcuaPlugin::rederived_component_identity(provisional.id, info, *adopted); + ASSERT_TRUE(rederived.has_value()) << "an adopted device with a nameplate must replace opcua-localhost"; + EXPECT_EQ(rederived->id, "siemens_ag_cpu_1505sp_f"); + EXPECT_EQ(rederived->name, "Siemens AG CPU 1505SP F"); +} + +TEST(RederivedComponentIdentity, KeepsTheIdentityWhenNothingChanged) { + // Same device on a later reconnect: no rename, so no entity churn and no INFO + // line claiming an identity change that did not happen. + OpcuaClient::DeviceInfo info; + info.di_manufacturer = "Siemens AG"; + info.di_model = "CPU 1505SP F"; + EXPECT_FALSE(OpcuaPlugin::rederived_component_identity("siemens_ag_cpu_1505sp_f", info, "opc.tcp://192.168.1.10:4840") + .has_value()); + + // A nameplate-less server on an adopted endpoint still moves off the + // fallback host it was provisionally named after. + const auto host_derived = OpcuaPlugin::rederived_component_identity("opcua-localhost", OpcuaClient::DeviceInfo{}, + "opc.tcp://192.168.1.10:4840"); + ASSERT_TRUE(host_derived.has_value()); + EXPECT_EQ(host_derived->id, "opcua-192_168_1_10"); +} + +// --------------------------------------------------------------------------- +// ClearFault: a link-state clear does not cascade +// --------------------------------------------------------------------------- + +TEST(MakeClearFaultRequest, LinkStateClearSkipsTheCorrelationCascade) { + // The connect-time PLC_COMMS_LOST clear says the link came back. A + // correlation rule may name PLC_COMMS_LOST as the root cause of every symptom + // the outage produced, and clearing those is an operator's call, not a link + // event's. + const auto link_state = OpcuaPlugin::make_clear_fault_request(kCommsLostFaultCode, /*link_state=*/true); + EXPECT_EQ(link_state.fault_code, kCommsLostFaultCode); + EXPECT_TRUE(link_state.skip_correlation_auto_clear); + + // Positive control on the same request builder: an operator-driven clear (the + // SOVD DELETE route) leaves the cascade alone, so the flag above is the + // link-state rule and not a hardcoded true. + const auto operator_clear = OpcuaPlugin::make_clear_fault_request("PLC_TANK_HIGH", /*link_state=*/false); + EXPECT_EQ(operator_clear.fault_code, "PLC_TANK_HIGH"); + EXPECT_FALSE(operator_clear.skip_correlation_auto_clear); +} + +// --------------------------------------------------------------------------- +// Pending fault dispatch buffer: reports outrank clears +// --------------------------------------------------------------------------- + +namespace { + +OpcuaPlugin::PendingFaultDispatch report_entry(const std::string & code) { + return {OpcuaPlugin::PendingFaultDispatch::Kind::Report, code, []() {}}; +} + +OpcuaPlugin::PendingFaultDispatch clear_entry(const std::string & code) { + return {OpcuaPlugin::PendingFaultDispatch::Kind::Clear, code, []() {}}; +} + +size_t count_kind(const std::vector & buffer, + OpcuaPlugin::PendingFaultDispatch::Kind kind) { + return static_cast( + std::count_if(buffer.begin(), buffer.end(), [kind](const OpcuaPlugin::PendingFaultDispatch & entry) { + return entry.kind == kind; + })); +} + +} // namespace + +TEST(EnqueuePendingDispatch, ReconnectClearsNeverEvictABufferedAlarmReport) { + // A flapping link with no fault_manager: 300 reconnects, each enqueueing a + // connect-time clear, while ten real alarm reports wait to be flushed. The + // reports are one-shot edges from the PLC; the clears are re-derivable. + std::vector buffer; + for (int i = 0; i < 10; ++i) { + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, + report_entry("PLC_ALARM_" + std::to_string(i))); + } + for (int i = 0; i < 300; ++i) { + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, clear_entry(kCommsLostFaultCode)); + } + + EXPECT_EQ(count_kind(buffer, OpcuaPlugin::PendingFaultDispatch::Kind::Report), 10u) + << "connect-time clears evicted buffered alarm reports"; + EXPECT_EQ(count_kind(buffer, OpcuaPlugin::PendingFaultDispatch::Kind::Clear), 1u) + << "at most one clear per fault code may be pending"; + for (int i = 0; i < 10; ++i) { + EXPECT_EQ(buffer[static_cast(i)].fault_code, "PLC_ALARM_" + std::to_string(i)); + } +} + +TEST(EnqueuePendingDispatch, AFullReportBufferRefusesAClearInsteadOfDroppingAReport) { + std::vector buffer; + for (size_t i = 0; i < OpcuaPlugin::kMaxPendingDispatches; ++i) { + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, + report_entry("PLC_ALARM_" + std::to_string(i))); + } + ASSERT_EQ(buffer.size(), OpcuaPlugin::kMaxPendingDispatches); + + EXPECT_EQ(OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, + clear_entry(kCommsLostFaultCode)), + OpcuaPlugin::PendingEnqueueOutcome::Refused); + EXPECT_EQ(count_kind(buffer, OpcuaPlugin::PendingFaultDispatch::Kind::Report), OpcuaPlugin::kMaxPendingDispatches); + EXPECT_EQ(buffer.front().fault_code, "PLC_ALARM_0") << "the oldest report must survive an incoming clear"; + + // A report arriving at a full buffer still drops the oldest one: reports do + // not outrank each other, so the bound still holds. + EXPECT_EQ( + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, report_entry("PLC_ALARM_NEW")), + OpcuaPlugin::PendingEnqueueOutcome::EvictedReport); + EXPECT_EQ(buffer.size(), OpcuaPlugin::kMaxPendingDispatches); + EXPECT_EQ(buffer.front().fault_code, "PLC_ALARM_1"); + EXPECT_EQ(buffer.back().fault_code, "PLC_ALARM_NEW"); +} + +TEST(EnqueuePendingDispatch, AFullBufferGivesUpAPendingClearBeforeAReport) { + std::vector buffer; + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, clear_entry("PLC_OLD_CLEAR")); + for (size_t i = 1; i < OpcuaPlugin::kMaxPendingDispatches; ++i) { + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, + report_entry("PLC_ALARM_" + std::to_string(i))); + } + ASSERT_EQ(buffer.size(), OpcuaPlugin::kMaxPendingDispatches); + + EXPECT_EQ( + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, report_entry("PLC_ALARM_NEW")), + OpcuaPlugin::PendingEnqueueOutcome::EvictedClear); + EXPECT_EQ(count_kind(buffer, OpcuaPlugin::PendingFaultDispatch::Kind::Clear), 0u); + EXPECT_EQ(buffer.front().fault_code, "PLC_ALARM_1") << "the clear went, not the oldest report"; +} + +TEST(EnqueuePendingDispatch, ARequeuedClearMovesToTheBackSoOrderStillHolds) { + // Report-then-clear for one code must still flush in that order after the + // clear is re-enqueued, or the flush would leave the fault standing. + std::vector buffer; + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, clear_entry("PLC_FLAP")); + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, report_entry("PLC_FLAP")); + EXPECT_EQ(OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, clear_entry("PLC_FLAP")), + OpcuaPlugin::PendingEnqueueOutcome::ReplacedClear); + + ASSERT_EQ(buffer.size(), 2u); + EXPECT_EQ(buffer[0].kind, OpcuaPlugin::PendingFaultDispatch::Kind::Report); + EXPECT_EQ(buffer[1].kind, OpcuaPlugin::PendingFaultDispatch::Kind::Clear) + << "the newest clear must flush after the report it supersedes"; + // Clears for DIFFERENT codes are independent. + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, clear_entry("PLC_OTHER")); + EXPECT_EQ(count_kind(buffer, OpcuaPlugin::PendingFaultDispatch::Kind::Clear), 2u); +} + TEST(AdoptRediscoveredEndpoint, AdoptsOnlyADifferentNonEmptyUrl) { const std::string current = "opc.tcp://localhost:4840"; From 48a2632a7f219ce317723e56895fd84a26cfe788 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Sun, 6 Sep 2026 16:35:01 +0200 Subject: [PATCH 06/14] test(gateway): assert the freeze-frame capture path from a real capture The two Frame::source constants had no assertion from a capture: only a hand-built frame in the merge helper's test named one, so swapping the DataProvider and route values left the whole suite green. The route and DataProvider loss-of-comms tests now each assert the constant their own path must produce, plus the literal wire value - symbol against symbol stays equal when the two constants are swapped, and that string is what every x-medkit.source consumer reads. The merge test that omits the key keeps its place as a helper contract for a frame a caller built without naming a path - the capture paths always name one - and says so instead of standing in as a control for them. The peer-node count test listed only two of the three helper nodes the gateway creates in its own process, so a lone gateway with a lifecycle reader would have counted a peer and skipped the empty-graph warning. Document the entity-frame source field in the REST fault snapshot reference: an entity frame carries no topic or message type, so source is the only provenance a consumer gets. --- docs/api/rest.rst | 20 ++++++++++------ .../test/test_entity_freeze_frame_capture.cpp | 23 ++++++++++++++++--- .../test/test_gateway_node.cpp | 4 ++++ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/docs/api/rest.rst b/docs/api/rest.rst index 0321f22fc..0faa882f1 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -1501,13 +1501,19 @@ Query and manage faults. - ``freeze_frame``: Data captured at fault confirmation. Entity frames for faults that were already confirmed when the gateway started are captured at gateway start instead and carry ``"capture_origin": "startup"`` in - their ``x-medkit`` block. For a plugin-backed entity that reports its - link down, the values are the plugin's last known ones and may predate - the confirmation by the length of the outage; such entries carry - ``connected`` (the payload's link flag, ``false`` for the loss-of-comms - case) and ``source_timestamp`` (the payload's own timestamp, verbatim) - in ``x-medkit``, both only when the plugin's payload reports them. - ``captured_at`` always dates the capture, not the values. + their ``x-medkit`` block. An entity frame also carries ``source`` in + ``x-medkit``, naming the path that read the values + (``plugin_data_provider`` for the owning plugin's DataProvider, + ``plugin_x_plc_data_route`` for its ``x-plc-data`` route). These values + are not a ROS message, so ``topic`` and ``message_type`` are empty and + ``source`` is the only field saying where the numbers came from. For a + plugin-backed entity that reports its link down, the values are the + plugin's last known ones and may predate the confirmation by the length of + the outage; such entries carry ``connected`` (the payload's link flag, + ``false`` for the loss-of-comms case) and ``source_timestamp`` (the + payload's own timestamp, verbatim) in ``x-medkit``, both only when the + plugin's payload reports them. ``captured_at`` always dates the capture, + not the values. - ``rosbag``: Recording file available via bulk-data endpoint **Response codes:** diff --git a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp index 152297ba0..8ef14dbc4 100644 --- a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp @@ -573,6 +573,15 @@ TEST_F(EntityFreezeFrameCaptureTest, DisconnectedEntityWithLastKnownValuesIsCapt ASSERT_TRUE(frames[0].connected.has_value()); EXPECT_FALSE(*frames[0].connected); EXPECT_EQ(frames[0].source_timestamp, 1234567890); + // Which path read the values. This capture had no DataProvider and went + // through the route fallback, so the frame must name that path; the + // DataProvider flavour of the same case asserts the other constant, which is + // what stops the two from being swapped at their call sites unnoticed. + EXPECT_EQ(frames[0].source, EntityFreezeFrameCapture::kSourceXPlcDataRoute); + // The wire value itself, not just the symbol: swapping what the two constants + // hold is an API break for every consumer of x-medkit.source, and comparing + // symbol against symbol would not see it. + EXPECT_EQ(frames[0].source, "plugin_x_plc_data_route"); } /// @verifies REQ_INTEROP_088 @@ -642,6 +651,11 @@ TEST_F(EntityFreezeFrameCaptureTest, DisconnectedDataProviderWithLastKnownValues ASSERT_TRUE(frames[0].connected.has_value()); EXPECT_FALSE(*frames[0].connected); EXPECT_TRUE(frames[0].source_timestamp.is_null()); // provider content has no timestamp field + // The provider path names itself, and the route path (same case, above) names + // the other constant: the pair is what makes a swap of the two call sites + // visible. The literal pins the wire value the API reference documents. + EXPECT_EQ(frames[0].source, EntityFreezeFrameCapture::kSourceDataProvider); + EXPECT_EQ(frames[0].source, "plugin_data_provider"); } /// @verifies REQ_INTEROP_088 @@ -805,9 +819,12 @@ TEST(MergeEntityFreezeFrames, CarriesCapturePathAsSource) { EXPECT_EQ(snap["message_type"], ""); } -TEST(MergeEntityFreezeFrames, OmitsSourceWhenTheCaptureNamedNoPath) { - // Absence control for the test above, on the same harness: a frame whose - // capture path is unknown must not have one invented for it. +TEST(MergeEntityFreezeFrames, OmitsSourceForAFrameThatNamesNoPath) { + // A merge-helper contract, not a control for the capture tests: both capture + // paths always name themselves (asserted from real captures in + // Disconnected{Entity,DataProvider}WithLastKnownValuesIsCaptured), so this + // frame is one only a caller can build. The helper must then leave the key + // out rather than invent a provenance the wire consumer would trust. json env_data = {{"snapshots", json::array()}}; EntityFreezeFrameCapture::Frame frame; frame.entity_id = "plc_app"; diff --git a/src/ros2_medkit_gateway/test/test_gateway_node.cpp b/src/ros2_medkit_gateway/test/test_gateway_node.cpp index 2f81dbc6c..557aef176 100644 --- a/src/ros2_medkit_gateway/test/test_gateway_node.cpp +++ b/src/ros2_medkit_gateway/test/test_gateway_node.cpp @@ -1058,10 +1058,14 @@ TEST(GatewayStartupSummary, CountPeerNodesExcludesOwnAndHidden) { } TEST(GatewayStartupSummary, CountPeerNodesZeroWhenOnlyOwnNodes) { + // Every helper the gateway creates inside its own process. A gateway alone on + // the graph must report zero peers, so each helper has to be recognized - + // including the lifecycle reader, which the list previously omitted. const std::vector> nodes = { {"ros2_medkit_gateway", "/"}, {"ros2_medkit_gateway_sub", "/"}, {"ros2_medkit_gateway_fault_clients", "/"}, + {"ros2_medkit_gateway_lifecycle_state_reader", "/"}, }; // Zero peers is the condition that triggers the empty-graph warning. EXPECT_EQ(ros2_medkit_gateway::GatewayNode::count_peer_nodes(nodes, "/ros2_medkit_gateway"), 0u); From d1e61d7716718511f22bb953c70ae95c7c010992 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Sun, 6 Sep 2026 18:51:20 +0200 Subject: [PATCH 07/14] fix(opcua): close the scoped-clear hole and rank the pending buffer by what is re-derivable The per-entity SOVD route DELETE /{entity}/faults/{code} lands on FaultProvider::clear_fault for a plugin-owned entity, which is the branch the gateway takes instead of its own. The gateway sets skip_correlation_auto_clear there so an operator scoped to one entity cannot cascade-clear correlated symptoms reported by apps in other entities, and the ClearFault contract documents that guarantee, but this plugin sent the flag off and reopened the hole wherever a PLC is involved. It now sets the flag, and the reason each call site sets or clears it travels with the call: a ClearOrigin says whether the device reported the condition inactive (a real resolution, cascade kept), the link came back, or an operator cleared through the scoped route. The pending-dispatch buffer used that same distinction too bluntly. It gave up any clear before any report, but only the link-state clear is re-derivable - the next reconnect sends it again. A device alarm's inactive edge is as one-shot as its raise, so evicting it left the flush replaying the raise with nothing behind it and the fault standing while the device said inactive. Ranking is now by re-derivability: the link-state clear is what a full buffer gives up first, everything else ages out oldest-first as it did before. The start-up discovery sweep claimed to be cancellable through the shutdown flag, but nothing can set that flag while it runs: it happens inside set_context(), during node construction, before the executor the gateway shuts down from ever spins. A SIGTERM during a wide sweep therefore waited the sweep out. Both sweeps now ask one predicate that also reads rclcpp::ok(), which rclcpp's own signal handler turns false, so the start-up sweep ends on the signal and the rescan keeps ending on shutdown() as well. Also: a rescan sweep that throws now stamps the cadence on its way out, or the next poll iteration would immediately start another one; the start-up log line no longer states the wrong reason for reporting in full; and the package changelog records this branch. Measuring that start-up sweep also showed the gateway logging nothing at all while it ran: the discovery report is buffered so a repeated rescan can be reported at DEBUG, and the "scanning [subnets]" announcement had been swept up with it. A minutes-long sweep with no output reads as a hung process, so the announcement is sent before the sweep again, with the same first-pass INFO / rescan DEBUG levelling on its own. --- docs/api/rest.rst | 2 +- .../ros2_medkit_gateway/gateway_node.hpp | 4 +- .../test/test_entity_freeze_frame_capture.cpp | 2 +- .../ros2_medkit_opcua/CHANGELOG.rst | 10 + .../ros2_medkit_opcua/README.md | 13 +- .../docker/scripts/run_discovery_race_test.sh | 8 +- .../ros2_medkit_opcua/opcua_plugin.hpp | 122 +++++-- .../ros2_medkit_opcua/src/opcua_plugin.cpp | 139 +++++--- .../test/test_network_discovery.cpp | 4 +- .../test/test_opcua_plugin.cpp | 333 ++++++++++++++++-- 10 files changed, 507 insertions(+), 130 deletions(-) diff --git a/docs/api/rest.rst b/docs/api/rest.rst index 0faa882f1..590414c62 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -1509,7 +1509,7 @@ Query and manage faults. ``source`` is the only field saying where the numbers came from. For a plugin-backed entity that reports its link down, the values are the plugin's last known ones and may predate the confirmation by the length of - the outage; such entries carry ``connected`` (the payload's link flag, + the outage. Such entries carry ``connected`` (the payload's link flag, ``false`` for the loss-of-comms case) and ``source_timestamp`` (the payload's own timestamp, verbatim) in ``x-medkit``, both only when the plugin's payload reports them. ``captured_at`` always dates the capture, diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp index d4585d99a..89a153d1b 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp @@ -514,7 +514,7 @@ class GatewayNode : public rclcpp::Node { * `_monitor` or `2`, and dropping a real node is the worse error. * * @param node_fqn Fully qualified node name to test ("/ns/node") - * @param self_fqn The gateway node's own FQN; an empty value matches nothing + * @param self_fqn The gateway node's own FQN. An empty value matches nothing */ bool is_own_gateway_node(const std::string & node_fqn, const std::string & self_fqn); @@ -533,7 +533,7 @@ bool is_own_gateway_node(const std::string & node_fqn, const std::string & self_ * * @param apps App vector to filter in place * @param peer_routing_table Maps entity_id -> peer_name for remote entities - * @param self_fqn The gateway node's own FQN; empty disables the self check + * @param self_fqn The gateway node's own FQN. Empty disables the self check * @return Number of apps removed */ size_t filter_internal_node_apps(std::vector & apps, diff --git a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp index 8ef14dbc4..b97372530 100644 --- a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp @@ -574,7 +574,7 @@ TEST_F(EntityFreezeFrameCaptureTest, DisconnectedEntityWithLastKnownValuesIsCapt EXPECT_FALSE(*frames[0].connected); EXPECT_EQ(frames[0].source_timestamp, 1234567890); // Which path read the values. This capture had no DataProvider and went - // through the route fallback, so the frame must name that path; the + // through the route fallback, so the frame must name that path. The // DataProvider flavour of the same case asserts the other constant, which is // what stops the two from being swapped at their call sites unnoticed. EXPECT_EQ(frames[0].source, EntityFreezeFrameCapture::kSourceXPlcDataRoute); diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/CHANGELOG.rst b/src/ros2_medkit_plugins/ros2_medkit_opcua/CHANGELOG.rst index 9f37ccc7f..1db5561be 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/CHANGELOG.rst +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/CHANGELOG.rst @@ -2,6 +2,16 @@ Changelog for package ros2_medkit_opcua ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Forthcoming +----------- +* **Breaking:** ``discovery.interval_s`` now distinguishes unset from an explicit ``0``. Leaving the key out (and leaving ``OPCUA_DISCOVERY_INTERVAL_S`` unset) keeps the built-in 30 s re-scan cadence, while an explicit ``0`` means "discovery on, start-up scan only" instead of selecting that same default. A deployment that wrote ``interval_s: 0`` meaning "use the default" stops re-scanning, and dropping the key restores the previous behaviour. A negative value is refused with a warning and leaves the cadence unset +* While no OPC UA session is established, the reconnect loop re-scans on that cadence and adopts a server that appeared after start-up, so a gateway that booted before its PLC finds it without a restart. The cadence is measured from the end of the previous sweep, the reconnect backoff is capped at the cadence so the documented interval is the real one, and a sweep is cancellable (``SIGINT`` / ``SIGTERM`` during start-up, the plugin's ``shutdown()`` for a re-scan) instead of having to run to completion. A re-scan whose outcome has not changed is reported at DEBUG rather than repeating the whole report every interval +* With no node map, the SOVD component identity is re-derived from the device on the first session after such an adoption, so a component named after the fallback endpoint (because nothing answered at start-up) stops being served under that placeholder once the real PLC is adopted +* ``PLC_COMMS_LOST`` is cleared on every successful connect, including the first, so a fault ``fault_manager`` persisted before a restart does not stand against a healthy link. That clear, and the per-entity ``DELETE /{entity}/faults/{code}`` route when it is served by this plugin, set ``skip_correlation_auto_clear``: neither is an operator resolving a root cause, so neither may cascade-clear correlated symptom faults reported by apps in other entities. A clear reported by the device itself still cascades +* Fault dispatches buffered while ``fault_manager`` is unreachable no longer lose one-shot events. Only the link-state ``PLC_COMMS_LOST`` clear is re-derivable (the next reconnect sends it again), so it is what the bounded buffer gives up first. Alarm reports, a device alarm's inactive edge and an operator's scoped clear age out oldest-first as before, and at most one clear per fault code is pending at a time +* Gateway-side changes that land on this plugin's entities: a freeze-frame captured from a plugin entity now names the path that read the values in ``x-medkit.source`` (``plugin_data_provider`` or ``plugin_x_plc_data_route``, the only provenance an entity frame carries since it has no ROS topic), and the gateway no longer lists its own in-process helper nodes among the discovered apps +* Contributors: @bburda + 0.7.0 (2026-08-27) ------------------ * Config-less discovery. A read-only network scan finds the OPC UA server instead of requiring its endpoint up front (`#509 `_), ``auto_browse`` walks the address space recursively and builds the SOVD tree from it (`#510 `_), and identity, writability and fault triggers are read from the device itself rather than declared in a node map (`#544 `_) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md index 65c3d2225..7b6abdde0 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md @@ -604,8 +604,8 @@ plugins.opcua.discovery: connect_timeout_ms: 600 # per-port TCP connect timeout scan_concurrency: 100 # bounded, polite concurrent connect count identify_timeout_ms: 6000 # per GetEndpoints identify - # re-scan cadence while disconnected. Omit the key for the built-in 30 s; - # set it to 0 to keep discovery on but never re-scan (start-up scan only). + # re-scan cadence while disconnected. Omit the key for the built-in 30 s, + # or set it to 0 to keep discovery on but never re-scan (start-up scan only). interval_s: 30 anonymous_none_only: true # only auto-connect None/Anonymous servers ``` @@ -613,7 +613,7 @@ plugins.opcua.discovery: Environment overrides (Docker / appliance): `OPCUA_DISCOVERY_ENABLED`, `OPCUA_DISCOVERY_SUBNETS` (comma-separated CIDRs), `OPCUA_DISCOVERY_INTERVAL_S`. Leaving `interval_s` (and `OPCUA_DISCOVERY_INTERVAL_S`) unset means "no cadence -stated" and takes the 30 s default; an explicit `0` is honoured as written and +stated" and takes the 30 s default. An explicit `0` is honoured as written and turns the recurring sweep off. A negative value is refused with a warning and leaves the cadence unset. @@ -656,8 +656,11 @@ Safety / OT posture: `interval_s` (default 30 s) for as long as it stays disconnected. Set `interval_s: 0` (or `OPCUA_DISCOVERY_INTERVAL_S=0`) to keep discovery on with the start-up scan only, or `enabled: false` to switch it off entirely. -- A sweep is cancelled when the plugin shuts down, so a stop does not have to - wait out a subnet the size of a /16. +- A sweep is cancellable, so a stop does not have to wait out a subnet the size + of a /16. The start-up sweep runs while the gateway node is still being + constructed, so what ends it is `SIGINT` / `SIGTERM`, which the plugin sees + through `rclcpp::ok()`. A re-scan sweep runs on the poll thread and is ended + by either that or the plugin's own `shutdown()`. - An explicitly configured `endpoint_url` (or `OPCUA_ENDPOINT_URL`) always wins; discovery then does nothing, so it never opens a second session on a PLC the plugin already polls. diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh index dc52d3823..5b807472b 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh @@ -37,8 +37,8 @@ CONFIG_DIR=/tmp/discovery_race_config # The fallback the plugin keeps when a scan selects nothing (OpcuaClientConfig). FALLBACK_ENDPOINT="opc.tcp://localhost:4840" # Config-less naming: with no node map the component id is derived from the -# device. Before any server exists that can only be the fallback endpoint's host; -# after adoption it is the test server's DI nameplate (Manufacturer "SelfPatch +# device. Before any server exists that can only be the fallback endpoint's +# host. After adoption it is the test server's DI nameplate (Manufacturer "SelfPatch # Devices" + Model "SPX-1000"), slugified. FALLBACK_COMPONENT_ID="opcua-localhost" DEVICE_COMPONENT_ID="selfpatch_devices_spx_1000" @@ -62,8 +62,8 @@ fail() { exit 1 } -# x-plc-status of a named component (the node-map pass pins the id; the -# config-less pass has to look it up first). +# x-plc-status of a named component. The node-map pass pins the id, the +# config-less pass has to look it up first. status_json_for() { curl -sf "http://localhost:${GATEWAY_PORT}/api/v1/components/$1/x-plc-status" || echo '{}' } diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp index a646bfccd..702c6cd8d 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp @@ -148,14 +148,19 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // Where one discovery pass reports to, plus the memory that keeps a repeated // identical pass quiet. A rescan runs every ``interval_s`` for the life of a - // disconnected process, so re-emitting the same scan line, per-server lines, - // summary and "no auto-connectable server" WARN each time buries every other - // message in the log. ``previous_outcome`` is owned by the caller (the plugin - // keeps one across rescans): when it is non-null and the pass reaches the same - // outcome as the pass before it, the whole report goes to ``debug`` instead. - // The first pass, and every pass whose outcome changed, is always reported at - // info/warn. A null ``previous_outcome`` (the startup scan, and tests that do - // not care) reports every pass. + // disconnected process, so re-emitting the same per-server lines, summary and + // "no auto-connectable server" WARN each time buries every other message in + // the log. ``previous_outcome`` is owned by the caller (the plugin keeps one + // across rescans): when it is non-null and the pass reaches the same outcome + // as the pass before it, the whole report goes to ``debug`` instead. The first + // pass, and every pass whose outcome changed, is always reported at info/warn. + // A null ``previous_outcome`` (tests that do not care) reports every pass. + // + // The "scanning [subnets]" announcement is NOT part of that report. It is sent + // before the sweep runs, because a wide subnet takes minutes and an operator + // watching start-up has to see the gateway working rather than hung. It says + // what the pass is about to do rather than what it found, so it carries the + // same first-pass / rescan levelling on its own. struct DiscoveryReporter { std::function info; std::function warn; @@ -243,15 +248,47 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, const OpcuaClient::DeviceInfo & info, const std::string & endpoint_url); - // Build the ClearFault request for one fault code. ``link_state`` marks a - // clear that only reports the OPC-UA link came back (the connect-time - // ``PLC_COMMS_LOST`` clear). Such a clear must not cascade: a correlation rule - // may name PLC_COMMS_LOST as the root cause of every symptom the outage - // produced, and the link returning is not an operator resolving those. An - // operator-driven clear (the SOVD DELETE route) leaves the flag off and keeps - // the cascade. Static so the wire field is assertable without a fault manager. + // Why a ClearFault is being sent. Two properties follow from it and nothing + // else does, so the origin travels instead of a pair of loose booleans: + // - whether the correlation cascade must be skipped + // (``clear_skips_correlation``), which goes on the wire, and + // - whether the clear is re-derivable (``clear_is_link_state``), which is + // what the pending buffer may give up first under pressure. + enum class ClearOrigin { + /// The device reported the condition inactive (an ``event_alarms`` / + /// ``auto_alarms`` condition, or a threshold rule going false). A one-shot + /// edge nothing will re-send, and a real resolution, so the cascade stands. + DeviceAlarm, + /// The OPC-UA session came back, so ``PLC_COMMS_LOST`` no longer holds. + /// Re-derived on the next reconnect if it is lost, and not an operator + /// resolving a root cause, so it must not cascade. + LinkState, + /// The SOVD per-entity ``DELETE /{entity}/faults/{code}`` route reached + /// FaultProvider::clear_fault. An operator scoped to one entity must not + /// cascade-clear symptoms reported by apps in other entities, which is the + /// same rule the gateway applies on its own (non-plugin) branch of that + /// route. One-shot: nothing re-derives an operator's decision. + ScopedOperator + }; + + // Whether this clear must leave the correlation engine's auto_clear_with_root + // cascade alone. True for everything except a device-reported clear. + static bool clear_skips_correlation(ClearOrigin origin) { + return origin != ClearOrigin::DeviceAlarm; + } + + // Whether this clear will be re-derived if it is dropped. Only the link-state + // clear will: the next reconnect sends it again. + static bool clear_is_link_state(ClearOrigin origin) { + return origin == ClearOrigin::LinkState; + } + + // Build the ClearFault request for one fault code. + // ``skip_correlation_auto_clear`` goes on the wire verbatim (see ClearOrigin + // for who sets it and why). Static so the wire field is assertable without a + // fault manager. static ros2_medkit_msgs::srv::ClearFault::Request make_clear_fault_request(const std::string & fault_code, - bool link_state); + bool skip_correlation_auto_clear); // One entry in the bounded buffer of fault dispatches held while the // fault_manager service is unmatched. @@ -259,28 +296,37 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, enum class Kind { Report, Clear }; Kind kind{Kind::Report}; std::string fault_code; ///< dedup key for a Clear; diagnostic for a Report + /// Clear only: this dispatch is re-derivable (ClearOrigin::LinkState), so + /// the buffer may drop it before anything that is not. + bool link_state{false}; std::function dispatch; }; // What ``enqueue_pending_dispatch`` did, so the caller can log it. enum class PendingEnqueueOutcome { - Buffered, ///< appended, nothing lost - ReplacedClear, ///< superseded the pending clear for the same fault code - EvictedClear, ///< buffer was full: dropped a pending clear to make room - EvictedReport, ///< buffer was full of reports and a report arrived - Refused ///< buffer was full of reports and a clear arrived + Buffered, ///< appended, nothing lost + ReplacedClear, ///< superseded the pending clear for the same fault code + EvictedLinkStateClear, ///< buffer was full: dropped a re-derivable clear to make room + EvictedOldest, ///< buffer was full with nothing re-derivable in it: dropped the oldest entry + Refused ///< buffer was full with nothing re-derivable and the incoming clear was }; // Enqueue policy for the bounded pending-dispatch buffer. // - // Reports outrank clears. A report is a one-shot edge from the PLC that - // nothing will re-send, while a clear is re-derivable: the link state is - // re-observed on the next reconnect. So at most ONE clear per fault code is - // ever pending (a newer one moves to the back, keeping report-then-clear - // order), a full buffer gives up its oldest pending clear first, and a clear - // arriving at a buffer full of reports is refused rather than evicting one. - // Without this a flapping link enqueued one connect-time clear per reconnect - // attempt and pushed real alarm reports out of the buffer. + // Only a link-state clear is re-derivable: the next reconnect sends it again. + // Everything else in the buffer is a one-shot edge nothing will re-send - a + // report, a device alarm going inactive, an operator's scoped clear - so those + // rank together and age out oldest-first, exactly as the buffer behaved before + // any of this. A link-state clear is what a full buffer gives up first, and an + // incoming one is refused rather than pushing a one-shot dispatch out. At most + // ONE clear per fault code is pending at a time (a newer one moves to the + // back, so an interleaved report-then-clear still flushes in that order). + // + // Without the link-state ranking a flapping link enqueued one connect-time + // clear per reconnect attempt and pushed real alarm reports out of the buffer. + // Without the "only link-state" part, a device alarm's inactive edge was + // evicted ahead of an older report and the flush replayed the raise with no + // clear behind it, leaving the fault standing while the device said inactive. static PendingEnqueueOutcome enqueue_pending_dispatch(std::vector & buffer, size_t max_size, PendingFaultDispatch entry); @@ -305,9 +351,9 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // Report/clear fault via ROS 2 service (private helpers, not the FaultProvider overrides) void send_report_fault(const std::string & entity_id, const std::string & fault_code, const std::string & severity_str, const std::string & message); - // ``link_state`` marks a clear that reports the OPC-UA link came back rather - // than an operator resolving a root cause; see make_clear_fault_request. - void send_clear_fault(const std::string & fault_code, bool link_state = false); + // ``origin`` says why the clear is being sent, which decides both the wire + // flag and how the pending buffer ranks it. See ClearOrigin. + void send_clear_fault(const std::string & fault_code, ClearOrigin origin = ClearOrigin::DeviceAlarm); // Clear PLC_COMMS_LOST after the initial connect in set_context() succeeded. // Unconditional on purpose: the fault manager keys faults by fault_code and @@ -379,6 +425,18 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // (null to report every pass in full). DiscoveryReporter discovery_reporter(std::string * previous_outcome) const; + // Abort predicate handed to a discovery sweep. Two independent stop signals, + // because the two sweeps run at different points in the process lifetime: + // - ``shutdown_requested_`` is set by shutdown(), which the gateway calls + // after its executor returns. That ends a RESCAN sweep, which runs on the + // poll thread long after start-up. + // - ``rclcpp::ok()`` turns false as soon as rclcpp's own SIGINT / SIGTERM + // handler runs. The STARTUP sweep runs inside set_context(), i.e. during + // node construction and before the executor spins, so shutdown() cannot + // be reached while it is in progress and the signal is the only thing + // that can end it. + bool discovery_cancelled() const; + // Poll-thread hook bound into PollerConfig::rediscover_endpoint whenever // discovery runs without a configured endpoint. Called from the poller's // reconnect arm, so only while no session is up, and rate-limited to one scan diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp index 355570fa4..158523936 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp @@ -1049,10 +1049,11 @@ void OpcuaPlugin::on_alarm_change(const std::string & entity_id, } else { log_info("Alarm cleared: " + signal.fault_code + " on " + entity_id); // The poller's own comms-lost clear on a successful reconnect is the same - // link-state clear as the connect-time one, so it does not cascade either. - // Every other code is a real alarm going inactive on the device and keeps - // the default correlation behaviour. - send_clear_fault(signal.fault_code, /*link_state=*/signal.fault_code == kCommsLostFaultCode); + // link-state event as the connect-time one. Every other code here is the + // device reporting its condition inactive, which is a real resolution and a + // one-shot edge, so it keeps the cascade and the buffer treats it as such. + send_clear_fault(signal.fault_code, + signal.fault_code == kCommsLostFaultCode ? ClearOrigin::LinkState : ClearOrigin::DeviceAlarm); } } @@ -1231,7 +1232,11 @@ void OpcuaPlugin::on_event_alarm(const AlarmEventDelivery & delivery) { break; case AlarmAction::ClearFault: log_info("AlarmCondition CLEARED: " + delivery.fault_code); - send_clear_fault(delivery.fault_code); + // The device itself reported the condition cleared (Part 9 lifecycle), so + // this IS a resolution at the source and the correlation engine may act on + // it. DeviceAlarm is also the default, spelled out here because this is + // the one call site where the cascade is deliberately kept. + send_clear_fault(delivery.fault_code, ClearOrigin::DeviceAlarm); break; case AlarmAction::NoOp: break; @@ -1266,34 +1271,29 @@ void OpcuaPlugin::send_report_fault(const std::string & entity_id, const std::st request->severity = ros2_medkit_msgs::msg::Fault::SEVERITY_INFO; } - send_or_buffer({PendingFaultDispatch::Kind::Report, fault_code, [this, request]() { + send_or_buffer({PendingFaultDispatch::Kind::Report, fault_code, /*link_state=*/false, [this, request]() { fault_clients_->report->async_send_request(request); }}); } ros2_medkit_msgs::srv::ClearFault::Request OpcuaPlugin::make_clear_fault_request(const std::string & fault_code, - bool link_state) { + bool skip_correlation_auto_clear) { ros2_medkit_msgs::srv::ClearFault::Request request; request.fault_code = fault_code; - // A link-state clear reports that the OPC-UA session came back. It is not an - // operator resolving a root cause, so it must not trip the correlation - // engine's auto_clear_with_root cascade: a rule naming PLC_COMMS_LOST as the - // root cause would otherwise clear every symptom fault the outage produced, - // none of which this plugin has any evidence about. - request.skip_correlation_auto_clear = link_state; + request.skip_correlation_auto_clear = skip_correlation_auto_clear; return request; } -void OpcuaPlugin::send_clear_fault(const std::string & fault_code, bool link_state) { +void OpcuaPlugin::send_clear_fault(const std::string & fault_code, ClearOrigin origin) { if (!fault_clients_->clear) { log_warn("ClearFault service client not available"); return; } - auto request = - std::make_shared(make_clear_fault_request(fault_code, link_state)); + auto request = std::make_shared( + make_clear_fault_request(fault_code, clear_skips_correlation(origin))); - send_or_buffer({PendingFaultDispatch::Kind::Clear, fault_code, [this, request]() { + send_or_buffer({PendingFaultDispatch::Kind::Clear, fault_code, clear_is_link_state(origin), [this, request]() { fault_clients_->clear->async_send_request(request); }}); } @@ -1305,8 +1305,14 @@ void OpcuaPlugin::clear_comms_lost_on_connect() { // ClearFault is idempotent from this side: send_clear_fault is // fire-and-forget, so a "Fault not found" answer for a code that was never // raised costs nothing here and is the normal case on a healthy start. + // + // LinkState: this says the session came back, not that an operator resolved + // anything, so a correlation rule naming PLC_COMMS_LOST as a root cause must + // not cascade-clear the symptoms the outage produced. It is also the one clear + // the next reconnect re-derives, so the pending buffer may drop it before + // anything one-shot. log_info(std::string("OPC-UA connection established; clearing any standing ") + kCommsLostFaultCode); - send_clear_fault(kCommsLostFaultCode, /*link_state=*/true); + send_clear_fault(kCommsLostFaultCode, ClearOrigin::LinkState); } OpcuaPlugin::PendingEnqueueOutcome OpcuaPlugin::enqueue_pending_dispatch(std::vector & buffer, @@ -1329,21 +1335,22 @@ OpcuaPlugin::PendingEnqueueOutcome OpcuaPlugin::enqueue_pending_dispatch(std::ve PendingEnqueueOutcome outcome = replaced ? PendingEnqueueOutcome::ReplacedClear : PendingEnqueueOutcome::Buffered; if (buffer.size() >= max_size) { - // A report is a one-shot edge from the PLC that nothing will re-send; a - // clear is re-derivable from the next reconnect. So a full buffer gives up a - // pending clear first, and refuses an incoming clear rather than evicting a - // report for it. - const auto oldest_clear = std::find_if(buffer.begin(), buffer.end(), [](const PendingFaultDispatch & pending) { - return pending.kind == PendingFaultDispatch::Kind::Clear; + // Only a link-state clear is re-derivable: the next reconnect sends it + // again. A full buffer gives that up first, and refuses an incoming one + // rather than pushing out a dispatch nothing will re-send. Everything else - + // reports, a device alarm's inactive edge, an operator's scoped clear - is + // one-shot and ages out oldest-first. + const auto oldest_link_state = std::find_if(buffer.begin(), buffer.end(), [](const PendingFaultDispatch & pending) { + return pending.kind == PendingFaultDispatch::Kind::Clear && pending.link_state; }); - if (oldest_clear != buffer.end()) { - buffer.erase(oldest_clear); - outcome = PendingEnqueueOutcome::EvictedClear; - } else if (is_clear) { + if (oldest_link_state != buffer.end()) { + buffer.erase(oldest_link_state); + outcome = PendingEnqueueOutcome::EvictedLinkStateClear; + } else if (is_clear && entry.link_state) { return PendingEnqueueOutcome::Refused; } else { buffer.erase(buffer.begin()); - outcome = PendingEnqueueOutcome::EvictedReport; + outcome = PendingEnqueueOutcome::EvictedOldest; } } @@ -1360,15 +1367,15 @@ void OpcuaPlugin::send_or_buffer(PendingFaultDispatch entry) { std::lock_guard lock(pending_reports_mutex_); outcome = enqueue_pending_dispatch(pending_reports_, kMaxPendingDispatches, std::move(entry)); } - if (outcome == PendingEnqueueOutcome::EvictedReport) { + if (outcome == PendingEnqueueOutcome::EvictedOldest) { log_warn("pending fault dispatch buffer full (" + std::to_string(kMaxPendingDispatches) + - "), dropping the oldest report"); - } else if (outcome == PendingEnqueueOutcome::EvictedClear) { + "), dropping the oldest dispatch"); + } else if (outcome == PendingEnqueueOutcome::EvictedLinkStateClear) { log_warn("pending fault dispatch buffer full (" + std::to_string(kMaxPendingDispatches) + - "), dropping the oldest pending clear"); + "), dropping a link-state clear the next reconnect re-derives"); } else if (outcome == PendingEnqueueOutcome::Refused) { - log_warn("pending fault dispatch buffer full of reports (" + std::to_string(kMaxPendingDispatches) + - "), dropping this clear instead of a report"); + log_warn("pending fault dispatch buffer full of one-shot dispatches (" + std::to_string(kMaxPendingDispatches) + + "), dropping this link-state clear instead"); } // Drains immediately (in order) if the sink is already matched. flush_pending_reports(); @@ -1464,7 +1471,7 @@ std::optional OpcuaPlugin::rederived_component_identity(const } void OpcuaPlugin::maybe_rederive_component_identity() { - // An explicit node map owns the component name; only the config-less path + // An explicit node map owns the component name. Only the config-less path // derives it from the device. if (!node_map_path_.empty() || !client_ || !client_->is_connected()) { return; @@ -1650,12 +1657,19 @@ std::optional OpcuaPlugin::rescan_step(int interval_s, if (now() - *last_scan_end < std::chrono::seconds(interval_s)) { return std::nullopt; } - const auto result = sweep(); // Stamp the END of the sweep: a legal /16 runs for minutes, and stamping its // start would make the next one due the moment this one returned - the poll // thread would sweep back to back and only attempt a reconnect once a sweep. - *last_scan_end = now(); - return result; + // A sweep that threw still consumed that time, so the stamp is owed either + // way, or the next poll iteration would start another one immediately. + try { + const auto result = sweep(); + *last_scan_end = now(); + return result; + } catch (...) { + *last_scan_end = now(); + throw; + } } std::optional OpcuaPlugin::discover_endpoint(const OpcuaDiscoveryConfig & config, bool endpoint_configured, @@ -1717,8 +1731,18 @@ std::optional OpcuaPlugin::discover_endpoint(const OpcuaDiscoveryCo for (const auto & s : subnets) { subnet_list += (subnet_list.empty() ? "" : ", ") + s; } - info_line("OPC-UA discovery: read-only active scan of [" + subnet_list + "] on " + - std::to_string(config.ports.size()) + " port(s)..."); + // The announcement goes out NOW, not through the buffered report: a sweep of a + // wide subnet runs for minutes, and an operator watching start-up has to see + // that the gateway is scanning rather than hung. It says what the pass is + // about to do, not what it found, so it stays out of the outcome digest and is + // levelled on its own - the first pass announces at INFO, a rescan at DEBUG so + // the recurring sweep does not repeat it every interval. + const bool first_pass = reporter.previous_outcome == nullptr || reporter.previous_outcome->empty(); + const auto & announce_sink = first_pass ? reporter.info : reporter.debug; + if (announce_sink) { + announce_sink("OPC-UA discovery: read-only active scan of [" + subnet_list + "] on " + + std::to_string(config.ports.size()) + " port(s)..."); + } const std::vector found = discovery.run(cancelled); @@ -1776,12 +1800,15 @@ void OpcuaPlugin::run_startup_discovery() { return; } - // The startup scan is always reported in full (no previous outcome to compare - // against) and always cancellable, so a shutdown during set_context does not - // wait out a whole sweep. + // The startup scan is the first pass, so its outcome digest is still empty and + // the report comes out in full. It is cancellable too, but not by shutdown(): + // this runs inside set_context(), i.e. during node construction and before the + // executor spins, so nothing can call shutdown() until this returns. What ends + // it is the SIGINT / SIGTERM that rclcpp's own handler turns into + // !rclcpp::ok() - see discovery_cancelled(). const auto chosen = discover_endpoint(discovery_config_, endpoint_configured_, discovery_scan_fn_, discovery_identify_fn_, discovery_reporter(&last_discovery_outcome_), [this]() { - return shutdown_requested_.load(); + return discovery_cancelled(); }); // Stamp when the sweep FINISHED: the rescan cadence is measured from the end // of the previous sweep, so a long sweep is not immediately followed by @@ -1812,6 +1839,16 @@ void OpcuaPlugin::run_startup_discovery() { log_info("OPC-UA discovery: auto-selected endpoint " + *chosen + " - handing to the connect + introspect path."); } +bool OpcuaPlugin::discovery_cancelled() const { + // Either stop signal ends a sweep. shutdown() is what ends a RESCAN (it runs + // on the poll thread, long after start-up). rclcpp::ok() going false is what + // ends the STARTUP sweep, which runs during node construction where shutdown() + // is not reachable yet. Checking both in one predicate keeps the two sweeps + // from drifting apart, and rclcpp::ok() only reads the default context's + // atomic shutdown flag, so it is safe to call from either thread. + return shutdown_requested_.load() || !rclcpp::ok(); +} + OpcuaPlugin::DiscoveryReporter OpcuaPlugin::discovery_reporter(std::string * previous_outcome) const { DiscoveryReporter reporter; reporter.info = [this](const std::string & m) { @@ -1831,7 +1868,7 @@ std::optional OpcuaPlugin::rescan_endpoint_for_reconnect() { // A sweep is a bounded but multi-second blocking call on the poll thread, and // stop() has to wait for whatever it is in the middle of. Do not start one the // shutdown is going to throw away. - if (shutdown_requested_.load()) { + if (discovery_cancelled()) { return std::nullopt; } const int interval_s = effective_rescan_interval_s(discovery_config_, endpoint_configured_); @@ -1845,7 +1882,7 @@ std::optional OpcuaPlugin::rescan_endpoint_for_reconnect() { [this]() { return discover_endpoint(discovery_config_, endpoint_configured_, discovery_scan_fn_, discovery_identify_fn_, discovery_reporter(&last_discovery_outcome_), [this]() { - return shutdown_requested_.load(); + return discovery_cancelled(); }); }); // The live client config, not client_config_: this runs on the poll thread @@ -2358,7 +2395,13 @@ tl::expected OpcuaPlugin::clear_f return tl::make_unexpected(FaultProviderErrorInfo{FaultProviderError::Internal, "plugin not initialized", 503}); } - send_clear_fault(fault_code); + // This is the per-entity SOVD route DELETE /{entity}/faults/{code}. The + // gateway sets skip_correlation_auto_clear on its own branch of that route so + // an operator scoped to one entity cannot cascade-clear symptoms reported by + // apps in other entities, and a plugin-owned entity must not be the hole in + // that rule: the request goes through this provider instead, so the same flag + // has to be set here. + send_clear_fault(fault_code, ClearOrigin::ScopedOperator); return dto::FaultClearResult{ nlohmann::json{{"status", "cleared"}, {"fault_code", fault_code}, {"entity_id", entity_id}}}; } diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_network_discovery.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_network_discovery.cpp index 96c32e339..7ef7c0e4f 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_network_discovery.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_network_discovery.cpp @@ -408,8 +408,8 @@ TEST(NetworkDiscoveryRun, IdentifyFailureRecordedAsLead) { // run(cancelled): a shutdown must not wait out a whole sweep // --------------------------------------------------------------------------- // TEST(NetworkDiscoveryRun, CancelStopsTheSweepInsteadOfProbingEveryHost) { - // A /24 is 254 probes and a legal /16 is 65k; the caller runs them on the poll - // thread a shutdown has to join. With scan_concurrency 1 the sweep is + // A /24 is 254 probes and a legal /16 is 65k, and the caller runs them on the + // poll thread a shutdown has to join. With scan_concurrency 1 the sweep is // sequential, so the probe count is exactly what the cancel predicate allowed. std::atomic probes{0}; std::atomic stop{false}; diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp index 9984cef85..9d5a0af9a 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp @@ -822,6 +822,39 @@ TEST(RescanStep, SpacesSweepsFromTheEndOfThePreviousOne) { EXPECT_EQ(sweeps, 2); } +TEST(RescanStep, AThrowingSweepStillStampsTheCadence) { + // A sweep that throws still consumed its minutes. If the stamp were owed only + // on the normal path, the next poll iteration would find the cadence due and + // start another sweep immediately, so a server that makes the identify throw + // would turn the poll thread into a continuous scanner. + const auto t0 = std::chrono::steady_clock::time_point{}; + auto clock_now = t0 + std::chrono::seconds(30); + const auto now = [&clock_now]() { + return clock_now; + }; + int sweeps = 0; + const auto throwing_sweep = [&sweeps, &clock_now]() -> std::optional { + ++sweeps; + clock_now += std::chrono::seconds(120); + throw std::runtime_error("identify blew up mid-sweep"); + }; + + auto last_end = t0; + EXPECT_THROW(OpcuaPlugin::rescan_step(30, now, &last_end, throwing_sweep), std::runtime_error); + EXPECT_EQ(sweeps, 1); + EXPECT_EQ(last_end, clock_now) << "a sweep that threw still has to stamp the cadence"; + + // Inside the interval after that failed sweep: not due, so no second sweep. + clock_now += std::chrono::seconds(29); + EXPECT_NO_THROW(OpcuaPlugin::rescan_step(30, now, &last_end, throwing_sweep)); + EXPECT_EQ(sweeps, 1) << "a failed sweep let the next one start inside the interval"; + + // Positive control: one full interval later it is due again (and throws again). + clock_now += std::chrono::seconds(1); + EXPECT_THROW(OpcuaPlugin::rescan_step(30, now, &last_end, throwing_sweep), std::runtime_error); + EXPECT_EQ(sweeps, 2); +} + TEST(RescanStep, DoesNothingWithoutACadence) { const auto t0 = std::chrono::steady_clock::time_point{}; auto last_end = t0; @@ -929,6 +962,86 @@ TEST(DiscoverEndpoint, AnUnchangedRescanReportsAtDebugInsteadOfRepeatingItself) EXPECT_GT(info.size(), first_info) << "a changed outcome must be reported at INFO"; } +TEST(DiscoverEndpoint, APredicateThatFlipsMidSweepEndsThePass) { + // What a stop signal does to a sweep in progress. The plugin hands + // discover_endpoint a predicate that answers for both stop signals (the + // shutdown flag and rclcpp::ok()). Here it flips after a handful of probes, + // as either would mid-sweep. + std::atomic probes{0}; + std::atomic stop{false}; + auto stopping_scan = [&probes, &stop](const std::string & ip, uint16_t port, int) { + if (probes.fetch_add(1) + 1 >= 5) { + stop.store(true); + } + return ip == "192.168.1.10" && port == 4840; // the PLC IS there to be found + }; + + OpcuaDiscoveryConfig cfg = rescan_cfg(); + cfg.scan_concurrency = 1; // sequential, so the probe count is the predicate's doing + const auto chosen = OpcuaPlugin::discover_endpoint(cfg, /*endpoint_configured=*/false, stopping_scan, + fake_identify({{"opc.tcp://192.168.1.10:4840", plc_identity()}}), + silent_reporter(), [&stop]() { + return stop.load(); + }); + + EXPECT_FALSE(chosen.has_value()) << "a cancelled pass must not hand back a partial result"; + EXPECT_LE(probes.load(), 6) << "the sweep ran on after the stop signal"; + + // Positive control on the same fakes: without the predicate the very same + // sweep visits all 254 hosts and selects the PLC. + probes.store(0); + stop.store(false); + const auto uncancelled = OpcuaPlugin::discover_endpoint( + cfg, /*endpoint_configured=*/false, + [&probes](const std::string & ip, uint16_t port, int) { + probes.fetch_add(1); + return ip == "192.168.1.10" && port == 4840; + }, + fake_identify({{"opc.tcp://192.168.1.10:4840", plc_identity()}}), silent_reporter()); + ASSERT_TRUE(uncancelled.has_value()); + EXPECT_EQ(*uncancelled, "opc.tcp://192.168.1.10:4840"); + EXPECT_EQ(probes.load(), 254); +} + +TEST(DiscoverEndpoint, TheScanIsAnnouncedBeforeTheSweepRuns) { + // A /16 sweep runs for minutes. If the announcement waited for the report at + // the end of the pass, start-up would log nothing while it swept and an + // operator would read that as a hung gateway. + std::vector info; + std::vector debug; + std::string announced_before_first_probe; + std::string outcome; + OpcuaPlugin::DiscoveryReporter reporter; + reporter.info = [&info](const std::string & m) { + info.push_back(m); + }; + reporter.warn = kSilent; + reporter.debug = [&debug](const std::string & m) { + debug.push_back(m); + }; + reporter.previous_outcome = &outcome; + + auto scan_recording_the_log = [&info, &announced_before_first_probe](const std::string &, uint16_t, int) { + if (announced_before_first_probe.empty() && !info.empty()) { + announced_before_first_probe = info.front(); + } + return false; + }; + OpcuaPlugin::discover_endpoint(rescan_cfg(), /*endpoint_configured=*/false, scan_recording_the_log, fake_identify({}), + reporter); + EXPECT_NE(announced_before_first_probe.find("read-only active scan of"), std::string::npos) + << "the sweep started before the operator was told anything (first INFO line: '" + << (info.empty() ? std::string("") : info.front()) << "')"; + + // On a rescan the announcement drops to DEBUG: the sweep repeats every + // interval_s for the life of the outage and must not narrate every pass. + const size_t info_after_first = info.size(); + OpcuaPlugin::discover_endpoint(rescan_cfg(), /*endpoint_configured=*/false, fake_scan({}), fake_identify({}), + reporter); + EXPECT_EQ(info.size(), info_after_first) << "the rescan announced itself at INFO again"; + EXPECT_FALSE(debug.empty()); +} + TEST(DiscoverEndpoint, WithNoRepeatMemoryEveryPassIsReported) { // Positive control for the test above: the same two identical passes with no // previous_outcome (the startup scan's own reporter) report in full twice, so @@ -1001,38 +1114,61 @@ TEST(RederivedComponentIdentity, KeepsTheIdentityWhenNothingChanged) { } // --------------------------------------------------------------------------- -// ClearFault: a link-state clear does not cascade +// ClearFault: only a clear the device itself reported may cascade // --------------------------------------------------------------------------- -TEST(MakeClearFaultRequest, LinkStateClearSkipsTheCorrelationCascade) { - // The connect-time PLC_COMMS_LOST clear says the link came back. A - // correlation rule may name PLC_COMMS_LOST as the root cause of every symptom - // the outage produced, and clearing those is an operator's call, not a link - // event's. - const auto link_state = OpcuaPlugin::make_clear_fault_request(kCommsLostFaultCode, /*link_state=*/true); - EXPECT_EQ(link_state.fault_code, kCommsLostFaultCode); - EXPECT_TRUE(link_state.skip_correlation_auto_clear); - - // Positive control on the same request builder: an operator-driven clear (the - // SOVD DELETE route) leaves the cascade alone, so the flag above is the - // link-state rule and not a hardcoded true. - const auto operator_clear = OpcuaPlugin::make_clear_fault_request("PLC_TANK_HIGH", /*link_state=*/false); - EXPECT_EQ(operator_clear.fault_code, "PLC_TANK_HIGH"); - EXPECT_FALSE(operator_clear.skip_correlation_auto_clear); +TEST(ClearOrigin, OnlyADeviceReportedClearKeepsTheCorrelationCascade) { + using Origin = OpcuaPlugin::ClearOrigin; + // The link coming back is not an operator resolving a root cause, and neither + // is an operator scoped to ONE entity: a correlation rule naming + // PLC_COMMS_LOST as the root cause would otherwise clear symptom faults + // reported by apps in entities that operator cannot even see. The gateway + // applies exactly this rule on its own branch of the same DELETE route. + EXPECT_TRUE(OpcuaPlugin::clear_skips_correlation(Origin::LinkState)); + EXPECT_TRUE(OpcuaPlugin::clear_skips_correlation(Origin::ScopedOperator)); + // Positive control on the same predicate: the device reporting its own + // condition inactive IS a resolution at the source, so that clear cascades. + // Without this case the rule above would be indistinguishable from a + // hardcoded true. + EXPECT_FALSE(OpcuaPlugin::clear_skips_correlation(Origin::DeviceAlarm)); + + // The buffer's ranking is a different question from the wire flag: only the + // link-state clear is re-derivable, the operator's scoped clear is as + // one-shot as an alarm report. + EXPECT_TRUE(OpcuaPlugin::clear_is_link_state(Origin::LinkState)); + EXPECT_FALSE(OpcuaPlugin::clear_is_link_state(Origin::ScopedOperator)); + EXPECT_FALSE(OpcuaPlugin::clear_is_link_state(Origin::DeviceAlarm)); +} + +TEST(MakeClearFaultRequest, CarriesTheSkipFlagAndCodeVerbatim) { + const auto skipping = OpcuaPlugin::make_clear_fault_request(kCommsLostFaultCode, true); + EXPECT_EQ(skipping.fault_code, kCommsLostFaultCode); + EXPECT_TRUE(skipping.skip_correlation_auto_clear); + + const auto cascading = OpcuaPlugin::make_clear_fault_request("PLC_TANK_HIGH", false); + EXPECT_EQ(cascading.fault_code, "PLC_TANK_HIGH"); + EXPECT_FALSE(cascading.skip_correlation_auto_clear); } // --------------------------------------------------------------------------- -// Pending fault dispatch buffer: reports outrank clears +// Pending fault dispatch buffer: only what is re-derivable may be dropped first // --------------------------------------------------------------------------- namespace { OpcuaPlugin::PendingFaultDispatch report_entry(const std::string & code) { - return {OpcuaPlugin::PendingFaultDispatch::Kind::Report, code, []() {}}; + return {OpcuaPlugin::PendingFaultDispatch::Kind::Report, code, /*link_state=*/false, []() {}}; +} + +// A clear the next reconnect will send again (PLC_COMMS_LOST). +OpcuaPlugin::PendingFaultDispatch link_state_clear_entry(const std::string & code) { + return {OpcuaPlugin::PendingFaultDispatch::Kind::Clear, code, /*link_state=*/true, []() {}}; } -OpcuaPlugin::PendingFaultDispatch clear_entry(const std::string & code) { - return {OpcuaPlugin::PendingFaultDispatch::Kind::Clear, code, []() {}}; +// A clear nothing will re-send: the device reported its condition inactive, or +// an operator cleared through the scoped SOVD route. +OpcuaPlugin::PendingFaultDispatch device_clear_entry(const std::string & code) { + return {OpcuaPlugin::PendingFaultDispatch::Kind::Clear, code, /*link_state=*/false, []() {}}; } size_t count_kind(const std::vector & buffer, @@ -1048,14 +1184,15 @@ size_t count_kind(const std::vector & buffer, TEST(EnqueuePendingDispatch, ReconnectClearsNeverEvictABufferedAlarmReport) { // A flapping link with no fault_manager: 300 reconnects, each enqueueing a // connect-time clear, while ten real alarm reports wait to be flushed. The - // reports are one-shot edges from the PLC; the clears are re-derivable. + // reports are one-shot edges from the PLC, the clears are re-derivable. std::vector buffer; for (int i = 0; i < 10; ++i) { OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, report_entry("PLC_ALARM_" + std::to_string(i))); } for (int i = 0; i < 300; ++i) { - OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, clear_entry(kCommsLostFaultCode)); + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, + link_state_clear_entry(kCommsLostFaultCode)); } EXPECT_EQ(count_kind(buffer, OpcuaPlugin::PendingFaultDispatch::Kind::Report), 10u) @@ -1067,7 +1204,7 @@ TEST(EnqueuePendingDispatch, ReconnectClearsNeverEvictABufferedAlarmReport) { } } -TEST(EnqueuePendingDispatch, AFullReportBufferRefusesAClearInsteadOfDroppingAReport) { +TEST(EnqueuePendingDispatch, AFullOneShotBufferRefusesALinkStateClearInsteadOfDroppingOne) { std::vector buffer; for (size_t i = 0; i < OpcuaPlugin::kMaxPendingDispatches; ++i) { OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, @@ -1076,24 +1213,25 @@ TEST(EnqueuePendingDispatch, AFullReportBufferRefusesAClearInsteadOfDroppingARep ASSERT_EQ(buffer.size(), OpcuaPlugin::kMaxPendingDispatches); EXPECT_EQ(OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, - clear_entry(kCommsLostFaultCode)), + link_state_clear_entry(kCommsLostFaultCode)), OpcuaPlugin::PendingEnqueueOutcome::Refused); EXPECT_EQ(count_kind(buffer, OpcuaPlugin::PendingFaultDispatch::Kind::Report), OpcuaPlugin::kMaxPendingDispatches); - EXPECT_EQ(buffer.front().fault_code, "PLC_ALARM_0") << "the oldest report must survive an incoming clear"; + EXPECT_EQ(buffer.front().fault_code, "PLC_ALARM_0") << "the oldest report must survive an incoming link-state clear"; - // A report arriving at a full buffer still drops the oldest one: reports do - // not outrank each other, so the bound still holds. + // A report arriving at the same full buffer still drops the oldest entry: + // one-shot dispatches do not outrank each other, so the bound still holds. EXPECT_EQ( OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, report_entry("PLC_ALARM_NEW")), - OpcuaPlugin::PendingEnqueueOutcome::EvictedReport); + OpcuaPlugin::PendingEnqueueOutcome::EvictedOldest); EXPECT_EQ(buffer.size(), OpcuaPlugin::kMaxPendingDispatches); EXPECT_EQ(buffer.front().fault_code, "PLC_ALARM_1"); EXPECT_EQ(buffer.back().fault_code, "PLC_ALARM_NEW"); } -TEST(EnqueuePendingDispatch, AFullBufferGivesUpAPendingClearBeforeAReport) { +TEST(EnqueuePendingDispatch, AFullBufferGivesUpALinkStateClearBeforeAReport) { std::vector buffer; - OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, clear_entry("PLC_OLD_CLEAR")); + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, + link_state_clear_entry(kCommsLostFaultCode)); for (size_t i = 1; i < OpcuaPlugin::kMaxPendingDispatches; ++i) { OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, report_entry("PLC_ALARM_" + std::to_string(i))); @@ -1102,18 +1240,55 @@ TEST(EnqueuePendingDispatch, AFullBufferGivesUpAPendingClearBeforeAReport) { EXPECT_EQ( OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, report_entry("PLC_ALARM_NEW")), - OpcuaPlugin::PendingEnqueueOutcome::EvictedClear); + OpcuaPlugin::PendingEnqueueOutcome::EvictedLinkStateClear); EXPECT_EQ(count_kind(buffer, OpcuaPlugin::PendingFaultDispatch::Kind::Clear), 0u); - EXPECT_EQ(buffer.front().fault_code, "PLC_ALARM_1") << "the clear went, not the oldest report"; + EXPECT_EQ(buffer.front().fault_code, "PLC_ALARM_1") << "the re-derivable clear went, not the oldest report"; +} + +TEST(EnqueuePendingDispatch, ADeviceAlarmClearIsNotEvictedAheadOfAnOlderReport) { + // The device says an alarm went inactive while the fault_manager is + // unreachable. That edge is as one-shot as the raise: drop it and the flush + // replays the raise with nothing behind it, so the fault stands while the + // device reports it clear. Only the link-state clear is re-derivable. + std::vector buffer; + for (int i = 0; i < 100; ++i) { + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, + report_entry("PLC_ALARM_" + std::to_string(i))); + } + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, report_entry("PLC_TANK_HIGH")); + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, + device_clear_entry("PLC_TANK_HIGH")); + for (size_t i = buffer.size(); i < OpcuaPlugin::kMaxPendingDispatches; ++i) { + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, + report_entry("PLC_FILLER_" + std::to_string(i))); + } + ASSERT_EQ(buffer.size(), OpcuaPlugin::kMaxPendingDispatches); + + EXPECT_EQ( + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, report_entry("PLC_ALARM_NEW")), + OpcuaPlugin::PendingEnqueueOutcome::EvictedOldest); + EXPECT_NE(buffer.front().fault_code, "PLC_ALARM_0") << "the oldest entry is what ages out"; + const auto device_clear = + std::find_if(buffer.begin(), buffer.end(), [](const OpcuaPlugin::PendingFaultDispatch & entry) { + return entry.kind == OpcuaPlugin::PendingFaultDispatch::Kind::Clear && entry.fault_code == "PLC_TANK_HIGH"; + }); + ASSERT_NE(device_clear, buffer.end()) << "a device alarm's inactive edge was evicted ahead of an older report"; + // ... and it still flushes after the raise it supersedes. + const auto raise = std::find_if(buffer.begin(), buffer.end(), [](const OpcuaPlugin::PendingFaultDispatch & entry) { + return entry.kind == OpcuaPlugin::PendingFaultDispatch::Kind::Report && entry.fault_code == "PLC_TANK_HIGH"; + }); + ASSERT_NE(raise, buffer.end()); + EXPECT_LT(raise - buffer.begin(), device_clear - buffer.begin()); } TEST(EnqueuePendingDispatch, ARequeuedClearMovesToTheBackSoOrderStillHolds) { // Report-then-clear for one code must still flush in that order after the // clear is re-enqueued, or the flush would leave the fault standing. std::vector buffer; - OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, clear_entry("PLC_FLAP")); + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, link_state_clear_entry("PLC_FLAP")); OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, report_entry("PLC_FLAP")); - EXPECT_EQ(OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, clear_entry("PLC_FLAP")), + EXPECT_EQ(OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, + link_state_clear_entry("PLC_FLAP")), OpcuaPlugin::PendingEnqueueOutcome::ReplacedClear); ASSERT_EQ(buffer.size(), 2u); @@ -1121,7 +1296,7 @@ TEST(EnqueuePendingDispatch, ARequeuedClearMovesToTheBackSoOrderStillHolds) { EXPECT_EQ(buffer[1].kind, OpcuaPlugin::PendingFaultDispatch::Kind::Clear) << "the newest clear must flush after the report it supersedes"; // Clears for DIFFERENT codes are independent. - OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, clear_entry("PLC_OTHER")); + OpcuaPlugin::enqueue_pending_dispatch(buffer, OpcuaPlugin::kMaxPendingDispatches, device_clear_entry("PLC_OTHER")); EXPECT_EQ(count_kind(buffer, OpcuaPlugin::PendingFaultDispatch::Kind::Clear), 2u); } @@ -1629,4 +1804,92 @@ component_id: race_runtime << "flush_pending_reports never dispatched - swap-vs-push path not covered"; } +// The SOVD per-entity route DELETE /{entity}/faults/{code} lands on +// FaultProvider::clear_fault for a plugin-owned entity, which is the branch the +// gateway takes INSTEAD of its own (where it sets skip_correlation_auto_clear +// itself). So the flag has to be set here or the documented guarantee - an +// operator scoped to one entity cannot cascade-clear symptoms reported by apps +// in other entities - has a hole exactly where a PLC is involved. This drives +// the real route entry point and reads the field off the wire. +TEST(OpcuaPluginScopedClear, SovdDeleteSkipsTheCorrelationCascade) { + ScopedRclcpp rclcpp_scope; + auto node = std::make_shared("opcua_scoped_clear_flag"); + auto fault_manager = std::make_shared("opcua_scoped_clear_faultmgr"); + + std::mutex received_mutex; + std::vector received; + auto report_srv = fault_manager->create_service( + "/fault_manager/report_fault", [](const std::shared_ptr, + std::shared_ptr res) { + res->accepted = true; + }); + auto clear_srv = fault_manager->create_service( + "/fault_manager/clear_fault", + [&received, &received_mutex](const std::shared_ptr req, + std::shared_ptr res) { + { + std::lock_guard lock(received_mutex); + received.push_back(*req); + } + res->success = true; + }); + + const std::string yaml_path = "/tmp/test_opcua_scoped_clear_nodemap.yaml"; + { + std::ofstream f(yaml_path); + f << R"( +area_id: scoped_plc +component_id: scoped_runtime +nodes: + - node_id: "ns=2;i=1" + entity_id: tank + data_name: level + data_type: float +)"; + } + + OpcuaPlugin plugin; + nlohmann::json config; + config["node_map_path"] = yaml_path; + config["endpoint_url"] = "opc.tcp://127.0.0.1:1"; // nothing listening; the fault sink is the subject + config["poll_interval_ms"] = 100; + plugin.configure(config); + + RealNodePluginContext ctx(node.get()); + ctx.entities["tank"] = {SovdEntityType::APP, "tank", "/scoped_plc", "/scoped_plc/scoped_runtime/tank"}; + plugin.set_context(ctx); + + ScopedExecutorSpin spinner({node, fault_manager}); + auto probe = node->create_client("/fault_manager/clear_fault"); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!probe->service_is_ready() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + ASSERT_TRUE(probe->service_is_ready()) << "stub ClearFault server never became discoverable"; + + // The route's own entry point, not a helper it happens to call. + const auto result = plugin.clear_fault("tank", "PLC_TANK_HIGH"); + ASSERT_TRUE(result.has_value()); + + const auto flush_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + bool delivered = false; + while (!delivered && std::chrono::steady_clock::now() < flush_deadline) { + { + std::lock_guard lock(received_mutex); + delivered = !received.empty(); + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + + spinner.stop(); + plugin.shutdown(); + std::remove(yaml_path.c_str()); + + std::lock_guard lock(received_mutex); + ASSERT_FALSE(received.empty()) << "the scoped DELETE never reached the fault manager"; + EXPECT_EQ(received.front().fault_code, "PLC_TANK_HIGH"); + EXPECT_TRUE(received.front().skip_correlation_auto_clear) + << "a per-entity DELETE served by the plugin cascade-cleared correlated symptoms"; +} + } // namespace ros2_medkit_gateway From bfecdd4b1ab33587a6625684dc76559d17531689 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Sun, 6 Sep 2026 20:24:12 +0200 Subject: [PATCH 08/14] test(opcua): pin the clear-origin sites and the sweep predicate on the wire Four call sites decide whether a ClearFault may cascade, and only the scoped SOVD DELETE was pinned: swapping the origin at any of the other three left the suite green. The two that need a live session are now driven against the test_alarm_server fixture with a real fault manager on the other end, so the flag is read off the wire. A successful connect must clear PLC_COMMS_LOST without cascading, and the same test drives the fixture's own CLI to fire and clear a condition, because a clear the device itself reported is a resolution at the source and must keep the cascade. Having both cases in one test is what makes each flag a decision rather than a constant. The fixture harness gained a stdin pipe to send those commands, the way the docker scenario already drives it through a FIFO. The poller's own clear travels the same callback as every device alarm, so the rule that tells them apart moved into clear_origin_for_signal and is tested on both branches, exact code match included. The sweep's cancel predicate had the same shape of hole: it is private and no test reached it, so reverting it to the shutdown flag alone left everything green while a SIGTERM during the start-up sweep would again have to wait the sweep out. The rule is now the static discovery_cancelled_for, tested on both inputs, with the member reduced to reading the two values off the process. A second test shows the input is real by shutting a private rclcpp context down and reading rclcpp::ok() back. The comment on the cancellation test no longer claims it exercises the plugin's own predicate, which it never did. Also: the remaining prose semicolons on this branch (two comments and five operator-visible log strings) are periods and commas now, and the Refused outcome's doc comment says what it means. --- .../ros2_medkit_opcua/opcua_plugin.hpp | 42 ++- .../ros2_medkit_opcua/src/opcua_plugin.cpp | 26 +- .../test/test_opcua_identity.cpp | 253 ++++++++++++++++++ .../test/test_opcua_plugin.cpp | 54 +++- 4 files changed, 343 insertions(+), 32 deletions(-) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp index 702c6cd8d..cfe7a5bed 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp @@ -283,6 +283,30 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, return origin == ClearOrigin::LinkState; } + // Whether a discovery sweep must stop now, given the two independent stop + // signals. Static and pure so both inputs are testable: the member + // ``discovery_cancelled()`` only reads them off the process and hands them + // here, so this is the whole rule. + // + // - ``shutdown_requested`` is set by shutdown(), which the gateway calls + // after its executor returns. That ends a RESCAN sweep, which runs on the + // poll thread long after start-up. + // - ``rclcpp_ok`` is false once rclcpp's own SIGINT / SIGTERM handler has + // run. The START-UP sweep runs inside set_context(), during node + // construction and before the executor spins, so shutdown() cannot be + // reached while it is in progress and the signal is the only thing that + // can end it. + static bool discovery_cancelled_for(bool shutdown_requested, bool rclcpp_ok) { + return shutdown_requested || !rclcpp_ok; + } + + // Which kind of clear a fault-detection signal going inactive is. The poller + // emits the component-scoped ``PLC_COMMS_LOST`` clear through the same + // callback as every device alarm, and only that one is a link-state event. + static ClearOrigin clear_origin_for_signal(const std::string & fault_code) { + return fault_code == kCommsLostFaultCode ? ClearOrigin::LinkState : ClearOrigin::DeviceAlarm; + } + // Build the ClearFault request for one fault code. // ``skip_correlation_auto_clear`` goes on the wire verbatim (see ClearOrigin // for who sets it and why). Static so the wire field is assertable without a @@ -295,7 +319,7 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, struct PendingFaultDispatch { enum class Kind { Report, Clear }; Kind kind{Kind::Report}; - std::string fault_code; ///< dedup key for a Clear; diagnostic for a Report + std::string fault_code; ///< dedup key for a Clear, diagnostic for a Report /// Clear only: this dispatch is re-derivable (ClearOrigin::LinkState), so /// the buffer may drop it before anything that is not. bool link_state{false}; @@ -308,7 +332,8 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, ReplacedClear, ///< superseded the pending clear for the same fault code EvictedLinkStateClear, ///< buffer was full: dropped a re-derivable clear to make room EvictedOldest, ///< buffer was full with nothing re-derivable in it: dropped the oldest entry - Refused ///< buffer was full with nothing re-derivable and the incoming clear was + Refused ///< buffer was full with nothing re-derivable in it and the incoming + ///< dispatch was itself a re-derivable clear, so it was dropped instead }; // Enqueue policy for the bounded pending-dispatch buffer. @@ -425,16 +450,9 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // (null to report every pass in full). DiscoveryReporter discovery_reporter(std::string * previous_outcome) const; - // Abort predicate handed to a discovery sweep. Two independent stop signals, - // because the two sweeps run at different points in the process lifetime: - // - ``shutdown_requested_`` is set by shutdown(), which the gateway calls - // after its executor returns. That ends a RESCAN sweep, which runs on the - // poll thread long after start-up. - // - ``rclcpp::ok()`` turns false as soon as rclcpp's own SIGINT / SIGTERM - // handler runs. The STARTUP sweep runs inside set_context(), i.e. during - // node construction and before the executor spins, so shutdown() cannot - // be reached while it is in progress and the signal is the only thing - // that can end it. + // Abort predicate handed to a discovery sweep: reads the two stop signals off + // the process and applies ``discovery_cancelled_for``, which holds the rule + // and the reasoning behind it. bool discovery_cancelled() const; // Poll-thread hook bound into PollerConfig::rediscover_endpoint whenever diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp index 158523936..bbdfc11d8 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp @@ -1052,8 +1052,7 @@ void OpcuaPlugin::on_alarm_change(const std::string & entity_id, // link-state event as the connect-time one. Every other code here is the // device reporting its condition inactive, which is a real resolution and a // one-shot edge, so it keeps the cascade and the buffer treats it as such. - send_clear_fault(signal.fault_code, - signal.fault_code == kCommsLostFaultCode ? ClearOrigin::LinkState : ClearOrigin::DeviceAlarm); + send_clear_fault(signal.fault_code, clear_origin_for_signal(signal.fault_code)); } } @@ -1311,7 +1310,7 @@ void OpcuaPlugin::clear_comms_lost_on_connect() { // not cascade-clear the symptoms the outage produced. It is also the one clear // the next reconnect re-derives, so the pending buffer may drop it before // anything one-shot. - log_info(std::string("OPC-UA connection established; clearing any standing ") + kCommsLostFaultCode); + log_info(std::string("OPC-UA connection established, clearing any standing ") + kCommsLostFaultCode); send_clear_fault(kCommsLostFaultCode, ClearOrigin::LinkState); } @@ -1723,7 +1722,7 @@ std::optional OpcuaPlugin::discover_endpoint(const OpcuaDiscoveryCo const auto subnets = discovery.resolve_subnets(); if (subnets.empty()) { - warn_line("OPC-UA discovery: no subnet configured and could not derive a local /24; nothing to scan."); + warn_line("OPC-UA discovery: no subnet configured and could not derive a local /24, nothing to scan."); emit(); return std::nullopt; } @@ -1779,7 +1778,7 @@ std::optional OpcuaPlugin::discover_endpoint(const OpcuaDiscoveryCo const DiscoveredEndpoint * chosen = NetworkDiscovery::select_auto_endpoint(found, config.anonymous_none_only); if (chosen == nullptr) { warn_line( - "OPC-UA discovery: no auto-connectable None/Anonymous data server found; leaving the endpoint unchanged. " + "OPC-UA discovery: no auto-connectable None/Anonymous data server found, leaving the endpoint unchanged. " "Secured-only servers require operator credentials."); emit(); return std::nullopt; @@ -1796,7 +1795,7 @@ void OpcuaPlugin::run_startup_discovery() { } if (endpoint_configured_) { log_info("OPC-UA discovery enabled but endpoint_url is explicitly configured (" + client_config_.endpoint_url + - "); skipping auto-discovery to avoid a second session."); + "). Skipping auto-discovery to avoid a second session."); return; } @@ -1824,11 +1823,11 @@ void OpcuaPlugin::run_startup_discovery() { // which of the two they configured. const int startup_interval_s = effective_rescan_interval_s(discovery_config_, endpoint_configured_); if (startup_interval_s > 0) { - log_info("OPC-UA discovery: startup scan selected no endpoint; the reconnect loop rescans every " + + log_info("OPC-UA discovery: startup scan selected no endpoint. The reconnect loop rescans every " + std::to_string(startup_interval_s) + "s while down."); } else { log_warn( - "OPC-UA discovery: startup scan selected no endpoint and re-scanning is off (interval_s: 0); the endpoint " + "OPC-UA discovery: startup scan selected no endpoint and re-scanning is off (interval_s: 0). The endpoint " "stays at " + client_config_.endpoint_url + " until the plugin is restarted."); } @@ -1840,13 +1839,10 @@ void OpcuaPlugin::run_startup_discovery() { } bool OpcuaPlugin::discovery_cancelled() const { - // Either stop signal ends a sweep. shutdown() is what ends a RESCAN (it runs - // on the poll thread, long after start-up). rclcpp::ok() going false is what - // ends the STARTUP sweep, which runs during node construction where shutdown() - // is not reachable yet. Checking both in one predicate keeps the two sweeps - // from drifting apart, and rclcpp::ok() only reads the default context's - // atomic shutdown flag, so it is safe to call from either thread. - return shutdown_requested_.load() || !rclcpp::ok(); + // rclcpp::ok() only reads the default context's atomic shutdown flag, so it is + // safe to call from the set_context thread and the poll thread alike. The rule + // itself, and why both signals are needed, lives in discovery_cancelled_for. + return discovery_cancelled_for(shutdown_requested_.load(), rclcpp::ok()); } OpcuaPlugin::DiscoveryReporter OpcuaPlugin::discovery_reporter(std::string * previous_outcome) const { diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp index 93b84ee1f..f96daf0e3 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp @@ -45,7 +45,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -53,6 +55,8 @@ #include #include +#include +#include #include "ros2_medkit_gateway/plugins/ros_plugin_context.hpp" @@ -193,17 +197,28 @@ class AlarmServer { if (pipe(pipefd) != 0) { return false; } + int stdin_pipe[2]; + if (pipe(stdin_pipe) != 0) { + close(pipefd[0]); + close(pipefd[1]); + return false; + } pid_ = fork(); if (pid_ < 0) { close(pipefd[0]); close(pipefd[1]); + close(stdin_pipe[0]); + close(stdin_pipe[1]); return false; } if (pid_ == 0) { dup2(pipefd[1], STDOUT_FILENO); dup2(pipefd[1], STDERR_FILENO); + dup2(stdin_pipe[0], STDIN_FILENO); close(pipefd[0]); close(pipefd[1]); + close(stdin_pipe[0]); + close(stdin_pipe[1]); std::string port_str = std::to_string(port); std::vector argv_vec{binary.c_str(), "--port", port_str.c_str()}; for (const auto & arg : extra_args) { @@ -214,11 +229,27 @@ class AlarmServer { _exit(127); } close(pipefd[1]); + close(stdin_pipe[0]); read_fd_ = pipefd[0]; + write_fd_ = stdin_pipe[1]; return wait_for_ready(15000); } + // One CLI command ("fire Overpressure 750", "clear Overpressure", ...). The + // fixture reads them line by line off stdin. + bool send(const std::string & command) { + if (write_fd_ < 0) { + return false; + } + const std::string line = command + "\n"; + return write(write_fd_, line.c_str(), line.size()) == static_cast(line.size()); + } + void stop() { + if (write_fd_ >= 0) { + close(write_fd_); + write_fd_ = -1; + } if (pid_ > 0) { kill(pid_, SIGTERM); int status = 0; @@ -258,6 +289,7 @@ class AlarmServer { pid_t pid_{-1}; int read_fd_{-1}; + int write_fd_{-1}; }; std::string fixture_binary() { @@ -631,4 +663,225 @@ TEST_F(OpcuaIdentityE2ETest, SuccessfulConnectClearsCommsLostNeverRaisedHere) { << "comms-lost must not be raised while the connection is up"; } +namespace { + +// RAII rclcpp init/shutdown, tearing down only what it started. +struct ScopedRclcpp { + const bool owned_; + ScopedRclcpp() : owned_(!rclcpp::ok()) { + if (owned_) { + rclcpp::init(0, nullptr); + } + } + ~ScopedRclcpp() { + if (owned_ && rclcpp::ok()) { + rclcpp::shutdown(); + } + } + ScopedRclcpp(const ScopedRclcpp &) = delete; + ScopedRclcpp & operator=(const ScopedRclcpp &) = delete; +}; + +// The plugin only builds its fault-service clients when the context hands it a +// real node, which is what makes the ClearFault request observable on the wire. +class RealNodePluginContext : public FakePluginContext { + public: + explicit RealNodePluginContext(rclcpp::Node * node) : node_(node) { + } + rclcpp::Node * node() const override { + return node_; + } + + private: + rclcpp::Node * node_; +}; + +} // namespace + +// The connect-time clear, read off the wire. clear_comms_lost_on_connect() is +// only reachable through a connect that SUCCEEDS, so it needs the live fixture, +// and the flag it sets is only observable with a real fault-manager service on +// the other end. A correlation rule may name PLC_COMMS_LOST as the root cause of +// every symptom an outage produced, and the link coming back is not an operator +// resolving those, so this clear must not cascade. +TEST_F(OpcuaIdentityE2ETest, ConnectTimeCommsLostClearSkipsTheCorrelationCascade) { + ScopedRclcpp rclcpp_scope; + auto node = std::make_shared("opcua_identity_connect_clear"); + auto fault_manager = std::make_shared("opcua_identity_connect_clear_faultmgr"); + + std::mutex received_mutex; + std::vector cleared_requests; + auto report_srv = fault_manager->create_service( + "/fault_manager/report_fault", [](const std::shared_ptr, + std::shared_ptr res) { + res->accepted = true; + }); + auto clear_srv = fault_manager->create_service( + "/fault_manager/clear_fault", + [&cleared_requests, &received_mutex](const std::shared_ptr req, + std::shared_ptr res) { + { + std::lock_guard lock(received_mutex); + cleared_requests.push_back(*req); + } + res->success = true; + }); + + rclcpp::executors::MultiThreadedExecutor executor; + executor.add_node(node); + executor.add_node(fault_manager); + std::thread spin_thread([&executor]() { + executor.spin(); + }); + + const std::string yaml_path = write_minimal_node_map(); + OpcuaPlugin plugin; + nlohmann::json config; + config["endpoint_url"] = endpoint_; + config["node_map_path"] = yaml_path; + config["poll_interval_ms"] = 100; + plugin.configure(config); + + RealNodePluginContext ctx(node.get()); + ctx.entities["tank"] = {SovdEntityType::APP, "tank", "/test_plc", "/test_plc/test_runtime/tank"}; + // The connect inside set_context() succeeds against the fixture, which is the + // only way to reach the connect-time clear. + plugin.set_context(ctx); + + // The clear may be buffered until the stub service is DDS-matched. The poll + // thread drains the buffer on its next cycle. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(20); + bool delivered = false; + while (!delivered && std::chrono::steady_clock::now() < deadline) { + { + std::lock_guard lock(received_mutex); + delivered = !cleared_requests.empty(); + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + executor.cancel(); + if (spin_thread.joinable()) { + spin_thread.join(); + } + plugin.shutdown(); + std::remove(yaml_path.c_str()); + + std::lock_guard lock(received_mutex); + ASSERT_FALSE(cleared_requests.empty()) << "a successful connect sent no ClearFault at all"; + EXPECT_EQ(cleared_requests.front().fault_code, std::string(kCommsLostFaultCode)); + EXPECT_TRUE(cleared_requests.front().skip_correlation_auto_clear) + << "the connect-time clear cascade-cleared the symptoms of the outage it ended"; +} + +// The other side of the same rule, also on the wire: when the DEVICE reports its +// condition inactive, that IS a resolution at the source, so the correlation +// engine may act on it and the flag stays off. Only a live AlarmCondition +// lifecycle reaches on_event_alarm's ClearFault arm, so this drives the +// fixture's own CLI to fire and then clear a condition. +TEST_F(OpcuaIdentityE2ETest, DeviceReportedAlarmClearKeepsTheCorrelationCascade) { + ScopedRclcpp rclcpp_scope; + auto node = std::make_shared("opcua_identity_device_clear"); + auto fault_manager = std::make_shared("opcua_identity_device_clear_faultmgr"); + + std::mutex received_mutex; + std::vector reported; + std::vector cleared_requests; + auto report_srv = fault_manager->create_service( + "/fault_manager/report_fault", + [&reported, &received_mutex](const std::shared_ptr req, + std::shared_ptr res) { + { + std::lock_guard lock(received_mutex); + reported.push_back(req->fault_code); + } + res->accepted = true; + }); + auto clear_srv = fault_manager->create_service( + "/fault_manager/clear_fault", + [&cleared_requests, &received_mutex](const std::shared_ptr req, + std::shared_ptr res) { + { + std::lock_guard lock(received_mutex); + cleared_requests.push_back(*req); + } + res->success = true; + }); + + rclcpp::executors::MultiThreadedExecutor executor; + executor.add_node(node); + executor.add_node(fault_manager); + std::thread spin_thread([&executor]() { + executor.spin(); + }); + + OpcuaPlugin plugin; + nlohmann::json config; + config["endpoint_url"] = endpoint_; + config["poll_interval_ms"] = 100; + // Zero-config native A&C on the Server EventNotifier, with auto_clear so the + // condition going inactive clears the fault without an operator ack/confirm. + config["auto_alarms"] = nlohmann::json{{"enabled", true}, {"auto_clear", true}}; + plugin.configure(config); + + RealNodePluginContext ctx(node.get()); + plugin.set_context(ctx); + + const auto reported_count = [&received_mutex, &reported]() { + std::lock_guard lock(received_mutex); + return reported.size(); + }; + // The connect-time PLC_COMMS_LOST clear also lands here (this connect + // succeeded), so a clear is looked up by the code it names. + const auto clear_for = [&received_mutex, &cleared_requests](const std::string & code) -> std::optional { + std::lock_guard lock(received_mutex); + for (const auto & req : cleared_requests) { + if (req.fault_code == code) { + return req.skip_correlation_auto_clear; + } + } + return std::nullopt; + }; + + // Fire until the event subscription is up and a report lands. The retry is the + // subscription handshake, not flakiness in the assertion: an event fired + // before the subscribe simply is not delivered. + const auto fire_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (reported_count() == 0 && std::chrono::steady_clock::now() < fire_deadline) { + ASSERT_TRUE(server_.send("fire Overpressure 750")); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + ASSERT_GT(reported_count(), 0u) << "the fixture's AlarmCondition never reached the fault manager"; + + std::string alarm_code; + { + std::lock_guard lock(received_mutex); + alarm_code = reported.front(); + } + ASSERT_TRUE(server_.send("clear Overpressure")); + const auto clear_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (!clear_for(alarm_code).has_value() && std::chrono::steady_clock::now() < clear_deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + executor.cancel(); + if (spin_thread.joinable()) { + spin_thread.join(); + } + plugin.shutdown(); + + const auto device_clear_skips = clear_for(alarm_code); + ASSERT_TRUE(device_clear_skips.has_value()) + << "the device reporting condition " << alarm_code << " inactive sent no ClearFault"; + EXPECT_FALSE(*device_clear_skips) << "a clear the device itself reported must keep the correlation cascade"; + + // The connect-time clear travelled the same wire in the same test, and it is + // the opposite case: not an operator resolving anything, so it does not + // cascade. Having both here is what makes the flag above a decision rather + // than a constant. + const auto link_state_clear_skips = clear_for(kCommsLostFaultCode); + ASSERT_TRUE(link_state_clear_skips.has_value()) << "the connect-time clear never arrived"; + EXPECT_TRUE(*link_state_clear_skips); +} + } // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp index 9d5a0af9a..0f53ddb05 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp @@ -963,10 +963,11 @@ TEST(DiscoverEndpoint, AnUnchangedRescanReportsAtDebugInsteadOfRepeatingItself) } TEST(DiscoverEndpoint, APredicateThatFlipsMidSweepEndsThePass) { - // What a stop signal does to a sweep in progress. The plugin hands - // discover_endpoint a predicate that answers for both stop signals (the - // shutdown flag and rclcpp::ok()). Here it flips after a handful of probes, - // as either would mid-sweep. + // What a stop signal does to a sweep in progress. This predicate is the + // test's own, standing in for the one the plugin passes: it flips after a + // handful of probes, as either of the plugin's two stop signals would + // mid-sweep. The plugin's own predicate is pinned separately, by + // DiscoveryCancelledFor. std::atomic probes{0}; std::atomic stop{false}; auto stopping_scan = [&probes, &stop](const std::string & ip, uint16_t port, int) { @@ -1113,10 +1114,53 @@ TEST(RederivedComponentIdentity, KeepsTheIdentityWhenNothingChanged) { EXPECT_EQ(host_derived->id, "opcua-192_168_1_10"); } +// --------------------------------------------------------------------------- +// The stop signals a discovery sweep watches +// --------------------------------------------------------------------------- + +TEST(DiscoveryCancelledFor, EitherStopSignalEndsASweep) { + // The plugin's own predicate is this rule applied to two values it reads off + // the process, so this is the whole of it. + EXPECT_FALSE(OpcuaPlugin::discovery_cancelled_for(/*shutdown_requested=*/false, /*rclcpp_ok=*/true)) + << "a running process must not cancel its own sweep"; + // shutdown() ends a rescan sweep on the poll thread. + EXPECT_TRUE(OpcuaPlugin::discovery_cancelled_for(/*shutdown_requested=*/true, /*rclcpp_ok=*/true)); + // SIGINT / SIGTERM ends the start-up sweep, which runs during node + // construction where shutdown() cannot be reached at all. + EXPECT_TRUE(OpcuaPlugin::discovery_cancelled_for(/*shutdown_requested=*/false, /*rclcpp_ok=*/false)) + << "a signal during the start-up sweep left it running"; + EXPECT_TRUE(OpcuaPlugin::discovery_cancelled_for(true, false)); +} + +TEST(DiscoveryCancelledFor, RclcppOkIsTheSignalTheStartUpSweepWatches) { + // The second input is not hypothetical: rclcpp's shutdown is what a SIGTERM + // turns into, and it is observable exactly this way. A private context keeps + // the process-wide default one (which other tests here initialise) untouched. + auto context = std::make_shared(); + context->init(0, nullptr); + ASSERT_TRUE(rclcpp::ok(context)); + EXPECT_FALSE(OpcuaPlugin::discovery_cancelled_for(/*shutdown_requested=*/false, rclcpp::ok(context))); + + context->shutdown("simulated SIGTERM"); + ASSERT_FALSE(rclcpp::ok(context)) << "rclcpp::ok did not follow the shutdown a signal performs"; + EXPECT_TRUE(OpcuaPlugin::discovery_cancelled_for(/*shutdown_requested=*/false, rclcpp::ok(context))); +} + // --------------------------------------------------------------------------- // ClearFault: only a clear the device itself reported may cascade // --------------------------------------------------------------------------- +TEST(ClearOriginForSignal, OnlyTheCommsLostCodeIsALinkStateClear) { + // The poller emits its component-scoped comms-lost clear through the same + // callback as every device alarm going inactive, so the fault code is the only + // thing that tells the two apart on that path. + EXPECT_EQ(OpcuaPlugin::clear_origin_for_signal(kCommsLostFaultCode), OpcuaPlugin::ClearOrigin::LinkState); + EXPECT_EQ(OpcuaPlugin::clear_origin_for_signal("PLC_TANK_HIGH"), OpcuaPlugin::ClearOrigin::DeviceAlarm); + EXPECT_EQ(OpcuaPlugin::clear_origin_for_signal(std::string(kCommsLostFaultCode) + "_UPSTREAM"), + OpcuaPlugin::ClearOrigin::DeviceAlarm) + << "the match must be the exact code, not a prefix"; +} + TEST(ClearOrigin, OnlyADeviceReportedClearKeepsTheCorrelationCascade) { using Origin = OpcuaPlugin::ClearOrigin; // The link coming back is not an operator resolving a root cause, and neither @@ -1851,7 +1895,7 @@ component_id: scoped_runtime OpcuaPlugin plugin; nlohmann::json config; config["node_map_path"] = yaml_path; - config["endpoint_url"] = "opc.tcp://127.0.0.1:1"; // nothing listening; the fault sink is the subject + config["endpoint_url"] = "opc.tcp://127.0.0.1:1"; // nothing listening, the fault sink is the subject config["poll_interval_ms"] = 100; plugin.configure(config); From e8b9b0a4571df4f69ac3fed4f0ba58c47d10d1f1 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Tue, 8 Sep 2026 22:21:19 +0200 Subject: [PATCH 09/14] feat(gateway): keep an entity freeze-frame across a gateway restart The frames captured for plugin-backed entities lived in process memory, so a restart threw them away and the startup catch-up re-read the plant as it is now. What came back was the current values under the original fault, stamped with the restart and marked capture_origin: startup. The values at fault time, which are the whole point of a freeze-frame, were gone, and a consumer had no way to tell that the numbers under a fault from last night were read this morning. EntityFreezeFrameCapture now takes an EntityFreezeFrameStore. It writes a fault's frames through on every capture, under the same lock as the map, and loads them back in its constructor, so a reloaded frame is served exactly as it was captured: its original captured_at, its own capture_origin (absent for a confirm-edge frame, startup for one the catch-up took), and the connected and source_timestamp provenance it carried. The startup catch-up then skips any fault that already has a frame and re-reads only the ones with none, so a fault that confirmed while the gateway was down still gets its startup frame. Two backends sit behind the interface, SQLite for the gateway and an in-memory one for tests. entity_freeze_frame.storage.path names the file. Empty puts it next to triggers.storage.path, and with neither set the frames stay in memory, exactly as they were before. A store that cannot be opened or written is reported and the capture keeps working from memory. Bounds. The retained-frame bound of 256 faults counts reloaded and freshly captured frames together, so a reloaded frame no longer spends catch-up budget it does not need, and an evicted fault loses its rows as well as its map entry so the bound means something across a restart. At startup, frames whose fault the fault manager no longer holds in any status are dropped. A fault reported as cleared keeps its frame, and when the fault manager cannot be asked at all nothing is dropped. --- docs/config/server.rst | 13 + docs/tutorials/snapshots.rst | 25 +- src/ros2_medkit_gateway/CMakeLists.txt | 5 + .../core/entity_freeze_frame_store.hpp | 120 +++++++ .../core/sqlite_entity_freeze_frame_store.hpp | 62 ++++ .../entity_freeze_frame_capture.hpp | 79 ++++- .../ros2_medkit_gateway/gateway_node.hpp | 6 + .../core/sqlite_entity_freeze_frame_store.cpp | 259 ++++++++++++++ .../src/entity_freeze_frame_capture.cpp | 275 ++++++++++++++- src/ros2_medkit_gateway/src/gateway_node.cpp | 67 ++++ .../test/test_entity_freeze_frame_capture.cpp | 324 ++++++++++++++++++ .../test/test_entity_freeze_frame_store.cpp | 225 ++++++++++++ 12 files changed, 1438 insertions(+), 22 deletions(-) create mode 100644 src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/entity_freeze_frame_store.hpp create mode 100644 src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/sqlite_entity_freeze_frame_store.hpp create mode 100644 src/ros2_medkit_gateway/src/core/sqlite_entity_freeze_frame_store.cpp create mode 100644 src/ros2_medkit_gateway/test/test_entity_freeze_frame_store.cpp diff --git a/docs/config/server.rst b/docs/config/server.rst index 0171b268a..15ec5c10a 100644 --- a/docs/config/server.rst +++ b/docs/config/server.rst @@ -384,6 +384,19 @@ Configure how the gateway connects to the fault manager services and event topic values, marked ``connected: false`` in the snapshot's ``x-medkit`` block. Explicit snapshot config in the fault manager always wins when present. Only active when plugins are loaded. + * - ``entity_freeze_frame.storage.path`` + - string + - ``""`` + - SQLite file the captured frames are persisted in, so a restart serves the + values frozen at fault time instead of re-reading the plant. When empty, + the frames go in ``entity_freeze_frames.db`` next to + ``triggers.storage.path``; with that empty too they stay in memory and are + lost on restart. A reloaded frame keeps its original ``captured_at`` and + its ``capture_origin``, and the startup catch-up then runs only for faults + that have no stored frame. The retained-frame bound of 256 faults counts + reloaded and freshly captured frames together, dropping the oldest first, + and a frame whose fault the fault manager no longer holds at all is + dropped at startup. When ``fault_manager.namespace`` is set, the gateway also subscribes to the matching fault event topic (for example ``/robot1/fault_manager/events`` instead of the default diff --git a/docs/tutorials/snapshots.rst b/docs/tutorials/snapshots.rst index c23e845ef..d0aac6d87 100644 --- a/docs/tutorials/snapshots.rst +++ b/docs/tutorials/snapshots.rst @@ -201,21 +201,34 @@ and may predate the confirmation by the length of the outage; the entry's payload includes one, ``source_timestamp`` (the payload's own timestamp) alongside ``captured_at``. +With ``entity_freeze_frame.storage.path`` set, a captured frame survives a +gateway restart: it is reloaded at start and served exactly as it was +captured, with its original ``captured_at`` and no ``capture_origin`` marker. +Set the path to a file on a volume that outlives the container, or leave it +empty and the frames go next to the trigger store +(``triggers.storage.path``); with neither set they are process memory only and +a restart loses them. + Faults that are already confirmed when the gateway starts are caught up at startup: the gateway lists the confirmed faults and captures a frame for each -plugin-backed one, so a device standing in fault across a gateway restart -still gets a frame. Catch-up frames carry ``"capture_origin": "startup"`` in -their ``x-medkit`` block because their values were read at gateway start, not -when the fault confirmed (which may be long before, since the fault manager +plugin-backed one that does not already have a stored frame, so a device +standing in fault across a gateway restart still gets a frame, and one whose +frame was already taken keeps the values from its own confirm edge instead of +today's. Catch-up frames carry ``"capture_origin": "startup"`` in their +``x-medkit`` block because their values were read at gateway start, not when +the fault confirmed (which may be long before, since the fault manager persists faults); ``captured_at`` always stamps the moment the values were -read. Frames without the marker were captured on the confirm edge. Disable -with: +read. Frames without the marker were captured on the confirm edge, and a +reloaded frame keeps whichever marker it was captured with. Disable with: .. code-block:: bash ros2 run ros2_medkit_gateway gateway_node --ros-args \ -p entity_freeze_frame.enabled:=false +A plugin entity keeps exactly one frame per fault: a re-confirm re-samples the +plugin and replaces it, on disk as in memory. + A plugin entity's values are not a ROS message, so ``topic`` and ``message_type`` are empty on these frames. ``x-medkit.source`` names the capture path instead, so a consumer can still tell where the values came diff --git a/src/ros2_medkit_gateway/CMakeLists.txt b/src/ros2_medkit_gateway/CMakeLists.txt index 23b20ac4d..527921338 100644 --- a/src/ros2_medkit_gateway/CMakeLists.txt +++ b/src/ros2_medkit_gateway/CMakeLists.txt @@ -840,6 +840,11 @@ if(BUILD_TESTING) target_link_libraries(test_entity_freeze_frame_capture gateway_ros2) medkit_target_dependencies(test_entity_freeze_frame_capture rclcpp ros2_medkit_msgs) + # Entity freeze-frame persistence (both store backends). Links gateway_core + # only (ROS-neutral). + medkit_add_gtest(test_entity_freeze_frame_store test/test_entity_freeze_frame_store.cpp) + target_link_libraries(test_entity_freeze_frame_store gateway_core) + # Add update manager tests medkit_add_gtest(test_update_manager test/test_update_manager.cpp) target_link_libraries(test_update_manager gateway_ros2) diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/entity_freeze_frame_store.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/entity_freeze_frame_store.hpp new file mode 100644 index 000000000..e76c0d08c --- /dev/null +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/entity_freeze_frame_store.hpp @@ -0,0 +1,120 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace ros2_medkit_gateway { + +/// One persisted entity freeze-frame: the row shape of the store, keyed by +/// (fault_code, entity_id). +/// +/// `frame` holds everything that is not already a column of its own - the +/// compact {resource_id: value} dict under "values", plus the payload +/// provenance the capture recorded ("connected", "source_timestamp") when the +/// plugin reported it. Keeping those in the blob rather than in columns is +/// what lets a reloaded frame be served byte for byte as it was captured. +struct StoredEntityFreezeFrame { + std::string fault_code; + std::string entity_id; + nlohmann::json frame; ///< {"values": {...}, "connected"?: bool, "source_timestamp"?: any} + int64_t captured_at_ns{0}; + std::string source; ///< capture path that read the values + std::string capture_origin; ///< "startup" for a catch-up frame, empty on a confirm edge +}; + +/// Persistence for the gateway's entity freeze-frames. +/// +/// The gateway's frames are process memory, so a restart re-derives them from +/// whatever the plant reads *now* - the values at fault time are gone and the +/// re-read is stamped with the restart. This store is what makes the captured +/// frame outlive the process. +/// +/// Writes are per fault code and wholesale: a re-confirm replaces every row +/// for that code, mirroring the in-memory map, whose entry for a code is +/// likewise replaced as a unit. Implementations must be thread-safe. +class EntityFreezeFrameStore { + public: + virtual ~EntityFreezeFrameStore() = default; + + /// Replace every row for @p fault_code with @p frames (one row per entity). + /// An empty vector leaves no rows for the code. + virtual tl::expected replace_frames(const std::string & fault_code, + const std::vector & frames) = 0; + + /// Drop every row for @p fault_code. Removing a code that has no rows is + /// not an error: the caller evicts by code and does not track what is on disk. + virtual tl::expected erase_frames(const std::string & fault_code) = 0; + + /// Every row, oldest capture first. The order is what lets a caller honour a + /// retained-frame bound by keeping the newest codes. + virtual tl::expected, std::string> load_all() = 0; +}; + +/// In-memory backend: the store contract without a file, for tests and for +/// callers that want the interface without persistence. +class InMemoryEntityFreezeFrameStore : public EntityFreezeFrameStore { + public: + tl::expected replace_frames(const std::string & fault_code, + const std::vector & frames) override { + std::lock_guard lock(mutex_); + if (frames.empty()) { + rows_.erase(fault_code); + return {}; + } + rows_[fault_code] = frames; + return {}; + } + + tl::expected erase_frames(const std::string & fault_code) override { + std::lock_guard lock(mutex_); + rows_.erase(fault_code); + return {}; + } + + tl::expected, std::string> load_all() override { + std::lock_guard lock(mutex_); + std::vector all; + for (const auto & entry : rows_) { + all.insert(all.end(), entry.second.begin(), entry.second.end()); + } + // Same total order the SQLite backend serves, so a caller's bound-keeping + // behaves identically on both. + std::stable_sort(all.begin(), all.end(), [](const StoredEntityFreezeFrame & a, const StoredEntityFreezeFrame & b) { + if (a.captured_at_ns != b.captured_at_ns) { + return a.captured_at_ns < b.captured_at_ns; + } + if (a.fault_code != b.fault_code) { + return a.fault_code < b.fault_code; + } + return a.entity_id < b.entity_id; + }); + return all; + } + + private: + mutable std::mutex mutex_; + std::map> rows_; +}; + +} // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/sqlite_entity_freeze_frame_store.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/sqlite_entity_freeze_frame_store.hpp new file mode 100644 index 000000000..5b076b62a --- /dev/null +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/sqlite_entity_freeze_frame_store.hpp @@ -0,0 +1,62 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "ros2_medkit_gateway/core/entity_freeze_frame_store.hpp" + +namespace ros2_medkit_gateway { + +/// SQLite-backed entity freeze-frame persistence. +/// +/// Thread-safe via internal mutex. The table is created on first open. Use +/// ":memory:" for an ephemeral database. +class SqliteEntityFreezeFrameStore : public EntityFreezeFrameStore { + public: + /// Open (or create) the database at `db_path`. + /// @throws std::runtime_error on SQLite open/init failure. + explicit SqliteEntityFreezeFrameStore(const std::string & db_path); + + ~SqliteEntityFreezeFrameStore() override; + + // Non-copyable, non-movable (owns SQLite connection) + SqliteEntityFreezeFrameStore(const SqliteEntityFreezeFrameStore &) = delete; + SqliteEntityFreezeFrameStore & operator=(const SqliteEntityFreezeFrameStore &) = delete; + SqliteEntityFreezeFrameStore(SqliteEntityFreezeFrameStore &&) = delete; + SqliteEntityFreezeFrameStore & operator=(SqliteEntityFreezeFrameStore &&) = delete; + + tl::expected replace_frames(const std::string & fault_code, + const std::vector & frames) override; + tl::expected erase_frames(const std::string & fault_code) override; + tl::expected, std::string> load_all() override; + + private: + /// Create the table if it does not exist. + void initialize_schema(); + + /// Delete every row for a code. Caller holds mutex_. + tl::expected delete_code_locked(const std::string & fault_code); + + std::string db_path_; + sqlite3 * db_{nullptr}; + mutable std::mutex mutex_; +}; + +} // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp index d3c041ee4..5871e733a 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp @@ -31,6 +31,7 @@ #include #include "rclcpp/rclcpp.hpp" +#include "ros2_medkit_gateway/core/entity_freeze_frame_store.hpp" #include "ros2_medkit_gateway/core/providers/data_provider.hpp" #include "ros2_medkit_gateway/ros2_common/ros2_subscription_slot.hpp" #include "ros2_medkit_msgs/msg/fault_event.hpp" @@ -57,6 +58,13 @@ namespace ros2_medkit_gateway { * kept across EVENT_CLEARED (the confirmed-state record stays attached to * the cleared fault's detail) and overwritten on every EVENT_CONFIRMED, so * a re-occurrence re-samples the plugin at its own confirm time. + * + * With an EntityFreezeFrameStore the frames also survive the process. Without + * one a restart loses them, and the startup catch-up then re-reads the plant + * as it is *now* and stamps that re-read with the restart - the values at + * fault time are gone. With one, the stored frame is reloaded before the + * catch-up and served exactly as it was captured, and the catch-up re-reads + * only the faults that have no frame. */ class EntityFreezeFrameCapture { public: @@ -67,6 +75,13 @@ class EntityFreezeFrameCapture { static constexpr const char * kSourceDataProvider = "plugin_data_provider"; static constexpr const char * kSourceXPlcDataRoute = "plugin_x_plc_data_route"; + /// Persisted marker for a startup catch-up frame, served as + /// ``x-medkit.capture_origin``. Stored so a reloaded frame keeps saying + /// which clock its captured_at came from: the property is intrinsic to the + /// frame, so a restart must not launder a catch-up frame into a confirm-edge + /// one. A confirm-edge frame stores the empty string and stays unmarked. + static constexpr const char * kCaptureOriginStartup = "startup"; + /// One captured frame: the entity's data values at fault-confirm time. /// captured_at_ns dates the capture, not the values - a disconnected entity /// serves its last known values, whose age is bounded only by the outage. @@ -114,6 +129,15 @@ class EntityFreezeFrameCapture { /// that is what lets the destructor's join interrupt the wait. using StandingFaultLister = std::function(const std::function & should_abort)>; + /// Reports every fault code the fault_manager still holds, whatever its + /// status, so a reloaded frame for a fault that is gone can be dropped. + /// Returns nullopt when the fault_manager could not be asked - the caller + /// then drops nothing, because "could not tell" must never read as "gone". + /// Called once, on the capture thread, right after the standing-fault + /// lister has already waited for the services. + using KnownFaultCodeLister = + std::function>(const std::function & should_abort)>; + /** * @param node ROS 2 node used to resolve the fault-events topic name and logger * @param exec shared subscription executor; the fault-events subscription is @@ -123,11 +147,19 @@ class EntityFreezeFrameCapture { * @param resolver entity-to-DataProvider resolver (typically wraps PluginManager) * @param route_fetcher x-plc-data route fallback for entities whose plugin * has no DataProvider (the commercial PLC bridges); may be null - * @param max_faults retained-frame bound; oldest fault's frames evicted past it + * @param max_faults retained-frame bound; oldest fault's frames evicted past it. + * Counts reloaded and freshly captured frames together. + * @param standing_lister lists the faults already confirmed at startup + * @param store frame persistence; null keeps the frames in process memory only + * @param known_code_lister reports the fault codes the fault_manager still + * holds, so reloaded frames for faults that are gone can be dropped; + * null keeps every reloaded frame */ EntityFreezeFrameCapture(rclcpp::Node * node, ros2_common::Ros2SubscriptionExecutor & exec, DataProviderResolver resolver, RouteDataFetcher route_fetcher = nullptr, - size_t max_faults = 256, StandingFaultLister standing_lister = nullptr); + size_t max_faults = 256, StandingFaultLister standing_lister = nullptr, + std::shared_ptr store = nullptr, + KnownFaultCodeLister known_code_lister = nullptr); ~EntityFreezeFrameCapture(); @@ -214,6 +246,35 @@ class EntityFreezeFrameCapture { /// clear/re-report cycle; one line per code is enough for an operator). void log_fallback_failure_once(const std::string & fault_code, const std::string & message); + /// Fill frames_ from the store. Runs in the constructor, so a frame captured + /// before the last shutdown is already being served when the first request + /// arrives, and the catch-up (which starts later, on capture_thread_) + /// already sees it. Honours max_faults_ by keeping the newest codes: a store + /// larger than the bound must not evict what it just loaded. + void load_persisted_frames(); + + /// Serialize a frame for the store. The columns carry entity, timestamp, + /// source and origin; the blob carries the values and the payload + /// provenance, so a reload reproduces the frame exactly. + static StoredEntityFreezeFrame to_stored(const std::string & fault_code, const Frame & frame); + + /// Inverse of to_stored. Returns nullopt for a row whose blob is not shaped + /// like a frame (a hand-edited or half-written file). + static std::optional from_stored(const StoredEntityFreezeFrame & row); + + /// Write the code's frames through to the store, replacing what was there. + /// Caller holds mutex_, so the file and the map cannot disagree. + void persist_frames_locked(const std::string & fault_code, const std::vector & frames); + + /// Drop the code's rows from the store. Caller holds mutex_. + void erase_persisted_locked(const std::string & fault_code); + + /// Drop reloaded frames whose fault the fault_manager no longer holds (its + /// store was replaced or wiped under ours). Codes reported in any status, + /// cleared included, are kept: a cleared fault keeps its frame. Does nothing + /// without a known-code lister, or when the lister cannot answer. + void prune_frames_for_unknown_faults(const std::function & should_abort); + std::unique_ptr subscription_slot_; DataProviderResolver resolver_; RouteDataFetcher route_fetcher_; @@ -230,6 +291,14 @@ class EntityFreezeFrameCapture { std::unordered_map> frames_; std::deque insertion_order_; ///< eviction order (FIFO) std::unordered_set fallback_logged_; ///< fault codes already warned about (bounded) + /// Codes whose frames came from the store and have not been re-captured + /// since. Only these are eligible for the unknown-fault prune: a frame this + /// process captured is by definition for a fault the fault_manager just + /// confirmed, whatever a stale list reply says. + std::unordered_set reloaded_codes_; + /// Store-write failures already warned about, so a read-only or full volume + /// costs one line, not one per capture. + bool store_write_warned_{false}; /// Confirm events pending capture: fed by the subscription worker, drained /// by capture_thread_. Bounded - oldest event dropped when full. @@ -241,6 +310,12 @@ class EntityFreezeFrameCapture { /// Lists faults already confirmed at construction; run once on that thread. StandingFaultLister standing_lister_; + + /// Frame persistence; null keeps the frames in process memory only. + std::shared_ptr store_; + /// Reports the codes the fault_manager still holds; run once, on the + /// capture thread, after the standing-fault lister. + KnownFaultCodeLister known_code_lister_; }; } // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp index 89a153d1b..25813eee0 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp @@ -328,6 +328,12 @@ class GatewayNode : public rclcpp::Node { void refresh_cache(); void start_rest_server(); + /// Open the entity freeze-frame store from `entity_freeze_frame.storage.path`, + /// falling back to a file next to the trigger store. Returns nullptr when no + /// path resolves or the file cannot be opened - the frames are then process + /// memory only, which is what they were before the store existed. + std::shared_ptr open_entity_freeze_frame_store(); + /// Log a one-time discovery summary shortly after startup: discovered node / /// topic / entity counts, the REST URL and a sample curl. When no application /// nodes are visible, also warn loudly with the active ROS environment diff --git a/src/ros2_medkit_gateway/src/core/sqlite_entity_freeze_frame_store.cpp b/src/ros2_medkit_gateway/src/core/sqlite_entity_freeze_frame_store.cpp new file mode 100644 index 000000000..0eaccfc6a --- /dev/null +++ b/src/ros2_medkit_gateway/src/core/sqlite_entity_freeze_frame_store.cpp @@ -0,0 +1,259 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ros2_medkit_gateway/core/sqlite_entity_freeze_frame_store.hpp" + +#include +#include +#include + +namespace ros2_medkit_gateway { + +namespace { + +/// RAII wrapper for SQLite prepared statements (mirrors SqliteTriggerStore's, +/// plus the 64-bit binds a nanosecond timestamp needs). +class SqliteStatement { + public: + SqliteStatement(sqlite3 * db, const char * sql) : db_(db) { + if (sqlite3_prepare_v2(db, sql, -1, &stmt_, nullptr) != SQLITE_OK) { + throw std::runtime_error(std::string("Failed to prepare statement: ") + sqlite3_errmsg(db)); + } + } + + ~SqliteStatement() { + if (stmt_) { + sqlite3_finalize(stmt_); + } + } + + SqliteStatement(const SqliteStatement &) = delete; + SqliteStatement & operator=(const SqliteStatement &) = delete; + SqliteStatement(SqliteStatement &&) = delete; + SqliteStatement & operator=(SqliteStatement &&) = delete; + + void bind_text(int index, const std::string & value) { + const auto size = value.size(); + if (size > static_cast(std::numeric_limits::max())) { + throw std::runtime_error("bind_text: value exceeds SQLite int length limit"); + } + if (sqlite3_bind_text(stmt_, index, value.c_str(), static_cast(size), SQLITE_TRANSIENT) != SQLITE_OK) { + throw std::runtime_error(std::string("Failed to bind text: ") + sqlite3_errmsg(db_)); + } + } + + void bind_int64(int index, int64_t value) { + if (sqlite3_bind_int64(stmt_, index, value) != SQLITE_OK) { + throw std::runtime_error(std::string("Failed to bind int64: ") + sqlite3_errmsg(db_)); + } + } + + int step() { + return sqlite3_step(stmt_); + } + + std::string column_text(int index) { + const auto * text = reinterpret_cast(sqlite3_column_text(stmt_, index)); + return text ? std::string(text) : std::string(); + } + + int64_t column_int64(int index) { + return sqlite3_column_int64(stmt_, index); + } + + private: + sqlite3 * db_; + sqlite3_stmt * stmt_{nullptr}; +}; + +} // namespace + +// --------------------------------------------------------------------------- +// Lifecycle +// --------------------------------------------------------------------------- + +SqliteEntityFreezeFrameStore::SqliteEntityFreezeFrameStore(const std::string & db_path) : db_path_(db_path) { + int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX; + if (sqlite3_open_v2(db_path.c_str(), &db_, flags, nullptr) != SQLITE_OK) { + std::string error = db_ ? sqlite3_errmsg(db_) : "Unknown error"; + if (db_) { + sqlite3_close(db_); + db_ = nullptr; + } + throw std::runtime_error("Failed to open entity freeze-frame database '" + db_path + "': " + error); + } + + char * err_msg = nullptr; + if (sqlite3_exec(db_, "PRAGMA journal_mode=WAL;", nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + sqlite3_close(db_); + db_ = nullptr; + throw std::runtime_error("Failed to enable WAL mode: " + error); + } + + sqlite3_busy_timeout(db_, 5000); + initialize_schema(); +} + +SqliteEntityFreezeFrameStore::~SqliteEntityFreezeFrameStore() { + if (db_) { + sqlite3_close(db_); + } +} + +// --------------------------------------------------------------------------- +// Schema +// --------------------------------------------------------------------------- + +void SqliteEntityFreezeFrameStore::initialize_schema() { + // One row per (fault_code, entity_id): a fault reported by two entities + // freezes both, and a re-confirm replaces the code's rows as a unit. + const char * create_table = R"( + CREATE TABLE IF NOT EXISTS entity_freeze_frames ( + fault_code TEXT NOT NULL, + entity_id TEXT NOT NULL, + frame TEXT NOT NULL, + captured_at_ns INTEGER NOT NULL, + source TEXT NOT NULL DEFAULT '', + capture_origin TEXT NOT NULL DEFAULT '', + PRIMARY KEY (fault_code, entity_id) + ); + )"; + + char * err_msg = nullptr; + if (sqlite3_exec(db_, create_table, nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + throw std::runtime_error("Failed to create entity_freeze_frames table: " + error); + } +} + +// --------------------------------------------------------------------------- +// Writes +// --------------------------------------------------------------------------- + +tl::expected SqliteEntityFreezeFrameStore::delete_code_locked(const std::string & fault_code) { + SqliteStatement del(db_, "DELETE FROM entity_freeze_frames WHERE fault_code = ?"); + del.bind_text(1, fault_code); + if (del.step() != SQLITE_DONE) { + return tl::make_unexpected(std::string("Failed to delete entity freeze-frames: ") + sqlite3_errmsg(db_)); + } + return {}; +} + +tl::expected +SqliteEntityFreezeFrameStore::replace_frames(const std::string & fault_code, + const std::vector & frames) { + std::lock_guard lock(mutex_); + + try { + // Delete-then-insert in one transaction: a re-confirm that no longer + // reports an entity must not leave that entity's stale row behind, and a + // reader must never see the code half-written. + char * err_msg = nullptr; + if (sqlite3_exec(db_, "BEGIN IMMEDIATE", nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + return tl::make_unexpected("replace_frames: BEGIN failed: " + error); + } + + const auto rollback = [this] { + sqlite3_exec(db_, "ROLLBACK", nullptr, nullptr, nullptr); + }; + + auto deleted = delete_code_locked(fault_code); + if (!deleted) { + rollback(); + return deleted; + } + + for (const auto & frame : frames) { + SqliteStatement stmt(db_, + "INSERT OR REPLACE INTO entity_freeze_frames " + "(fault_code, entity_id, frame, captured_at_ns, source, capture_origin) " + "VALUES (?,?,?,?,?,?)"); + stmt.bind_text(1, fault_code); + stmt.bind_text(2, frame.entity_id); + stmt.bind_text(3, frame.frame.dump()); + stmt.bind_int64(4, frame.captured_at_ns); + stmt.bind_text(5, frame.source); + stmt.bind_text(6, frame.capture_origin); + if (stmt.step() != SQLITE_DONE) { + std::string error = sqlite3_errmsg(db_); + rollback(); + return tl::make_unexpected("Failed to save entity freeze-frame: " + error); + } + } + + if (sqlite3_exec(db_, "COMMIT", nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + rollback(); + return tl::make_unexpected("replace_frames: COMMIT failed: " + error); + } + return {}; + } catch (const std::exception & e) { + sqlite3_exec(db_, "ROLLBACK", nullptr, nullptr, nullptr); + return tl::make_unexpected(std::string("replace_frames: ") + e.what()); + } +} + +tl::expected SqliteEntityFreezeFrameStore::erase_frames(const std::string & fault_code) { + std::lock_guard lock(mutex_); + try { + return delete_code_locked(fault_code); + } catch (const std::exception & e) { + return tl::make_unexpected(std::string("erase_frames: ") + e.what()); + } +} + +// --------------------------------------------------------------------------- +// load_all +// --------------------------------------------------------------------------- + +tl::expected, std::string> SqliteEntityFreezeFrameStore::load_all() { + std::lock_guard lock(mutex_); + + try { + SqliteStatement stmt(db_, + "SELECT fault_code, entity_id, frame, captured_at_ns, source, capture_origin " + "FROM entity_freeze_frames " + "ORDER BY captured_at_ns ASC, fault_code ASC, entity_id ASC"); + + std::vector result; + while (stmt.step() == SQLITE_ROW) { + StoredEntityFreezeFrame row; + row.fault_code = stmt.column_text(0); + row.entity_id = stmt.column_text(1); + auto parsed = nlohmann::json::parse(stmt.column_text(2), nullptr, false); + if (parsed.is_discarded()) { + // One unreadable row must not cost the operator every other frame: + // skip it, the caller's catch-up re-reads that fault if it is still + // standing. + continue; + } + row.frame = std::move(parsed); + row.captured_at_ns = stmt.column_int64(3); + row.source = stmt.column_text(4); + row.capture_origin = stmt.column_text(5); + result.push_back(std::move(row)); + } + return result; + } catch (const std::exception & e) { + return tl::make_unexpected(std::string("load_all: ") + e.what()); + } +} + +} // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp index 01bde9674..14e418178 100644 --- a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp @@ -14,8 +14,10 @@ #include "ros2_medkit_gateway/entity_freeze_frame_capture.hpp" +#include #include #include +#include #include "ros2_medkit_gateway/fault_manager_paths.hpp" @@ -45,12 +47,20 @@ std::string string_field(const nlohmann::json & item, const char * field) { EntityFreezeFrameCapture::EntityFreezeFrameCapture(rclcpp::Node * node, ros2_common::Ros2SubscriptionExecutor & exec, DataProviderResolver resolver, RouteDataFetcher route_fetcher, - size_t max_faults, StandingFaultLister standing_lister) + size_t max_faults, StandingFaultLister standing_lister, + std::shared_ptr store, + KnownFaultCodeLister known_code_lister) : resolver_(std::move(resolver)) , route_fetcher_(std::move(route_fetcher)) , logger_(node->get_logger()) , max_faults_(max_faults > 0 ? max_faults : 1) - , standing_lister_(std::move(standing_lister)) { + , standing_lister_(std::move(standing_lister)) + , store_(std::move(store)) + , known_code_lister_(std::move(known_code_lister)) { + // Before anything can serve or capture: a frame taken before the last + // shutdown is the one the operator is owed, and the catch-up must see it so + // it re-reads only the faults that have none. + load_persisted_frames(); // Resolve the topic from the gateway node (it owns fault_manager.namespace); // the subscription itself is created on the executor's dedicated _sub node so // it never races rcl's hash-map on the main node (issue #375). @@ -104,6 +114,204 @@ EntityFreezeFrameCapture::frames_for(const std::string & fault_code) const { return it != frames_.end() ? it->second : std::vector{}; } +StoredEntityFreezeFrame EntityFreezeFrameCapture::to_stored(const std::string & fault_code, const Frame & frame) { + StoredEntityFreezeFrame row; + row.fault_code = fault_code; + row.entity_id = frame.entity_id; + row.frame = nlohmann::json::object(); + row.frame["values"] = frame.values; + // Written only when the capture had them, so the reload reproduces the + // frame's own "reported nothing" as absence rather than as a null. + if (frame.connected.has_value()) { + row.frame["connected"] = *frame.connected; + } + if (!frame.source_timestamp.is_null()) { + row.frame["source_timestamp"] = frame.source_timestamp; + } + row.captured_at_ns = frame.captured_at_ns; + row.source = frame.source; + row.capture_origin = frame.startup_catchup ? kCaptureOriginStartup : ""; + return row; +} + +std::optional +EntityFreezeFrameCapture::from_stored(const StoredEntityFreezeFrame & row) { + if (row.entity_id.empty() || !row.frame.is_object()) { + return std::nullopt; + } + const auto values = row.frame.find("values"); + if (values == row.frame.end()) { + return std::nullopt; + } + Frame frame; + frame.entity_id = row.entity_id; + frame.values = *values; + frame.captured_at_ns = row.captured_at_ns; + frame.source = row.source; + // Reloading must not launder a catch-up frame into a confirm-edge one: its + // captured_at is still a gateway start, so the marker still belongs on it. + frame.startup_catchup = row.capture_origin == kCaptureOriginStartup; + const auto connected = row.frame.find("connected"); + if (connected != row.frame.end() && connected->is_boolean()) { + frame.connected = connected->get(); + } + const auto source_timestamp = row.frame.find("source_timestamp"); + if (source_timestamp != row.frame.end()) { + frame.source_timestamp = *source_timestamp; + } + return frame; +} + +void EntityFreezeFrameCapture::persist_frames_locked(const std::string & fault_code, + const std::vector & frames) { + if (!store_) { + return; + } + std::vector rows; + rows.reserve(frames.size()); + for (const auto & frame : frames) { + rows.push_back(to_stored(fault_code, frame)); + } + auto written = store_->replace_frames(fault_code, rows); + if (!written && !store_write_warned_) { + // One line for the life of the process: a read-only or full volume would + // otherwise log once per confirm, and the frames still work in memory. + store_write_warned_ = true; + RCLCPP_WARN(logger_, "Entity freeze-frame store write failed, frames are process-local until restart: %s", + written.error().c_str()); + } +} + +void EntityFreezeFrameCapture::erase_persisted_locked(const std::string & fault_code) { + if (!store_) { + return; + } + auto erased = store_->erase_frames(fault_code); + if (!erased && !store_write_warned_) { + store_write_warned_ = true; + RCLCPP_WARN(logger_, "Entity freeze-frame store delete failed: %s", erased.error().c_str()); + } +} + +void EntityFreezeFrameCapture::load_persisted_frames() { + if (!store_) { + return; + } + auto rows = store_->load_all(); + if (!rows) { + RCLCPP_WARN(logger_, "Entity freeze-frame store unreadable, starting with no reloaded frames: %s", + rows.error().c_str()); + return; + } + + std::unordered_map> loaded; + std::unordered_map newest; + size_t unreadable = 0; + for (const auto & row : *rows) { + auto frame = from_stored(row); + if (!frame) { + ++unreadable; + continue; + } + auto it = newest.find(row.fault_code); + if (it == newest.end()) { + newest.emplace(row.fault_code, row.captured_at_ns); + } else { + it->second = std::max(it->second, row.captured_at_ns); + } + loaded[row.fault_code].push_back(std::move(*frame)); + } + + // Oldest code first, so the retained-frame bound drops what a restart can + // least afford to keep rather than what it just read. + std::vector codes; + codes.reserve(loaded.size()); + for (const auto & entry : loaded) { + codes.push_back(entry.first); + } + std::sort(codes.begin(), codes.end(), [&newest](const std::string & a, const std::string & b) { + if (newest.at(a) != newest.at(b)) { + return newest.at(a) < newest.at(b); + } + return a < b; + }); + const size_t over_cap = codes.size() > max_faults_ ? codes.size() - max_faults_ : 0; + + std::lock_guard lock(mutex_); + for (size_t i = 0; i < codes.size(); ++i) { + const auto & code = codes[i]; + if (i < over_cap) { + // Past the bound: drop the rows too, or every start re-reads frames it + // can never serve and the file grows without one. + erase_persisted_locked(code); + continue; + } + insertion_order_.push_back(code); + frames_[code] = std::move(loaded[code]); + reloaded_codes_.insert(code); + } + if (!frames_.empty()) { + RCLCPP_INFO(logger_, "Entity freeze-frame: reloaded frames for %zu fault(s) from the store", frames_.size()); + } + if (over_cap > 0) { + RCLCPP_WARN(logger_, + "Entity freeze-frame store held %zu fault(s) beyond the retained-frame bound of %zu; " + "the oldest were dropped", + over_cap, max_faults_); + } + if (unreadable > 0) { + RCLCPP_WARN(logger_, "Entity freeze-frame store: %zu unreadable row(s) skipped", unreadable); + } +} + +void EntityFreezeFrameCapture::prune_frames_for_unknown_faults(const std::function & should_abort) { + if (!known_code_lister_) { + return; + } + { + std::lock_guard lock(mutex_); + if (reloaded_codes_.empty()) { + return; + } + } + std::optional> known; + try { + known = known_code_lister_(should_abort); + } catch (const std::exception & e) { + RCLCPP_WARN(logger_, "Entity freeze-frame prune skipped: known-fault lister threw: %s", e.what()); + return; + } catch (...) { + RCLCPP_WARN(logger_, "Entity freeze-frame prune skipped: known-fault lister threw"); + return; + } + if (!known) { + return; // could not ask: "cannot tell" must never read as "the fault is gone" + } + + size_t dropped = 0; + { + std::lock_guard lock(mutex_); + for (auto it = reloaded_codes_.begin(); it != reloaded_codes_.end();) { + if (known->count(*it) != 0) { + ++it; // reported in any status, cleared included: the frame stays + continue; + } + const std::string code = *it; + it = reloaded_codes_.erase(it); + frames_.erase(code); + insertion_order_.erase(std::remove(insertion_order_.begin(), insertion_order_.end(), code), + insertion_order_.end()); + erase_persisted_locked(code); + ++dropped; + } + } + if (dropped > 0) { + RCLCPP_INFO(logger_, + "Entity freeze-frame: dropped %zu reloaded frame(s) for fault(s) the fault manager no longer holds", + dropped); + } +} + nlohmann::json EntityFreezeFrameCapture::values_from_list_content(const nlohmann::json & content) { if (!content.contains("items") || !content["items"].is_array()) { return content; @@ -288,7 +496,7 @@ void EntityFreezeFrameCapture::wait_for_events_publisher(const std::function standing; - try { - standing = standing_lister_(should_abort); - } catch (const std::exception & e) { - RCLCPP_WARN(logger_, "Entity freeze-frame startup catch-up failed: standing-fault lister threw: %s", e.what()); - return; - } catch (...) { - RCLCPP_WARN(logger_, "Entity freeze-frame startup catch-up failed: standing-fault lister threw"); + if (standing_lister_) { + try { + standing = standing_lister_(should_abort); + } catch (const std::exception & e) { + RCLCPP_WARN(logger_, "Entity freeze-frame startup catch-up failed: standing-fault lister threw: %s", e.what()); + return; + } catch (...) { + RCLCPP_WARN(logger_, "Entity freeze-frame startup catch-up failed: standing-fault lister threw"); + return; + } + } + // Runs after the lister has already waited the fault services out, so the + // extra query costs a round trip rather than a second startup stall. + prune_frames_for_unknown_faults(should_abort); + if (!standing_lister_ || should_abort()) { return; } // Codes with a confirm already queued belong to the drain loop: capturing @@ -327,7 +543,18 @@ void EntityFreezeFrameCapture::capture_standing_faults() { queued_codes.insert(queued->fault.fault_code); } } - size_t framed = 0; + // Frames reloaded from the store already answer for their faults, and the + // bound counts them: they are not budget this catch-up gets to spend twice. + std::unordered_set already_framed; + { + std::lock_guard lock(mutex_); + already_framed.reserve(frames_.size()); + for (const auto & entry : frames_) { + already_framed.insert(entry.first); + } + } + size_t framed = already_framed.size(); + size_t captured = 0; size_t over_cap = 0; for (const auto & fault : standing) { if (should_abort()) { @@ -339,6 +566,13 @@ void EntityFreezeFrameCapture::capture_standing_faults() { if (queued_codes.count(fault.fault_code) != 0) { continue; } + // The stored frame is the one from this fault's own confirm edge. Re-reading + // the plant now would replace it with today's values under a "startup" + // marker, which is exactly what persisting the frame is here to stop. Sits + // before the bound check so a reloaded frame spends no catch-up budget. + if (already_framed.count(fault.fault_code) != 0) { + continue; + } if (framed >= max_faults_) { ++over_cap; // storing more would FIFO-evict this catch-up's own frames continue; @@ -349,6 +583,7 @@ void EntityFreezeFrameCapture::capture_standing_faults() { event.fault.reporting_sources = fault.reporting_sources; if (capture_for_event(event, /*startup_catchup=*/true)) { ++framed; + ++captured; } } if (over_cap > 0) { @@ -357,8 +592,8 @@ void EntityFreezeFrameCapture::capture_standing_faults() { "bound of %zu", over_cap, max_faults_); } - if (framed > 0) { - RCLCPP_INFO(logger_, "Entity freeze-frame: captured %zu fault(s) that were already confirmed at startup", framed); + if (captured > 0) { + RCLCPP_INFO(logger_, "Entity freeze-frame: captured %zu fault(s) that were already confirmed at startup", captured); } } @@ -432,11 +667,23 @@ bool EntityFreezeFrameCapture::capture_for_event(const ros2_medkit_msgs::msg::Fa if (frames_.find(fault_code) == frames_.end()) { insertion_order_.push_back(fault_code); while (frames_.size() >= max_faults_ && !insertion_order_.empty()) { - frames_.erase(insertion_order_.front()); + const std::string evicted = insertion_order_.front(); + frames_.erase(evicted); insertion_order_.pop_front(); + // The store follows the map out: an evicted frame that stayed on disk + // would come back on the next start and the bound would mean nothing + // across restarts. + reloaded_codes_.erase(evicted); + erase_persisted_locked(evicted); } } + // A capture on this fault's own edge supersedes whatever was reloaded for it, + // so the code is no longer a candidate for the reloaded-frame prune. + reloaded_codes_.erase(fault_code); frames_[fault_code] = std::move(frames); + // Under the same lock as the map, so the file and what is being served + // cannot disagree about what was frozen. + persist_frames_locked(fault_code, frames_[fault_code]); RCLCPP_DEBUG(logger_, "Captured entity freeze-frame(s) for fault '%s'", fault_code.c_str()); return true; diff --git a/src/ros2_medkit_gateway/src/gateway_node.cpp b/src/ros2_medkit_gateway/src/gateway_node.cpp index 0d4e8c5ff..e89699fc1 100644 --- a/src/ros2_medkit_gateway/src/gateway_node.cpp +++ b/src/ros2_medkit_gateway/src/gateway_node.cpp @@ -21,7 +21,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -41,6 +43,7 @@ #include "ros2_medkit_gateway/plugins/ros_plugin_context.hpp" #include "ros2_medkit_gateway/core/http/handlers/sse_transport_provider.hpp" +#include "ros2_medkit_gateway/core/sqlite_entity_freeze_frame_store.hpp" #include "ros2_medkit_gateway/core/sqlite_trigger_store.hpp" using namespace std::chrono_literals; @@ -232,6 +235,7 @@ GatewayNode::GatewayNode(const rclcpp::NodeOptions & options) : Node("ros2_medki // Zero-config freeze-frames for plugin-backed entities (opt-out) declare_parameter("entity_freeze_frame.enabled", true); + declare_parameter("entity_freeze_frame.storage.path", ""); // Locking parameters declare_parameter("locking.enabled", true); @@ -1871,6 +1875,7 @@ void GatewayNode::init_entity_freeze_frame_capture(ros2_common::Ros2Subscription if (!get_parameter("entity_freeze_frame.enabled").as_bool() || !plugin_mgr_ || !plugin_mgr_->has_plugins()) { return; } + auto frame_store = open_entity_freeze_frame_store(); entity_freeze_frame_capture_ = std::make_unique( this, exec, [this](const std::string & entity_id) { @@ -1927,9 +1932,71 @@ void GatewayNode::init_entity_freeze_frame_capture(ros2_common::Ros2Subscription RCLCPP_INFO(get_logger(), "Standing-fault freeze-frame catch-up: no confirmed faults at startup"); } return std::move(*parsed); + }, + std::move(frame_store), + // Every code the fault manager still holds, in any status. Frames for + // faults it no longer has (its own store was replaced under ours) are + // dropped; nullopt means it could not be asked, and then nothing is. + [this](const std::function & should_abort) -> std::optional> { + if (!fault_service_transport_ || should_abort()) { + return std::nullopt; + } + // Non-blocking: the standing-fault lister has already waited the + // services out, so a miss here means they are genuinely absent and + // the reply would say nothing about what still exists. + if (!fault_service_transport_->is_available()) { + return std::nullopt; + } + auto result = fault_service_transport_->list_faults("", true, true, true, true, true, false); + if (!result.success) { + return std::nullopt; + } + const auto faults = result.data.find("faults"); + if (faults == result.data.end() || !faults->is_array()) { + return std::nullopt; + } + std::unordered_set codes; + for (const auto & item : *faults) { + if (item.is_object()) { + const auto code = item.find("fault_code"); + if (code != item.end() && code->is_string()) { + codes.insert(code->get()); + } + } + } + return codes; }); } +std::shared_ptr GatewayNode::open_entity_freeze_frame_store() { + // An explicit path wins. With none, the frames go next to the trigger store, + // which is where an operator already points a persistent volume; with no + // trigger store either there is nowhere to put them and they stay in memory, + // as they were before the store existed. + std::string path = get_parameter("entity_freeze_frame.storage.path").as_string(); + if (path.empty()) { + const std::string trigger_path = get_parameter("triggers.storage.path").as_string(); + if (trigger_path.empty()) { + RCLCPP_INFO(get_logger(), + "Entity freeze-frames are not persisted (no entity_freeze_frame.storage.path and no " + "triggers.storage.path); they are lost on restart"); + return nullptr; + } + path = (std::filesystem::path(trigger_path).parent_path() / "entity_freeze_frames.db").string(); + } + try { + auto store = std::make_shared(path); + RCLCPP_INFO(get_logger(), "Entity freeze-frames persisted in %s", path.c_str()); + return store; + } catch (const std::exception & e) { + // A store the gateway cannot open must not stop it from capturing: the + // frames stay in memory, exactly as they did before persistence existed. + RCLCPP_ERROR(get_logger(), "Entity freeze-frame store '%s' could not be opened, frames stay in memory: %s", + path.c_str(), e.what()); + return nullptr; + } +} + EntityFreezeFrameCapture * GatewayNode::get_entity_freeze_frame_capture() const { return entity_freeze_frame_capture_.get(); } diff --git a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp index b97372530..94983463f 100644 --- a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp @@ -18,15 +18,19 @@ #include #include #include +#include #include +#include #include #include #include #include #include #include +#include #include +#include "ros2_medkit_gateway/core/entity_freeze_frame_store.hpp" #include "ros2_medkit_gateway/entity_freeze_frame_capture.hpp" #include "ros2_medkit_gateway/http/handlers/fault_handlers.hpp" #include "ros2_medkit_gateway/ros2_common/ros2_subscription_executor.hpp" @@ -38,6 +42,8 @@ using ros2_medkit_gateway::DataProvider; using ros2_medkit_gateway::DataProviderError; using ros2_medkit_gateway::DataProviderErrorInfo; using ros2_medkit_gateway::EntityFreezeFrameCapture; +using ros2_medkit_gateway::InMemoryEntityFreezeFrameStore; +using ros2_medkit_gateway::StoredEntityFreezeFrame; using ros2_medkit_gateway::handlers::FaultHandlers; using ros2_medkit_gateway::ros2_common::Ros2SubscriptionExecutor; using ros2_medkit_msgs::msg::Fault; @@ -676,6 +682,324 @@ TEST_F(EntityFreezeFrameCaptureTest, OldestFaultEvictedPastMaxFaults) { EXPECT_FALSE(capture.frames_for("PLC_EVICT_C").empty()); } +// =========================================================================== +// Persistence: the frame outlives the process, so a restart serves what was +// frozen at fault time instead of re-reading the plant as it is now. +// =========================================================================== + +namespace { + +/// One stored row, shaped as the capture writes them. +StoredEntityFreezeFrame make_stored_row(const std::string & fault_code, const std::string & entity_id, + int64_t captured_at_ns, double level = 7.0, + const std::string & capture_origin = "") { + StoredEntityFreezeFrame row; + row.fault_code = fault_code; + row.entity_id = entity_id; + row.frame = json{{"values", {{"level", level}}}, {"connected", false}, {"source_timestamp", "2026-09-08T17:51:40Z"}}; + row.captured_at_ns = captured_at_ns; + row.source = EntityFreezeFrameCapture::kSourceXPlcDataRoute; + row.capture_origin = capture_origin; + return row; +} + +/// Route fetcher that serves a fixed level and records which entities it read, +/// so a test can prove the plant was NOT re-read for a fault that already has +/// a frame. +class CountingRouteFetcher { + public: + explicit CountingRouteFetcher(double level) : level_(level) { + } + + std::optional operator()(const std::string & entity_id) { + { + std::lock_guard lock(mutex_); + reads_[entity_id] += 1; + } + return json{{"connected", true}, {"items", json::array({{{"name", "level"}, {"value", level_}}})}}; + } + + int reads(const std::string & entity_id) { + std::lock_guard lock(mutex_); + auto it = reads_.find(entity_id); + return it == reads_.end() ? 0 : it->second; + } + + private: + double level_; + std::mutex mutex_; + std::map reads_; +}; + +} // namespace + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, CaptureWritesTheFrameThroughToTheStore) { + auto store = std::make_shared(); + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [this](const std::string & entity_id) -> DataProvider * { + return entity_id == "plc_app" ? provider_.get() : nullptr; + }, + nullptr, 256, nullptr, store); + + ASSERT_TRUE(publish_and_wait(capture, make_confirmed_event("PLC_PERSIST", {"plc_app"}))); + const auto served = capture.frames_for("PLC_PERSIST"); + ASSERT_EQ(served.size(), 1u); + + auto rows = store->load_all(); + ASSERT_TRUE(rows.has_value()); + ASSERT_EQ(rows->size(), 1u); + EXPECT_EQ((*rows)[0].fault_code, "PLC_PERSIST"); + EXPECT_EQ((*rows)[0].entity_id, "plc_app"); + EXPECT_EQ((*rows)[0].captured_at_ns, served[0].captured_at_ns); + EXPECT_EQ((*rows)[0].frame["values"], served[0].values); + EXPECT_EQ((*rows)[0].source, EntityFreezeFrameCapture::kSourceDataProvider); + EXPECT_EQ((*rows)[0].capture_origin, ""); // captured on the confirm edge +} + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, ReConfirmOverwritesTheStoredRow) { + auto store = std::make_shared(); + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [this](const std::string & entity_id) -> DataProvider * { + return entity_id == "plc_app" ? provider_.get() : nullptr; + }, + nullptr, 256, nullptr, store); + + ASSERT_TRUE(publish_and_wait(capture, make_confirmed_event("PLC_RECONFIRM", {"plc_app"}))); + + // The store must follow the map: today's semantics are one frame per fault, + // so a stale row would resurrect the previous occurrence's values on restart. + provider_->set_temperature(99.0); + const auto reconfirm = make_confirmed_event("PLC_RECONFIRM", {"plc_app"}); + const auto deadline = std::chrono::steady_clock::now() + 5s; + bool overwritten = false; + while (std::chrono::steady_clock::now() < deadline && !overwritten) { + publisher_->publish(reconfirm); + std::this_thread::sleep_for(20ms); + auto rows = store->load_all(); + overwritten = rows.has_value() && rows->size() == 1u && + std::abs((*rows)[0].frame["values"].value("temperature", 0.0) - 99.0) < 1e-9; + } + EXPECT_TRUE(overwritten); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, AReloadedFrameIsServedExactlyAsItWasCaptured) { + auto store = std::make_shared(); + json first_wire; + int64_t captured_at_ns = 0; + { + CountingRouteFetcher fetcher(41.0); + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + [&fetcher](const std::string & entity_id) { + return fetcher(entity_id); + }, + 256, nullptr, store); + ASSERT_TRUE(publish_and_wait(capture, make_confirmed_event("PLC_RESTART", {"route_plc_app"}))); + const auto frames = capture.frames_for("PLC_RESTART"); + ASSERT_EQ(frames.size(), 1u); + captured_at_ns = frames[0].captured_at_ns; + first_wire = FaultHandlers::merge_entity_freeze_frames(json{{"snapshots", json::array()}}, frames); + } + + // A second gateway life on the same store, with the plant now reading + // something else entirely: the served frame must still be the frozen one. + CountingRouteFetcher moved_on(999.0); + EntityFreezeFrameCapture restarted( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + [&moved_on](const std::string & entity_id) { + return moved_on(entity_id); + }, + 256, nullptr, store); + + const auto reloaded = restarted.frames_for("PLC_RESTART"); + ASSERT_EQ(reloaded.size(), 1u); + EXPECT_EQ(reloaded[0].captured_at_ns, captured_at_ns); + EXPECT_FALSE(reloaded[0].startup_catchup); + const auto second_wire = FaultHandlers::merge_entity_freeze_frames(json{{"snapshots", json::array()}}, reloaded); + EXPECT_EQ(second_wire, first_wire); // byte for byte, marker included + ASSERT_EQ(second_wire["snapshots"].size(), 1u); + EXPECT_FALSE(second_wire["snapshots"][0].contains("capture_origin")); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, CatchUpSkipsAReloadedCodeAndFramesOneWithoutARow) { + auto store = std::make_shared(); + ASSERT_TRUE( + store->replace_frames("PLC_HAS_FRAME", {make_stored_row("PLC_HAS_FRAME", "route_stored_app", 4242)}).has_value()); + + CountingRouteFetcher fetcher(7.0); + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + [&fetcher](const std::string & entity_id) { + return fetcher(entity_id); + }, + 256, + [](const std::function &) -> std::vector { + return {{"PLC_HAS_FRAME", {"route_stored_app"}}, {"PLC_NO_FRAME", {"route_fresh_app"}}}; + }, + store); + + // Positive control on the same harness: a standing fault with no stored row + // still gets its startup frame, so an empty PLC_HAS_FRAME below would be a + // broken catch-up rather than a working skip. + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (capture.frames_for("PLC_NO_FRAME").empty() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(20ms); + } + const auto fresh = capture.frames_for("PLC_NO_FRAME"); + ASSERT_EQ(fresh.size(), 1u); + EXPECT_TRUE(fresh[0].startup_catchup); + + const auto stored = capture.frames_for("PLC_HAS_FRAME"); + ASSERT_EQ(stored.size(), 1u); + EXPECT_EQ(stored[0].captured_at_ns, 4242); // the frozen moment, not this start + EXPECT_FALSE(stored[0].startup_catchup); + EXPECT_EQ(fetcher.reads("route_stored_app"), 0); // the plant was never re-read for it +} + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, ReloadDropsAFaultTheManagerNoLongerHoldsAndKeepsAClearedOne) { + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames("PLC_GONE", {make_stored_row("PLC_GONE", "route_a", 100)}).has_value()); + ASSERT_TRUE(store->replace_frames("PLC_CLEARED", {make_stored_row("PLC_CLEARED", "route_b", 200)}).has_value()); + + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + nullptr, 256, + [](const std::function &) -> std::vector { + return {}; // nothing standing: only the reload and the prune run + }, + store, + // The fault manager reports the cleared fault and knows nothing of the + // other: its own store was replaced under ours. + [](const std::function &) -> std::optional> { + return std::unordered_set{"PLC_CLEARED"}; + }); + + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (!capture.frames_for("PLC_GONE").empty() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(20ms); + } + EXPECT_TRUE(capture.frames_for("PLC_GONE").empty()); + // A cleared fault keeps its frame: the gateway's retention across a clear is + // exactly what persisting it is meant to preserve. + EXPECT_FALSE(capture.frames_for("PLC_CLEARED").empty()); + auto rows = store->load_all(); + ASSERT_TRUE(rows.has_value()); + ASSERT_EQ(rows->size(), 1u); + EXPECT_EQ((*rows)[0].fault_code, "PLC_CLEARED"); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, AnUnanswerableKnownFaultListerDropsNothing) { + // Absence assertion, controlled by the test above: the same seeding with a + // lister that CAN answer drops PLC_GONE, so "nothing dropped" here is the + // "could not tell" rule and not a prune that never runs. + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames("PLC_GONE", {make_stored_row("PLC_GONE", "route_a", 100)}).has_value()); + ASSERT_TRUE(store->replace_frames("PLC_CLEARED", {make_stored_row("PLC_CLEARED", "route_b", 200)}).has_value()); + + std::atomic asked{false}; + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + nullptr, 256, + [](const std::function &) -> std::vector { + return {}; + }, + store, + [&asked](const std::function &) -> std::optional> { + asked.store(true); + return std::nullopt; // fault manager unreachable + }); + + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (!asked.load() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(20ms); + } + ASSERT_TRUE(asked.load()); + std::this_thread::sleep_for(300ms); // would be enough for a prune to land + EXPECT_FALSE(capture.frames_for("PLC_GONE").empty()); + EXPECT_FALSE(capture.frames_for("PLC_CLEARED").empty()); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, TheRetainedFrameBoundCountsReloadedFrames) { + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames("PLC_LOADED_A", {make_stored_row("PLC_LOADED_A", "route_a", 100)}).has_value()); + ASSERT_TRUE(store->replace_frames("PLC_LOADED_B", {make_stored_row("PLC_LOADED_B", "route_b", 200)}).has_value()); + + const auto standing = [](const std::function &) -> std::vector { + return {{"PLC_FRESH", {"route_fresh"}}}; + }; + CountingRouteFetcher fetcher(7.0); + const auto route = [&fetcher](const std::string & entity_id) { + return fetcher(entity_id); + }; + const auto no_provider = [](const std::string &) -> DataProvider * { + return nullptr; + }; + + { + // Bound of 2, already met by the two reloaded frames: catching PLC_FRESH up + // would FIFO-evict one of them, so it is refused instead. + EntityFreezeFrameCapture capped(node_.get(), *sub_exec_, no_provider, route, /*max_faults=*/2, standing, store); + std::this_thread::sleep_for(1s); // enough for an uncapped catch-up to land + EXPECT_TRUE(capped.frames_for("PLC_FRESH").empty()); + EXPECT_FALSE(capped.frames_for("PLC_LOADED_A").empty()); + EXPECT_FALSE(capped.frames_for("PLC_LOADED_B").empty()); + } + + // Positive control on the same harness: one more slot and the same catch-up + // frames it. + EntityFreezeFrameCapture roomy(node_.get(), *sub_exec_, no_provider, route, /*max_faults=*/3, standing, store); + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (roomy.frames_for("PLC_FRESH").empty() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(20ms); + } + EXPECT_FALSE(roomy.frames_for("PLC_FRESH").empty()); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, AnEvictedFaultLosesItsStoredRowToo) { + // Otherwise the bound holds only within a process: an evicted frame would + // come back on the next start and the file would grow without a limit. + auto store = std::make_shared(); + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [this](const std::string & entity_id) -> DataProvider * { + return entity_id == "plc_app" ? provider_.get() : nullptr; + }, + nullptr, /*max_faults=*/1, nullptr, store); + + ASSERT_TRUE(publish_and_wait(capture, make_confirmed_event("PLC_EVICT_FIRST", {"plc_app"}))); + ASSERT_TRUE(publish_and_wait(capture, make_confirmed_event("PLC_EVICT_SECOND", {"plc_app"}))); + + auto rows = store->load_all(); + ASSERT_TRUE(rows.has_value()); + ASSERT_EQ(rows->size(), 1u); + EXPECT_EQ((*rows)[0].fault_code, "PLC_EVICT_SECOND"); +} + TEST(ContentHasLiveData, GatesOnItemsNotOnTheLinkFlag) { using Capture = EntityFreezeFrameCapture; EXPECT_TRUE(Capture::content_has_live_data( diff --git a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_store.cpp b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_store.cpp new file mode 100644 index 000000000..00e564137 --- /dev/null +++ b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_store.cpp @@ -0,0 +1,225 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include + +#include +#include +#include +#include + +#include "ros2_medkit_gateway/core/entity_freeze_frame_store.hpp" +#include "ros2_medkit_gateway/core/sqlite_entity_freeze_frame_store.hpp" + +using json = nlohmann::json; +using ros2_medkit_gateway::EntityFreezeFrameStore; +using ros2_medkit_gateway::InMemoryEntityFreezeFrameStore; +using ros2_medkit_gateway::SqliteEntityFreezeFrameStore; +using ros2_medkit_gateway::StoredEntityFreezeFrame; + +namespace { + +StoredEntityFreezeFrame make_row(const std::string & fault_code, const std::string & entity_id, int64_t captured_at_ns, + json frame = json{{"values", {{"temperature", 42.5}, {"pressure", 3.2}}}, + {"connected", false}, + {"source_timestamp", "2026-09-08T17:51:40.387Z"}}) { + StoredEntityFreezeFrame row; + row.fault_code = fault_code; + row.entity_id = entity_id; + row.frame = std::move(frame); + row.captured_at_ns = captured_at_ns; + row.source = "plugin_x_plc_data_route"; + row.capture_origin = ""; + return row; +} + +/// Both backends must behave identically: the in-memory one is what tests and +/// a path-less gateway get, and a difference between them would only show up +/// on the box. +enum class Backend { InMemory, Sqlite }; + +class EntityFreezeFrameStoreTest : public ::testing::TestWithParam { + protected: + void SetUp() override { + db_path_ = std::filesystem::temp_directory_path() / + ("test_entity_freeze_frame_store_" + std::to_string(::getpid()) + ".db"); + std::filesystem::remove(db_path_); + store_ = open(); + } + + void TearDown() override { + store_.reset(); + std::filesystem::remove(db_path_); + } + + std::unique_ptr open() { + if (GetParam() == Backend::InMemory) { + return std::make_unique(); + } + return std::make_unique(db_path_.string()); + } + + std::filesystem::path db_path_; + std::unique_ptr store_; +}; + +} // namespace + +/// @verifies REQ_INTEROP_088 +TEST_P(EntityFreezeFrameStoreTest, RoundTripsEveryFieldOfARow) { + auto row = make_row("JAM_INFEED", "plc_app", 1757353900387000000); + row.capture_origin = "startup"; + ASSERT_TRUE(store_->replace_frames("JAM_INFEED", {row}).has_value()); + + auto loaded = store_->load_all(); + ASSERT_TRUE(loaded.has_value()); + ASSERT_EQ(loaded->size(), 1u); + const auto & got = (*loaded)[0]; + EXPECT_EQ(got.fault_code, "JAM_INFEED"); + EXPECT_EQ(got.entity_id, "plc_app"); + EXPECT_EQ(got.frame, row.frame); + EXPECT_EQ(got.captured_at_ns, 1757353900387000000); // nanoseconds need the full 64 bits + EXPECT_EQ(got.source, "plugin_x_plc_data_route"); + EXPECT_EQ(got.capture_origin, "startup"); +} + +/// @verifies REQ_INTEROP_088 +TEST_P(EntityFreezeFrameStoreTest, ReplaceDropsEntitiesTheNewCaptureNoLongerReports) { + ASSERT_TRUE(store_ + ->replace_frames("JAM_INFEED", + {make_row("JAM_INFEED", "plc_app", 10), make_row("JAM_INFEED", "second_app", 20)}) + .has_value()); + // A re-confirm that only frames one entity must not leave the other's row + // behind: the served frames are replaced as a unit, so the rows are too. + ASSERT_TRUE(store_->replace_frames("JAM_INFEED", {make_row("JAM_INFEED", "plc_app", 30)}).has_value()); + + auto loaded = store_->load_all(); + ASSERT_TRUE(loaded.has_value()); + ASSERT_EQ(loaded->size(), 1u); + EXPECT_EQ((*loaded)[0].entity_id, "plc_app"); + EXPECT_EQ((*loaded)[0].captured_at_ns, 30); +} + +/// @verifies REQ_INTEROP_088 +TEST_P(EntityFreezeFrameStoreTest, OneCodePerFaultRowsAreKeptApart) { + ASSERT_TRUE(store_->replace_frames("JAM_INFEED", {make_row("JAM_INFEED", "plc_app", 10)}).has_value()); + ASSERT_TRUE(store_->replace_frames("SAFETY_CURTAIN", {make_row("SAFETY_CURTAIN", "plc_app", 20)}).has_value()); + + ASSERT_TRUE(store_->erase_frames("JAM_INFEED").has_value()); + + auto loaded = store_->load_all(); + ASSERT_TRUE(loaded.has_value()); + ASSERT_EQ(loaded->size(), 1u); + EXPECT_EQ((*loaded)[0].fault_code, "SAFETY_CURTAIN"); +} + +/// @verifies REQ_INTEROP_088 +TEST_P(EntityFreezeFrameStoreTest, LoadAllServesOldestCaptureFirst) { + // The caller keeps the newest codes when the store holds more than its + // retained-frame bound, so the order is part of the contract, not a detail. + ASSERT_TRUE(store_->replace_frames("NEWEST", {make_row("NEWEST", "plc_app", 300)}).has_value()); + ASSERT_TRUE(store_->replace_frames("OLDEST", {make_row("OLDEST", "plc_app", 100)}).has_value()); + ASSERT_TRUE(store_->replace_frames("MIDDLE", {make_row("MIDDLE", "plc_app", 200)}).has_value()); + + auto loaded = store_->load_all(); + ASSERT_TRUE(loaded.has_value()); + ASSERT_EQ(loaded->size(), 3u); + EXPECT_EQ((*loaded)[0].fault_code, "OLDEST"); + EXPECT_EQ((*loaded)[1].fault_code, "MIDDLE"); + EXPECT_EQ((*loaded)[2].fault_code, "NEWEST"); +} + +/// @verifies REQ_INTEROP_088 +TEST_P(EntityFreezeFrameStoreTest, ErasingACodeWithNoRowsIsNotAnError) { + // The caller evicts by fault code and does not track what reached the file. + EXPECT_TRUE(store_->erase_frames("NEVER_STORED").has_value()); +} + +/// @verifies REQ_INTEROP_088 +TEST_P(EntityFreezeFrameStoreTest, AnEmptyReplaceLeavesNoRows) { + ASSERT_TRUE(store_->replace_frames("JAM_INFEED", {make_row("JAM_INFEED", "plc_app", 10)}).has_value()); + ASSERT_TRUE(store_->replace_frames("JAM_INFEED", {}).has_value()); + + auto loaded = store_->load_all(); + ASSERT_TRUE(loaded.has_value()); + EXPECT_TRUE(loaded->empty()); +} + +INSTANTIATE_TEST_SUITE_P(Backends, EntityFreezeFrameStoreTest, ::testing::Values(Backend::InMemory, Backend::Sqlite), + [](const ::testing::TestParamInfo & param_info) { + return param_info.param == Backend::InMemory ? "InMemory" : "Sqlite"; + }); + +// =========================================================================== +// SQLite only: the point of the file is that it outlives the process. +// =========================================================================== + +/// @verifies REQ_INTEROP_088 +TEST(SqliteEntityFreezeFrameStoreFile, RowsSurviveReopen) { + const auto path = std::filesystem::temp_directory_path() / + ("test_entity_freeze_frame_reopen_" + std::to_string(::getpid()) + ".db"); + std::filesystem::remove(path); + + { + SqliteEntityFreezeFrameStore store(path.string()); + ASSERT_TRUE( + store.replace_frames("JAM_INFEED", {make_row("JAM_INFEED", "plc_app", 1757353900387000000)}).has_value()); + } + + SqliteEntityFreezeFrameStore reopened(path.string()); + auto loaded = reopened.load_all(); + ASSERT_TRUE(loaded.has_value()); + ASSERT_EQ(loaded->size(), 1u); + EXPECT_EQ((*loaded)[0].fault_code, "JAM_INFEED"); + EXPECT_EQ((*loaded)[0].captured_at_ns, 1757353900387000000); + EXPECT_EQ((*loaded)[0].frame["values"]["temperature"], 42.5); + EXPECT_EQ((*loaded)[0].frame["connected"], false); + + std::filesystem::remove(path); +} + +/// @verifies REQ_INTEROP_088 +TEST(SqliteEntityFreezeFrameStoreFile, AnUnreadableRowCostsOnlyItself) { + const auto path = std::filesystem::temp_directory_path() / + ("test_entity_freeze_frame_corrupt_" + std::to_string(::getpid()) + ".db"); + std::filesystem::remove(path); + + { + SqliteEntityFreezeFrameStore store(path.string()); + ASSERT_TRUE(store.replace_frames("GOOD", {make_row("GOOD", "plc_app", 20)}).has_value()); + } + // Hand-edit one blob into something that is not JSON, as a half-written file + // or a fat-fingered sqlite3 session would. + { + sqlite3 * db = nullptr; + ASSERT_EQ(sqlite3_open(path.c_str(), &db), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(db, + "INSERT INTO entity_freeze_frames " + "(fault_code, entity_id, frame, captured_at_ns, source, capture_origin) " + "VALUES ('BROKEN','plc_app','{not json',10,'','')", + nullptr, nullptr, nullptr), + SQLITE_OK); + sqlite3_close(db); + } + + SqliteEntityFreezeFrameStore reopened(path.string()); + auto loaded = reopened.load_all(); + ASSERT_TRUE(loaded.has_value()); + ASSERT_EQ(loaded->size(), 1u); + EXPECT_EQ((*loaded)[0].fault_code, "GOOD"); + + std::filesystem::remove(path); +} From 18ccd7dde12c7c74a8b5a6ca49453e346220beba Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 9 Sep 2026 13:32:15 +0200 Subject: [PATCH 10/14] fix(gateway): re-read a freeze-frame whose fault confirmed again while the gateway was down A persisted frame is reloaded at start and the catch-up skips any fault that already has one. That is right while the fault stands, and wrong the moment it cleared and confirmed again in the meantime. The stored frame then holds the previous incident's values, and the gateway served them under the new occurrence with no marker at all, so nothing on the wire said the numbers came from a different event than the fault being read. The standing-fault list now carries each fault's first_occurred, which the fault manager resets only when a CLEARED fault reactivates. A reloaded frame whose captured_at predates it belongs to an occurrence that has ended, so the row is dropped before the catch-up decides what is already framed. The fault is then re-read now and the result marked capture_origin: startup, exactly what an unframed standing fault has always got. last_occurred cannot serve this. It moves on every FAILED report, so on a fault that keeps failing it is always newer than the row and every standing fault would be re-read at every restart, which is the behaviour persisting the frame is here to replace. A value the reply does not carry reads as "cannot tell" and keeps the stored frame. Also announces the parameter in the changelog, and folds the one-line "exactly one frame per fault" paragraph into the persistence paragraph it belongs to. --- docs/config/server.rst | 4 +- docs/tutorials/snapshots.rst | 9 +- src/ros2_medkit_gateway/CHANGELOG.rst | 5 + .../entity_freeze_frame_capture.hpp | 19 +++ .../src/entity_freeze_frame_capture.cpp | 57 ++++++++ .../test/test_entity_freeze_frame_capture.cpp | 130 ++++++++++++++++++ 6 files changed, 219 insertions(+), 5 deletions(-) diff --git a/docs/config/server.rst b/docs/config/server.rst index 15ec5c10a..5f548e817 100644 --- a/docs/config/server.rst +++ b/docs/config/server.rst @@ -396,7 +396,9 @@ Configure how the gateway connects to the fault manager services and event topic that have no stored frame. The retained-frame bound of 256 faults counts reloaded and freshly captured frames together, dropping the oldest first, and a frame whose fault the fault manager no longer holds at all is - dropped at startup. + dropped at startup. A frame belonging to an occurrence that has since + been cleared and confirmed again is re-read at startup and marked + ``capture_origin: startup`` rather than served as the current one. When ``fault_manager.namespace`` is set, the gateway also subscribes to the matching fault event topic (for example ``/robot1/fault_manager/events`` instead of the default diff --git a/docs/tutorials/snapshots.rst b/docs/tutorials/snapshots.rst index d0aac6d87..c522d6e4b 100644 --- a/docs/tutorials/snapshots.rst +++ b/docs/tutorials/snapshots.rst @@ -207,7 +207,11 @@ captured, with its original ``captured_at`` and no ``capture_origin`` marker. Set the path to a file on a volume that outlives the container, or leave it empty and the frames go next to the trigger store (``triggers.storage.path``); with neither set they are process memory only and -a restart loses them. +a restart loses them. A plugin entity keeps exactly one frame per fault: a +re-confirm re-samples the plugin and replaces it, on disk as in memory, and a +re-confirm the gateway was down for is re-read at startup and marked +``capture_origin: startup``, so a new occurrence never serves the previous +one's values. Faults that are already confirmed when the gateway starts are caught up at startup: the gateway lists the confirmed faults and captures a frame for each @@ -226,9 +230,6 @@ reloaded frame keeps whichever marker it was captured with. Disable with: ros2 run ros2_medkit_gateway gateway_node --ros-args \ -p entity_freeze_frame.enabled:=false -A plugin entity keeps exactly one frame per fault: a re-confirm re-samples the -plugin and replaces it, on disk as in memory. - A plugin entity's values are not a ROS message, so ``topic`` and ``message_type`` are empty on these frames. ``x-medkit.source`` names the capture path instead, so a consumer can still tell where the values came diff --git a/src/ros2_medkit_gateway/CHANGELOG.rst b/src/ros2_medkit_gateway/CHANGELOG.rst index 6c4dd39ee..9491d0d78 100644 --- a/src/ros2_medkit_gateway/CHANGELOG.rst +++ b/src/ros2_medkit_gateway/CHANGELOG.rst @@ -2,6 +2,11 @@ Changelog for package ros2_medkit_gateway ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Forthcoming +----------- +* Entity freeze-frames survive a restart when ``entity_freeze_frame.storage.path`` names a database. Without one the frames lived in process memory, so a restart threw them away and the startup catch-up re-read the plant as it is now, serving today's values under the original fault and marking them ``x-medkit.capture_origin: startup`` - the values at fault time, which are the point of a freeze-frame, were gone. A reloaded frame is served exactly as it was captured: its original ``captured_at``, its own ``capture_origin`` (absent on a confirm-edge frame), and the ``connected`` / ``source_timestamp`` provenance it carried, and the startup catch-up then re-reads only the faults that have no frame. Leaving the path empty puts the store in ``entity_freeze_frames.db`` next to ``triggers.storage.path``; with neither set the frames stay in memory as before, and a store that cannot be opened or written is reported while the capture keeps working. The retained-frame bound of 256 faults counts reloaded and freshly captured frames together, a frame whose fault the fault_manager no longer holds in any status is dropped at startup (one reported as cleared keeps its frame), and a frame belonging to an occurrence that has since been cleared and re-confirmed is re-read at startup and marked ``capture_origin: startup`` rather than served as the current one +* Contributors: @bburda + 0.7.0 (2026-08-27) ------------------ * Rosbag bulk-data is addressed by recording id instead of fault code, so a fault holding several recordings can expose each one. ``GET /{entity}/bulk-data/rosbags`` now emits one descriptor per recording rather than one per fault - a burst that shares a bag used to appear as several entries each reporting the full bag size - and the covered faults move into ``x-medkit.fault_codes`` (was the scalar ``x-medkit.fault_code``). Old URLs keep working: an id that is not a recording is resolved as a fault code and serves that fault's newest recording, which is what it returned before. Authorization is unchanged in effect - a download is allowed when any fault the recording covers is in the entity's source scope, which is exactly the set that could reach it previously (`#623 `_, `#620 `_) diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp index 5871e733a..e42f9baa2 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp @@ -119,6 +119,15 @@ class EntityFreezeFrameCapture { struct StandingFault { std::string fault_code; std::vector reporting_sources; + /// When THIS occurrence of the fault started, from the list reply's + /// `first_occurred` (seconds on the wire, nanoseconds here). The + /// fault_manager resets it only when a CLEARED fault reactivates, so it is + /// what tells a stored frame from a previous occurrence apart from one + /// belonging to the occurrence being served now. `last_occurred` cannot do + /// this: it moves on every report, so a fault that keeps failing would look + /// re-occurred at every restart. 0 when the reply did not report it, which + /// reads as "cannot tell" and keeps the stored frame. + int64_t first_occurred_ns{0}; }; /// Lists the faults that are already confirmed when this object starts, so @@ -275,6 +284,16 @@ class EntityFreezeFrameCapture { /// without a known-code lister, or when the lister cannot answer. void prune_frames_for_unknown_faults(const std::function & should_abort); + /// Drop reloaded frames that belong to an earlier occurrence of their fault: + /// the fault cleared and confirmed again while the gateway was down, so the + /// stored frame holds the previous incident's values and serving it unmarked + /// would present them as this occurrence's. Dropping the row makes the + /// catch-up treat the fault as unframed, so it re-reads the plugin now and + /// marks the result `startup`, which is what an unframed standing fault has + /// always got. Only reloaded codes are eligible: a frame this process + /// captured is by definition this occurrence's. + void drop_reloaded_frames_from_earlier_occurrences(const std::vector & standing); + std::unique_ptr subscription_slot_; DataProviderResolver resolver_; RouteDataFetcher route_fetcher_; diff --git a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp index 14e418178..e7d27f56e 100644 --- a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp @@ -312,6 +312,48 @@ void EntityFreezeFrameCapture::prune_frames_for_unknown_faults(const std::functi } } +void EntityFreezeFrameCapture::drop_reloaded_frames_from_earlier_occurrences( + const std::vector & standing) { + size_t dropped = 0; + { + std::lock_guard lock(mutex_); + if (reloaded_codes_.empty()) { + return; + } + for (const auto & fault : standing) { + if (fault.first_occurred_ns <= 0 || reloaded_codes_.count(fault.fault_code) == 0) { + continue; + } + const auto entry = frames_.find(fault.fault_code); + if (entry == frames_.end()) { + continue; + } + // The newest of the code's frames dates the stored capture: they are all + // written by one capture, so if even that one predates the occurrence the + // whole set belongs to an incident that has since been cleared. + int64_t newest = 0; + for (const auto & frame : entry->second) { + newest = std::max(newest, frame.captured_at_ns); + } + if (fault.first_occurred_ns <= newest) { + continue; // same occurrence: the stored frame is the one to serve + } + frames_.erase(entry); + insertion_order_.erase(std::remove(insertion_order_.begin(), insertion_order_.end(), fault.fault_code), + insertion_order_.end()); + reloaded_codes_.erase(fault.fault_code); + erase_persisted_locked(fault.fault_code); + ++dropped; + } + } + if (dropped > 0) { + RCLCPP_INFO(logger_, + "Entity freeze-frame: %zu reloaded frame(s) belong to an earlier occurrence of their fault and were " + "dropped; the catch-up re-reads those entities", + dropped); + } +} + nlohmann::json EntityFreezeFrameCapture::values_from_list_content(const nlohmann::json & content) { if (!content.contains("items") || !content["items"].is_array()) { return content; @@ -389,6 +431,17 @@ EntityFreezeFrameCapture::standing_faults_from_list_reply(const nlohmann::json & fault.reporting_sources.push_back(src.get()); } } + // Seconds on the wire (fault_msg_conversions), nanoseconds here so it can + // be compared with a frame's captured_at_ns without converting per row. + // Anything that is not a positive number leaves it at 0, which reads as + // "the reply did not say" and never costs a stored frame. + const auto first_occurred = item.find("first_occurred"); + if (first_occurred != item.end() && first_occurred->is_number()) { + const double seconds = first_occurred->get(); + if (seconds > 0.0) { + fault.first_occurred_ns = static_cast(seconds * 1e9); + } + } standing.push_back(std::move(fault)); } return standing; @@ -531,6 +584,10 @@ void EntityFreezeFrameCapture::capture_standing_faults() { if (!standing_lister_ || should_abort()) { return; } + // Before anything reads frames_ as "already answered for": a fault that + // cleared and confirmed again while the gateway was down must not be served + // the previous incident's values. + drop_reloaded_frames_from_earlier_occurrences(standing); // Codes with a confirm already queued belong to the drain loop: capturing // them here too would read the plugin twice for one confirm. std::unordered_set queued_codes; diff --git a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp index 94983463f..b02560674 100644 --- a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp @@ -1000,6 +1000,112 @@ TEST_F(EntityFreezeFrameCaptureTest, AnEvictedFaultLosesItsStoredRowToo) { EXPECT_EQ((*rows)[0].fault_code, "PLC_EVICT_SECOND"); } +namespace { + +/// A capture whose store already holds one frame for PLC_REOCCUR, taken at +/// `stored_at_ns` with level 10.0, against a plant that now reads 99.0. The +/// standing lister reports the fault with `first_occurred_ns`, which is what +/// decides whether the stored frame belongs to the occurrence being served. +struct ReoccurrenceHarness { + std::shared_ptr store = std::make_shared(); + CountingRouteFetcher plant{99.0}; + static constexpr int64_t kStoredAtNs = 1'000'000'000'000'000'000; +}; + +} // namespace + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, AReloadedFrameFromAnEarlierOccurrenceIsReReadAndMarkedStartup) { + // The fault cleared and confirmed again while the gateway was down, so the + // stored frame holds the PREVIOUS incident's values. Serving it unmarked + // would present last week's numbers as this occurrence's. + ReoccurrenceHarness h; + ASSERT_TRUE( + h.store->replace_frames("PLC_REOCCUR", {make_stored_row("PLC_REOCCUR", "route_stored_app", h.kStoredAtNs, 10.0)}) + .has_value()); + + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + [&h](const std::string & entity_id) { + return h.plant(entity_id); + }, + 256, + [](const std::function &) -> std::vector { + EntityFreezeFrameCapture::StandingFault fault; + fault.fault_code = "PLC_REOCCUR"; + fault.reporting_sources = {"route_stored_app"}; + fault.first_occurred_ns = ReoccurrenceHarness::kStoredAtNs + 60'000'000'000; // a minute after the frame + return std::vector{fault}; + }, + h.store); + + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (std::chrono::steady_clock::now() < deadline) { + const auto now = capture.frames_for("PLC_REOCCUR"); + if (!now.empty() && std::abs(now[0].values.value("level", 0.0) - 99.0) < 1e-9) { + break; + } + std::this_thread::sleep_for(20ms); + } + const auto served = capture.frames_for("PLC_REOCCUR"); + ASSERT_EQ(served.size(), 1u); + EXPECT_DOUBLE_EQ(served[0].values.value("level", 0.0), 99.0); // this occurrence, not the last one + EXPECT_TRUE(served[0].startup_catchup); // read at start, so it says so + EXPECT_GT(served[0].captured_at_ns, ReoccurrenceHarness::kStoredAtNs); + + auto rows = h.store->load_all(); + ASSERT_TRUE(rows.has_value()); + ASSERT_EQ(rows->size(), 1u); + EXPECT_DOUBLE_EQ((*rows)[0].frame["values"].value("level", 0.0), 99.0); // replaced on disk too + EXPECT_EQ((*rows)[0].capture_origin, "startup"); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, AReloadedFrameFromTheSameOccurrenceIsKeptUnmarked) { + // Control for the test above on the same harness: the fault never cleared, so + // its first_occurred predates the frame and the stored values are the ones + // this occurrence froze. Re-reading the plant here is the whole defect. + ReoccurrenceHarness h; + ASSERT_TRUE( + h.store->replace_frames("PLC_REOCCUR", {make_stored_row("PLC_REOCCUR", "route_stored_app", h.kStoredAtNs, 10.0)}) + .has_value()); + + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + [&h](const std::string & entity_id) { + return h.plant(entity_id); + }, + 256, + [](const std::function &) -> std::vector { + EntityFreezeFrameCapture::StandingFault fault; + fault.fault_code = "PLC_REOCCUR"; + fault.reporting_sources = {"route_stored_app"}; + fault.first_occurred_ns = ReoccurrenceHarness::kStoredAtNs - 60'000'000'000; // a minute before the frame + return std::vector{fault}; + }, + h.store); + + std::this_thread::sleep_for(1s); // enough for a catch-up read to land + const auto served = capture.frames_for("PLC_REOCCUR"); + ASSERT_EQ(served.size(), 1u); + EXPECT_DOUBLE_EQ(served[0].values.value("level", 0.0), 10.0); + EXPECT_EQ(served[0].captured_at_ns, ReoccurrenceHarness::kStoredAtNs); + EXPECT_FALSE(served[0].startup_catchup); + EXPECT_EQ(h.plant.reads("route_stored_app"), 0); // the plant was never re-read + + auto rows = h.store->load_all(); + ASSERT_TRUE(rows.has_value()); + ASSERT_EQ(rows->size(), 1u); + EXPECT_DOUBLE_EQ((*rows)[0].frame["values"].value("level", 0.0), 10.0); // row untouched + EXPECT_EQ((*rows)[0].captured_at_ns, ReoccurrenceHarness::kStoredAtNs); +} + TEST(ContentHasLiveData, GatesOnItemsNotOnTheLinkFlag) { using Capture = EntityFreezeFrameCapture; EXPECT_TRUE(Capture::content_has_live_data( @@ -1081,6 +1187,30 @@ TEST(StandingFaultsFromListReply, ParsesWellFormedReply) { EXPECT_TRUE((*standing)[1].reporting_sources.empty()); } +TEST(StandingFaultsFromListReply, FirstOccurredIsReadInSecondsAndKeptInNanoseconds) { + // The wire carries seconds (fault_msg_conversions), the comparison against a + // frame's captured_at_ns needs nanoseconds. Anything that is not a positive + // number leaves 0, which the caller reads as "cannot tell" and never lets + // cost a stored frame. + const json data = { + {"faults", + json::array({json{{"fault_code", "SECONDS"}, + {"reporting_sources", json::array({"a"})}, + {"first_occurred", 1788948705.5}}, + json{{"fault_code", "ABSENT"}, {"reporting_sources", json::array({"a"})}}, + json{{"fault_code", "NOT_A_NUMBER"}, + {"reporting_sources", json::array({"a"})}, + {"first_occurred", "yesterday"}}, + json{{"fault_code", "ZERO"}, {"reporting_sources", json::array({"a"})}, {"first_occurred", 0}}})}}; + const auto standing = EntityFreezeFrameCapture::standing_faults_from_list_reply(data); + ASSERT_TRUE(standing.has_value()); + ASSERT_EQ(standing->size(), 4u); + EXPECT_EQ((*standing)[0].first_occurred_ns, 1788948705500000000); + EXPECT_EQ((*standing)[1].first_occurred_ns, 0); + EXPECT_EQ((*standing)[2].first_occurred_ns, 0); + EXPECT_EQ((*standing)[3].first_occurred_ns, 0); +} + TEST(StandingFaultsFromListReply, RepliesNotShapedLikeListFaultsYieldNullopt) { // nullopt (vs empty vector) is what lets the caller warn on a malformed or // renamed reply instead of silently disabling the catch-up. From 47153026b8f48f2aec15ad1f8904abf784ed14b5 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 9 Sep 2026 13:32:15 +0200 Subject: [PATCH 11/14] feat(opcua): persist the demo gateway's entity freeze-frames The docker demo set neither entity_freeze_frame.storage.path nor triggers.storage.path, so the gateway reported that the frames are not persisted and kept them in process memory. The start and test scripts already create /var/lib/ros2_medkit for the fault manager, so the store goes there and a restart of the demo gateway serves the values frozen when the alarm confirmed instead of re-reading the PLC as it is now. --- .../ros2_medkit_opcua/docker/gateway_params.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml index a331166cb..88285032c 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml @@ -12,3 +12,10 @@ ros2_medkit_gateway: allowed_origins: ["*"] plugins: ["opcua"] plugins.opcua.poll_interval_ms: 1000 + # Freeze-frames for the PLC entities are written next to the rest of the + # gateway's state, so a restart serves the values frozen when the fault + # confirmed instead of re-reading the PLC as it is now. The directory is + # the one the start / test scripts already create. + entity_freeze_frame: + storage: + path: "/var/lib/ros2_medkit/entity_freeze_frames.db" From 28a4697ca01f743816fcb35ac0d4abec07a27b51 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 9 Sep 2026 14:39:05 +0200 Subject: [PATCH 12/14] fix(gateway): re-read a stale freeze-frame before discarding it, and say when it goes The re-occurrence fix erased the stale row from memory and from the store and only then let the catch-up try to re-read the entity. When the entity could not answer, which is a restart while the PLC link is down, the fault ended with no frame in memory, no row on disk and not one line about it. The route path returns nothing without a message, the failed capture was neither counted nor logged, and the drop's own INFO claimed the catch-up re-reads those entities as a fact it never checked. The order is now read, then decide. Stale codes are identified without touching anything, the catch-up re-reads them like any unframed fault, and a successful read replaces the frame and its row in one write, marked startup. Only a read that yields nothing drops the row, and then it warns with the fault code, the entity and why, because the operator is losing evidence and the log is the only place they can learn it. The tail INFO counts what happened, how many were re-read and how many went with no replacement. Two things that follow from the new order. A fault with no reporting sources was stale to the comparison and invisible to the catch-up, so its row was dropped by one test and never re-read by the other. It is now a re-read that cannot be attempted and takes the same path and the same warning. A stale code also jumps the queued-confirm skip: that skip saves one plugin read per confirm, and here the alternative is leaving a dead occurrence's frame in place on the chance the drain loop succeeds. Also guards the seconds-to-nanoseconds conversion of first_occurred against a double outside int64's range, where the cast is undefined, and documents the two windows the comparison deliberately leaves alone: a fault that re-failed without re-confirming is not in the confirmed list and keeps its frame until it confirms, and a HEALED to FAILED cycle does not reset first_occurred. --- docs/config/server.rst | 6 +- docs/tutorials/snapshots.rst | 16 +- src/ros2_medkit_gateway/CHANGELOG.rst | 2 +- .../entity_freeze_frame_capture.hpp | 29 ++- .../src/entity_freeze_frame_capture.cpp | 160 +++++++++++----- .../test/test_entity_freeze_frame_capture.cpp | 179 +++++++++++++++++- .../ros2_medkit_opcua/README.md | 20 +- .../docker/gateway_params.yaml | 8 +- .../ros2_medkit_opcua/docker/scripts/start.sh | 14 ++ .../ros2_medkit_opcua/docker/scripts/stop.sh | 10 + 10 files changed, 369 insertions(+), 75 deletions(-) diff --git a/docs/config/server.rst b/docs/config/server.rst index 5f548e817..bc8e79e84 100644 --- a/docs/config/server.rst +++ b/docs/config/server.rst @@ -390,7 +390,7 @@ Configure how the gateway connects to the fault manager services and event topic - SQLite file the captured frames are persisted in, so a restart serves the values frozen at fault time instead of re-reading the plant. When empty, the frames go in ``entity_freeze_frames.db`` next to - ``triggers.storage.path``; with that empty too they stay in memory and are + ``triggers.storage.path``. With that empty too they stay in memory and are lost on restart. A reloaded frame keeps its original ``captured_at`` and its ``capture_origin``, and the startup catch-up then runs only for faults that have no stored frame. The retained-frame bound of 256 faults counts @@ -398,7 +398,9 @@ Configure how the gateway connects to the fault manager services and event topic and a frame whose fault the fault manager no longer holds at all is dropped at startup. A frame belonging to an occurrence that has since been cleared and confirmed again is re-read at startup and marked - ``capture_origin: startup`` rather than served as the current one. + ``capture_origin: startup`` rather than served as the current one. When + that re-read cannot answer, the stale frame is discarded with a warning + naming the fault code and the entity. When ``fault_manager.namespace`` is set, the gateway also subscribes to the matching fault event topic (for example ``/robot1/fault_manager/events`` instead of the default diff --git a/docs/tutorials/snapshots.rst b/docs/tutorials/snapshots.rst index c522d6e4b..2f7ef6b2f 100644 --- a/docs/tutorials/snapshots.rst +++ b/docs/tutorials/snapshots.rst @@ -206,12 +206,20 @@ gateway restart: it is reloaded at start and served exactly as it was captured, with its original ``captured_at`` and no ``capture_origin`` marker. Set the path to a file on a volume that outlives the container, or leave it empty and the frames go next to the trigger store -(``triggers.storage.path``); with neither set they are process memory only and -a restart loses them. A plugin entity keeps exactly one frame per fault: a +(``triggers.storage.path``). With neither set they are process memory only +and a restart loses them. A plugin entity keeps exactly one frame per fault: a re-confirm re-samples the plugin and replaces it, on disk as in memory, and a re-confirm the gateway was down for is re-read at startup and marked -``capture_origin: startup``, so a new occurrence never serves the previous -one's values. +``capture_origin: startup``, so a confirmed occurrence never serves the +previous one's values. The startup comparison is against the fault's +``first_occurred``, which the fault manager resets on reactivation from +``CLEARED``, so it holds for exactly the faults the catch-up sees: a fault +that re-failed but has not re-confirmed yet is not in the confirmed list and +keeps its frame until it does confirm, and a ``HEALED`` to ``FAILED`` cycle +does not reset ``first_occurred`` at all (healing is off by default). If the +re-read cannot answer, because the entity is unreachable or serves nothing +usable, the stale frame is discarded rather than served, and the gateway warns +with the fault code and the entity so the missing evidence is not silent. Faults that are already confirmed when the gateway starts are caught up at startup: the gateway lists the confirmed faults and captures a frame for each diff --git a/src/ros2_medkit_gateway/CHANGELOG.rst b/src/ros2_medkit_gateway/CHANGELOG.rst index 9491d0d78..4c802ae73 100644 --- a/src/ros2_medkit_gateway/CHANGELOG.rst +++ b/src/ros2_medkit_gateway/CHANGELOG.rst @@ -4,7 +4,7 @@ Changelog for package ros2_medkit_gateway Forthcoming ----------- -* Entity freeze-frames survive a restart when ``entity_freeze_frame.storage.path`` names a database. Without one the frames lived in process memory, so a restart threw them away and the startup catch-up re-read the plant as it is now, serving today's values under the original fault and marking them ``x-medkit.capture_origin: startup`` - the values at fault time, which are the point of a freeze-frame, were gone. A reloaded frame is served exactly as it was captured: its original ``captured_at``, its own ``capture_origin`` (absent on a confirm-edge frame), and the ``connected`` / ``source_timestamp`` provenance it carried, and the startup catch-up then re-reads only the faults that have no frame. Leaving the path empty puts the store in ``entity_freeze_frames.db`` next to ``triggers.storage.path``; with neither set the frames stay in memory as before, and a store that cannot be opened or written is reported while the capture keeps working. The retained-frame bound of 256 faults counts reloaded and freshly captured frames together, a frame whose fault the fault_manager no longer holds in any status is dropped at startup (one reported as cleared keeps its frame), and a frame belonging to an occurrence that has since been cleared and re-confirmed is re-read at startup and marked ``capture_origin: startup`` rather than served as the current one +* Entity freeze-frames survive a restart when ``entity_freeze_frame.storage.path`` names a database. Without one the frames lived in process memory, so a restart threw them away and the startup catch-up re-read the plant as it is now, serving today's values under the original fault and marking them ``x-medkit.capture_origin: startup`` - the values at fault time, which are the point of a freeze-frame, were gone. A reloaded frame is served exactly as it was captured: its original ``captured_at``, its own ``capture_origin`` (absent on a confirm-edge frame), and the ``connected`` / ``source_timestamp`` provenance it carried, and the startup catch-up then re-reads only the faults that have no frame. Leaving the path empty puts the store in ``entity_freeze_frames.db`` next to ``triggers.storage.path``. With neither set the frames stay in memory as before, and a store that cannot be opened or written is reported while the capture keeps working. The retained-frame bound of 256 faults counts reloaded and freshly captured frames together, a frame whose fault the fault_manager no longer holds in any status is dropped at startup (one reported as cleared keeps its frame), and a frame belonging to an occurrence that has since been cleared and re-confirmed is re-read at startup and marked ``capture_origin: startup`` rather than served as the current one * Contributors: @bburda 0.7.0 (2026-08-27) diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp index e42f9baa2..f86369767 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp @@ -284,15 +284,26 @@ class EntityFreezeFrameCapture { /// without a known-code lister, or when the lister cannot answer. void prune_frames_for_unknown_faults(const std::function & should_abort); - /// Drop reloaded frames that belong to an earlier occurrence of their fault: - /// the fault cleared and confirmed again while the gateway was down, so the - /// stored frame holds the previous incident's values and serving it unmarked - /// would present them as this occurrence's. Dropping the row makes the - /// catch-up treat the fault as unframed, so it re-reads the plugin now and - /// marks the result `startup`, which is what an unframed standing fault has - /// always got. Only reloaded codes are eligible: a frame this process - /// captured is by definition this occurrence's. - void drop_reloaded_frames_from_earlier_occurrences(const std::vector & standing); + /// Reloaded codes whose stored frame belongs to an EARLIER occurrence of + /// their fault: it cleared and confirmed again while the gateway was down, so + /// the frame holds the previous incident's values and serving it unmarked + /// would present them as this occurrence's. + /// + /// Read-only on purpose. The row stays until a re-read has actually been + /// tried, so a fault whose entity answers gets its frame replaced rather than + /// deleted and then not re-taken. It is dropped only by drop_stale_frame(), + /// after the attempt failed. Only reloaded codes are eligible: a frame this + /// process captured is by definition this occurrence's. + std::unordered_set stale_reloaded_codes(const std::vector & standing) const; + + /// Last resort for a stale-occurrence code the catch-up could not re-read: + /// erase it from memory and from the store, and say so. Keeping it would + /// serve the previous incident's values unmarked, which is the defect this + /// path exists to fix, so the frame goes. Never silently, though: the + /// operator is losing evidence and only the log can tell them. + /// @p entities names the reporting sources that were tried (empty when the + /// fault named none), @p reason why no frame could be taken. + void drop_stale_frame(const std::string & fault_code, const std::string & entities, const char * reason); std::unique_ptr subscription_slot_; DataProviderResolver resolver_; diff --git a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp index e7d27f56e..88ea3416c 100644 --- a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp @@ -43,6 +43,19 @@ std::string string_field(const nlohmann::json & item, const char * field) { return it != item.end() && it->is_string() ? it->get() : std::string(); } +/// Reporting sources as one comma-separated string, for a log line that has to +/// name the entities an operator would go and look at. +std::string join_sources(const std::vector & sources) { + std::string joined; + for (const auto & source : sources) { + if (!joined.empty()) { + joined += ", "; + } + joined += source; + } + return joined; +} + } // namespace EntityFreezeFrameCapture::EntityFreezeFrameCapture(rclcpp::Node * node, ros2_common::Ros2SubscriptionExecutor & exec, @@ -312,46 +325,52 @@ void EntityFreezeFrameCapture::prune_frames_for_unknown_faults(const std::functi } } -void EntityFreezeFrameCapture::drop_reloaded_frames_from_earlier_occurrences( - const std::vector & standing) { - size_t dropped = 0; - { - std::lock_guard lock(mutex_); - if (reloaded_codes_.empty()) { - return; +std::unordered_set +EntityFreezeFrameCapture::stale_reloaded_codes(const std::vector & standing) const { + std::unordered_set stale; + std::lock_guard lock(mutex_); + if (reloaded_codes_.empty()) { + return stale; + } + for (const auto & fault : standing) { + if (fault.first_occurred_ns <= 0 || reloaded_codes_.count(fault.fault_code) == 0) { + continue; } - for (const auto & fault : standing) { - if (fault.first_occurred_ns <= 0 || reloaded_codes_.count(fault.fault_code) == 0) { - continue; - } - const auto entry = frames_.find(fault.fault_code); - if (entry == frames_.end()) { - continue; - } - // The newest of the code's frames dates the stored capture: they are all - // written by one capture, so if even that one predates the occurrence the - // whole set belongs to an incident that has since been cleared. - int64_t newest = 0; - for (const auto & frame : entry->second) { - newest = std::max(newest, frame.captured_at_ns); - } - if (fault.first_occurred_ns <= newest) { - continue; // same occurrence: the stored frame is the one to serve - } - frames_.erase(entry); - insertion_order_.erase(std::remove(insertion_order_.begin(), insertion_order_.end(), fault.fault_code), - insertion_order_.end()); - reloaded_codes_.erase(fault.fault_code); - erase_persisted_locked(fault.fault_code); - ++dropped; + const auto entry = frames_.find(fault.fault_code); + if (entry == frames_.end()) { + continue; } + // The newest of the code's frames dates the stored capture: they are all + // written by one capture, so if even that one predates the occurrence the + // whole set belongs to an incident that has since been cleared. + int64_t newest = 0; + for (const auto & frame : entry->second) { + newest = std::max(newest, frame.captured_at_ns); + } + if (fault.first_occurred_ns <= newest) { + continue; // same occurrence: the stored frame is the one to serve + } + stale.insert(fault.fault_code); } - if (dropped > 0) { - RCLCPP_INFO(logger_, - "Entity freeze-frame: %zu reloaded frame(s) belong to an earlier occurrence of their fault and were " - "dropped; the catch-up re-reads those entities", - dropped); - } + return stale; +} + +void EntityFreezeFrameCapture::drop_stale_frame(const std::string & fault_code, const std::string & entities, + const char * reason) { + { + std::lock_guard lock(mutex_); + frames_.erase(fault_code); + insertion_order_.erase(std::remove(insertion_order_.begin(), insertion_order_.end(), fault_code), + insertion_order_.end()); + reloaded_codes_.erase(fault_code); + erase_persisted_locked(fault_code); + } + // The operator is losing evidence here. Keeping the frame would serve the + // previous incident's values as this one's, so it goes, but never silently. + RCLCPP_WARN(logger_, + "Entity freeze-frame for fault '%s': the stored frame is from an earlier occurrence and entity '%s' " + "could not be re-read (%s). The stored frame was discarded, so this occurrence has no freeze-frame.", + fault_code.c_str(), entities.c_str(), reason); } nlohmann::json EntityFreezeFrameCapture::values_from_list_content(const nlohmann::json & content) { @@ -438,8 +457,14 @@ EntityFreezeFrameCapture::standing_faults_from_list_reply(const nlohmann::json & const auto first_occurred = item.find("first_occurred"); if (first_occurred != item.end() && first_occurred->is_number()) { const double seconds = first_occurred->get(); - if (seconds > 0.0) { - fault.first_occurred_ns = static_cast(seconds * 1e9); + // The range is checked on the nanosecond product, before the cast: a + // double outside int64's range makes the conversion undefined, and a + // NaN fails every comparison so it lands here too. Untrusted input on + // this path is a malformed or hostile reply, not just a stale clock. + const double nanoseconds = seconds * 1e9; + constexpr double kMaxRepresentableNs = 9.2e18; // below int64 max, with room for the ulp + if (nanoseconds > 0.0 && nanoseconds < kMaxRepresentableNs) { + fault.first_occurred_ns = static_cast(nanoseconds); } } standing.push_back(std::move(fault)); @@ -584,10 +609,6 @@ void EntityFreezeFrameCapture::capture_standing_faults() { if (!standing_lister_ || should_abort()) { return; } - // Before anything reads frames_ as "already answered for": a fault that - // cleared and confirmed again while the gateway was down must not be served - // the previous incident's values. - drop_reloaded_frames_from_earlier_occurrences(standing); // Codes with a confirm already queued belong to the drain loop: capturing // them here too would read the plugin twice for one confirm. std::unordered_set queued_codes; @@ -600,27 +621,54 @@ void EntityFreezeFrameCapture::capture_standing_faults() { queued_codes.insert(queued->fault.fault_code); } } + // Reloaded frames from an occurrence that has since been cleared and + // re-confirmed. Identified here, still on disk: whether the row goes is + // decided below, by whether the re-read could replace it. + const auto stale = stale_reloaded_codes(standing); // Frames reloaded from the store already answer for their faults, and the // bound counts them: they are not budget this catch-up gets to spend twice. + // A stale one answers for nothing, so it counts as absent while it is being + // re-read and its replacement takes the slot it already held. std::unordered_set already_framed; { std::lock_guard lock(mutex_); already_framed.reserve(frames_.size()); for (const auto & entry : frames_) { - already_framed.insert(entry.first); + if (stale.count(entry.first) == 0) { + already_framed.insert(entry.first); + } } } size_t framed = already_framed.size(); size_t captured = 0; + size_t re_read = 0; + size_t discarded = 0; size_t over_cap = 0; for (const auto & fault : standing) { if (should_abort()) { return; } - if (fault.fault_code.empty() || fault.reporting_sources.empty()) { + if (fault.fault_code.empty()) { + continue; + } + const bool is_stale = stale.count(fault.fault_code) != 0; + // Every path below that gives up on a stale code has to drop its row: the + // whole point of calling it stale is that serving it unmarked is wrong. + if (fault.reporting_sources.empty()) { + // The drop test and the re-read test must agree on this, or a fault with + // no entity is stale to one and invisible to the other, and its row + // survives to be served. + if (is_stale) { + drop_stale_frame(fault.fault_code, "", "the fault reports no entity to read"); + ++discarded; + } continue; } - if (queued_codes.count(fault.fault_code) != 0) { + // A stale code jumps the queued-confirm skip. That skip exists so one + // confirm costs one plugin read, but here the alternative is leaving a + // frame from a dead occurrence in place on the chance the drain loop + // succeeds. One extra read is the cheaper mistake. + if (!is_stale && queued_codes.count(fault.fault_code) != 0) { continue; } // The stored frame is the one from this fault's own confirm edge. Re-reading @@ -632,15 +680,29 @@ void EntityFreezeFrameCapture::capture_standing_faults() { } if (framed >= max_faults_) { ++over_cap; // storing more would FIFO-evict this catch-up's own frames + if (is_stale) { + drop_stale_frame(fault.fault_code, join_sources(fault.reporting_sources), + "the retained-frame bound was already reached"); + ++discarded; + } continue; } ros2_medkit_msgs::msg::FaultEvent event; event.event_type = ros2_medkit_msgs::msg::FaultEvent::EVENT_CONFIRMED; event.fault.fault_code = fault.fault_code; event.fault.reporting_sources = fault.reporting_sources; + // capture_for_event replaces the code's frames and its rows as one + // delete-then-insert, so a successful re-read swaps the stale frame out + // without a window in which the fault has none. if (capture_for_event(event, /*startup_catchup=*/true)) { ++framed; ++captured; + if (is_stale) { + ++re_read; + } + } else if (is_stale) { + drop_stale_frame(fault.fault_code, join_sources(fault.reporting_sources), "the entity served no usable values"); + ++discarded; } } if (over_cap > 0) { @@ -652,6 +714,12 @@ void EntityFreezeFrameCapture::capture_standing_faults() { if (captured > 0) { RCLCPP_INFO(logger_, "Entity freeze-frame: captured %zu fault(s) that were already confirmed at startup", captured); } + if (re_read > 0 || discarded > 0) { + RCLCPP_INFO(logger_, + "Entity freeze-frame: %zu reloaded frame(s) from an earlier occurrence re-read, %zu discarded with no " + "replacement", + re_read, discarded); + } } void EntityFreezeFrameCapture::capture_worker() { diff --git a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp index b02560674..57f8a5300 100644 --- a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp @@ -703,6 +703,45 @@ StoredEntityFreezeFrame make_stored_row(const std::string & fault_code, const st return row; } +/// Store that counts what the capture asks of it, so a test can pin the ORDER +/// of a replacement. A stale frame must be swapped out by one write, never +/// erased first and re-taken afterwards if the plant happens to answer. +class CountingEntityFreezeFrameStore : public ros2_medkit_gateway::EntityFreezeFrameStore { + public: + tl::expected replace_frames(const std::string & fault_code, + const std::vector & frames) override { + replaces_.fetch_add(1); + return inner_.replace_frames(fault_code, frames); + } + + tl::expected erase_frames(const std::string & fault_code) override { + erases_.fetch_add(1); + return inner_.erase_frames(fault_code); + } + + tl::expected, std::string> load_all() override { + return inner_.load_all(); + } + + /// Forget the writes the test itself made while seeding. + void reset_counts() { + replaces_.store(0); + erases_.store(0); + } + + int replaces() const { + return replaces_.load(); + } + int erases() const { + return erases_.load(); + } + + private: + InMemoryEntityFreezeFrameStore inner_; + std::atomic replaces_{0}; + std::atomic erases_{0}; +}; + /// Route fetcher that serves a fixed level and records which entities it read, /// so a test can prove the plant was NOT re-read for a fault that already has /// a frame. @@ -1007,11 +1046,45 @@ namespace { /// standing lister reports the fault with `first_occurred_ns`, which is what /// decides whether the stored frame belongs to the occurrence being served. struct ReoccurrenceHarness { - std::shared_ptr store = std::make_shared(); + std::shared_ptr store = std::make_shared(); CountingRouteFetcher plant{99.0}; static constexpr int64_t kStoredAtNs = 1'000'000'000'000'000'000; }; +/// Route fetcher whose entity never answers: the plugin-unreachable shape, and +/// the one the whole feature exists for (a restart while the link is down). +class UnreachableRouteFetcher { + public: + std::optional operator()(const std::string & entity_id) { + std::lock_guard lock(mutex_); + reads_[entity_id] += 1; + return std::nullopt; + } + + int reads(const std::string & entity_id) { + std::lock_guard lock(mutex_); + auto it = reads_.find(entity_id); + return it == reads_.end() ? 0 : it->second; + } + + private: + std::mutex mutex_; + std::map reads_; +}; + +/// Standing lister reporting one fault whose occurrence began after the frame +/// the store holds for it. +EntityFreezeFrameCapture::StandingFaultLister reoccurred_lister(const std::string & code, + const std::vector & sources) { + return [code, sources](const std::function &) { + EntityFreezeFrameCapture::StandingFault fault; + fault.fault_code = code; + fault.reporting_sources = sources; + fault.first_occurred_ns = ReoccurrenceHarness::kStoredAtNs + 60'000'000'000; + return std::vector{fault}; + }; +} + } // namespace /// @verifies REQ_INTEROP_088 @@ -1023,6 +1096,7 @@ TEST_F(EntityFreezeFrameCaptureTest, AReloadedFrameFromAnEarlierOccurrenceIsReRe ASSERT_TRUE( h.store->replace_frames("PLC_REOCCUR", {make_stored_row("PLC_REOCCUR", "route_stored_app", h.kStoredAtNs, 10.0)}) .has_value()); + h.store->reset_counts(); EntityFreezeFrameCapture capture( node_.get(), *sub_exec_, @@ -1032,15 +1106,7 @@ TEST_F(EntityFreezeFrameCaptureTest, AReloadedFrameFromAnEarlierOccurrenceIsReRe [&h](const std::string & entity_id) { return h.plant(entity_id); }, - 256, - [](const std::function &) -> std::vector { - EntityFreezeFrameCapture::StandingFault fault; - fault.fault_code = "PLC_REOCCUR"; - fault.reporting_sources = {"route_stored_app"}; - fault.first_occurred_ns = ReoccurrenceHarness::kStoredAtNs + 60'000'000'000; // a minute after the frame - return std::vector{fault}; - }, - h.store); + 256, reoccurred_lister("PLC_REOCCUR", {"route_stored_app"}), h.store); const auto deadline = std::chrono::steady_clock::now() + 15s; while (std::chrono::steady_clock::now() < deadline) { @@ -1061,6 +1127,99 @@ TEST_F(EntityFreezeFrameCaptureTest, AReloadedFrameFromAnEarlierOccurrenceIsReRe ASSERT_EQ(rows->size(), 1u); EXPECT_DOUBLE_EQ((*rows)[0].frame["values"].value("level", 0.0), 99.0); // replaced on disk too EXPECT_EQ((*rows)[0].capture_origin, "startup"); + + // The order, not just the outcome: the plant is read first and the row is + // then swapped by ONE write. Erasing first and re-taking afterwards leaves + // the fault with nothing whenever the entity cannot answer, which is exactly + // the case this feature is for. + EXPECT_EQ(h.store->erases(), 0); + EXPECT_EQ(h.store->replaces(), 1); + EXPECT_EQ(h.plant.reads("route_stored_app"), 1); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, AStaleFrameWhoseEntityCannotBeReReadIsDroppedWithAWarning) { + // The headline case: the gateway restarts while the PLC link is down, and the + // fault re-confirmed in the meantime. The stored frame is from the previous + // occurrence so it must not be served, and the re-read cannot replace it, so + // the fault ends with no frame. That is a real loss of evidence and the log + // is the only place the operator can learn about it. + auto store = std::make_shared(); + UnreachableRouteFetcher plant; + ASSERT_TRUE(store + ->replace_frames("PLC_LINK_DOWN", {make_stored_row("PLC_LINK_DOWN", "route_stored_app", + ReoccurrenceHarness::kStoredAtNs, 10.0)}) + .has_value()); + + testing::internal::CaptureStderr(); + { + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + [&plant](const std::string & entity_id) { + return plant(entity_id); + }, + 256, reoccurred_lister("PLC_LINK_DOWN", {"route_stored_app"}), store); + + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (std::chrono::steady_clock::now() < deadline && plant.reads("route_stored_app") == 0) { + std::this_thread::sleep_for(20ms); + } + std::this_thread::sleep_for(300ms); // let the drop that follows the failed read land + EXPECT_GE(plant.reads("route_stored_app"), 1); // the re-read WAS attempted + EXPECT_TRUE(capture.frames_for("PLC_LINK_DOWN").empty()); + } + const auto logs = testing::internal::GetCapturedStderr(); + + auto rows = store->load_all(); + ASSERT_TRUE(rows.has_value()); + EXPECT_TRUE(rows->empty()); // and the row is gone, not left to be served + EXPECT_NE(logs.find("PLC_LINK_DOWN"), std::string::npos) << logs; + EXPECT_NE(logs.find("route_stored_app"), std::string::npos) << logs; + EXPECT_NE(logs.find("no freeze-frame"), std::string::npos) << logs; +} + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, AStaleFrameWhoseFaultNamesNoEntityIsDroppedWithAWarning) { + // The two eligibility tests have to agree. A standing fault with no reporting + // sources is stale to the comparison and unreadable to the catch-up, so it is + // a re-read that cannot even be attempted, and it gets the same treatment and + // the same line rather than disappearing quietly. + auto store = std::make_shared(); + CountingRouteFetcher plant{99.0}; + ASSERT_TRUE(store + ->replace_frames("PLC_NO_ENTITY", {make_stored_row("PLC_NO_ENTITY", "route_stored_app", + ReoccurrenceHarness::kStoredAtNs, 10.0)}) + .has_value()); + + testing::internal::CaptureStderr(); + { + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + [&plant](const std::string & entity_id) { + return plant(entity_id); + }, + 256, reoccurred_lister("PLC_NO_ENTITY", {}), store); + + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (std::chrono::steady_clock::now() < deadline && !capture.frames_for("PLC_NO_ENTITY").empty()) { + std::this_thread::sleep_for(20ms); + } + EXPECT_TRUE(capture.frames_for("PLC_NO_ENTITY").empty()); + EXPECT_EQ(plant.reads("route_stored_app"), 0); // nothing to read, and none was invented + } + const auto logs = testing::internal::GetCapturedStderr(); + + auto rows = store->load_all(); + ASSERT_TRUE(rows.has_value()); + EXPECT_TRUE(rows->empty()); + EXPECT_NE(logs.find("PLC_NO_ENTITY"), std::string::npos) << logs; + EXPECT_NE(logs.find("no entity to read"), std::string::npos) << logs; } /// @verifies REQ_INTEROP_088 diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md index 7b6abdde0..e3f142d3f 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md @@ -871,10 +871,28 @@ curl -s http://localhost:8080/api/v1/apps/tank_process/x-plc-data | jq . # Automated tests (16 assertions) bash scripts/run_integration_tests.sh -# Stop +# Stop (keeps the gateway's state) bash scripts/stop.sh ``` +`start.sh` mounts the named volume `ros2-medkit-opcua-state` at +`/var/lib/ros2_medkit`, so the gateway's state survives `stop.sh` and the next +`start.sh`: the entity freeze frames, `faults.db` and the rosbags. That is what +lets a fault raised before the stop still serve the values frozen when it +confirmed, with its original `captured_at` and no `x-medkit.capture_origin`, +rather than a fresh read of the PLC as it is after the restart. `stop.sh` +removes the container, so state left in the container's writable layer would +not survive it. + +To start from a clean slate, purge the volume: + +```bash +bash scripts/stop.sh +docker volume rm ros2-medkit-opcua-state +``` + +Set `OPCUA_DEMO_STATE_VOLUME` to use a different volume name. + A separate scenario covers the config-less discovery start-up race, which the suite above cannot see because it pins `OPCUA_ENDPOINT_URL` and so short-circuits discovery. It starts the gateway before any server, with diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml index 88285032c..f03bc668a 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml @@ -14,8 +14,12 @@ ros2_medkit_gateway: plugins.opcua.poll_interval_ms: 1000 # Freeze-frames for the PLC entities are written next to the rest of the # gateway's state, so a restart serves the values frozen when the fault - # confirmed instead of re-reading the PLC as it is now. The directory is - # the one the start / test scripts already create. + # confirmed instead of re-reading the PLC as it is now. scripts/start.sh + # mounts the named volume "ros2-medkit-opcua-state" here, which is what + # carries the frames across scripts/stop.sh too: that script removes the + # container, so the writable layer would not survive it. The test scripts + # start their own containers without a volume and keep the frames inside + # the container, which is all a single-run suite needs. entity_freeze_frame: storage: path: "/var/lib/ros2_medkit/entity_freeze_frames.db" diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/start.sh b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/start.sh index f80d00fbc..bac6fc6ca 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/start.sh +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/start.sh @@ -2,8 +2,14 @@ # Start OpenPLC + medkit gateway for manual testing. # Usage: from the ros2_medkit repo root, run # bash src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/start.sh +# +# The gateway's state (entity freeze frames, faults.db, rosbags) is kept on the +# named volume below, so it survives stop.sh and a later start.sh. Purge it with +# docker volume rm ros2-medkit-opcua-state set -eo pipefail +STATE_VOLUME="${OPCUA_DEMO_STATE_VOLUME:-ros2-medkit-opcua-state}" + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DOCKER_DIR="$(dirname "$SCRIPT_DIR")" PLUGIN_DIR="$(dirname "$DOCKER_DIR")" @@ -23,6 +29,12 @@ echo "" echo "=== Starting containers ===" docker rm -f openplc gateway 2>/dev/null || true docker network create plc-demo 2>/dev/null || true +# The gateway's state goes on a named volume rather than the container's +# writable layer, because stop.sh removes the container. Without this a +# freeze-frame captured when the alarm confirmed would be destroyed by the only +# stop procedure the demo ships, and the next start would re-read the PLC as it +# is then instead of serving the values frozen at fault time. +docker volume create "$STATE_VOLUME" >/dev/null docker run -d --name openplc --network plc-demo -p 4840:4840 openplc-tank echo "OpenPLC starting..." @@ -35,6 +47,7 @@ for _ in $(seq 1 45); do done docker run -d --name gateway --network plc-demo -p 8080:8080 \ + -v "$STATE_VOLUME":/var/lib/ros2_medkit \ -e ROS_DOMAIN_ID=60 \ -e OPCUA_ENDPOINT_URL="opc.tcp://openplc:4840/openplc/opcua" \ -e OPCUA_NODE_MAP_PATH="/config/tank_nodes.yaml" \ @@ -62,6 +75,7 @@ for _ in $(seq 1 30); do echo "" echo "Stop: bash scripts/stop.sh" echo "Tests: bash scripts/run_integration_tests.sh" + echo "State: volume '$STATE_VOLUME' (kept across stop/start)" exit 0 fi sleep 2 diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/stop.sh b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/stop.sh index fb3fa3de6..2ad4d0bda 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/stop.sh +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/stop.sh @@ -1,4 +1,14 @@ #!/usr/bin/env bash +# Stop the OpenPLC + medkit gateway demo. +# +# The gateway's state volume is deliberately LEFT IN PLACE. It holds the entity +# freeze frames, faults.db and the rosbags, and a freeze-frame is only worth +# anything if it outlives the process that took it. This script removes the +# container, so state kept in the container's writable layer would go with it. +STATE_VOLUME="${OPCUA_DEMO_STATE_VOLUME:-ros2-medkit-opcua-state}" + docker rm -f gateway openplc 2>/dev/null docker network rm plc-demo 2>/dev/null echo "Stopped." +echo "State kept in volume '$STATE_VOLUME'. Purge it with:" +echo " docker volume rm $STATE_VOLUME" From 302725cc6eb957d017d0970cd01bae779f101400 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 9 Sep 2026 18:32:28 +0200 Subject: [PATCH 13/14] fix(gateway): settle the stale freeze-frames before the bound decides anything Re-reading a stale frame in the same pass as the faults that have none put the retained-frame bound in front of a read that cannot cost anything, and made the occupancy it works from wrong for everyone else. A stale code already holds a slot in frames_, and capture_for_event evicts only when the code is new to the map, so its replacement can neither exceed the bound nor push another frame out. Counting it as absent therefore did two things at once at a full bound. A fault with no frame at all was admitted against the under-count and the FIFO evicted the front of the insertion order, which is the OLDEST live reloaded frame, from memory and from disk, without naming it. And the stale code itself then hit the bound check and was discarded with a warning saying its entity could not be re-read, when the entity had never been asked. The catch-up now runs in two passes. First every stale code is re-read in place, with no bound check and no eviction, replaced on success and dropped with the existing warning on failure, which frees the slot it was holding. Only then is the occupancy read, from what frames_ really holds, and the faults with no frame are admitted against that. Eviction can no longer reach a code that is stale or mid-re-read, and the bound branch for stale codes is gone because it can no longer be reached. The truncation warning now names the fault codes it refused (up to ten) instead of only counting them. A count leaves an operator knowing that some fault details lack their context but not which. Also gives the range guard on the seconds-to-nanoseconds conversion a test. Reverting it left the suite green, so it guarded nothing that was checked: 1e19 seconds and +inf both reach the cast, and both come back as INT64_MIN on this box, which reads as "older than every frame" and would discard every reloaded frame the reply mentions. NaN is refused by the sign test either way. --- .../src/entity_freeze_frame_capture.cpp | 124 ++++++---- .../test/test_entity_freeze_frame_capture.cpp | 216 +++++++++++++++++- 2 files changed, 287 insertions(+), 53 deletions(-) diff --git a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp index 88ea3416c..90ddbe15b 100644 --- a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp @@ -36,6 +36,10 @@ constexpr size_t kMaxLoggedFaultCodes = 1024; /// the standing-fault snapshot; past it the catch-up proceeds best-effort. constexpr std::chrono::seconds kEventsMatchTimeout{10}; +/// How many over-the-bound fault codes the truncation warning names before it +/// stops. Enough for an operator to act on, short of a 256-code log line. +constexpr size_t kMaxNamedOverCapCodes = 10; + /// Read a string field totally: json::value() throws type_error.302 when the /// key is present but not a string, and plugin content is untrusted. std::string string_field(const nlohmann::json & item, const char * field) { @@ -625,65 +629,97 @@ void EntityFreezeFrameCapture::capture_standing_faults() { // re-confirmed. Identified here, still on disk: whether the row goes is // decided below, by whether the re-read could replace it. const auto stale = stale_reloaded_codes(standing); - // Frames reloaded from the store already answer for their faults, and the - // bound counts them: they are not budget this catch-up gets to spend twice. - // A stale one answers for nothing, so it counts as absent while it is being - // re-read and its replacement takes the slot it already held. - std::unordered_set already_framed; - { - std::lock_guard lock(mutex_); - already_framed.reserve(frames_.size()); - for (const auto & entry : frames_) { - if (stale.count(entry.first) == 0) { - already_framed.insert(entry.first); - } - } - } - size_t framed = already_framed.size(); + size_t captured = 0; size_t re_read = 0; size_t discarded = 0; size_t over_cap = 0; + + // ---- Pass 1: the stale codes, in place ---------------------------------- + // Each already owns a slot in frames_, and capture_for_event evicts only when + // the code is new to the map, so a re-read can neither exceed the + // retained-frame bound nor push anyone else out. That makes the bound check + // wrong here in both directions. It would refuse a read that costs nothing, + // and while these codes are counted as absent a bare code admitted against + // that under-count would FIFO-evict a live frame that is still wanted. So the + // stale codes are settled first, and only then is the real occupancy known. for (const auto & fault : standing) { if (should_abort()) { return; } - if (fault.fault_code.empty()) { + if (fault.fault_code.empty() || stale.count(fault.fault_code) == 0) { continue; } - const bool is_stale = stale.count(fault.fault_code) != 0; - // Every path below that gives up on a stale code has to drop its row: the - // whole point of calling it stale is that serving it unmarked is wrong. + // A stale code also jumps the queued-confirm skip. That skip exists so one + // confirm costs one plugin read, but here the alternative is leaving a + // frame from a dead occurrence in place on the chance the drain loop + // succeeds. One extra read is the cheaper mistake. if (fault.reporting_sources.empty()) { - // The drop test and the re-read test must agree on this, or a fault with - // no entity is stale to one and invisible to the other, and its row + // The staleness test and the re-read test must agree on this, or a fault + // with no entity is stale to one and invisible to the other, and its row // survives to be served. - if (is_stale) { - drop_stale_frame(fault.fault_code, "", "the fault reports no entity to read"); - ++discarded; - } + drop_stale_frame(fault.fault_code, "", "the fault reports no entity to read"); + ++discarded; continue; } - // A stale code jumps the queued-confirm skip. That skip exists so one - // confirm costs one plugin read, but here the alternative is leaving a - // frame from a dead occurrence in place on the chance the drain loop - // succeeds. One extra read is the cheaper mistake. - if (!is_stale && queued_codes.count(fault.fault_code) != 0) { + ros2_medkit_msgs::msg::FaultEvent event; + event.event_type = ros2_medkit_msgs::msg::FaultEvent::EVENT_CONFIRMED; + event.fault.fault_code = fault.fault_code; + event.fault.reporting_sources = fault.reporting_sources; + // capture_for_event replaces the code's frames and its rows as one + // delete-then-insert, so a successful re-read swaps the stale frame out + // without a window in which the fault has none. + if (capture_for_event(event, /*startup_catchup=*/true)) { + ++captured; + ++re_read; + } else { + drop_stale_frame(fault.fault_code, join_sources(fault.reporting_sources), "the entity served no usable values"); + ++discarded; // the slot it held is now free for the pass below + } + } + + // ---- Pass 2: the faults that have no frame at all ------------------------ + // Occupancy is read after the stale pass, so it is what frames_ really holds: + // every stale code has by now been replaced in place or dropped. A bare code + // is therefore admitted only against a slot that is genuinely free, and the + // FIFO inside capture_for_event can only reach codes that are neither stale + // nor mid-re-read. + std::vector over_cap_codes; + std::unordered_set already_framed; + size_t framed = 0; + { + std::lock_guard lock(mutex_); + already_framed.reserve(frames_.size()); + for (const auto & entry : frames_) { + already_framed.insert(entry.first); + } + framed = frames_.size(); + } + for (const auto & fault : standing) { + if (should_abort()) { + return; + } + if (fault.fault_code.empty() || fault.reporting_sources.empty()) { + continue; + } + // Codes with a confirm already queued belong to the drain loop: capturing + // them here too would read the plugin twice for one confirm. + if (queued_codes.count(fault.fault_code) != 0) { continue; } // The stored frame is the one from this fault's own confirm edge. Re-reading // the plant now would replace it with today's values under a "startup" - // marker, which is exactly what persisting the frame is here to stop. Sits - // before the bound check so a reloaded frame spends no catch-up budget. + // marker, which is exactly what persisting the frame is here to stop. This + // also covers a stale code the pass above just replaced. if (already_framed.count(fault.fault_code) != 0) { continue; } if (framed >= max_faults_) { - ++over_cap; // storing more would FIFO-evict this catch-up's own frames - if (is_stale) { - drop_stale_frame(fault.fault_code, join_sources(fault.reporting_sources), - "the retained-frame bound was already reached"); - ++discarded; + ++over_cap; // storing more would FIFO-evict a frame that is still wanted + // Named, not just counted: "3 faults went unframed" leaves an operator + // with no way to tell which fault details are missing their context. + if (over_cap_codes.size() < kMaxNamedOverCapCodes) { + over_cap_codes.push_back(fault.fault_code); } continue; } @@ -691,25 +727,17 @@ void EntityFreezeFrameCapture::capture_standing_faults() { event.event_type = ros2_medkit_msgs::msg::FaultEvent::EVENT_CONFIRMED; event.fault.fault_code = fault.fault_code; event.fault.reporting_sources = fault.reporting_sources; - // capture_for_event replaces the code's frames and its rows as one - // delete-then-insert, so a successful re-read swaps the stale frame out - // without a window in which the fault has none. if (capture_for_event(event, /*startup_catchup=*/true)) { ++framed; ++captured; - if (is_stale) { - ++re_read; - } - } else if (is_stale) { - drop_stale_frame(fault.fault_code, join_sources(fault.reporting_sources), "the entity served no usable values"); - ++discarded; } } if (over_cap > 0) { RCLCPP_WARN(logger_, "Entity freeze-frame startup catch-up truncated: %zu standing fault(s) beyond the retained-frame " - "bound of %zu", - over_cap, max_faults_); + "bound of %zu, so they have no freeze-frame: %s%s", + over_cap, max_faults_, join_sources(over_cap_codes).c_str(), + over_cap > over_cap_codes.size() ? ", ..." : ""); } if (captured > 0) { RCLCPP_INFO(logger_, "Entity freeze-frame: captured %zu fault(s) that were already confirmed at startup", captured); diff --git a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp index 57f8a5300..79a109c17 100644 --- a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp @@ -14,16 +14,19 @@ #include +#include #include #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -1085,6 +1088,88 @@ EntityFreezeFrameCapture::StandingFaultLister reoccurred_lister(const std::strin }; } +/// Standing lister over an explicit list, so a test can fix the reply's order +/// and each fault's occurrence start independently. +EntityFreezeFrameCapture::StandingFaultLister +listed_faults(std::vector faults) { + return [faults](const std::function &) { + return faults; + }; +} + +EntityFreezeFrameCapture::StandingFault standing_fault(const std::string & code, const std::vector & srcs, + int64_t first_occurred_ns) { + EntityFreezeFrameCapture::StandingFault fault; + fault.fault_code = code; + fault.reporting_sources = srcs; + fault.first_occurred_ns = first_occurred_ns; + return fault; +} + +/// Route fetcher that answers for every entity except the named ones, so one +/// entity out of several can be the unreachable one. +class SelectiveRouteFetcher { + public: + SelectiveRouteFetcher(double level, std::set unreachable) + : level_(level), unreachable_(std::move(unreachable)) { + } + + std::optional operator()(const std::string & entity_id) { + { + std::lock_guard lock(mutex_); + reads_[entity_id] += 1; + } + if (unreachable_.count(entity_id) != 0) { + return std::nullopt; + } + return json{{"connected", true}, {"items", json::array({{{"name", "level"}, {"value", level_}}})}}; + } + + int reads(const std::string & entity_id) { + std::lock_guard lock(mutex_); + auto it = reads_.find(entity_id); + return it == reads_.end() ? 0 : it->second; + } + + private: + double level_; + std::set unreachable_; + std::mutex mutex_; + std::map reads_; +}; + +/// Fault codes present in the store, sorted, for a whole-store assertion. +std::vector stored_codes(const std::shared_ptr & store) { + auto rows = store->load_all(); + std::vector codes; + if (rows) { + for (const auto & row : *rows) { + codes.push_back(row.fault_code); + } + } + std::sort(codes.begin(), codes.end()); + return codes; +} + +// The bound probe both tests below run. Two slots, one live reloaded frame +// (PLC_KEEP, the older of the two so it is the FIFO front), one stale reloaded +// frame (PLC_STALE) and one standing fault with no frame at all (PLC_NEW). +constexpr int64_t kKeepAtNs = ReoccurrenceHarness::kStoredAtNs; +constexpr int64_t kStaleAtNs = ReoccurrenceHarness::kStoredAtNs + 10'000'000'000; + +std::vector bound_probe_standing() { + return {standing_fault("PLC_NEW", {"route_new"}, kStaleAtNs + 30'000'000'000), + standing_fault("PLC_STALE", {"route_stale"}, kStaleAtNs + 60'000'000'000), // after its frame, so stale + standing_fault("PLC_KEEP", {"route_keep"}, kKeepAtNs - 60'000'000'000)}; // before its frame, so live +} + +void seed_bound_probe(const std::shared_ptr & store) { + ASSERT_TRUE( + store->replace_frames("PLC_KEEP", {make_stored_row("PLC_KEEP", "route_keep", kKeepAtNs, 10.0)}).has_value()); + ASSERT_TRUE( + store->replace_frames("PLC_STALE", {make_stored_row("PLC_STALE", "route_stale", kStaleAtNs, 11.0)}).has_value()); +} + } // namespace /// @verifies REQ_INTEROP_088 @@ -1265,6 +1350,109 @@ TEST_F(EntityFreezeFrameCaptureTest, AReloadedFrameFromTheSameOccurrenceIsKeptUn EXPECT_EQ((*rows)[0].captured_at_ns, ReoccurrenceHarness::kStoredAtNs); } +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, AStaleReReadAtTheBoundNeverCostsALiveFrame) { + // Two slots, both taken by reloaded frames, and a third standing fault with + // none. The stale one owns its slot and its replacement cannot exceed the + // bound, so it must be re-read in place. The bare one has no slot, so it goes + // unframed - and must not be let in against an occupancy that counts the + // stale code as absent, because the FIFO would then evict PLC_KEEP, which is + // a live frame for a fault that is still standing. + auto store = std::make_shared(); + seed_bound_probe(store); + SelectiveRouteFetcher plant(99.0, {}); + + testing::internal::CaptureStderr(); + { + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + [&plant](const std::string & entity_id) { + return plant(entity_id); + }, + /*max_faults=*/2, listed_faults(bound_probe_standing()), store); + + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (std::chrono::steady_clock::now() < deadline && plant.reads("route_stale") == 0) { + std::this_thread::sleep_for(20ms); + } + std::this_thread::sleep_for(500ms); // enough for an unbounded pass to admit PLC_NEW + + const auto keep = capture.frames_for("PLC_KEEP"); + ASSERT_EQ(keep.size(), 1u) << "the live reloaded frame was evicted"; + EXPECT_DOUBLE_EQ(keep[0].values.value("level", 0.0), 10.0); + EXPECT_EQ(keep[0].captured_at_ns, kKeepAtNs); + EXPECT_FALSE(keep[0].startup_catchup); + + const auto stale = capture.frames_for("PLC_STALE"); + ASSERT_EQ(stale.size(), 1u); + EXPECT_DOUBLE_EQ(stale[0].values.value("level", 0.0), 99.0); // re-read in place + EXPECT_TRUE(stale[0].startup_catchup); + EXPECT_GT(stale[0].captured_at_ns, kStaleAtNs); + + EXPECT_TRUE(capture.frames_for("PLC_NEW").empty()); // no slot for it + + EXPECT_EQ(plant.reads("route_keep"), 0); // a live frame is never re-read + EXPECT_EQ(plant.reads("route_stale"), 1); // the stale one is, exactly once + EXPECT_EQ(plant.reads("route_new"), 0); // refused before the plugin was touched + } + const auto logs = testing::internal::GetCapturedStderr(); + + EXPECT_EQ(stored_codes(store), (std::vector{"PLC_KEEP", "PLC_STALE"})); + EXPECT_NE(logs.find("PLC_NEW"), std::string::npos) << logs; // the truncation names it +} + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, AFailedStaleReReadFreesItsSlotForAFaultWithNoFrame) { + // Same probe, but the stale code's entity cannot answer. Its row is dropped, + // which genuinely frees a slot, so the bare fault now fits - and PLC_KEEP is + // still not the one that pays for it. + auto store = std::make_shared(); + seed_bound_probe(store); + SelectiveRouteFetcher plant(99.0, {"route_stale"}); + + testing::internal::CaptureStderr(); + { + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + [&plant](const std::string & entity_id) { + return plant(entity_id); + }, + /*max_faults=*/2, listed_faults(bound_probe_standing()), store); + + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (std::chrono::steady_clock::now() < deadline && capture.frames_for("PLC_NEW").empty()) { + std::this_thread::sleep_for(20ms); + } + + const auto keep = capture.frames_for("PLC_KEEP"); + ASSERT_EQ(keep.size(), 1u) << "the live reloaded frame was evicted"; + EXPECT_EQ(keep[0].captured_at_ns, kKeepAtNs); + EXPECT_FALSE(keep[0].startup_catchup); + + EXPECT_TRUE(capture.frames_for("PLC_STALE").empty()); // dropped, not served + + const auto fresh = capture.frames_for("PLC_NEW"); + ASSERT_EQ(fresh.size(), 1u) << "the freed slot was not reused"; + EXPECT_TRUE(fresh[0].startup_catchup); + + EXPECT_EQ(plant.reads("route_keep"), 0); + EXPECT_GE(plant.reads("route_stale"), 1); // it WAS asked before being dropped + EXPECT_EQ(plant.reads("route_new"), 1); + } + const auto logs = testing::internal::GetCapturedStderr(); + + EXPECT_EQ(stored_codes(store), (std::vector{"PLC_KEEP", "PLC_NEW"})); + EXPECT_NE(logs.find("PLC_STALE"), std::string::npos) << logs; + EXPECT_NE(logs.find("route_stale"), std::string::npos) << logs; + EXPECT_NE(logs.find("no freeze-frame"), std::string::npos) << logs; +} + TEST(ContentHasLiveData, GatesOnItemsNotOnTheLinkFlag) { using Capture = EntityFreezeFrameCapture; EXPECT_TRUE(Capture::content_has_live_data( @@ -1360,14 +1548,32 @@ TEST(StandingFaultsFromListReply, FirstOccurredIsReadInSecondsAndKeptInNanosecon json{{"fault_code", "NOT_A_NUMBER"}, {"reporting_sources", json::array({"a"})}, {"first_occurred", "yesterday"}}, - json{{"fault_code", "ZERO"}, {"reporting_sources", json::array({"a"})}, {"first_occurred", 0}}})}}; + json{{"fault_code", "ZERO"}, {"reporting_sources", json::array({"a"})}, {"first_occurred", 0}}, + // Past what int64 holds once multiplied out, so the cast + // itself is undefined. On x86-64 it lands on INT64_MIN, + // which is below every captured_at and would look like + // "this fault re-occurred", discarding a good frame. + json{{"fault_code", "HUGE"}, {"reporting_sources", json::array({"a"})}, {"first_occurred", 1e19}}, + json{{"fault_code", "INFINITE"}, + {"reporting_sources", json::array({"a"})}, + {"first_occurred", std::numeric_limits::infinity()}}, + json{{"fault_code", "NAN_SECONDS"}, + {"reporting_sources", json::array({"a"})}, + {"first_occurred", std::numeric_limits::quiet_NaN()}}})}}; const auto standing = EntityFreezeFrameCapture::standing_faults_from_list_reply(data); ASSERT_TRUE(standing.has_value()); - ASSERT_EQ(standing->size(), 4u); + ASSERT_EQ(standing->size(), 7u); + EXPECT_EQ((*standing)[0].fault_code, "SECONDS"); EXPECT_EQ((*standing)[0].first_occurred_ns, 1788948705500000000); - EXPECT_EQ((*standing)[1].first_occurred_ns, 0); - EXPECT_EQ((*standing)[2].first_occurred_ns, 0); - EXPECT_EQ((*standing)[3].first_occurred_ns, 0); + EXPECT_EQ((*standing)[1].first_occurred_ns, 0); // absent + EXPECT_EQ((*standing)[2].first_occurred_ns, 0); // not a number + EXPECT_EQ((*standing)[3].first_occurred_ns, 0); // zero + EXPECT_EQ((*standing)[4].fault_code, "HUGE"); + EXPECT_EQ((*standing)[4].first_occurred_ns, 0); + EXPECT_EQ((*standing)[5].fault_code, "INFINITE"); + EXPECT_EQ((*standing)[5].first_occurred_ns, 0); + EXPECT_EQ((*standing)[6].fault_code, "NAN_SECONDS"); + EXPECT_EQ((*standing)[6].first_occurred_ns, 0); } TEST(StandingFaultsFromListReply, RepliesNotShapedLikeListFaultsYieldNullopt) { From b4bf815dd3190b28b7a4384f50ba52c0be66b62d Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 9 Sep 2026 19:06:36 +0200 Subject: [PATCH 14/14] fix(gateway): ask a stale freeze-frame's entity once per catch-up, not twice Splitting the catch-up into two passes left a code the first pass had dropped looking, to the second, exactly like a fault that never had a frame. It is absent from frames_ for that very reason, so with a slot free the second pass called the plugin again for the same entity: a second blocking read of a link that had just failed to answer, once per unreachable stale entity at every start. When the link came back between the two reads it was worse than wasteful. The fault ended up holding a fresh frame in memory and on disk while the warning had already told the operator the stored frame was discarded and this occurrence had none, the summary still counted it as discarded with no replacement, and the slot the drop had freed went back to the code that lost it instead of to the fault still waiting for one. The first pass now records every code it settles, replaced or dropped alike, and the second skips them. One read per fault per catch-up, a dropped frame stays dropped for that catch-up, and its slot goes to the next fault with no frame, so the counters and the log lines say what actually happened. The branch's own test hid this behind EXPECT_GE on the read count, and only passed at all because in that one shape the other fault reached the second pass first and the bound then turned the stale code away. It now pins the count exactly, alongside the two shapes that show the defect directly. Also corrects a comment that had the failure mode backwards. An out-of-range cast is undefined, and it is the platform's answer that decides what happens: x86-64 gives INT64_MIN, which the non-positive check rejects, so the frame survives by luck, while a saturating target gives a large positive value that passes that check, sits above every captured_at and discards a good frame. The range guard is what makes the two behave the same. --- .../src/entity_freeze_frame_capture.cpp | 23 ++- .../test/test_entity_freeze_frame_capture.cpp | 137 +++++++++++++++++- 2 files changed, 152 insertions(+), 8 deletions(-) diff --git a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp index 90ddbe15b..d7de8333d 100644 --- a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp @@ -643,6 +643,13 @@ void EntityFreezeFrameCapture::capture_standing_faults() { // and while these codes are counted as absent a bare code admitted against // that under-count would FIFO-evict a live frame that is still wanted. So the // stale codes are settled first, and only then is the real occupancy known. + // + // Every code this pass touches is recorded, replaced or dropped alike. A + // replaced one is back in frames_ and pass 2 would skip it anyway, but a + // dropped one is not, and without this pass 2 would see a fault with no frame + // and a free slot and ask the same unreachable entity a second time. One + // blocking plugin read per catch-up per fault, and no more. + std::unordered_set settled; for (const auto & fault : standing) { if (should_abort()) { return; @@ -659,9 +666,11 @@ void EntityFreezeFrameCapture::capture_standing_faults() { // with no entity is stale to one and invisible to the other, and its row // survives to be served. drop_stale_frame(fault.fault_code, "", "the fault reports no entity to read"); + settled.insert(fault.fault_code); ++discarded; continue; } + settled.insert(fault.fault_code); ros2_medkit_msgs::msg::FaultEvent event; event.event_type = ros2_medkit_msgs::msg::FaultEvent::EVENT_CONFIRMED; event.fault.fault_code = fault.fault_code; @@ -674,7 +683,7 @@ void EntityFreezeFrameCapture::capture_standing_faults() { ++re_read; } else { drop_stale_frame(fault.fault_code, join_sources(fault.reporting_sources), "the entity served no usable values"); - ++discarded; // the slot it held is now free for the pass below + ++discarded; // the slot it held is now free for the NEXT fault, not for this one again } } @@ -709,9 +718,15 @@ void EntityFreezeFrameCapture::capture_standing_faults() { } // The stored frame is the one from this fault's own confirm edge. Re-reading // the plant now would replace it with today's values under a "startup" - // marker, which is exactly what persisting the frame is here to stop. This - // also covers a stale code the pass above just replaced. - if (already_framed.count(fault.fault_code) != 0) { + // marker, which is exactly what persisting the frame is here to stop. + // + // `settled` is the other half of that: pass 1 has already had its one go at + // every stale code, and a code it dropped is absent from frames_ precisely + // because its entity could not answer. Asking again in the same catch-up + // would be a second blocking read of an entity that just failed, and if it + // answered this time the fault would end up holding a frame the warning has + // already said it discarded. + if (already_framed.count(fault.fault_code) != 0 || settled.count(fault.fault_code) != 0) { continue; } if (framed >= max_faults_) { diff --git a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp index 79a109c17..cbce202ef 100644 --- a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp @@ -1138,6 +1138,38 @@ class SelectiveRouteFetcher { std::map reads_; }; +/// Route fetcher whose named entity fails its first N calls and answers after +/// that: a link that comes back between two reads inside one catch-up. +class FlakyRouteFetcher { + public: + FlakyRouteFetcher(double level, std::string flaky, int failures) + : level_(level), flaky_(std::move(flaky)), failures_left_(failures) { + } + + std::optional operator()(const std::string & entity_id) { + std::lock_guard lock(mutex_); + reads_[entity_id] += 1; + if (entity_id == flaky_ && failures_left_ > 0) { + --failures_left_; + return std::nullopt; + } + return json{{"connected", true}, {"items", json::array({{{"name", "level"}, {"value", level_}}})}}; + } + + int reads(const std::string & entity_id) { + std::lock_guard lock(mutex_); + auto it = reads_.find(entity_id); + return it == reads_.end() ? 0 : it->second; + } + + private: + double level_; + std::string flaky_; + int failures_left_; + std::mutex mutex_; + std::map reads_; +}; + /// Fault codes present in the store, sorted, for a whole-store assertion. std::vector stored_codes(const std::shared_ptr & store) { auto rows = store->load_all(); @@ -1442,7 +1474,7 @@ TEST_F(EntityFreezeFrameCaptureTest, AFailedStaleReReadFreesItsSlotForAFaultWith EXPECT_TRUE(fresh[0].startup_catchup); EXPECT_EQ(plant.reads("route_keep"), 0); - EXPECT_GE(plant.reads("route_stale"), 1); // it WAS asked before being dropped + EXPECT_EQ(plant.reads("route_stale"), 1); // asked once before being dropped, never twice EXPECT_EQ(plant.reads("route_new"), 1); } const auto logs = testing::internal::GetCapturedStderr(); @@ -1453,6 +1485,97 @@ TEST_F(EntityFreezeFrameCaptureTest, AFailedStaleReReadFreesItsSlotForAFaultWith EXPECT_NE(logs.find("no freeze-frame"), std::string::npos) << logs; } +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, AStaleCodeIsAskedOnceEvenWithASlotToSpare) { + // Three slots for three faults, so nothing competes and the bound never + // speaks. A stale code whose entity cannot answer is dropped, which leaves a + // fault with no frame and a free slot: exactly the shape in which a second + // pass would go back and block on the same dead entity all over again. + auto store = std::make_shared(); + seed_bound_probe(store); + SelectiveRouteFetcher plant(99.0, {"route_stale"}); + + testing::internal::CaptureStderr(); + { + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + [&plant](const std::string & entity_id) { + return plant(entity_id); + }, + /*max_faults=*/3, listed_faults(bound_probe_standing()), store); + + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (std::chrono::steady_clock::now() < deadline && capture.frames_for("PLC_NEW").empty()) { + std::this_thread::sleep_for(20ms); + } + std::this_thread::sleep_for(500ms); // enough for a second pass at route_stale to land + + EXPECT_EQ(plant.reads("route_stale"), 1); // one blocking read per catch-up, whatever the capacity + EXPECT_EQ(plant.reads("route_keep"), 0); + EXPECT_EQ(plant.reads("route_new"), 1); + EXPECT_TRUE(capture.frames_for("PLC_STALE").empty()); // dropped, and it stays dropped + ASSERT_EQ(capture.frames_for("PLC_KEEP").size(), 1u); + EXPECT_EQ(capture.frames_for("PLC_KEEP")[0].captured_at_ns, kKeepAtNs); + ASSERT_EQ(capture.frames_for("PLC_NEW").size(), 1u); + } + const auto logs = testing::internal::GetCapturedStderr(); + + EXPECT_EQ(stored_codes(store), (std::vector{"PLC_KEEP", "PLC_NEW"})); + EXPECT_NE(logs.find("PLC_STALE"), std::string::npos) << logs; +} + +/// @verifies REQ_INTEROP_088 +TEST_F(EntityFreezeFrameCaptureTest, AStaleCodeThatWouldAnswerOnASecondAskIsStillOnlyAskedOnce) { + // The entity fails once and would answer if asked again. Asking again is what + // must not happen: the warning has already told the operator the frame was + // discarded and this occurrence has none, so a frame appearing anyway would + // make the log a lie, the summary would count a discard that did not stick, + // and the slot freed by the drop would go back to the code that just lost it + // instead of to the fault still waiting for one. + auto store = std::make_shared(); + seed_bound_probe(store); + FlakyRouteFetcher plant(99.0, "route_stale", /*failures=*/1); + // STALE first in the reply, so nothing else can reach the bound before it. + auto standing = bound_probe_standing(); + std::swap(standing[0], standing[1]); + + testing::internal::CaptureStderr(); + { + EntityFreezeFrameCapture capture( + node_.get(), *sub_exec_, + [](const std::string &) -> DataProvider * { + return nullptr; + }, + [&plant](const std::string & entity_id) { + return plant(entity_id); + }, + /*max_faults=*/2, listed_faults(standing), store); + + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (std::chrono::steady_clock::now() < deadline && capture.frames_for("PLC_NEW").empty()) { + std::this_thread::sleep_for(20ms); + } + std::this_thread::sleep_for(500ms); + + EXPECT_EQ(plant.reads("route_stale"), 1); + EXPECT_TRUE(capture.frames_for("PLC_STALE").empty()) << "the discarded frame came back"; + ASSERT_EQ(capture.frames_for("PLC_KEEP").size(), 1u); + EXPECT_EQ(capture.frames_for("PLC_KEEP")[0].captured_at_ns, kKeepAtNs); + ASSERT_EQ(capture.frames_for("PLC_NEW").size(), 1u) << "the freed slot did not reach the waiting fault"; + } + const auto logs = testing::internal::GetCapturedStderr(); + + EXPECT_EQ(stored_codes(store), (std::vector{"PLC_KEEP", "PLC_NEW"})); + EXPECT_NE(logs.find("PLC_STALE"), std::string::npos) << logs; + EXPECT_NE(logs.find("no freeze-frame"), std::string::npos) << logs; + // The summary has to describe what actually happened, not what was attempted. + EXPECT_NE(logs.find("0 reloaded frame(s) from an earlier occurrence re-read, 1 discarded"), std::string::npos) + << logs; +} + TEST(ContentHasLiveData, GatesOnItemsNotOnTheLinkFlag) { using Capture = EntityFreezeFrameCapture; EXPECT_TRUE(Capture::content_has_live_data( @@ -1550,9 +1673,15 @@ TEST(StandingFaultsFromListReply, FirstOccurredIsReadInSecondsAndKeptInNanosecon {"first_occurred", "yesterday"}}, json{{"fault_code", "ZERO"}, {"reporting_sources", json::array({"a"})}, {"first_occurred", 0}}, // Past what int64 holds once multiplied out, so the cast - // itself is undefined. On x86-64 it lands on INT64_MIN, - // which is below every captured_at and would look like - // "this fault re-occurred", discarding a good frame. + // itself is undefined. What the platform then hands back + // decides the behaviour, which is the whole problem. On + // x86-64 it is INT64_MIN, which the non-positive check in + // stale_reloaded_codes happens to reject, so the frame + // survives by luck. A saturating target hands back a large + // POSITIVE value instead, which passes that check and sits + // above every captured_at, so the fault reads as re-occurred + // and a good frame is discarded. The range guard is what + // makes the outcome the same on both. json{{"fault_code", "HUGE"}, {"reporting_sources", json::array({"a"})}, {"first_occurred", 1e19}}, json{{"fault_code", "INFINITE"}, {"reporting_sources", json::array({"a"})},