From 3f01f493268e81484a0edcfabf3e26db7f257053 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 14:26:53 +0100 Subject: [PATCH 01/52] fault_manager: key every fault record by fault code and owner A fault record is identified by (fault_code, owner), where the owner is the source_id the report arrived with. Status, debounce, severity, occurrence count, freeze frames, snapshots, rosbag links, near misses and audit rows are per record, and clears, heals and every per-record service call act on one record. ClearFault, GetFault, GetSnapshots and GetRosbag gain a trailing source_id; an empty one is refused as ambiguous when several records share the code. MutedFaultInfo gains source_id. The SQLite store migrates on open. --- .github/copilot-instructions.md | 2 +- docs/api/messages.rst | 42 +- docs/config/fault-manager.rst | 111 +- docs/tutorials/docker.rst | 2 +- docs/tutorials/snapshots.rst | 6 +- .../action_status_bridge_node.hpp | 5 +- .../src/action_status_bridge_node.cpp | 11 +- .../test/test_action_status_bridge.cpp | 2 +- .../test/test_integration.test.py | 36 +- src/ros2_medkit_fault_manager/CMakeLists.txt | 9 + src/ros2_medkit_fault_manager/README.md | 162 +- .../config/fault_manager.yaml | 6 +- .../config/snapshots.yaml | 20 +- .../design/index.rst | 76 +- .../capture_thread_pool.hpp | 13 +- .../core/fault_manager_core.hpp | 57 +- .../correlation/correlation_engine.hpp | 78 +- .../fault_manager_node.hpp | 2 +- .../fault_storage.hpp | 255 ++-- .../ros2/ros_capture_hooks.hpp | 22 +- .../rosbag_capture.hpp | 63 +- .../snapshot_capture.hpp | 35 +- .../sqlite_fault_storage.hpp | 110 +- .../src/capture_thread_pool.cpp | 17 +- .../src/core/fault_manager_core.cpp | 272 ++-- .../src/core/fault_manager_startup.cpp | 4 +- .../src/correlation/correlation_engine.cpp | 119 +- .../src/fault_manager_node.cpp | 110 +- .../src/fault_storage.cpp | 210 +-- .../src/rosbag_capture.cpp | 117 +- .../src/snapshot_capture.cpp | 61 +- .../src/sqlite_fault_storage.cpp | 1101 +++++++++----- ..._fault_code_thresholds_integration.test.py | 156 +- .../test/integration/test_integration.test.py | 211 ++- .../integration/test_rosbag_history.test.py | 6 +- .../test_severity_consistency.test.py | 365 +++++ .../test/test_capture_thread_pool.cpp | 87 +- .../test/test_correlation_engine.cpp | 300 ++-- .../test/test_entity_thresholds.cpp | 43 +- .../test/test_fault_code_thresholds.cpp | 44 +- .../test/test_fault_manager.cpp | 1354 ++++++++++++++--- .../test_fault_manager_core_concurrency.cpp | 23 +- .../test/test_fault_severity.cpp | 893 +++++++++++ .../test/test_fault_thresholds.yaml | 3 +- .../test/test_rosbag_capture.cpp | 481 +++--- .../test/test_rosbag_storage_parity.cpp | 74 +- .../test/test_snapshot_capture.cpp | 116 +- .../test/test_sqlite_storage.cpp | 1154 +++++++++++--- .../test/test_integration.test.py | 43 +- src/ros2_medkit_gateway/README.md | 4 +- .../test_fault_severity_consistency.test.py | 287 ++++ src/ros2_medkit_msgs/README.md | 26 +- src/ros2_medkit_msgs/msg/Fault.msg | 15 +- src/ros2_medkit_msgs/msg/FaultEvent.msg | 3 +- src/ros2_medkit_msgs/msg/MutedFaultInfo.msg | 15 +- src/ros2_medkit_msgs/srv/ClearFault.srv | 11 +- src/ros2_medkit_msgs/srv/GetFault.srv | 8 +- src/ros2_medkit_msgs/srv/GetRosbag.srv | 9 +- src/ros2_medkit_msgs/srv/GetSnapshots.srv | 8 +- .../srv/ListFaultsForEntity.srv | 3 +- src/ros2_medkit_msgs/srv/ListRosbags.srv | 2 +- src/ros2_medkit_msgs/srv/ReportFault.srv | 10 +- 62 files changed, 6637 insertions(+), 2253 deletions(-) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py create mode 100644 src/ros2_medkit_fault_manager/test/test_fault_severity.cpp create mode 100644 src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index b8b36b43d..697495548 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -13,7 +13,7 @@ Multi-package colcon workspace under `src/`: | Package | Purpose | |---------|---------| | `ros2_medkit_gateway` | HTTP gateway - REST server, discovery, entity management, handlers, plugin framework | -| `ros2_medkit_fault_manager` | Fault aggregation with SQLite, AUTOSAR DEM-style debounce, rosbag/snapshot capture | +| `ros2_medkit_fault_manager` | Fault records per (code, reporting source) in SQLite, AUTOSAR DEM-style debounce, rosbag/snapshot capture | | `ros2_medkit_fault_reporter` | Client library for nodes to report faults | | `ros2_medkit_diagnostic_bridge` | Bridges `/diagnostics` topic to fault manager | | `ros2_medkit_serialization` | Runtime JSON <-> ROS 2 message serialization via dynmsg | diff --git a/docs/api/messages.rst b/docs/api/messages.rst index 1b796d66e..a4b6b4860 100644 --- a/docs/api/messages.rst +++ b/docs/api/messages.rst @@ -15,7 +15,12 @@ Messages Fault.msg ~~~~~~~~~ -Core fault data model representing an aggregated fault condition. +Core fault data model representing one fault record. + +A record is identified by the pair (``fault_code``, owning reporting source). The owner is +the ``source_id`` a ``ReportFault`` call carried. Two sources reporting one ``fault_code`` +are two records, each with its own status, debounce counter, ``occurrence_count``, severity +and timestamps, and each cleared on its own. .. code-block:: text @@ -49,7 +54,8 @@ Core fault data model representing an aggregated fault condition. # Current fault status (PREFAILED, PREPASSED, CONFIRMED, HEALED, CLEARED) string status - # List of source identifiers that have reported this fault + # The reporting source that owns this record, as a one-element list. Together + # with fault_code it identifies the record. string[] reporting_sources **Severity Constants:** @@ -146,7 +152,10 @@ prefix (for example ``/robot1/fault_manager/events``). MutedFaultInfo.msg ~~~~~~~~~~~~~~~~~~ -Information about correlated (muted) symptom faults. +Information about a correlated (muted) symptom record. One entry per muted RECORD: +a root cause mutes the symptoms of its own reporting source only, so two sources +muted on one ``fault_code`` produce two entries carrying that code, told apart by +``source_id``. .. code-block:: text @@ -154,6 +163,7 @@ Information about correlated (muted) symptom faults. string root_cause_code # Root cause that triggered muting string rule_id # Correlation rule ID that matched uint32 delay_ms # Time delay from root cause [ms] + string source_id # Reporting source that owns the muted record ClusterInfo.msg ~~~~~~~~~~~~~~~ @@ -189,7 +199,8 @@ Report a fault event to the FaultManager. uint8 event_type # EVENT_FAILED (0) or EVENT_PASSED (1) uint8 severity # Fault.SEVERITY_* constant (for FAILED events) string description # Human-readable description - string source_id # Fully qualified node name (e.g., "/powertrain/temp_sensor") + string source_id # Fully qualified node name (e.g., "/powertrain/temp_sensor"). + # Owns the record: (fault_code, source_id) is the record identity. **Response:** @@ -215,7 +226,7 @@ Report a fault event to the FaultManager. ClearFault.srv ~~~~~~~~~~~~~~ -Clear/acknowledge a fault. +Clear/acknowledge one fault record. **Request:** @@ -223,15 +234,23 @@ Clear/acknowledge a fault. string fault_code # Fault code to clear bool skip_correlation_auto_clear # Opt out of correlation cascade clear + string source_id # Reporting source that owns the record **Response:** .. code-block:: text - bool success # True if fault was found and cleared + bool success # True if the record was found and cleared string message # Status message or error description string[] auto_cleared_codes # Symptoms auto-cleared with root cause +``source_id`` names the owner of the record to clear. Leaving it empty is unscoped: the +call applies only when exactly one record carries the ``fault_code``, and fails otherwise +with ``success=false`` and a ``message`` beginning ``ambiguous:`` that lists the owners. +An ambiguous call clears nothing. ``GetFault``, ``GetSnapshots`` and ``GetRosbag`` carry +the same field and resolve the same way. On ``GetRosbag`` it scopes the ``fault_code`` +lookup only, and the ``recording_id`` path ignores it. + When ``skip_correlation_auto_clear`` is ``false`` (default), clearing a root-cause fault also clears every symptom that the correlation engine attributes to it via ``auto_clear_with_root`` rules; the cleared @@ -246,11 +265,12 @@ cluster-wide clearing still works. .. note:: - Added in ``ros2_medkit_msgs`` post-0.4.0. Adding a request field - changes the service type hash, so out-of-tree callers that invoke - ``/fault_manager/clear_fault`` directly via ``ros2 service call`` or - a generated client must rebuild against the new ``ros2_medkit_msgs`` - release to keep talking to ``fault_manager``. + ``skip_correlation_auto_clear`` was added in ``ros2_medkit_msgs`` post-0.4.0, + and ``source_id`` after it (the same field was added to ``GetFault``, + ``GetSnapshots`` and ``GetRosbag``). Adding a request field changes the + service type hash, so out-of-tree callers that invoke those services directly + via ``ros2 service call`` or a generated client must rebuild against the new + ``ros2_medkit_msgs`` release to keep talking to ``fault_manager``. ListFaults.srv ~~~~~~~~~~~~~~ diff --git a/docs/config/fault-manager.rst b/docs/config/fault-manager.rst index e7590a081..2075dfab4 100644 --- a/docs/config/fault-manager.rst +++ b/docs/config/fault-manager.rst @@ -1,7 +1,8 @@ Fault Manager Configuration =========================== -The ``ros2_medkit_fault_manager`` node aggregates and manages faults from multiple sources. +The ``ros2_medkit_fault_manager`` node keeps and manages one fault record per +(``fault_code``, reporting source) pair. This page documents all configuration parameters. .. contents:: Table of Contents @@ -199,6 +200,21 @@ The fault manager uses AUTOSAR DEM-style debounce filtering to prevent fault fla which is also the default. ``0`` is rejected: the threshold must be strictly negative, and the node falls back to ``-1`` with a warning. Faults with ``SEVERITY_CRITICAL`` always bypass debounce regardless of this setting. + The bypass (``critical_immediate_confirm`` in the debounce config, on by default and not a ROS + parameter) reads the **stored** severity of the fault record, not the severity of the report. + A record keeps the highest severity its source ever reported, across occurrences, so after a + CRITICAL occurrence is cleared, one WARN report from the same source confirms the next + occurrence at once, and the record shows CRITICAL. The same holds for a FAILED report on a + HEALED CRITICAL record: the latch does not hold it back. Another source's record of the same + code has its own severity and is not affected. + +.. note:: + + A FAILED report whose severity differs from the stored severity of its record is logged as a + warning naming the code, both severities and the source. One source should report one code + with one severity, so the warning points at a misconfigured reporter. It is logged once per + code, source and reported severity until the node restarts, so a reporter that repeats the + report at a high rate produces one line. .. important:: @@ -235,25 +251,26 @@ A **near miss** is a FAILED report that moved the debounce counter without the f CONFIRMED - the fault nearly happened. PASSED reports move the counter in the healing direction (the fault receding) and are not near misses. -The fault manager appends one entry per near miss to a per-fault-code series, holding the -timestamp, the counter value after the report, the confirmation threshold, the severity, the -reporting source and the fault status the report left behind. +The fault manager appends one entry per near miss to the series of the fault record it moved (one +series per fault code and reporting source), holding the timestamp, the counter value after the +report, the confirmation threshold, the severity the report carried (not the stored severity of the +record), the reporting source and the fault status the report left behind. The status matters when reading the series. The HEALED latch holds the status all the way from the healing threshold down to the confirmation threshold, so reports on the way back into a fault that does confirm are also near misses by the definition above. Entries recording ``PREFAILED`` are approaches from a resting state; entries recording ``HEALED`` are a counter walking back down under -the latch. The recorded confirmation threshold belongs to the reporting source, while the counter -is shared by all sources of that fault code, so with per-entity thresholds it is not on its own the -distance to confirmation. The series is **retained when the fault is cleared**, because acknowledging one -fault cycle must not erase how often that code approached confirmation across cycles. +the latch. The recorded confirmation threshold and the counter both belong to the record's own +reporting source, so with per-entity thresholds each entry still reads as the record's distance to +confirmation. The series is **retained when the fault is cleared**, because acknowledging one +fault cycle must not erase how often that record approached confirmation across cycles. .. code-block:: yaml fault_manager: ros__parameters: near_miss: - max_per_fault: 200 # Entries kept per fault code (0 = unlimited) + max_per_fault: 200 # Entries kept per fault record (0 = unlimited) .. list-table:: :header-rows: 1 @@ -264,7 +281,7 @@ fault cycle must not erase how often that code approached confirmation across cy - Description * - ``near_miss.max_per_fault`` - ``200`` - - Near-miss entries retained per fault code. When the bound is reached the **oldest** + - Near-miss entries retained per fault record. When the bound is reached the **oldest** entries are evicted, the same direction as ``snapshots.max_per_fault`` and the rosbag cap: a series frozen at boot says nothing about whether the rate of near misses is changing. Set to 0 for unlimited, accepting growth with the reporting rate. @@ -329,8 +346,8 @@ threshold overrides: - The ``source_id`` is the identifier passed in ``ReportFault`` service requests, typically the fully qualified name of the reporting ROS 2 node (e.g., ``/sensors/lidar/front_node``). - You can inspect actual ``source_id`` values in the ``reporting_sources`` field of existing - faults via ``GET /api/v1/faults``. + It also owns the record it creates, so you can inspect actual ``source_id`` values in the + one-element ``reporting_sources`` field of existing faults via ``GET /api/v1/faults``. - The ``source_id`` from ``ReportFault`` requests is matched against configured prefixes. - The **longest matching prefix** wins. For example, ``/sensors/lidar/front`` matches ``/sensors/lidar`` over ``/sensors``. @@ -340,12 +357,11 @@ threshold overrides: .. note:: - When multiple entities report the same ``fault_code``, each event applies the - thresholds resolved from that event's ``source_id``. This means the debounce - behavior follows the reporting entity, not the fault. The debounce counter, - however, belongs to the fault code, so the two entities share one counter under - two policies - see `Per-Fault-Code Thresholds`_ for what that does and how to - settle it. The node warns when it happens. + When multiple entities report the same ``fault_code``, each of them owns its own + record and each event applies the thresholds resolved from that event's + ``source_id``. The debounce counter belongs to the same record, so an entity's + configured band governs exactly the counter its own reports move: one entity's + reports can neither confirm nor heal another entity's fault. ``auto_confirm_after_sec`` is global-only and cannot be overridden per-entity. Critical faults skip debounce and confirm on their first occurrence; that is @@ -355,18 +371,18 @@ Per-Fault-Code Thresholds ~~~~~~~~~~~~~~~~~~~~~~~~~ Per-entity thresholds are the right tool when you know which subsystems are noisy but not -which codes they will emit. They have one limit: the debounce counter is kept per -``fault_code``, while the override is chosen per ``source_id``. Two entities reporting one -code therefore share a counter and debounce it under two policies, and the report that -happens to arrive decides the transition: +which codes they will emit. They have one limit: the override is chosen per ``source_id``, so +two entities reporting one code debounce it under two different policies, each on its own +record's counter: .. code-block:: text - motor (confirmation_threshold=-5) reports OVERHEAT -> counter=-1, PREFAILED - lidar (confirmation_threshold=-1) reports OVERHEAT -> counter=-1, CONFIRMED + motor (confirmation_threshold=-5) reports OVERHEAT -> motor's record: counter=-1, PREFAILED + lidar (confirmation_threshold=-1) reports OVERHEAT -> lidar's record: counter=-1, CONFIRMED -The motor's policy was bypassed. Per-fault_code thresholds are the layer that removes -that: they are matched on the code itself, so they resolve the same whoever reports. +When the policy should follow the code and not the reporter, per-fault_code thresholds are +the layer that does it: they are matched on the code itself, so they resolve the same +whoever reports. .. code-block:: yaml @@ -415,35 +431,14 @@ override carries: - Like the entity file, this one is loaded once at node startup. Changes require a restart. - At startup, a HEALED fault is reclassified to CLEARED when its code has healing disabled. The code's entry in this file decides, and the global ``healing_enabled`` applies when the - entry does not set it. Entity overrides are not used here, because a stored fault belongs - to its code and can have several sources. + entry does not set it. Entity overrides are not used here: the pass runs before any source + has reported, so it has no ``source_id`` to match a prefix against. .. note:: An override that names only some fields settles only those. If ``MOTOR_OVERHEAT`` pins ``confirmation_threshold`` but not ``healing_threshold``, the healing direction still - follows whichever entity reported, and the node still reports the conflict below. Pin - all three fields to settle a code completely. - -**When one code is debounced two ways** - -The node resolves the policy for every report, and warns the first time two sources resolve -different policies for one code: - -.. code-block:: text - - [WARN] Fault code 'OVERHEAT' is debounced two ways: '/powertrain/motor/left' resolves - confirmation=-5 healing_enabled=false healing=10, '/sensors/lidar/front' resolves - confirmation=-1 healing_enabled=true healing=1. The debounce counter belongs to the - fault code, so whichever source reports decides the transition and the other policy is - bypassed. Pin confirmation_threshold, healing_enabled and healing_threshold for this code - in fault_thresholds.config_file. - -The warning names both sources and both resolved policies, and is emitted **once per fault -code** for the life of the node, so a busy reporter does not turn it into a log storm. It -is a diagnostic, not an error: the configuration is legal and the fault is still debounced, -just not under a policy an operator chose. Giving the code an entry in -``fault_thresholds.config_file`` that pins all three fields ends it. + follows whichever entity reported. Pin all three fields to settle a code completely. Snapshot Configuration ---------------------- @@ -464,8 +459,8 @@ Basic Snapshot Settings max_message_size: 65536 # Max message size in bytes (64KB) default_topics: [] # Topics to capture for all faults config_file: "" # Path to YAML config file - recapture_cooldown_sec: 60.0 # Min seconds between snapshot captures per fault - max_per_fault: 10 # Max snapshots stored per fault code (0 = unlimited) + recapture_cooldown_sec: 60.0 # Min seconds between snapshot captures per fault record + max_per_fault: 10 # Max snapshots stored per fault record (0 = unlimited) capture_pool_size: 2 # Max concurrent capture threads (1-64) capture_queue_depth: 16 # Max pending captures before policy applies (1-65536) capture_queue_full_policy: reject_newest # reject_newest | drop_oldest @@ -506,11 +501,11 @@ Basic Snapshot Settings - Path to YAML file with fault-specific snapshot configurations. * - ``snapshots.recapture_cooldown_sec`` - ``60.0`` - - Minimum seconds between snapshot captures for the same fault code. + - Minimum seconds between snapshot captures for the same fault record. Prevents snapshot storms when a fault is reported repeatedly. Set to 0 to disable. * - ``snapshots.max_per_fault`` - ``10`` - - Maximum number of snapshot rows stored per fault code. One confirmation + - Maximum number of snapshot rows stored per fault record. One confirmation writes one row per configured topic, and those rows are evicted together: past the limit the OLDEST capture set is dropped whole. A capture larger than the cap is kept anyway rather than torn, since half a freeze frame is @@ -563,7 +558,7 @@ Capture continuous rosbag recordings around fault events. max_buffer_mb: 256 # Ring-buffer RAM cap max_bag_size_mb: 50 # Max size per bag file max_total_storage_mb: 500 # Max total storage - max_bags_per_fault: 1 # Recordings kept per fault code + max_bags_per_fault: 1 # Recordings kept per fault record auto_cleanup: true # Auto-delete old bags .. list-table:: @@ -645,7 +640,7 @@ Capture continuous rosbag recordings around fault events. limit until a newer recording replaces it. * - ``rosbag.max_bags_per_fault`` - ``1`` - - How many recordings one fault code keeps. Past the cap the oldest is + - How many recordings one fault record keeps. Past the cap the oldest is unlinked, so the default reproduces the historical behaviour exactly: a new recording replaces the previous one. ``0`` means unlimited, bounded only by ``max_total_storage_mb``. ``3`` is a reasonable value for a fault @@ -784,6 +779,10 @@ An append-only, hash-chained record of fault state transitions, for deployments that have to show afterwards what the fault state was and when it changed. Off by default: with it off there is no table, no file and no write cost. +Every row records the stored severity of the fault record at the transition, +the value ``GET /faults`` and the fault events show. The severity a single +report carried is in the near-miss series and in the severity mismatch warning. + .. code-block:: yaml fault_manager: diff --git a/docs/tutorials/docker.rst b/docs/tutorials/docker.rst index 4e898f8b9..853f2c3f9 100644 --- a/docs/tutorials/docker.rst +++ b/docs/tutorials/docker.rst @@ -52,7 +52,7 @@ Images are available for all supported ROS 2 distributions: Each image includes the gateway and all open-core packages: - ``ros2_medkit_gateway`` - HTTP REST server -- ``ros2_medkit_fault_manager`` - Fault aggregation and management +- ``ros2_medkit_fault_manager`` - Fault record keeping and lifecycle management - ``ros2_medkit_fault_reporter`` - Client library for fault reporting - ``ros2_medkit_diagnostic_bridge`` - Bridges ``/diagnostics`` to fault manager - ``ros2_medkit_serialization`` - Runtime JSON/ROS 2 serialization diff --git a/docs/tutorials/snapshots.rst b/docs/tutorials/snapshots.rst index aa2dc16b6..91d272535 100644 --- a/docs/tutorials/snapshots.rst +++ b/docs/tutorials/snapshots.rst @@ -122,11 +122,11 @@ Configure snapshot capture via fault manager parameters: appears after the fault manager starts is subscribed once it does. * - ``snapshots.recapture_cooldown_sec`` - ``60.0`` - - Minimum seconds between snapshot captures for the same fault code. + - Minimum seconds between snapshot captures for the same fault record. Prevents snapshot storms when a fault is reported repeatedly. Set to 0 to disable. * - ``snapshots.max_per_fault`` - ``10`` - - Maximum number of snapshot rows stored per fault code. One confirmation + - Maximum number of snapshot rows stored per fault record. One confirmation writes one row per configured topic, and those rows are evicted together: past the limit the OLDEST capture set is dropped whole. A capture larger than the cap is kept anyway rather than torn, since half a freeze frame @@ -593,7 +593,7 @@ Rosbag Configuration Options burst's bag at a time. * - ``snapshots.rosbag.max_bags_per_fault`` - ``1`` - - Recordings kept per fault code; ``0`` means unlimited. Past the cap the + - Recordings kept per fault record (``0`` means unlimited). Past the cap the fault's oldest recording is dropped, and a bag is deleted only once no fault still references it (a burst shares one recording). ``1`` is the historical behaviour - each re-confirmation replaces the previous bag; diff --git a/src/ros2_medkit_action_status_bridge/include/ros2_medkit_action_status_bridge/action_status_bridge_node.hpp b/src/ros2_medkit_action_status_bridge/include/ros2_medkit_action_status_bridge/action_status_bridge_node.hpp index 8fe9f89f6..b7f23ec02 100644 --- a/src/ros2_medkit_action_status_bridge/include/ros2_medkit_action_status_bridge/action_status_bridge_node.hpp +++ b/src/ros2_medkit_action_status_bridge/include/ros2_medkit_action_status_bridge/action_status_bridge_node.hpp @@ -142,8 +142,9 @@ class ActionStatusBridgeNode : public rclcpp::Node { /// Get (creating on first use) the FaultReporter for an action. The reporter's /// source_id is fixed when first created: the resolved server FQN if discovery /// has settled, otherwise the action name as a fallback so the fault still fires - /// on time. It is never re-attributed afterwards (reporting_sources is - /// append-only and the per-entity scope filter is strict-AND). + /// on time. It is never re-attributed afterwards: the fault manager keeps one + /// record per (fault code, source), so a swapped source would open a second + /// record and leave the provisional one raised with nothing left to heal it. ros2_medkit_fault_reporter::FaultReporter * reporter_for(const std::string & action_name); /// Resolve the action server's node FQN from its status-topic publisher, for diff --git a/src/ros2_medkit_action_status_bridge/src/action_status_bridge_node.cpp b/src/ros2_medkit_action_status_bridge/src/action_status_bridge_node.cpp index c3c665686..493a36ee1 100644 --- a/src/ros2_medkit_action_status_bridge/src/action_status_bridge_node.cpp +++ b/src/ros2_medkit_action_status_bridge/src/action_status_bridge_node.cpp @@ -535,11 +535,12 @@ ros2_medkit_fault_reporter::FaultReporter * ActionStatusBridgeNode::reporter_for // so the fault still fires on time. Reporting the fault on time takes priority // over entity attribution when discovery is slow. // - // The source is NOT re-attributed later: reporting_sources is append-only on the - // manager side and the per-entity /faults scope filter is strict-AND, so a - // provisional action-name source cannot be swapped for the FQN afterwards. - // Correct attribution for the slow-discovery case is a separate concern (it - // needs a way to supersede a provisional source). + // The source is NOT re-attributed later: the fault manager keeps one record per + // (fault code, source), so swapping the provisional action-name source for the + // FQN would open a second record under the FQN and leave the provisional one + // raised, with no reporter left to send its PASSED. Correct attribution for the + // slow-discovery case is a separate concern (it needs a way to supersede a + // provisional source). const std::string fqn = server_fqn_for_action(action_name); const std::string & source_id = fqn.empty() ? action_name : fqn; auto reporter = std::make_unique(this->shared_from_this(), source_id); diff --git a/src/ros2_medkit_action_status_bridge/test/test_action_status_bridge.cpp b/src/ros2_medkit_action_status_bridge/test/test_action_status_bridge.cpp index 7bf4aa249..fdbbb589b 100644 --- a/src/ros2_medkit_action_status_bridge/test/test_action_status_bridge.cpp +++ b/src/ros2_medkit_action_status_bridge/test/test_action_status_bridge.cpp @@ -283,7 +283,7 @@ TEST_F(ActionStatusBridgeTest, ReporterFor_StickyCreatedOnceNeverSwapped) { ActionStatusBridgeTestAccess access(node.get()); // No publisher exists, so the server FQN is unresolved and the reporter falls // back to the action name. It must be created once and reused, never swapped: - // reporting_sources is append-only, so a provisional source cannot be undone. + // a swapped source would open a second fault record and strand the first. const void * first = access.reporter_identity("/nav"); EXPECT_NE(first, nullptr); EXPECT_EQ(access.reporter_identity("/nav"), first); diff --git a/src/ros2_medkit_diagnostic_bridge/test/test_integration.test.py b/src/ros2_medkit_diagnostic_bridge/test/test_integration.test.py index 64931ef1e..d2b5e521e 100644 --- a/src/ros2_medkit_diagnostic_bridge/test/test_integration.test.py +++ b/src/ros2_medkit_diagnostic_bridge/test/test_integration.test.py @@ -153,7 +153,7 @@ def publish_diagnostic(self, name, level, message='Test message', hardware_id='t # Give time for message to be processed time.sleep(0.3) - def publish_until(self, name, level, expected_code, *, predicate=None, + def publish_until(self, name, level, expected_code, *, predicate=None, owner=None, message='Test message', hardware_id='test_hw', statuses=None, timeout=25.0): """ Republish a diagnostic until a matching fault satisfies *predicate*. @@ -167,6 +167,10 @@ def publish_until(self, name, level, expected_code, *, predicate=None, Re-emission also drives WARN/PASSED past their occurrence threshold within the filter window deterministically. + A record is the pair (fault code, reporting source), so one code can + name several records. With *owner* set, only the record that source + owns is considered. + Returns the matching fault once *predicate* holds; fails at the deadline. """ @@ -179,15 +183,17 @@ def predicate(_fault): self.publish_diagnostic(name, level, message, hardware_id) fault = next( (f for f in self.list_faults(statuses=statuses) - if f.fault_code == expected_code), + if f.fault_code == expected_code + and (owner is None or list(f.reporting_sources) == [owner])), None, ) if fault is not None: last = fault if predicate(fault): return fault + owned = f' owned by {owner}' if owner is not None else '' self.fail( - f'Fault {expected_code} not satisfied within {timeout}s ' + f'Fault {expected_code}{owned} not satisfied within {timeout}s ' f'(last seen: {last})' ) @@ -253,26 +259,30 @@ def test_05_fault_code_generation(self): diag_name, DiagnosticStatus.ERROR, expected_code, ) - def test_06_hardware_id_is_forwarded_as_source(self): - """Test that hardware_id is forwarded into fault reporting_sources.""" + def test_06_each_forwarded_hardware_id_owns_its_own_record(self): + """Test that each forwarded hardware_id reports the code as its own record.""" self.publish_until( 'shared_sensor', DiagnosticStatus.STALE, 'SHARED_SENSOR', message='No data from source A', hardware_id='/my_lidar_driver', - predicate=lambda f: '/my_lidar_driver' in f.reporting_sources, + owner='/my_lidar_driver', ) - - fault = self.publish_until( + self.publish_until( 'shared_sensor', DiagnosticStatus.STALE, 'SHARED_SENSOR', message='No data from source B', hardware_id='/my_camera_driver', - predicate=lambda f: '/my_lidar_driver' in f.reporting_sources - and '/my_camera_driver' in f.reporting_sources, + owner='/my_camera_driver', ) - self.assertEqual(fault.severity, Fault.SEVERITY_CRITICAL) - self.assertIn('/my_lidar_driver', fault.reporting_sources) - self.assertIn('/my_camera_driver', fault.reporting_sources) + # One diagnostic name forwarded under two hardware ids is two records + # of one code, each owned by the hardware id it came from. + records = [f for f in self.list_faults() if f.fault_code == 'SHARED_SENSOR'] + self.assertEqual( + sorted(list(f.reporting_sources) for f in records), + [['/my_camera_driver'], ['/my_lidar_driver']], + ) + for record in records: + self.assertEqual(record.severity, Fault.SEVERITY_CRITICAL, record.reporting_sources) def test_07_empty_hardware_id_falls_back_to_bridge_source(self): """Test that empty hardware_id falls back to the bridge node FQN.""" diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index 08649ed0f..ad71bde55 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -220,6 +220,11 @@ endif() target_link_libraries(test_rosbag_storage_parity fault_manager_lib) medkit_target_dependencies(test_rosbag_storage_parity rclcpp ros2_medkit_msgs) + # Severity contract of a fault record, run against both storage backends. + medkit_add_gtest(test_fault_severity test/test_fault_severity.cpp) + target_link_libraries(test_fault_severity fault_manager_lib) + medkit_target_dependencies(test_fault_severity rclcpp ros2_medkit_msgs) + # Fault audit log tests (hash chain, verify, rotation, reopen) medkit_add_gtest(test_fault_audit_log test/test_fault_audit_log.cpp) target_link_libraries(test_fault_audit_log fault_manager_lib) @@ -305,6 +310,10 @@ endif() medkit_add_launch_test(test_auto_confirm_visibility test/integration/test_auto_confirm_visibility.test.py TIMEOUT 120 LABELS "integration") + # Restarts the node twice on one SQLite database. + medkit_add_launch_test(test_severity_consistency test/integration/test_severity_consistency.test.py + TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_rosbag_entity_scope test/integration/test_rosbag_entity_scope.test.py TIMEOUT 120 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/README.md b/src/ros2_medkit_fault_manager/README.md index 47d2cc55e..38dbf66e1 100644 --- a/src/ros2_medkit_fault_manager/README.md +++ b/src/ros2_medkit_fault_manager/README.md @@ -4,9 +4,65 @@ Central fault manager node for the ros2_medkit fault management system. ## Overview -The FaultManager node provides a central point for fault aggregation and lifecycle management. -It receives fault reports from multiple sources, aggregates them by `fault_code`, and provides -query and clearing interfaces. +The FaultManager node provides a central point for fault record keeping and lifecycle management. +It receives fault reports from multiple sources, keeps one record per `(fault_code, reporting +source)` pair, and provides query and clearing interfaces. + +A record is identified by its `fault_code` and by the `source_id` the `ReportFault` call carried. +That source is the record's **owner**. Two sources reporting one `fault_code` are two records, each +with its own status, debounce counter, occurrence count, severity, timestamps, freeze frame, +snapshots, near-miss series and rosbag links. A clear or an auto-heal driven by one owner never +touches another owner's record. The services that act on a single record (`~/clear_fault`, +`~/get_fault`, `~/get_snapshots`, `~/get_rosbag`) carry a `source_id` request field naming the +owner. Leaving it empty is unscoped and applies only when exactly one record carries the +`fault_code`, otherwise the call fails with a message beginning `ambiguous:` that lists the +owners and changes nothing. + +### Upgrading a database written before the owner column + +A SQLite store written by an earlier release is keyed by the bare `fault_code` and is rebuilt on +the first open. The owner of each migrated record is the first entry of that row's legacy +`reporting_sources` value. A row that listed several sources folds onto its first one, because the +old schema recorded no per-source counter, status or timestamps to split it by, and inventing them +would put numbers in the store that no report ever produced. `confirmed_at_ns` is copied as it was, +except on a PREFAILED or PREPASSED row, where it becomes 0 (see Storage Backends below). + +The rebuild is one way. A fault manager from an earlier release opened on a migrated store aborts +on the first report of a fault code the store does not hold yet +(`NOT NULL constraint failed: faults.owner`), and on a fault code two sources share it writes one +source's data into the other source's row. If a rollback may be needed, keep a copy of `faults.db` +from before the upgrade and restore that copy instead of pointing the earlier release at the +migrated file. + +The recovery reads that column as text rather than requiring it to be valid JSON, because earlier +builds escaped only the quote, the backslash and `\b \f \n \r \t`, so a `source_id` carrying any +other control byte was written as text no JSON parser accepts. The mapping is exact: + +| stored `reporting_sources` | migrated owner | +|---|---| +| empty | `legacy` | +| `sensor_a` (a bare word, not an array) | `sensor_a` | +| `[]` | `legacy` | +| `["a<0x01>b"]` | `a<0x01>b` (the control byte is kept) | +| anything that is neither a JSON array nor a bare word (an object wrapper, a value behind a byte-order mark) | `legacy` | + +A migrated record never carries an empty owner. Where no source can be read the record gets the +synthetic owner `legacy` and one warning names it, so it stays addressable through `source_id` like +any other record: `ClearFault` and `GetFault` take `legacy` and act on it alone, and it appears +under that name in an `ambiguous:` refusal. New rows are always written as valid JSON, control +bytes escaped as `\u00XX`. + +The empty owner therefore means exactly one thing: an evidence row (freeze frame, snapshot, rosbag +link) not yet assigned to a record. Such a row takes the owner of its fault code when that code has +exactly one owner. When it has several, or none, the row keeps the empty owner and is named in a +warning rather than handed to an owner the database cannot prove it belongs to. That backfill runs +on any open that finds a row it can actually assign, not only on the open that adds the column, so +a store left half-migrated by an interrupted run is healed the next time it is opened, while a row +nothing can resolve is left alone instead of being retried forever. + +The table names `faults_new` and `freeze_frames_new` are reserved for the rebuild. Anything found +under those names is debris from a rebuild that did not finish and is dropped on open, with a +warning naming the table. ## Quick Start @@ -46,15 +102,18 @@ ros2 service call /fault_manager/clear_fault ros2_medkit_msgs/srv/ClearFault \ ## Features -- **Multi-source aggregation**: Same `fault_code` from different sources creates a single fault +- **Per-source records**: Same `fault_code` from different sources creates one record per source, + each filtered, cleared and healed on its own - **Occurrence tracking**: Counts outages, not reports - the count starts at one and rises only - when a cleared fault is raised again - and tracks all reporting sources -- **Severity escalation**: Fault severity is updated if a higher severity is reported + when a cleared record is raised again by its own owner +- **Severity escalation**: A record has one severity, the highest its owner has reported, kept + within an occurrence and across occurrences. A report with a different severity is logged (see + [Severity](#severity)) - **Persistent storage**: SQLite (default) or PostgreSQL backend ensures faults survive node restarts - **Debounce filtering** (optional): AUTOSAR DEM-style counter-based fault confirmation with per-entity and per-fault_code threshold overrides - **Snapshot capture**: Captures topic data when faults are confirmed for debugging (the value snapshots are deleted when the fault is cleared, unless `snapshots.retain_on_clear` is set) -- **Near-miss series**: Appends one entry per FAILED report that moved the debounce counter without confirming, bounded per fault code and retained when the fault is cleared -- **Freeze-frame retention**: One compact JSON freeze-frame per fault code, retained across `clear_fault` (see below) +- **Near-miss series**: Appends one entry per FAILED report that moved the debounce counter without confirming, bounded per record and retained when the record is cleared +- **Freeze-frame retention**: One compact JSON freeze-frame per record, retained across `clear_fault` (see below) - **Fault correlation** (optional): Root cause analysis with symptom muting and auto-clear - **Tamper-evident audit log** (optional): Append-only, hash-chained record of fault state transitions for verifiable history @@ -72,13 +131,13 @@ ros2 service call /fault_manager/clear_fault ros2_medkit_msgs/srv/ClearFault \ | `report_fault_queue_depth` | int | `100` | Requests `~/report_fault` holds until read (1-65536, read-only); past it they are dropped without an answer | | `entity_thresholds.config_file` | string | `""` | Path to YAML file with per-entity debounce threshold overrides | | `fault_thresholds.config_file` | string | `""` | Path to YAML file with per-fault_code debounce threshold overrides, applied on top of the entity ones | -| `near_miss.max_per_fault` | int | `200` | Near-miss entries retained per fault code, oldest evicted first (0 = unlimited) | +| `near_miss.max_per_fault` | int | `200` | Near-miss entries retained per fault record, oldest evicted first (0 = unlimited) | ### Snapshot Parameters Snapshots capture topic data when faults are confirmed for post-mortem debugging. -Each confirm also writes a **freeze-frame**: a single compact JSON object mapping every captured topic to its value at confirmation time, keyed by fault code. It differs from per-topic snapshots in two ways: snapshots are deleted when the fault is cleared, while the freeze-frame is retained across `clear_fault` (once the snapshots are gone, `~/get_fault` serves the retained frame so the confirmed-state record stays available after acknowledgement); and a re-confirm that captures nothing (e.g. source publishers down) never overwrites an existing non-empty frame. A fault code with no configured capture set gets no freeze-frame row; a configured capture that samples nothing on its first run records an empty `{}` frame. Freeze-frame storage is bounded by the number of distinct fault codes (one row per code, replaced in place) and rows are never evicted. +Each confirm also writes a **freeze-frame**: a single compact JSON object mapping every captured topic to its value at confirmation time, keyed by the record. Which topics are captured is decided by the fault CODE (`fault_specific` and `patterns` are configuration about what a code means), while what is written belongs to the record, so two owners confirming one code capture the same topics into two separate frames. It differs from per-topic snapshots in two ways: snapshots are deleted when the record is cleared, while the freeze-frame is retained across `clear_fault` (once the snapshots are gone, `~/get_fault` serves the retained frame so the confirmed-state record stays available after acknowledgement), and a re-confirm that captures nothing (e.g. source publishers down) never overwrites an existing non-empty frame. A fault code with no configured capture set gets no freeze-frame row. A configured capture that samples nothing on its first run records an empty `{}` frame. Freeze-frame storage is bounded by the number of distinct records (one row per record, replaced in place) and rows are never evicted. Under a fault storm, captures are bounded by a worker pool (`capture_pool_size`) draining a bounded queue (`capture_queue_depth`); excess captures are dropped per `capture_queue_full_policy` and logged (throttled). The pool is shared and is created when snapshots **or** rosbag is enabled, so these parameters bound both. `capture_pool_size` parallelizes freeze-frame snapshot capture only - rosbag stays single-writer regardless of pool size, and correlated faults confirming inside one post-roll window share a single recording. @@ -92,7 +151,7 @@ That single-writer property also shapes what each fault of a burst gets. Nothing | `snapshots.max_message_size` | int | `65536` | Maximum message size in bytes (larger messages skipped) | | `snapshots.default_topics` | string[] | `[]` | Topics to capture for all faults | | `snapshots.config_file` | string | `""` | Path to YAML config for `fault_specific` and `patterns` | -| `snapshots.recapture_cooldown_sec` | double | `60.0` | Min seconds between captures for the same fault code. | +| `snapshots.recapture_cooldown_sec` | double | `60.0` | Min seconds between captures for the same fault record. | | `snapshots.max_per_fault` | int | `10` | Max snapshots retained per fault. | | `snapshots.capture_pool_size` | int | `2` | Max concurrent capture threads under a fault storm (>= 1). Parallelizes snapshot capture only; rosbag stays single-writer. | | `snapshots.capture_queue_depth` | int | `16` | Max pending captures before the full-queue policy applies (>= 1). | @@ -137,6 +196,12 @@ This is the persisted fault store (`storage_type`), separate from the rosbag sto format used by black-box capture (`snapshots.rosbag.format`, see Rosbag Capture above). **SQLite (default)**: Faults are persisted to disk and survive node restarts. Uses WAL mode for optimal performance. +Each row also records when the current occurrence of its record confirmed (`confirmed_at_ns`, read from the +database file, not by any service). It is 0 until the occurrence confirms: a FAILED report on a CLEARED record +resets it, and the confirmation of that new occurrence sets it again. The owner migration keeps the value of a +CONFIRMED, HEALED or CLEARED row and writes 0 for a PREFAILED or PREPASSED one. A row that confirmed stays +CONFIRMED or HEALED until it is cleared, so a pending row with a value carries it from the occurrence before a +reactivation. **Memory**: Faults are stored in memory only. Useful for testing or when persistence is not required. @@ -152,6 +217,40 @@ The build downloads libpqxx 7.10.7 from GitHub at configure time and links it st A wrong configuration stops the node at startup with exit code 1: a connection string libpq cannot parse, or a server that accepts the connection but does not allow creating the schema, or a table that lacks a column the node uses. A server that cannot be reached (including a wrong password or a missing database) does not stop it: the node runs without storage, services with an error field answer `Fault storage unavailable`, and `ReportFault` answers `accepted=false`. After a failed connection attempt, requests fail at once for 5 s and the next request tries again. An attempt waits at most 2 s per host address unless `connect_timeout` or `PGCONNECT_TIMEOUT` says otherwise, and a request on an open connection to a server that went off the network fails after `tcp_user_timeout` (5 s). With a libpq service, these values come from the service file. See `docs/config/fault-manager.rst` for the full table. +## Severity + +A record has one severity. The stored severity is the highest any FAILED report from its owner has +carried, within an occurrence and across occurrences: `~/clear_fault` ends an occurrence but does +not lower the severity, and the SQLite store keeps it across restarts. Two reports of one code from +one source with different severities mean a misconfigured reporter, and keeping the maximum is the +conservative reading. Another source's record of the same code keeps its own severity: a CRITICAL +from one owner neither escalates nor confirms another owner's record. + +What each surface carries: + +| Surface | Severity | +|---|---| +| `~/list_faults`, `~/get_fault`, the gateway's `GET /faults` and fault detail | stored | +| `FaultEvent` on `~/events`, the gateway's `/faults/stream` frames and fault trigger payloads | stored, the same value a read of the record returns after the report | +| Audit rows (`occurred`, `confirmed`, `healed`, `cleared`) | stored, at the moment of the transition | +| Near-miss entries | the severity that report carried | + +The near-miss series is per-report history, so it keeps what the reporter sent. A lower report +stays visible there and in the mismatch warning below. + +`critical_immediate_confirm` reads the stored severity of the record. A FAILED report on a record +stored as `CRITICAL` confirms it at once, whatever severity the report carries, from a cleared +record and from the HEALED latch alike. So no report leaves a record `CRITICAL` in `PREFAILED` or +`PREPASSED`. A row an earlier release stored that way keeps its status through the migration until +its next FAILED report. + +When a FAILED report carries a severity other than the one stored for its record, the node logs a +warning naming the code, the reported and the stored severity and the source. It is logged once per +code, source and reported severity until the node restarts, so a reporter that repeats the same +report many times a second produces one line. The number of lines is bounded by the distinct +combinations, at most four per record, and not by the report rate. Each misconfigured source is +named on its own line. + ## Near-Miss Series A **near miss** is a FAILED report that moved the debounce counter without the fault ending up @@ -160,8 +259,8 @@ to any scoring sense used elsewhere. PASSED reports move the counter too, but in direction (the fault receding), so they are not near misses. The series is **append-only**: one entry per qualifying report, holding the timestamp, the counter -value after the report, the confirmation threshold it was measured against, the severity, the -reporting source, and the fault status the report left behind. Every field describes that one report, so a series spanning a threshold change +value after the report, the confirmation threshold it was measured against, the severity the +report carried, the reporting source, and the fault status the report left behind. Every field describes that one report, so a series spanning a threshold change or a new reporting source stays readable. Nothing is updated in place, so "this code approached confirmation five times this hour" stays answerable. @@ -175,12 +274,9 @@ counter walking back down under the latch. Without the field the two cannot be t rows written before the field existed. With per-entity thresholds the recorded `confirmation_threshold` is the one belonging to the -**reporting source**, while the debounce counter is shared by every source of that fault code. It -is therefore not by itself the distance to confirmation for the fault as a whole. An entry in -`fault_thresholds.config_file` that sets `confirmation_threshold` makes the two agree again: it -resolves the same for every source, so the recorded threshold is the fault's own. An entry with -only healing fields keeps the per-source threshold. The node warns -once per code when two sources resolve different policies for it. +**reporting source**. The counter it describes is that source's own record, so the pair is the +distance to confirmation for the record. The counter is no longer shared between sources of one +fault code: each source's reports move only its own record. Entries are kept and evicted in **arrival order**, not by their timestamps. Reporters carry their own clocks, so a report can arrive carrying a timestamp behind one already stored; ordering the @@ -197,11 +293,11 @@ retained snapshots, because it records the most recent confirmation while the sn to earlier ones. `~/get_snapshots` returns one entry per topic and serves the newest capture of that topic, whichever storage backend is in use. -The bound is **per fault code, not per database**. Fault codes are unbounded in cardinality, so a -reporter emitting a stream of distinct codes still grows the table; the bound caps what any single -code costs, not the total. +The bound is **per record, not per database**. Fault codes are unbounded in cardinality and so is +the set of reporting sources, so a reporter emitting a stream of distinct codes still grows the +table. The bound caps what any single record costs, not the total. -Retention is **bounded per fault code** by `near_miss.max_per_fault` (default 200), evicting the +Retention is **bounded per record** by `near_miss.max_per_fault` (default 200), evicting the **oldest** entries first. That is the same direction as `snapshots.max_per_fault` and the rosbag cap, and for the same reason: a series frozen at boot says nothing about whether the rate is changing, and the evidence a technician wants is the evidence from the fault happening now. Set it @@ -214,7 +310,7 @@ storage API or the database file. ## Advanced: Tamper-Evident Audit Log -An optional append-only, hash-chained audit log records every fault state transition (`occurred`, `confirmed`, `healed`, `cleared`) so the fault history is independently verifiable. Auto-recovery (a fault reaching the healing threshold via PASSED events) is recorded as a distinct `healed` row with source `auto_heal`, so the fault's END is in the timeline and is not confused with a manual `cleared`. The manager has no acknowledge action separate from clearing, so `~/clear_fault` is recorded as `cleared` (clear == ack); there is no `ack` kind. The log also records its own lifecycle with `logging_activated` / `logging_deactivated` markers at start and stop. It is **off by default** because it adds a write and storage cost per transition. +An optional append-only, hash-chained audit log records every fault state transition (`occurred`, `confirmed`, `healed`, `cleared`) so the fault history is independently verifiable. Auto-recovery (a record reaching the healing threshold via PASSED events) is recorded as a distinct `healed` row, so the record's END is in the timeline and is not confused with a manual `cleared`. Every row's `source` is the record's owner, the automatic transitions included: the `transition` column already says what moved the row, and with several owners per fault code the source has to answer whose record moved. The manager has no acknowledge action separate from clearing, so `~/clear_fault` is recorded as `cleared` (clear == ack), and there is no `ack` kind. The log also records its own lifecycle with `logging_activated` / `logging_deactivated` markers at start and stop. Every row records the record's stored severity at the transition (see [Severity](#severity)). It is **off by default** because it adds a write and storage cost per transition. Each transition appends one immutable row holding `record_hash = sha256(prev_hash + canonical(event))` (OpenSSL EVP SHA-256), the `prev_hash` it links to, and a monotonic `seq`. The hash is computed once at insert and never recomputed. A persisted chain head lets the chain resume across restarts. The log is stored in its own SQLite database (separate from the fault store) and is treated as append-only: the manager only ever inserts rows, and `BEFORE UPDATE` / `BEFORE DELETE` triggers reject out-of-band edits (the guarded rotation prune excepted). @@ -318,9 +414,9 @@ never confirms again, and `occurrence_count` does not move either. Prefer leavin `confirmation_threshold` at `-1` for such a reporter, and filter noisy samples in the reporter itself, where the samples are. -Two things ignore the counter. `SEVERITY_CRITICAL` confirms at once unless -`critical_immediate_confirm` is turned off in the debounce config, and that field is not exposed as -a ROS parameter. `auto_confirm_after_sec` promotes on elapsed time since the last FAILED, which is +Two things ignore the counter. A FAILED report on a record stored as `SEVERITY_CRITICAL` confirms +at once unless `critical_immediate_confirm` is turned off in the debounce config, and that field is +not exposed as a ROS parameter. `auto_confirm_after_sec` promotes on elapsed time since the last FAILED, which is not the same as observing that the condition is still there. ### Fault Lifecycle with Debounce @@ -369,7 +465,10 @@ Event types: `0` = EVENT_FAILED, `1` = EVENT_PASSED ### Immediate Confirmation -CRITICAL severity faults bypass debounce and are immediately CONFIRMED, regardless of threshold. +A FAILED report on a record whose stored severity is CRITICAL confirms it at once, regardless of +threshold and of the HEALED latch. The stored severity decides, not the report: after a CRITICAL +occurrence is cleared, one WARN report from the same source confirms the next occurrence, which +shows CRITICAL. This holds under a per-entity threshold override too. ## Advanced: Fault Correlation @@ -464,7 +563,9 @@ Response includes: - `muted_count`: Number of muted symptom faults - `cluster_count`: Number of active fault clusters -- `muted_faults[]`: Details of muted faults (when `include_muted=true`) +- `muted_faults[]`: Details of muted records (when `include_muted=true`), one entry per muted + record. Muting is per record, so a symptom muted under one owner's root cause never hides + another owner's record of the same code, and each entry names its owner in `source_id`. - `clusters[]`: Details of active clusters (when `include_clusters=true`) ### REST API (via Gateway) @@ -487,7 +588,8 @@ Response fields: "fault_code": "MOTOR_COMM_FL", "root_cause_code": "ESTOP_001", "rule_id": "estop_cascade", - "delay_ms": 50 + "delay_ms": 50, + "source_id": "/powertrain/motor_controller" } ], "clusters": [ diff --git a/src/ros2_medkit_fault_manager/config/fault_manager.yaml b/src/ros2_medkit_fault_manager/config/fault_manager.yaml index f6b25f5d6..ce1a3243f 100644 --- a/src/ros2_medkit_fault_manager/config/fault_manager.yaml +++ b/src/ros2_medkit_fault_manager/config/fault_manager.yaml @@ -17,8 +17,8 @@ fault_manager: # Near-miss series: one appended entry per FAILED report that moved the # debounce counter without confirming. Retained across clear_fault, bounded - # per fault code, oldest evicted first. 0 = unlimited (grows with the - # reporting rate). + # per record - one (fault_code, reporting source) pair - oldest evicted + # first. 0 = unlimited (grows with the reporting rate). # near_miss.max_per_fault: 200 # Healing OFF by default: a recovery signal (e.g. action SUCCEEDED) does not @@ -36,7 +36,7 @@ fault_manager: # to entity-scoped capture and is crash-safe (falls back / self-disables if no # storage backend is usable). Faults confirming during another fault's # post-roll (duration_after_sec) share that recording, one metadata entry per - # fault; at most 32 attach on top of the first, the rest get no entry (WARN). + # record. At most 32 attach on top of the first, the rest get no entry (WARN). snapshots.rosbag.enabled: false # Capture concurrency bound under fault storms (issue #441). These govern the diff --git a/src/ros2_medkit_fault_manager/config/snapshots.yaml b/src/ros2_medkit_fault_manager/config/snapshots.yaml index 2eba501c9..0a1e274d8 100644 --- a/src/ros2_medkit_fault_manager/config/snapshots.yaml +++ b/src/ros2_medkit_fault_manager/config/snapshots.yaml @@ -8,9 +8,13 @@ # 2. patterns - regex pattern match on fault code # 3. default_topics - fallback for all other faults # 4. entity-default - zero-config fallback when nothing above matches: -# the reporting source node's own published topics are captured +# the owning reporting source node's own published topics are captured # (parameter snapshots.entity_default, on by default) # +# Resolution is by fault CODE: this file says what a code means, not who reported +# it. What is written belongs to the RECORD, so two sources reporting one code +# capture the same topics into two separate freeze frames. +# # A fault code listed in fault_specific (or matched by a pattern) with an # EMPTY topic list is an explicit per-fault opt-out: nothing is captured and # the entity-default fallback is skipped for that code. @@ -105,7 +109,7 @@ rosbag: # Options: # "entity" - DEFAULT. Subscribe broadly for pre-roll, but on fault confirmation # write only the faulting source node's topics (+ /tf, /tf_static), - # resolved from the fault's reporting source. Useful black box with + # resolved from the record's owning reporting source. Useful black box with # zero per-topic config. Falls back to the full buffer if the source # cannot be resolved to a live node. # "config" - Use same topics as JSON snapshots (from this config file) @@ -154,7 +158,9 @@ rosbag: # Storage path for bag files (default: "" = system temp directory) # Empty string uses /tmp/rosbag_snapshots/ # Bag files are named: fault_{fault_code}_{timestamp}/ and that directory name is - # the recording's public id - the last segment of its bulk-data URL. + # the recording's public id - the last segment of its bulk-data URL. The name carries + # the code only: a burst of records sharing one window shares one recording, and each + # of them gets its own metadata row pointing at it. storage_path: "" # Maximum size per bag file in MB (default: 50) @@ -169,10 +175,10 @@ rosbag: # is larger: it is kept and a warning is logged. max_total_storage_mb: 500 - # Recordings kept per fault code (default: 1, 0 = unlimited) - # A fault that keeps re-confirming leaves a trail of black boxes instead of only - # the latest one. Past the cap the fault's OLDEST recording is dropped, and the bag - # is deleted once no fault still references it (a burst shares one recording). + # Recordings kept per record (default: 1, 0 = unlimited) + # A record that keeps re-confirming leaves a trail of black boxes instead of only + # the latest one. Past the cap the record's OLDEST recording is dropped, and the bag + # is deleted once no record still references it (a burst shares one recording). # # 1 is the historical behaviour: each re-confirmation replaces the previous bag. # 3 is a good starting point for an intermittent fault you are chasing. diff --git a/src/ros2_medkit_fault_manager/design/index.rst b/src/ros2_medkit_fault_manager/design/index.rst index 7f4031c32..e73ab81b1 100644 --- a/src/ros2_medkit_fault_manager/design/index.rst +++ b/src/ros2_medkit_fault_manager/design/index.rst @@ -185,15 +185,15 @@ Main Components - Future implementations can be added in Issue #8: Fault Persistence Options 3. **InMemoryFaultStorage** - Thread-safe in-memory implementation of FaultStorage - - Uses ``std::map`` keyed by ``fault_code`` for O(log n) lookups + - Uses ``std::map`` keyed by ``FaultId`` for O(log n) lookups, ordered by code first - Protected by ``std::mutex`` for concurrent service request handling - - Aggregates reports from multiple sources into single fault entries - - Implements severity escalation (higher severity overwrites lower) - - Tracks occurrence counts and all reporting sources + - Keeps one record per reporting source, filtered and cleared independently + - Implements severity escalation per record (higher severity overwrites lower, across occurrences too) + - Tracks occurrence counts per record -4. **FaultState** - Internal representation of a fault entry +4. **FaultState** - Internal representation of one fault record - Maps directly to ``ros2_medkit_msgs::msg::Fault`` via ``to_msg()`` - - Uses ``std::set`` for reporting_sources to ensure uniqueness + - Carries its owner and emits it as the single entry of reporting_sources - Tracks first and last occurrence timestamps - Manages fault status lifecycle with debounce (PREFAILED -> CONFIRMED -> CLEARED) @@ -250,8 +250,11 @@ Reports a new fault or updates an existing one. - **Input validation**: fault_code and source_id cannot be empty, event_type must be valid - **Event types**: FAILED (fault detected) or PASSED (fault condition cleared) - **Debounce**: FAILED events decrement counter, PASSED events increment counter -- **Aggregation**: Same fault_code from different sources creates a single fault entry -- **Severity escalation**: Fault severity is updated if a higher severity is reported +- **Record identity**: (fault_code, source_id). The same fault_code from different sources + creates one record per source, each debounced, cleared and healed on its own +- **Severity escalation**: A record's severity is updated if its own owner reports a higher one +- **Severity mismatch**: A FAILED report whose severity differs from the stored severity of its + record is logged as a warning, once per code, source and reported severity until restart - **Request queue**: Holds ``report_fault_queue_depth`` requests (default 100) until the executor reads them. The default service QoS holds 10. A reporter has one request in flight per fault code, so one reporter can fill the queue alone. Past the depth the middleware drops requests without an @@ -290,23 +293,49 @@ All ``FaultStorage`` public methods acquire a mutex lock to ensure thread safety when handling concurrent service requests. This is essential since ROS 2 service callbacks may execute on different threads. -Fault Aggregation -~~~~~~~~~~~~~~~~~ +Fault Records +~~~~~~~~~~~~~ -Multiple reports of the same ``fault_code`` (from same or different sources) are -aggregated into a single fault entry. This provides: +A fault record is identified by the pair ``(fault_code, source_id)``, where the +``source_id`` is the one the ``ReportFault`` call carried. That source is the record's +owner. Repeated reports from one source update that source's record. A report from a +source that owns no record for the code opens a new one. This provides: -- **Deduplication**: Prevents fault flooding from repeated reports -- **Source tracking**: Identifies all sources reporting the same fault -- **Occurrence counting**: Tracks how many times a fault was reported +- **Deduplication**: Prevents fault flooding from one source's repeated reports +- **Source attribution**: Each record names the one source that owns it +- **Occurrence counting**: Tracks how many times a record was raised + +Keeping the records apart is what makes the per-source state mean anything: the debounce +counter, the status, the severity, the occurrence count and the timestamps all belong to +one reporter, so one reporter recovering cannot walk another reporter's fault back from +confirmation, and one reporter's CRITICAL cannot escalate another's record. Severity Escalation ~~~~~~~~~~~~~~~~~~~ -When a fault is re-reported with a higher severity, the stored severity is updated. -This ensures the fault reflects the worst-case condition. Severity levels are ordered: +When a record is re-reported by its owner with a higher severity, the stored severity is +updated. This ensures the record reflects the worst-case condition its own reporter saw. +Severity levels are ordered: ``INFO(0) < WARN(1) < ERROR(2) < CRITICAL(3)``. +A record has one severity. The maximum is kept within an occurrence and across occurrences: a +FAILED report on a CLEARED record starts a new occurrence (``first_occurred`` and +``occurrence_count`` move) but does not lower the severity, and SQLite keeps it across restarts. +Different severities from one source for one code mean a misconfigured reporter, and the maximum +is the conservative choice. + +- **Reads, subscribers and the audit log** carry the stored severity of the record: + ``list_faults``, ``get_fault``, the ``FaultEvent`` payload, and through it the gateway's + ``/faults/stream`` frames and fault trigger payloads. An event therefore carries what a read of + the record returns right after the report. Every audit row records the stored severity at its + transition. +- **Near-miss entries** carry the reported severity: the series is per-report history. +- **Mismatch warning**: when a FAILED report carries another severity than the one stored for its + record, the node logs the code, both severities and the source. A set of + ``(fault_code, source_id, severity)`` keys limits it to one line per key for the process + lifetime, so the log volume does not follow the report rate. The source owns the record, so the + set holds at most four keys per record the storage already keeps. + Status Lifecycle (Debounce Model) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -320,7 +349,18 @@ Faults follow an AUTOSAR DEM-style debounce lifecycle: FAILED events decrement the debounce counter (towards confirmation). PASSED events increment the debounce counter (towards healing). -CRITICAL severity bypasses debounce and confirms immediately. +CRITICAL severity bypasses debounce and confirms immediately. The check reads the stored severity +of the record after the report is applied, so any FAILED report on a record stored as CRITICAL +confirms it, from CLEARED and from the HEALED latch alike, and no report leaves a record CRITICAL in +PREFAILED or PREPASSED. A row an earlier release stored that way keeps its status through the owner +migration until its next FAILED report. + +The SQLite store records when the current occurrence of a record confirmed (``confirmed_at_ns``). +A FAILED report on a CLEARED record resets it to 0, and the confirmation of the new occurrence sets +it. The owner migration keeps the value of a CONFIRMED, HEALED or CLEARED row and writes 0 for a +PREFAILED or PREPASSED one: under the latch a row that confirmed stays CONFIRMED or HEALED until a +clear, so a pending row with a value carries it from the occurrence before a reactivation. The +in-memory store keeps no confirmation time. The counter is always clamped to ``[confirmation_threshold, healing_threshold]`` so a long run of one-sided events cannot push it out to the integer limits and delay the opposite transition. diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/capture_thread_pool.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/capture_thread_pool.hpp index 76e70c078..83ea33dd7 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/capture_thread_pool.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/capture_thread_pool.hpp @@ -27,6 +27,7 @@ #include #include "ros2_medkit_fault_manager/core/log_sink.hpp" +#include "ros2_medkit_fault_manager/fault_storage.hpp" namespace ros2_medkit_fault_manager { @@ -46,7 +47,7 @@ enum class EnqueueResult { struct EnqueueOutcome { EnqueueResult result; - std::optional evicted_code; ///< Set only for kEvictedOldest. + std::optional evicted_id; ///< Set only for kEvictedOldest. }; /// Bounded worker pool that runs fault-capture jobs off the service thread. @@ -64,7 +65,7 @@ class CaptureThreadPool { /// for pool_size concurrent calls. Exceptions are caught and logged. /// @param logger_name Logger name of the lines written to @p log. CaptureThreadPool(std::size_t pool_size, std::size_t queue_depth, QueueFullPolicy full_policy, core::LogSink & log, - std::function capture_fn, std::string logger_name = "fault_manager"); + std::function capture_fn, std::string logger_name = "fault_manager"); ~CaptureThreadPool(); CaptureThreadPool(const CaptureThreadPool &) = delete; @@ -72,8 +73,8 @@ class CaptureThreadPool { CaptureThreadPool(CaptureThreadPool &&) = delete; CaptureThreadPool & operator=(CaptureThreadPool &&) = delete; - /// Enqueue a capture job. Non-blocking. Thread-safe. - EnqueueOutcome enqueue(const std::string & fault_code); + /// Enqueue a capture job for one fault record. Non-blocking. Thread-safe. + EnqueueOutcome enqueue(const FaultId & id); /// Stop accepting work, let in-flight jobs finish, discard pending, join all /// workers. Idempotent and noexcept. Called by the destructor. @@ -94,11 +95,11 @@ class CaptureThreadPool { const QueueFullPolicy full_policy_; core::LogSink & log_; std::string logger_name_; - std::function capture_fn_; + std::function capture_fn_; mutable std::mutex queue_mutex_; std::condition_variable cv_; - std::deque queue_; + std::deque queue_; bool stop_{false}; std::atomic dropped_captures_{0}; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp index 80b5e3697..36704260b 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp @@ -19,11 +19,13 @@ #include #include #include +#include #include #include #include +#include #include -#include +#include #include #include @@ -31,6 +33,7 @@ #include "ros2_medkit_fault_manager/core/fault_manager_config.hpp" #include "ros2_medkit_fault_manager/core/fault_manager_startup.hpp" #include "ros2_medkit_fault_manager/core/log_sink.hpp" +#include "ros2_medkit_fault_manager/fault_storage.hpp" namespace ros2_medkit_fault_manager::core { @@ -61,7 +64,7 @@ enum class CaptureEnqueue { struct CaptureResult { CaptureEnqueue result{CaptureEnqueue::kNone}; - std::string evicted_code; ///< Set only for kEvictedOldest. + FaultId evicted_id; ///< Set only for kEvictedOldest. }; /// Snapshot and rosbag capture the core triggers. Called with the core mutex held; must not call back into the core. @@ -74,10 +77,11 @@ class CaptureHooks { CaptureHooks & operator=(CaptureHooks &&) = delete; virtual ~CaptureHooks() = default; - /// Queue the snapshot and rosbag capture of @p code now. The core has already applied the recapture cooldown. - virtual CaptureResult capture_now(const std::string & code) = 0; - virtual void on_prefailed(const std::string & code) = 0; - virtual void on_cleared(const std::string & code) = 0; + /// Queue the snapshot and rosbag capture of the record @p id now. The core has already applied the recapture + /// cooldown. + virtual CaptureResult capture_now(const FaultId & id) = 0; + virtual void on_prefailed(const FaultId & id) = 0; + virtual void on_cleared(const FaultId & id) = 0; }; /// Wall-clock source, nanoseconds since the Unix epoch. @@ -125,6 +129,7 @@ struct ListQuery { /// MutedFaultInfo.msg field for field. struct MutedFault { std::string fault_code; + std::string source_id; ///< Owner of the muted record. std::string root_cause_code; std::string rule_id; uint32_t delay_ms{0}; @@ -196,9 +201,17 @@ class FaultManagerCore { ~FaultManagerCore() = default; ReportOutcome report(const fd::FaultReport & report); - ClearOutcome clear(const std::string & code, bool skip_correlation_auto_clear); + /// Clears one record. An empty @p source_id addresses the only record of @p code and is refused as + /// ambiguous when several owners hold it; a set one addresses exactly that owner's record. + ClearOutcome clear(const std::string & code, const std::string & source_id, bool skip_correlation_auto_clear); ListOutcome list(const ListQuery & query); - FaultDetail get(const std::string & code); + FaultDetail get(const std::string & code, const std::string & source_id); + + /// Resolves the record a single-record request addresses, under the core mutex. Empty @p source_id + /// picks the only record of @p fault_code; several records make it ambiguous. On failure returns + /// nullopt and sets @p error. + std::optional resolve_target(const std::string & fault_code, const std::string & source_id, + std::string & error); /// Throws FaultStorage::IgnorableConnectionException when the storage is unreachable. std::vector list_for_entity(const std::string & entity_id); @@ -251,25 +264,21 @@ class FaultManagerCore { } private: - /// The first debounce policy seen for a fault code, and who reported it. - struct DebouncePolicyWitness { - DebounceConfig config; - std::string source_id; - bool warned{false}; - }; - ReportOutcome report_locked(const fd::FaultReport & report); - ClearOutcome clear_locked(const std::string & code, bool skip_correlation_auto_clear); + ClearOutcome clear_locked(const std::string & code, const std::string & source_id, bool skip_correlation_auto_clear); ListOutcome list_locked(const ListQuery & query); /// @p rosbag_paths receives (snapshot index, bag path) for each rosbag entry, for sizing after unlock. - FaultDetail get_locked(const std::string & code, std::vector> & rosbag_paths); + FaultDetail get_locked(const std::string & code, const std::string & source_id, + std::vector> & rosbag_paths); + std::optional resolve_target_locked(const std::string & fault_code, const std::string & source_id, + std::string & error); void run_auto_confirm(int64_t wall_now_ns); - /// Queues a capture unless @p code is inside its recapture cooldown. - void capture_on_confirm(const std::string & code); + /// Queues a capture unless @p id is inside its recapture cooldown. + void capture_on_confirm(const FaultId & id); + /// Warns once per (code, source, severity) when a FAILED report disagrees with the stored severity. + void warn_severity_mismatch(const fd::FaultReport & report, uint8_t stored_severity); DebounceConfig resolve_config(const std::string & source_id, const std::string & fault_code) const; - void warn_on_conflicting_debounce_policy(const std::string & fault_code, const std::string & source_id, - const DebounceConfig & resolved); void audit_transition(const char * transition, const fd::FaultRecord & fault, const std::string & source_id, int64_t occurred_at_ns); void publish_event(const char * event_type, const fd::FaultRecord & fault, @@ -290,15 +299,15 @@ class FaultManagerCore { bool audit_fail_closed_{false}; std::mutex mutex_; - std::unordered_map debounce_policy_witness_; + std::set> severity_mismatch_logged_; /// Next due time of each tick job, ns since the epoch. int64_t auto_confirm_due_ns_{0}; int64_t cleanup_due_ns_{0}; int64_t last_tick_ns_{0}; std::atomic auto_confirm_runs_{0}; std::atomic cleanup_runs_{0}; - /// Recapture cooldown starts, per fault code. Empty when the cooldown is off. - std::unordered_map last_capture_times_; + /// Recapture cooldown starts, per record. Empty when the cooldown is off. + std::map last_capture_times_; std::atomic audit_dropped_writes_{0}; std::atomic audit_healthy_{true}; }; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp index ccb0810f5..aaa7c533f 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp @@ -20,10 +20,12 @@ #include #include #include +#include #include #include "ros2_medkit_fault_manager/correlation/pattern_matcher.hpp" #include "ros2_medkit_fault_manager/correlation/types.hpp" +#include "ros2_medkit_fault_manager/fault_storage.hpp" namespace ros2_medkit_fault_manager { namespace correlation { @@ -56,13 +58,20 @@ struct ProcessFaultResult { /// Result of clearing a fault struct ProcessClearResult { - /// List of symptom fault codes that should be auto-cleared - std::vector auto_cleared_codes; + /// The symptom RECORDS that should be auto-cleared. Records, not codes: a root + /// cause of one owner must never auto-clear another owner's record of the same + /// symptom code. + std::vector auto_cleared_symptoms; }; -/// Information about a muted fault (for ListFaults response) +/// Information about a muted record (for ListFaults response). +/// Carries both halves of the record identity: two owners muted on one code are two +/// entries of that code, told apart by owner. struct MutedFaultData { std::string fault_code; + /// Reporting source that owns the muted record. Filled from the map key, so it + /// cannot drift from the record this entry describes. + std::string owner; std::string root_cause_code; std::string rule_id; uint32_t delay_ms{0}; @@ -95,18 +104,24 @@ class CorrelationEngine { /// @param config Correlation configuration (must be enabled and valid) explicit CorrelationEngine(const CorrelationConfig & config); - /// Process an incoming fault - /// @param fault_code The fault code + /// Process an incoming fault record + /// + /// Rules match on the fault CODE - a rule says which codes are root causes and which + /// are symptoms, and says nothing about who reports them. Every RELATION the engine + /// then forms (root to symptom, muting, cluster membership, auto-clear cascade) is + /// between records of the SAME owner, so a root cause of owner X never mutes or + /// auto-clears a symptom of owner Y. + /// @param id The fault record /// @param severity The fault severity (for representative selection) /// @param timestamp When the fault occurred /// @return Processing result indicating whether to mute, correlations, etc. - ProcessFaultResult process_fault(const std::string & fault_code, const std::string & severity, + ProcessFaultResult process_fault(const FaultId & id, const std::string & severity, std::chrono::steady_clock::time_point timestamp = std::chrono::steady_clock::now()); - /// Process a fault being cleared - /// @param fault_code The fault code being cleared - /// @return Result with list of symptoms to auto-clear - ProcessClearResult process_clear(const std::string & fault_code); + /// Process a fault record being cleared + /// @param id The record being cleared + /// @return Result with the symptom records to auto-clear, all of the same owner + ProcessClearResult process_clear(const FaultId & id); /// Get all currently muted faults /// @return List of muted fault data @@ -115,10 +130,10 @@ class CorrelationEngine { /// Get count of muted faults uint32_t get_muted_count() const; - /// Whether a fault code is currently muted as a symptom. - /// @param fault_code Code to test - /// @return True while the code is suppressed by a root cause - bool is_muted(const std::string & fault_code) const; + /// Whether a fault record is currently muted as a symptom. + /// @param id Record to test + /// @return True while the record is suppressed by a root cause of the same owner + bool is_muted(const FaultId & id) const; /// Get all active clusters /// @return List of cluster data @@ -132,18 +147,18 @@ class CorrelationEngine { void cleanup_expired(); private: - /// Check if fault matches a root cause pattern in any hierarchical rule + /// Check if the record's CODE matches a root cause pattern in any hierarchical rule /// @return Rule ID if matched, empty optional otherwise std::optional try_as_root_cause(const std::string & fault_code); - /// Check if fault is a symptom of any pending root cause + /// Check if the record is a symptom of a pending root cause OF THE SAME OWNER /// @return ProcessFaultResult with correlation info if matched - std::optional try_as_symptom(const std::string & fault_code, - std::chrono::steady_clock::time_point timestamp); + std::optional try_as_symptom(const FaultId & id, std::chrono::steady_clock::time_point timestamp); - /// Check if fault matches an auto-cluster rule + /// Check if the record's code matches an auto-cluster rule. A cluster holds records + /// of one owner, so the pending-cluster key carries the owner alongside the rule id. /// @return ProcessFaultResult with cluster info if matched - std::optional try_auto_cluster(const std::string & fault_code, const std::string & severity, + std::optional try_auto_cluster(const FaultId & id, const std::string & severity, std::chrono::steady_clock::time_point timestamp); /// Generate unique cluster ID @@ -163,38 +178,41 @@ class CorrelationEngine { /// Active root causes waiting for symptoms struct PendingRootCause { - std::string fault_code; + FaultId fault_id; std::string rule_id; std::chrono::steady_clock::time_point timestamp; uint32_t window_ms; }; std::vector pending_root_causes_; - /// Mapping from root cause to its symptoms - std::map> root_to_symptoms_; + /// Mapping from a root cause RECORD to its symptom RECORDS, all of the same owner + std::map> root_to_symptoms_; - /// Muted faults (fault_code -> data) - std::map muted_faults_; + /// Muted records (record -> data) + std::map muted_faults_; /// Active clusters (cluster_id -> data) std::map active_clusters_; - /// Per-fault severities for active clusters (cluster_id -> (fault_code -> severity)) + /// Per-fault severities for active clusters (cluster_id -> (fault_code -> severity)). A cluster + /// holds the records of one owner, so a code appears once per cluster. std::map> active_cluster_severities_; - /// Mapping from fault code to cluster ID (for faults in clusters) - std::map fault_to_cluster_; + /// Mapping from a record to its cluster ID (for records in clusters) + std::map fault_to_cluster_; /// Pending cluster with steady_clock timestamp for window tracking struct PendingCluster { ClusterData data; + std::string owner; ///< Every member record of this cluster has this owner std::chrono::steady_clock::time_point steady_first_at; std::map fault_severities; ///< fault_code -> severity }; - /// Pending clusters being formed (rule_id -> cluster data) + /// Pending clusters being formed ((rule_id, owner) -> cluster data). Keyed by owner + /// too, so one rule forms one cluster per owner and membership never crosses owners. /// Once min_count is reached, moved to active_clusters_ - std::map pending_clusters_; + std::map, PendingCluster> pending_clusters_; /// Counter for cluster ID generation uint64_t cluster_counter_{0}; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_manager_node.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_manager_node.hpp index fa1f61e17..8f62e21d1 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_manager_node.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_manager_node.hpp @@ -179,7 +179,7 @@ class FaultManagerNode : public rclcpp::Node { /// Queue the snapshot and rosbag capture of a confirmed fault on the capture pool. The core calls it /// through the capture hooks after its cooldown check. - core::CaptureResult enqueue_capture(const std::string & fault_code); + core::CaptureResult enqueue_capture(const FaultId & id); /// Append a fault state-transition to the audit log through the core. /// @param transition One of the kTransition* constants. diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp index e82c01e9b..078abb4ae 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "ros2_medkit_fault_detection/fault_types.hpp" @@ -60,7 +61,8 @@ struct DebounceConfig { /// Default: 3 (3 more PASSED than FAILED events to heal). int32_t healing_threshold{3}; - /// Whether CRITICAL severity bypasses debounce and confirms immediately. + /// Whether a FAILED report on a record whose stored severity is CRITICAL confirms it immediately, + /// bypassing debounce and the HEALED latch. The stored severity decides, not the report's. bool critical_immediate_confirm{true}; /// Time-based auto-confirmation. If > 0, PREFAILED faults older than this are auto-confirmed. @@ -89,16 +91,42 @@ bool is_near_miss(bool is_failed_event, const std::string & resulting_status); /// defaults (-1 / 3). Returns true if the config was already valid. bool sanitize_debounce_config(DebounceConfig & config); -/// Internal fault state stored in memory. Times are wall-clock ns >= 0; 0 means never. +/// Identity of one fault record: a fault code plus the reporting source that owns it. +/// +/// The owner is the source_id the ReportFault call carried. Two sources reporting one +/// fault_code are two records, and every piece of per-fault state (status, debounce +/// counter, occurrence count, timestamps, severity, freeze frame, snapshots, near +/// misses, rosbag links) belongs to one record. A clear or a heal driven by one owner +/// never touches another owner's record of the same code. +/// +/// An aggregate on purpose: FaultId{code, owner} is the only way to build one, so no +/// call site can silently pass a bare code where a record identity is required. +struct FaultId { + std::string fault_code; + std::string owner; + + bool operator==(const FaultId & other) const { + return fault_code == other.fault_code && owner == other.owner; + } + + /// Ordered by code first, so the records of one code are contiguous in a map and + /// get_faults_by_code is a range scan rather than a full sweep. + bool operator<(const FaultId & other) const { + return std::tie(fault_code, owner) < std::tie(other.fault_code, other.owner); + } +}; + +/// Internal fault state of one record, stored in memory. Times are wall-clock ns >= 0; 0 means never. struct FaultState { std::string fault_code; + /// Reporting source that owns this record. Half of the record identity. + std::string owner; uint8_t severity{0}; std::string description; int64_t first_occurred_ns{0}; int64_t last_occurred_ns{0}; uint32_t occurrence_count{0}; ///< Count of genuine occurrences (new fault + each re-raise after CLEARED) std::string status; - std::set reporting_sources; // Debounce state (internal, not exposed in Fault.msg) int32_t debounce_counter{0}; ///< FAILED decrements (-1), PASSED increments (+1) @@ -115,6 +143,7 @@ struct FaultState { /// one moment and only mean anything together. struct SnapshotData { std::string fault_code; + std::string owner; ///< Reporting source that owns the record this reading belongs to std::string topic; std::string message_type; std::string data; ///< JSON-encoded message data @@ -126,13 +155,15 @@ struct SnapshotData { /// Compact freeze-frame captured when a fault confirms: a single JSON object mapping /// each captured topic to its latest value at confirmation time. Unlike per-topic -/// snapshots, a freeze-frame is keyed by fault_code (one row per code) and is RETAINED -/// across clear_fault, so the confirmed-state record persists after acknowledgement. -/// A row exists only for fault codes with a configured capture set; a fault code with -/// no capture configured gets no row at all (lookup returns nullopt, never an empty {}). +/// snapshots, a freeze-frame is keyed by the record identity (one row per +/// (fault_code, owner) pair) and is RETAINED across clear_fault, so the confirmed-state +/// record persists after acknowledgement. A row exists only for fault codes with a +/// configured capture set. A fault code with no capture configured gets no row at all +/// (lookup returns nullopt, never an empty {}). struct FreezeFrameData { std::string fault_code; - std::string data; ///< Compact JSON object: {"": , ...} + std::string owner; ///< Reporting source that owns the record this frame belongs to + std::string data; ///< Compact JSON object: {"": , ...} int64_t captured_at_ns{0}; }; @@ -146,7 +177,7 @@ struct FreezeFrameData { /// off the wire with its own copy of this rule; keep the two in step. std::string rosbag_recording_id(const std::string & file_path); -/// One entry of the near-miss series for a fault code. +/// One entry of the near-miss series of one fault record. /// /// A near miss is a FAILED report that moved the debounce counter WITHOUT the fault /// ending up CONFIRMED - the fault nearly happened. PASSED reports move the counter @@ -156,21 +187,21 @@ std::string rosbag_recording_id(const std::string & file_path); /// /// The series is append-only: one entry per qualifying report, never updated in place, /// and RETAINED across clear_fault, because acknowledging a fault cycle must not erase -/// the record of how often that code approached confirmation. It is bounded per fault -/// code (see set_max_near_misses_per_fault) and evicts the OLDEST entries first, so a +/// the record of how often that record approached confirmation. It is bounded per +/// record (see set_max_near_misses_per_fault) and evicts the OLDEST entries first, so a /// long-running appliance keeps the recent series rather than freezing it at boot. struct NearMissRecord { std::string fault_code; int64_t occurred_at_ns{0}; ///< Timestamp of the report that moved the counter int32_t debounce_counter{0}; ///< Counter value AFTER this report - /// Confirmation threshold the report was evaluated against. With per-entity overrides this is - /// the threshold of the REPORTING SOURCE, while the counter is shared by every source of the - /// code, so it is not by itself the distance to confirmation for the fault as a whole. + /// Confirmation threshold the report was evaluated against. The counter it belongs to is + /// this record's own, so with per-entity overrides both are the reporting source's and the + /// value is the distance to confirmation for the record. int32_t confirmation_threshold{0}; uint8_t severity{0}; ///< Severity carried by the report - std::string source_id; ///< Reporting source + std::string source_id; ///< Reporting source, which is the record's owner /// Fault status after the report was applied. Never CONFIRMED - that is what makes the report a /// near miss. It separates a counter climbing from a resting state (PREFAILED) from one walking @@ -186,6 +217,7 @@ struct NearMissRecord { /// only when its last row goes. struct RosbagFileInfo { std::string fault_code; + std::string owner; ///< Reporting source that owns the record holding this link std::string recording_id; ///< Basename of file_path; shared by every row of a burst std::string file_path; std::string format; ///< "sqlite3" or "mcap" @@ -226,11 +258,12 @@ class FaultStorage { /// @param kind Whether the report is a FAILED or a PASSED event /// @param severity Fault severity level (only used for FAILED events) /// @param description Human-readable description (only used for FAILED events) - /// @param source_id Reporting source identifier + /// @param source_id Reporting source identifier. Together with fault_code it names the + /// record this event applies to, and it creates that record when none exists. /// @param timestamp_ns Wall-clock time of the event, ns (>= 0) /// @param config Debounce configuration to apply for this event (resolved per-entity by the node) - /// @return true if this is a new occurrence (new fault or reactivated CLEARED fault), - /// false if existing active fault was updated + /// @return true if this is a new occurrence (new record or reactivated CLEARED record), + /// false if an existing active record was updated virtual bool report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) = 0; @@ -243,34 +276,44 @@ class FaultStorage { virtual std::vector list_faults(bool filter_by_severity, uint8_t severity, const std::vector & statuses) const = 0; - /// Get a single fault by fault_code - /// @param fault_code The fault code to look up + /// Get a single fault record + /// @param id The record to look up /// @return The fault if found, nullopt otherwise - virtual std::optional get_fault(const std::string & fault_code) const = 0; + virtual std::optional get_fault(const FaultId & id) const = 0; - /// Clear a fault by fault_code (manual acknowledgment). Drops the fault's per-topic snapshots; - /// the freeze-frame and the near-miss series are RETAINED, because they outlive a single fault - /// cycle and cannot be reconstructed afterwards. - /// @param fault_code The fault code to clear - /// @return true if fault was found and cleared, false if not found - virtual bool clear_fault(const std::string & fault_code) = 0; - - /// Get total number of stored faults + /// Every record carrying @p fault_code, one per owning reporting source. + /// + /// The resolution step behind an unscoped service call: with exactly one record the + /// call applies to it, with several it is ambiguous and the owners are the answer. + /// Returns them ordered by owner, so an ambiguity message is stable. + /// @param fault_code The fault code to look up + /// @return Every record carrying the code, empty when none does + virtual std::vector get_faults_by_code(const std::string & fault_code) const = 0; + + /// Clear one fault record (manual acknowledgment). Drops that record's per-topic snapshots. + /// The freeze-frame and the near-miss series are RETAINED, because they outlive a single fault + /// cycle and cannot be reconstructed afterwards. Another owner's record of the same code is + /// untouched, snapshots included. + /// @param id The record to clear + /// @return true if the record was found and cleared, false if not found + virtual bool clear_fault(const FaultId & id) = 0; + + /// Get total number of stored fault records virtual size_t size() const = 0; - /// Check if a fault exists - virtual bool contains(const std::string & fault_code) const = 0; + /// Check if a fault record exists + virtual bool contains(const FaultId & id) const = 0; - /// Check and confirm PREFAILED faults that have been pending too long (time-based confirmation) + /// Check and confirm PREFAILED records that have been pending too long (time-based confirmation) /// @param now_ns Wall-clock time for age calculation, ns - /// @return Fault codes that were confirmed by this call (so the caller can audit each). - virtual std::vector check_time_based_confirmation(int64_t now_ns) = 0; + /// @return The records that were confirmed by this call (so the caller can audit each). + virtual std::vector check_time_based_confirmation(int64_t now_ns) = 0; - /// Set maximum snapshots per fault code (0 = unlimited) + /// Set maximum snapshots per fault record (0 = unlimited) virtual void set_max_snapshots_per_fault(size_t /*max_count*/) { } - /// Cap on RECORDINGS retained per fault code (0 = unlimited). + /// Cap on RECORDINGS retained per fault record (0 = unlimited). /// /// Enforced inside store_rosbag_file(s), atomically with the insert: past the cap /// the fault's OLDEST recordings lose their row, and a bag whose last referencing @@ -318,16 +361,15 @@ class FaultStorage { /// @param snapshot The snapshot data to store virtual void store_snapshot(const SnapshotData & snapshot) = 0; - /// Get snapshots for a fault, NEWEST capture set first. + /// Get snapshots of one fault record, NEWEST capture set first. /// /// Ordered by capture_id descending, then captured_at_ns descending, in every /// backend. Readers fold rows into a per-topic map, so insertion order would let /// an older capture's values win on one backend and not the other. - /// @param fault_code The fault code to get snapshots for + /// @param id The record to get snapshots for /// @param topic_filter Optional topic filter (empty = all topics) - /// @return Vector of snapshots for the fault - virtual std::vector get_snapshots(const std::string & fault_code, - const std::string & topic_filter = "") const = 0; + /// @return Vector of snapshots of the record + virtual std::vector get_snapshots(const FaultId & id, const std::string & topic_filter = "") const = 0; /// Highest capture_id any stored snapshot holds, across every fault (0 when none). /// @@ -339,22 +381,23 @@ class FaultStorage { return 0; } - /// Store the compact freeze-frame captured for a fault (JSON dict of topic values). - /// Keyed by fault_code: a later capture for the same code replaces the frame. The frame + /// Store the compact freeze-frame captured for a fault record (JSON dict of topic values). + /// Keyed by the record identity carried on @p frame: a later capture for the same record + /// replaces the frame, and another owner's record of the same code keeps its own. The frame /// is retained across clear_fault so the confirmed-state record survives acknowledgement. - /// Storage is bounded by the number of distinct fault codes (one row per code, replaced - /// in place); rows are never evicted. Faults themselves are never deleted (clear_fault - /// only flips status), so there is currently no delete hook to tie eviction to. - /// @param frame The freeze-frame to store + /// Storage is bounded by the number of distinct records (one row per record, replaced in + /// place), and rows are never evicted. Records themselves are never deleted (clear_fault only + /// flips status), so there is currently no delete hook to tie eviction to. + /// @param frame The freeze-frame to store, carrying fault_code and owner virtual void store_freeze_frame(const FreezeFrameData & frame) = 0; - /// Get the freeze-frame captured for a fault, if any. - /// @param fault_code The fault code to look up - /// @return The freeze-frame if one was captured, nullopt otherwise (including fault - /// codes with no capture configured, which never get a row) - virtual std::optional get_freeze_frame(const std::string & fault_code) const = 0; + /// Get the freeze-frame captured for a fault record, if any. + /// @param id The record to look up + /// @return The freeze-frame if one was captured, nullopt otherwise (including records + /// with no capture configured, which never get a row) + virtual std::optional get_freeze_frame(const FaultId & id) const = 0; - /// Set the maximum number of near-miss entries retained per fault code. + /// Set the maximum number of near-miss entries retained per fault record. /// /// Entries beyond the bound are evicted oldest-first, including entries already stored when the /// bound is applied. 0 means unlimited, and so does any bound larger than the storage backend @@ -367,17 +410,18 @@ class FaultStorage { return 0; } - /// Get the near-miss series for a fault code, oldest entry first. - /// The series survives clear_fault; an unknown or never-near-missed code returns empty. - /// @param fault_code The fault code to look up + /// Get the near-miss series of one fault record, oldest entry first. + /// The series survives clear_fault. An unknown or never-near-missed record returns empty. + /// @param id The record to look up /// @return The retained near-miss entries in chronological order - virtual std::vector get_near_misses(const std::string & fault_code) const = 0; + virtual std::vector get_near_misses(const FaultId & id) const = 0; - /// Store rosbag file metadata for a fault - /// @param info The rosbag file info to store (replaces any existing entry for fault_code) + /// Store rosbag file metadata for a fault record + /// @param info The rosbag file info to store, carrying fault_code and owner (replaces any + /// existing link between that record and the same file_path) virtual void store_rosbag_file(const RosbagFileInfo & info) = 0; - /// Store one row per fault of a burst that shares a recording. + /// Store one row per record of a burst that shares a recording. /// /// Implementations MUST be all-or-nothing. The caller treats a throw as "no row /// was written" and discards the recording, so a batch that stored some rows and @@ -396,20 +440,20 @@ class FaultStorage { } } - /// The MOST RECENT recording of a fault, or nullopt. + /// The MOST RECENT recording of a fault record, or nullopt. /// - /// A fault can hold several recordings, so "the" recording is a choice: newest + /// A record can hold several recordings, so "the" recording is a choice: newest /// wins, because a black box is evidence about the machine you are about to /// inspect. Implementations must order deterministically - an unordered pick /// serves an arbitrary recording, which no test catches reliably. - /// @param fault_code The fault code to get rosbag for + /// @param id The record to get rosbag for /// @return Rosbag file info if exists, nullopt otherwise - virtual std::optional get_rosbag_file(const std::string & fault_code) const = 0; + virtual std::optional get_rosbag_file(const FaultId & id) const = 0; - /// Every recording of a fault, newest first. - virtual std::vector get_rosbag_files(const std::string & fault_code) const = 0; + /// Every recording of a fault record, newest first. + virtual std::vector get_rosbag_files(const FaultId & id) const = 0; - /// Every row of one recording - one per fault the recording covers. Backs the + /// Every row of one recording - one per record the recording covers. Backs the /// bulk-data download and the entity authorization scope check, both of which /// start from a recording id and need the faults behind it. virtual std::vector get_rosbag_files_by_recording(const std::string & recording_id) const = 0; @@ -420,23 +464,23 @@ class FaultStorage { /// @return number of rows removed virtual size_t delete_rosbag_recording(const std::string & recording_id) = 0; - /// Delete rosbag file record and the actual file for a fault. Faults from one - /// burst can share a recording; the file is unlinked only with the last record + /// Delete rosbag rows and the actual file for one fault record. Records from one + /// burst can share a recording, and the file is unlinked only with the last row /// that references it. - /// @param fault_code The fault code to delete rosbag for - /// @return true if record was deleted, false if not found - virtual bool delete_rosbag_file(const std::string & fault_code) = 0; + /// @param id The record to delete rosbags for + /// @return true if at least one row was deleted, false if none was found + virtual bool delete_rosbag_file(const FaultId & id) = 0; - /// Delete the records of several faults (typically the whole burst behind one + /// Delete the rows of several records (typically the whole burst behind one /// recording). Backends with real transactions (SQLite) remove the rows /// atomically and unlink the file only after the commit, so a crash mid-delete /// never leaves a row pointing at a removed bag. Default: plain loop. - /// @param fault_codes The fault codes to delete rosbag records for - /// @return Number of records actually deleted - virtual size_t delete_rosbag_files(const std::vector & fault_codes) { + /// @param ids The records to delete rosbag rows for + /// @return Number of records for which rows were actually deleted + virtual size_t delete_rosbag_files(const std::vector & ids) { size_t deleted = 0; - for (const auto & code : fault_codes) { - if (delete_rosbag_file(code)) { + for (const auto & id : ids) { + if (delete_rosbag_file(id)) { ++deleted; } } @@ -452,25 +496,27 @@ class FaultStorage { /// @return Vector of rosbag file info virtual std::vector get_all_rosbag_files() const = 0; - /// Get rosbags for all faults associated with an entity + /// Get rosbags of every record owned by an entity /// @param entity_fqn The entity's fully qualified name to filter by - /// @return Vector of rosbag file info for faults reported by this entity + /// @return Vector of rosbag file info for records this entity owns virtual std::vector list_rosbags_for_entity(const std::string & entity_fqn) const = 0; - /// Get all stored faults regardless of status (for filtering) - /// @return Vector of all faults in storage + /// Get all stored fault records regardless of status (for filtering) + /// @return Vector of all records in storage virtual std::vector get_all_faults() const = 0; - /// Predicate for reclassify_healed_as_cleared: true keeps that fault code HEALED. + /// Predicate for reclassify_healed_as_cleared: true keeps records of that fault code HEALED. using KeepHealedFn = std::function; - /// One-time startup cleanup: reclassify HEALED faults as CLEARED, so a HEALED row left by a + /// One-time startup cleanup: reclassify HEALED records as CLEARED, so a HEALED row left by a /// previous (healing-enabled) run does not behave inconsistently under the latch. /// Default is a no-op (in-memory storage starts empty). /// @param keep_healed Codes for which this returns true stay HEALED (their own healing is still - /// enabled). Empty means reclassify every HEALED fault. - /// @return fault codes of the reclassified faults, so the caller can audit each transition - virtual std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) { + /// enabled). Empty means reclassify every HEALED record. + /// @return the reclassified records, so the caller can audit each transition. Identities, + /// not codes: one code can hold a HEALED record for one owner and an untouched + /// CONFIRMED one for another, and auditing by code would claim both moved. + virtual std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) { (void)keep_healed; return {}; } @@ -498,15 +544,17 @@ class InMemoryFaultStorage : public FaultStorage { std::vector list_faults(bool filter_by_severity, uint8_t severity, const std::vector & statuses) const override; - std::optional get_fault(const std::string & fault_code) const override; + std::optional get_fault(const FaultId & id) const override; + + std::vector get_faults_by_code(const std::string & fault_code) const override; - bool clear_fault(const std::string & fault_code) override; + bool clear_fault(const FaultId & id) override; size_t size() const override; - bool contains(const std::string & fault_code) const override; + bool contains(const FaultId & id) const override; - std::vector check_time_based_confirmation(int64_t now_ns) override; + std::vector check_time_based_confirmation(int64_t now_ns) override; void set_max_snapshots_per_fault(size_t max_count) override; void set_retain_snapshots_on_clear(bool retain) override; @@ -514,32 +562,31 @@ class InMemoryFaultStorage : public FaultStorage { void store_snapshot(const SnapshotData & snapshot) override; void store_snapshots(const std::vector & snapshots) override; - std::vector get_snapshots(const std::string & fault_code, - const std::string & topic_filter = "") const override; + std::vector get_snapshots(const FaultId & id, const std::string & topic_filter = "") const override; int64_t get_max_capture_id() const override; void store_freeze_frame(const FreezeFrameData & frame) override; - std::optional get_freeze_frame(const std::string & fault_code) const override; + std::optional get_freeze_frame(const FaultId & id) const override; void set_max_rosbags_per_fault(size_t max_count) override; size_t set_max_near_misses_per_fault(size_t max_count) override; - std::vector get_near_misses(const std::string & fault_code) const override; + std::vector get_near_misses(const FaultId & id) const override; void store_rosbag_file(const RosbagFileInfo & info) override; /// All-or-nothing, as the base class requires: the batch is built beside the live /// map and swapped in, so a throw leaves the store exactly as it was. void store_rosbag_files(const std::vector & infos) override; - std::optional get_rosbag_file(const std::string & fault_code) const override; - std::vector get_rosbag_files(const std::string & fault_code) const override; + std::optional get_rosbag_file(const FaultId & id) const override; + std::vector get_rosbag_files(const FaultId & id) const override; std::vector get_rosbag_files_by_recording(const std::string & recording_id) const override; - bool delete_rosbag_file(const std::string & fault_code) override; + bool delete_rosbag_file(const FaultId & id) override; size_t delete_rosbag_recording(const std::string & recording_id) override; size_t get_total_rosbag_storage_bytes() const override; std::vector get_all_rosbag_files() const override; std::vector list_rosbags_for_entity(const std::string & entity_fqn) const override; std::vector get_all_faults() const override; - std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) override; + std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) override; private: /// Update fault status based on debounce counter and given config @@ -558,12 +605,12 @@ class InMemoryFaultStorage : public FaultStorage { bool path_referenced(const std::string & file_path) const; mutable std::mutex mutex_; - std::map faults_; + std::map faults_; std::vector snapshots_; - std::map freeze_frames_; ///< fault_code -> freeze-frame (retained across clear) + std::map freeze_frames_; ///< record -> freeze-frame (retained across clear) /// One entry per LINK, mirroring the flat SQLite table rather than a map keyed by - /// fault code - a fault holds several recordings now, and a recording several - /// faults. + /// the record - a record holds several recordings now, and a recording several + /// records. /// /// `seq` is the in-memory twin of SQLite's autoincrement id and is load-bearing, /// not decoration: a burst stamps ONE created_at_ns across every row it writes, so @@ -580,7 +627,7 @@ class InMemoryFaultStorage : public FaultStorage { /// A backend constructed directly (tests, embedders) therefore behaves exactly as /// it always did until someone opts into a history. 0 = unlimited. size_t max_rosbags_per_fault_{1}; - std::map> near_misses_; ///< fault_code -> series (retained across clear) + std::map> near_misses_; ///< record -> series (retained across clear) DebounceConfig config_; size_t max_snapshots_per_fault_{0}; ///< 0 = unlimited bool retain_snapshots_on_clear_{false}; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/ros2/ros_capture_hooks.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/ros2/ros_capture_hooks.hpp index 856d3b489..3f2112f4c 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/ros2/ros_capture_hooks.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/ros2/ros_capture_hooks.hpp @@ -26,27 +26,27 @@ namespace ros2_medkit_fault_manager::ros2 { /// the function given at construction; the node reads its capture members when it runs. class RosCaptureHooks : public core::CaptureHooks { public: - using CaptureFn = std::function; - using CodeFn = std::function; + using CaptureFn = std::function; + using RecordFn = std::function; - RosCaptureHooks(CaptureFn capture_now, CodeFn on_prefailed, CodeFn on_cleared) + RosCaptureHooks(CaptureFn capture_now, RecordFn on_prefailed, RecordFn on_cleared) : capture_now_(std::move(capture_now)), on_prefailed_(std::move(on_prefailed)), on_cleared_(std::move(on_cleared)) { } - core::CaptureResult capture_now(const std::string & code) override { - return capture_now_(code); + core::CaptureResult capture_now(const FaultId & id) override { + return capture_now_(id); } - void on_prefailed(const std::string & code) override { - on_prefailed_(code); + void on_prefailed(const FaultId & id) override { + on_prefailed_(id); } - void on_cleared(const std::string & code) override { - on_cleared_(code); + void on_cleared(const FaultId & id) override { + on_cleared_(id); } private: CaptureFn capture_now_; - CodeFn on_prefailed_; - CodeFn on_cleared_; + RecordFn on_prefailed_; + RecordFn on_cleared_; }; } // namespace ros2_medkit_fault_manager::ros2 diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/rosbag_capture.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/rosbag_capture.hpp index f380a8503..afa28afc6 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/rosbag_capture.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/rosbag_capture.hpp @@ -109,22 +109,24 @@ class RosbagCapture { /// Check if ring buffer is currently running bool is_running() const; - /// Called when a fault enters PREFAILED state (for lazy_start mode) - /// @param fault_code The fault code that entered PREFAILED - void on_fault_prefailed(const std::string & fault_code); - - /// Called when a fault is confirmed - flushes buffer to bag file. A fault that - /// confirms while the previous fault's post-roll is still running is attached - /// to that recording (same burst, same window) rather than losing its bag. - /// @param fault_code The fault code that was confirmed - void on_fault_confirmed(const std::string & fault_code); - - /// Called when a fault is cleared - deletes its bag record if auto_cleanup. - /// A shared bag survives until its last referencing fault clears; a fault + /// Called when a fault record enters PREFAILED state (for lazy_start mode) + /// @param id The record that entered PREFAILED + void on_fault_prefailed(const FaultId & id); + + /// Called when a fault record is confirmed - flushes buffer to bag file. A record + /// that confirms while the previous one's post-roll is still running is attached + /// to that recording (same burst, same window) rather than losing its bag. Two + /// owners confirming one code in a burst attach to the one in-flight recording, + /// and each gets its own rosbag_files row. + /// @param id The record that was confirmed + void on_fault_confirmed(const FaultId & id); + + /// Called when a fault record is cleared - deletes its bag record if auto_cleanup. + /// A shared bag survives until its last referencing record clears. A record /// cleared during its burst's post-roll is dropped from the in-flight - /// recording state and never gets a record. - /// @param fault_code The fault code that was cleared - void on_fault_cleared(const std::string & fault_code); + /// recording state and never gets a row. + /// @param id The record that was cleared + void on_fault_cleared(const FaultId & id); /// Get current configuration const RosbagConfig & config() const { @@ -244,10 +246,10 @@ class RosbagCapture { /// (falls back to SensorDataQoS when no publisher is known or qos_match is off) rclcpp::QoS resolve_topic_qos(const std::string & topic) const; - /// Compute the entity topic set for a fault (the faulting source node's + /// Compute the entity topic set for a fault record (its owning source node's /// pub/sub topics + /tf, intersected with the subscribed set). Empty set = /// scope unresolved. Never throws; failures degrade to an empty set. - std::set compute_entity_topics(const std::string & fault_code); + std::set compute_entity_topics(const FaultId & id); /// In "entity" mode, compute the set of topics to write for a confirmed fault /// (the faulting source node's pub/sub topics + /tf). Empty set = write all. @@ -256,7 +258,7 @@ class RosbagCapture { /// In "entity" mode, union an attached fault's entity topics into the active /// capture filter so its data reaches the shared bag from the attach onwards /// (empty resolution widens to all topics). Caller holds post_fault_timer_mutex_. - void widen_capture_filter_for(const std::string & fault_code, const std::set & topics); + void widen_capture_filter_for(const FaultId & id, const std::set & topics); /// Whether a topic should be written to the bag given the active entity filter bool should_capture_topic(const std::string & topic) const; @@ -295,9 +297,9 @@ class RosbagCapture { /// A post-fault recording whose writer is closed, on its way to the store worker. struct ClosedRecording { std::string bag_path; - /// Empty when the primary fault was cleared during the post-roll. - std::string fault_code; - std::set attached_fault_codes; + /// Empty fault_code when the primary record was cleared during the post-roll. + FaultId fault_id; + std::set attached_ids; double duration_sec{0.0}; int64_t created_at_ns{0}; }; @@ -317,7 +319,7 @@ class RosbagCapture { /// Make @p rows durable for the finished bag at @p bag_path, or discard the bag. /// /// Both finalisation paths end here, so a bag that cannot be looked up never - /// survives on disk: retrieval is keyed by fault code and the quota enumerates + /// survives on disk: retrieval goes through the rows and the quota enumerates /// rows, so a directory with no row is unreachable, uncounted, and can never be /// evicted to make room. /// @@ -339,7 +341,7 @@ class RosbagCapture { /// Whether the fault still gets a row: false only under auto_cleanup with no history, /// for a fault the store holds as cleared. Called under clear_rows_mutex_. - bool row_still_wanted(const std::string & code); + bool row_still_wanted(const FaultId & id); /// Whether the operator asked this fault code to keep a recording history. /// @@ -390,9 +392,9 @@ class RosbagCapture { /// Cheap, and checked before the entity scope is resolved: a level-triggered /// reporter re-confirming the same fault would otherwise pay for a fault-store read /// and a full graph enumeration on every repeat, none of which it can use. - bool is_current_recording_primary(const std::string & fault_code) const; + bool is_current_recording_primary(const FaultId & id) const; - bool attach_to_active_recording(const std::string & fault_code, const std::set & entity_topics); + bool attach_to_active_recording(const FaultId & id, const std::set & entity_topics); /// Try to subscribe to a single topic /// @param topic The topic to subscribe to @@ -443,16 +445,17 @@ class RosbagCapture { /// Running state std::atomic running_{false}; - /// Upper bound on how many extra faults one recording is registered for. + /// Upper bound on how many extra records one recording is registered for. static constexpr size_t kMaxAttachedFaults = 32; /// Post-fault recording state - std::string current_fault_code_; + FaultId current_fault_id_; std::string current_bag_path_; - /// Faults confirmed while the post-roll was already running. They share the + /// Records confirmed while the post-roll was already running. They share the /// recording window (one root cause, one burst), so each gets a metadata row - /// pointing at the same bag when it finalises. - std::set attached_fault_codes_; + /// pointing at the same bag when it finalises. Two owners of one code are two + /// entries here and two rows on finalise. + std::set attached_fault_ids_; /// Protects post_fault_timer_, the recording_post_fault_ transitions and the /// state above against concurrent access from on_fault_confirmed() (capture-pool /// thread) and post_fault_timer_callback() / stop() (executor thread). The diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp index c0705c55e..ce18563ec 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp @@ -89,9 +89,9 @@ struct RosbagConfig { /// (reliable/transient-local where offered) instead of forcing best-effort bool qos_match{true}; - /// Recordings retained per fault code (0 = unlimited, bounded only by - /// max_total_storage_mb). Keep-newest: past the cap the fault's OLDEST recording - /// loses its row, and the bag goes once no fault references it. 1 reproduces the + /// Recordings retained per fault record (0 = unlimited, bounded only by + /// max_total_storage_mb). Keep-newest: past the cap the record's OLDEST recording + /// loses its row, and the bag goes once no record references it. 1 reproduces the /// pre-#620 behaviour, where a re-confirm replaced the previous recording. size_t max_bags_per_fault{1}; @@ -165,17 +165,22 @@ class SnapshotCapture { SnapshotCapture(SnapshotCapture &&) = delete; SnapshotCapture & operator=(SnapshotCapture &&) = delete; - /// Capture snapshots for a fault that was just confirmed + /// Capture snapshots for a fault record that was just confirmed + /// + /// Topic resolution stays keyed by the fault CODE (fault_specific and patterns are + /// config about what a code means), while everything written is keyed by the RECORD: + /// the rows, the freeze frame and the mid-capture acknowledgement check. Two owners + /// confirming one code therefore capture the same topics into two separate frames. /// /// If the fault code resolves to no capture set (not in fault_specific, no pattern /// match, no default_topics), the entity-default fallback (when enabled) captures - /// the reporting source node's own published topics instead. A code explicitly + /// the owning source node's own published topics instead. A code explicitly /// listed in fault_specific or matched by a pattern never falls through - a /// present-but-empty topic list is a per-fault opt-out. Only when nothing /// resolves does capture return early: no freeze_frames row is written /// (no empty {} row) and FaultStorage::get_freeze_frame() returns nullopt for it. - /// @param fault_code The fault code that was confirmed - void capture(const std::string & fault_code); + /// @param id The fault record that was confirmed + void capture(const FaultId & id); /// Get current configuration const SnapshotConfig & config() const { @@ -196,12 +201,12 @@ class SnapshotCapture { /// entity-default capture. std::vector resolve_topics(const std::string & fault_code, bool & explicit_match) const; - /// Entity-default fallback: topics published by the fault's reporting source - /// node(s), excluding per-node noise (/rosout, /parameter_events). Non-FQN - /// sources (bare plugin entity ids - the gateway covers those) are skipped, - /// and empty is returned when no source resolves to a live node. Never + /// Entity-default fallback: topics published by the record's owning source + /// node, excluding per-node noise (/rosout, /parameter_events). A non-FQN + /// owner (a bare plugin entity id - the gateway covers those) is skipped, + /// and empty is returned when the owner resolves to no live node. Never /// throws; any failure degrades to empty. - std::vector resolve_entity_topics(const std::string & fault_code) const; + std::vector resolve_entity_topics(const FaultId & id) const; /// Capture a single topic on-demand (creates temporary subscription) /// On success also records the captured value into @p freeze_frame under the topic key. @@ -209,14 +214,14 @@ class SnapshotCapture { /// Appends to @p rows rather than storing: a capture is persisted as one set, /// so the per-fault cap can drop a whole old capture instead of truncating this /// one topic by topic. - bool capture_topic_on_demand(const std::string & fault_code, const std::string & topic, nlohmann::json & freeze_frame, + bool capture_topic_on_demand(const FaultId & id, const std::string & topic, nlohmann::json & freeze_frame, std::vector & rows); /// Capture a topic from background cache /// On success also records the cached value into @p freeze_frame under the topic key. /// @return true if data was available in cache - bool capture_topic_from_cache(const std::string & fault_code, const std::string & topic, - nlohmann::json & freeze_frame, std::vector & rows); + bool capture_topic_from_cache(const FaultId & id, const std::string & topic, nlohmann::json & freeze_frame, + std::vector & rows); /// Initialize background subscriptions for all configured topics void init_background_subscriptions(); diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp index 12fcd9b27..af15f551d 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp @@ -26,6 +26,16 @@ namespace ros2_medkit_fault_manager { +/// Owner given to a record migrated out of a database that recorded no readable +/// reporting source. +/// +/// A migrated record is never left with an empty owner. It would be unreachable +/// through the source_id every single-record service takes, and it would leave the +/// child backfill looking for work on every open. The empty owner means one thing +/// only, a child row (freeze frame, snapshot, rosbag link) not yet assigned to a +/// record, and keeping that meaning unambiguous is what this constant is for. +inline constexpr const char * kLegacyOwner = "legacy"; + /// SQLite-based fault storage implementation with persistence /// Thread-safe implementation using mutex protection class SqliteFaultStorage : public FaultStorage { @@ -55,15 +65,17 @@ class SqliteFaultStorage : public FaultStorage { std::vector list_faults(bool filter_by_severity, uint8_t severity, const std::vector & statuses) const override; - std::optional get_fault(const std::string & fault_code) const override; + std::optional get_fault(const FaultId & id) const override; + + std::vector get_faults_by_code(const std::string & fault_code) const override; - bool clear_fault(const std::string & fault_code) override; + bool clear_fault(const FaultId & id) override; size_t size() const override; - bool contains(const std::string & fault_code) const override; + bool contains(const FaultId & id) const override; - std::vector check_time_based_confirmation(int64_t now_ns) override; + std::vector check_time_based_confirmation(int64_t now_ns) override; void set_max_snapshots_per_fault(size_t max_count) override; void set_retain_snapshots_on_clear(bool retain) override; @@ -73,29 +85,28 @@ class SqliteFaultStorage : public FaultStorage { void store_snapshot(const SnapshotData & snapshot) override; void store_snapshots(const std::vector & snapshots) override; - std::vector get_snapshots(const std::string & fault_code, - const std::string & topic_filter = "") const override; + std::vector get_snapshots(const FaultId & id, const std::string & topic_filter = "") const override; int64_t get_max_capture_id() const override; void store_freeze_frame(const FreezeFrameData & frame) override; - std::optional get_freeze_frame(const std::string & fault_code) const override; + std::optional get_freeze_frame(const FaultId & id) const override; size_t set_max_near_misses_per_fault(size_t max_count) override; - std::vector get_near_misses(const std::string & fault_code) const override; + std::vector get_near_misses(const FaultId & id) const override; void store_rosbag_file(const RosbagFileInfo & info) override; void store_rosbag_files(const std::vector & infos) override; - std::optional get_rosbag_file(const std::string & fault_code) const override; - std::vector get_rosbag_files(const std::string & fault_code) const override; + std::optional get_rosbag_file(const FaultId & id) const override; + std::vector get_rosbag_files(const FaultId & id) const override; std::vector get_rosbag_files_by_recording(const std::string & recording_id) const override; - bool delete_rosbag_file(const std::string & fault_code) override; + bool delete_rosbag_file(const FaultId & id) override; size_t delete_rosbag_recording(const std::string & recording_id) override; - size_t delete_rosbag_files(const std::vector & fault_codes) override; + size_t delete_rosbag_files(const std::vector & ids) override; size_t get_total_rosbag_storage_bytes() const override; std::vector get_all_rosbag_files() const override; std::vector list_rosbags_for_entity(const std::string & entity_fqn) const override; std::vector get_all_faults() const override; - std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) override; + std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) override; /// Get the database path const std::string & db_path() const { @@ -122,16 +133,68 @@ class SqliteFaultStorage : public FaultStorage { /// C++ through rosbag_recording_id() so the basename rule has one implementation. void migrate_rosbag_files_add_recording_id(); - /// Whether a fault other than @p fault_code still references @p file_path. - /// One recording can back several faults of the same burst, so the bag must - /// only be unlinked once the last of them is gone. Caller holds mutex_. - - /// Whether any fault at all still references @p file_path. Caller holds mutex_. + /// Move a database whose fault identity is the bare fault_code onto (fault_code, owner). + /// + /// Detected per table by PRAGMA table_info lacking `owner`, which is what makes it safe + /// to re-run on every open. faults and freeze_frames are REBUILT (their identity is a + /// PRIMARY KEY, which ALTER TABLE cannot change and CREATE TABLE IF NOT EXISTS would + /// silently skip), while snapshots and rosbag_files only gain a column. The fault owner + /// comes from the legacy row's first reporting source, empty when none can be read: that + /// is the only per-source fact the old schema holds, so a row with several sources folds + /// onto its first one rather than inventing per-source counters, statuses and timestamps + /// that were never recorded. + /// + /// It never refuses to finish. A row whose content cannot be read leaves an empty owner + /// and a log line, because a migration that aborts on a row takes the whole database with + /// it: the rebuild rolls back and the next open reaches the same row again, forever. + void migrate_faults_add_owner(); + + /// Copy every legacy `faults` row into `faults_new`, deriving the owner in C++. + /// + /// In C++ rather than one INSERT ... SELECT because the derivation must not depend on the + /// column being valid JSON, and SQL's json_extract raises rather than returning NULL on + /// text it cannot parse. Caller holds the migration transaction. + void copy_legacy_fault_rows(); + + /// Give every ownerless child row the owner of its fault, where that is unambiguous. + /// + /// Runs on every open, not only when a column was just added: a child table can be left + /// with empty owners by an open interrupted between the faults rebuild and the backfill, + /// or by a database whose child table gained the column while faults was still legacy. + /// Such a row is invisible to every (code, owner) read, so leaving it is losing evidence. + /// A code carrying several owners is NOT resolved - guessing would hand one owner another + /// owner's evidence - and the row keeps its empty owner with a log line naming it. + void backfill_child_owners(); + + /// Whether any child row is waiting for an owner the database can actually prove, so the + /// backfill above has work to do. Deliberately the backfill's own condition and not just + /// "owner is empty": a row whose code has several owners or none can never be assigned, + /// and treating it as pending would re-enter the write transaction on every open forever. + /// Tables without the column yet are skipped: the migration adds it. + bool has_assignable_child_rows() const; + + /// Whether @p table exists, asked of sqlite_master. Used to tell a leftover scratch table + /// apart from a clean start, which DROP TABLE IF EXISTS on its own cannot report. + bool table_exists(const char * table) const; + + /// Drop one of the migration's scratch tables, warning first when it was really there. + /// `faults_new` and `freeze_frames_new` are reserved for this procedure, so debris under + /// those names is always an unfinished rebuild and never somebody's data. + void drop_scratch_table(const char * table); + + /// Whether @p table already carries the `owner` column. The four tables are created + /// by four independent CREATE TABLE IF NOT EXISTS statements, so a database can hold + /// a legacy one next to one this release just created: each is probed on its own. + bool table_has_owner(const char * table) const; + + /// Whether any row at all still references @p file_path. One recording can back + /// several records of the same burst, so the bag must only be unlinked once the + /// last of them is gone. Caller holds mutex_. bool path_referenced(const std::string & file_path) const; /// store_rosbag_file body without taking mutex_. Caller holds mutex_ and /// unlinks the returned replaced-bag path once the row change is durable. - /// @return file_paths whose last row for this fault the per-fault cap evicted. + /// @return file_paths whose last row for this record the per-record cap evicted. /// The caller unlinks each only after the commit, and only if path_referenced() /// still says nobody holds it. std::vector store_rosbag_file_locked(const RosbagFileInfo & info); @@ -149,7 +212,7 @@ class SqliteFaultStorage : public FaultStorage { /// @param debounce_counter Counter value after the report /// @param config Debounce config the report was evaluated against /// @param severity Severity carried by the report - /// @param source_id Reporting source + /// @param source_id Reporting source, which is the record's owner and scopes the series /// @param resulting_status Fault status after the report was applied void record_near_miss_locked(const std::string & fault_code, int64_t occurred_at_ns, int32_t debounce_counter, const DebounceConfig & config, uint8_t severity, const std::string & source_id, @@ -158,10 +221,9 @@ class SqliteFaultStorage : public FaultStorage { /// Run a plain SQL statement or throw with the SQLite error. Caller holds mutex_. void exec_or_throw(const char * sql); - /// Parse JSON array string to vector of strings - std::vector parse_json_array(const std::string & json_str) const; - - /// Serialize vector of strings to JSON array string + /// Serialize vector of strings to JSON array string. The faults table keeps + /// reporting_sources as the serialized form of owner, so a reader of the raw table + /// still sees the field it always saw. static std::string serialize_json_array(const std::vector & vec); std::string db_path_; diff --git a/src/ros2_medkit_fault_manager/src/capture_thread_pool.cpp b/src/ros2_medkit_fault_manager/src/capture_thread_pool.cpp index 8662855ce..f969c962c 100644 --- a/src/ros2_medkit_fault_manager/src/capture_thread_pool.cpp +++ b/src/ros2_medkit_fault_manager/src/capture_thread_pool.cpp @@ -20,7 +20,7 @@ namespace ros2_medkit_fault_manager { CaptureThreadPool::CaptureThreadPool(std::size_t pool_size, std::size_t queue_depth, QueueFullPolicy full_policy, - core::LogSink & log, std::function capture_fn, + core::LogSink & log, std::function capture_fn, std::string logger_name) : queue_depth_(queue_depth == 0 ? 1 : queue_depth) , full_policy_(full_policy) @@ -44,13 +44,13 @@ CaptureThreadPool::~CaptureThreadPool() { shutdown(); } -EnqueueOutcome CaptureThreadPool::enqueue(const std::string & fault_code) { +EnqueueOutcome CaptureThreadPool::enqueue(const FaultId & id) { std::lock_guard lock(queue_mutex_); if (stop_) { return {EnqueueResult::kRejectedShuttingDown, std::nullopt}; } if (queue_.size() < queue_depth_) { - queue_.push_back(fault_code); + queue_.push_back(id); cv_.notify_one(); return {EnqueueResult::kAccepted, std::nullopt}; } @@ -60,9 +60,9 @@ EnqueueOutcome CaptureThreadPool::enqueue(const std::string & fault_code) { return {EnqueueResult::kDroppedNewest, std::nullopt}; } // kDropOldest: evict the oldest pending job. - std::string evicted = std::move(queue_.front()); + FaultId evicted = std::move(queue_.front()); queue_.pop_front(); - queue_.push_back(fault_code); + queue_.push_back(id); dropped_captures_.fetch_add(1, std::memory_order_relaxed); cv_.notify_one(); return {EnqueueResult::kEvictedOldest, std::move(evicted)}; @@ -103,7 +103,7 @@ std::size_t CaptureThreadPool::pending_size() const { void CaptureThreadPool::worker_loop() { for (;;) { - std::string job; + FaultId job; { std::unique_lock lock(queue_mutex_); cv_.wait(lock, [this] { @@ -120,10 +120,11 @@ void CaptureThreadPool::worker_loop() { capture_fn_(job); } } catch (const std::exception & e) { - core::logf(log_, core::LogLevel::kError, logger_name_, "Capture job for '%s' threw: %s", job.c_str(), e.what()); + core::logf(log_, core::LogLevel::kError, logger_name_, "Capture job for '%s' threw: %s", job.fault_code.c_str(), + e.what()); } catch (...) { core::logf(log_, core::LogLevel::kError, logger_name_, "Capture job for '%s' threw unknown exception", - job.c_str()); + job.fault_code.c_str()); } } } diff --git a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp index e29364da9..99dd349db 100644 --- a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp +++ b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include namespace ros2_medkit_fault_manager::core { @@ -60,6 +60,27 @@ int64_t next_due_ns(int64_t due, int64_t now, int64_t period_ns) { return due + k * period_ns; } +/// Name of a Fault.msg SEVERITY_* value, as the constants spell it. +const char * severity_name(uint8_t severity) { + switch (severity) { + case fd::severity::kInfo: + return "INFO"; + case fd::severity::kWarn: + return "WARN"; + case fd::severity::kError: + return "ERROR"; + case fd::severity::kCritical: + return "CRITICAL"; + default: + return "UNKNOWN"; + } +} + +/// The owner a stored record names, empty when it names none. +std::string owner_of(const fd::FaultRecord & record) { + return record.reporting_sources.empty() ? std::string() : record.reporting_sources.front(); +} + std::string extract_topic_name(const std::string & topic_path) { auto pos = topic_path.rfind('/'); if (pos != std::string::npos && pos < topic_path.length() - 1) { @@ -125,9 +146,10 @@ ReportOutcome FaultManagerCore::report(const fd::FaultReport & report) { return report_locked(report); } -ClearOutcome FaultManagerCore::clear(const std::string & code, bool skip_correlation_auto_clear) { +ClearOutcome FaultManagerCore::clear(const std::string & code, const std::string & source_id, + bool skip_correlation_auto_clear) { std::lock_guard lock(mutex_); - return clear_locked(code, skip_correlation_auto_clear); + return clear_locked(code, source_id, skip_correlation_auto_clear); } ListOutcome FaultManagerCore::list(const ListQuery & query) { @@ -135,12 +157,12 @@ ListOutcome FaultManagerCore::list(const ListQuery & query) { return list_locked(query); } -FaultDetail FaultManagerCore::get(const std::string & code) { +FaultDetail FaultManagerCore::get(const std::string & code, const std::string & source_id) { std::vector> rosbag_paths; FaultDetail detail; { std::lock_guard lock(mutex_); - detail = get_locked(code, rosbag_paths); + detail = get_locked(code, source_id, rosbag_paths); } // Reads metadata.yaml per recording; outside the lock so fault reports do not wait on disk. for (const auto & [index, path] : rosbag_paths) { @@ -150,6 +172,54 @@ FaultDetail FaultManagerCore::get(const std::string & code) { return detail; } +std::optional FaultManagerCore::resolve_target(const std::string & fault_code, const std::string & source_id, + std::string & error) { + std::lock_guard lock(mutex_); + return resolve_target_locked(fault_code, source_id, error); +} + +std::optional FaultManagerCore::resolve_target_locked(const std::string & fault_code, + const std::string & source_id, std::string & error) { + error.clear(); + + try { + if (!source_id.empty()) { + FaultId id{fault_code, source_id}; + if (!storage_->contains(id)) { + error = "Fault not found: " + fault_code + " (source " + source_id + ")"; + return std::nullopt; + } + return id; + } + + // Unscoped. One record: the call means that record. Several: refuse rather than + // pick, because acting on an arbitrary owner's record is the failure this identity + // change exists to remove, and the caller cannot tell which one it got. + const auto candidates = storage_->get_faults_by_code(fault_code); + if (candidates.empty()) { + error = "Fault not found: " + fault_code; + return std::nullopt; + } + if (candidates.size() > 1) { + std::string owners; + for (const auto & candidate : candidates) { + if (!owners.empty()) { + owners += ", "; + } + owners += owner_of(candidate); + } + error = "ambiguous: " + fault_code + " is reported by " + std::to_string(candidates.size()) + " sources (" + + owners + "). Set source_id to pick one."; + return std::nullopt; + } + return FaultId{fault_code, owner_of(candidates.front())}; + } catch (const FaultStorage::IgnorableConnectionException & e) { + logf(log_, LogLevel::kWarn, config_.logger_name, "Failed to connect with the fault storage server: %s", e.what()); + error = storage_unavailable(e); + return std::nullopt; + } +} + std::vector FaultManagerCore::list_for_entity(const std::string & entity_id) { std::lock_guard lock(mutex_); std::vector result; @@ -198,7 +268,7 @@ void FaultManagerCore::run_auto_confirm(int64_t wall_now_ns) { logf(log_, LogLevel::kWarn, logger, "Failed to connect with the fault storage server: %s", e.what()); }; - std::vector confirmed; + std::vector confirmed; try { confirmed = storage_->check_time_based_confirmation(wall_now_ns); } catch (const FaultStorage::IgnorableConnectionException & e) { @@ -209,10 +279,10 @@ void FaultManagerCore::run_auto_confirm(int64_t wall_now_ns) { return; } // Audit every timer-driven PREFAILED->CONFIRMED transition, or the audit log's verify() cannot see it. - for (const auto & fault_code : confirmed) { + for (const auto & id : confirmed) { std::optional fault; try { - fault = storage_->get_fault(fault_code); + fault = storage_->get_fault(id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); return; @@ -220,18 +290,18 @@ void FaultManagerCore::run_auto_confirm(int64_t wall_now_ns) { if (!fault) { continue; } - audit_transition(kTransitionConfirmed, *fault, "auto_confirm_timer", wall_now_ns); + audit_transition(kTransitionConfirmed, *fault, id.owner, wall_now_ns); // Same rules as the report path: a symptom muted by a root cause is not announced, and the // capture is not gated by muting. - if (!correlation_ || !correlation_->is_muted(fault_code)) { + if (!correlation_ || !correlation_->is_muted(id)) { publish_event(fd::fault_event_type::kConfirmed, *fault); } - capture_on_confirm(fault_code); + capture_on_confirm(id); } logf(log_, LogLevel::kInfo, logger, "Auto-confirmed %zu PREFAILED fault(s) due to time threshold", confirmed.size()); } -void FaultManagerCore::capture_on_confirm(const std::string & code) { +void FaultManagerCore::capture_on_confirm(const FaultId & id) { const double cooldown_sec = config_.snapshots_recapture_cooldown_sec; const bool cooldown_enabled = cooldown_sec > 0.0; const auto cooldown = std::chrono::duration(cooldown_sec); @@ -246,23 +316,24 @@ void FaultManagerCore::capture_on_confirm(const std::string & code) { ++it; } } - if (last_capture_times_.find(code) != last_capture_times_.end()) { - logf(log_, LogLevel::kDebug, config_.logger_name, "Skipping capture for '%s' - cooldown active", code.c_str()); + if (last_capture_times_.find(id) != last_capture_times_.end()) { + logf(log_, LogLevel::kDebug, config_.logger_name, "Skipping capture for '%s' (source %s) - cooldown active", + id.fault_code.c_str(), id.owner.c_str()); return; } } - const CaptureResult outcome = hooks_.capture_now(code); + const CaptureResult outcome = hooks_.capture_now(id); if (!cooldown_enabled) { return; } switch (outcome.result) { case CaptureEnqueue::kAccepted: - last_capture_times_[code] = now; + last_capture_times_[id] = now; break; case CaptureEnqueue::kEvictedOldest: - last_capture_times_[code] = now; - last_capture_times_.erase(outcome.evicted_code); // keeps the evicted fault retriable + last_capture_times_[id] = now; + last_capture_times_.erase(outcome.evicted_id); // keeps the evicted record retriable break; case CaptureEnqueue::kDroppedNewest: // not recorded: a later re-confirmation may still capture case CaptureEnqueue::kShuttingDown: @@ -322,10 +393,14 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { logf(log_, LogLevel::kWarn, logger, "Failed to connect with the fault storage server: %s", e.what()); }; - // Get status before update (if fault exists) + // The record this report addresses. The source_id owns it, so a report from a second source of the + // same code creates a second record instead of merging. + const FaultId id{report.fault_code, report.source_id}; + + // Get status before update (if the record exists) std::optional fault_before; try { - fault_before = storage_->get_fault(report.fault_code); + fault_before = storage_->get_fault(id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); return outcome; @@ -333,7 +408,8 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { std::string status_before = fault_before ? fault_before->status : ""; // Resolve the debounce config: global, then the entity override matching - // source_id, then the fault code's own override. + // source_id, then the fault code's own override. The record and the config share + // an owner, so the resolved band always governs the counter it is applied to. auto resolved_config = resolve_config(report.source_id, report.fault_code); const int64_t event_time_ns = clock_.now_ns(); @@ -348,25 +424,24 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { outcome.accepted = true; - // Get updated fault state to publish event + if (fault_before && failed && report.severity != fault_before->severity) { + warn_severity_mismatch(report, fault_before->severity); + } + + // Get updated record state to publish event std::optional fault_after; try { - fault_after = storage_->get_fault(report.fault_code); + fault_after = storage_->get_fault(id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); return outcome; } if (fault_after) { - // Only for codes the store holds: a PASSED for an unknown code writes nothing, so it must - // not add an entry here either (the diagnostic bridge sends PASSED for every OK status). - warn_on_conflicting_debounce_policy(report.fault_code, report.source_id, resolved_config); - // Process through correlation engine (if enabled). Only FAILED events correlate. bool should_mute = false; if (correlation_ && failed) { - auto correlation_result = - correlation_->process_fault(report.fault_code, correlation::severity_to_string(report.severity)); + auto correlation_result = correlation_->process_fault(id, correlation::severity_to_string(report.severity)); should_mute = correlation_result.should_mute; @@ -404,8 +479,9 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { } just_confirmed = true; } else if (!is_new && fault_after->status == fd::fault_status::kConfirmed) { - // Fault was already CONFIRMED, data updated (last_occurred, severity, sources). - // Not occurrence_count: a re-report inside one occurrence does not touch it. + // Record was already CONFIRMED, data updated (last_occurred, severity). Not + // reporting_sources: it names the one owner and never grows. Not + // occurrence_count: a re-report inside one occurrence does not touch it. if (!should_mute) { publish_event(fd::fault_event_type::kUpdated, *fault_after); } @@ -436,11 +512,11 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { audit_transition(kTransitionConfirmed, *fault_after, report.source_id, event_time_ns); } if (just_healed) { - audit_transition(kTransitionHealed, *fault_after, "auto_heal", event_time_ns); + audit_transition(kTransitionHealed, *fault_after, id.owner, event_time_ns); } if (just_confirmed) { - capture_on_confirm(report.fault_code); + capture_on_confirm(id); } // PREFAILED starts the lazy_start rosbag capture @@ -448,7 +524,7 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { (!is_new && status_before != fd::fault_status::kPrefailed && fault_after->status == fd::fault_status::kPrefailed); if (just_prefailed) { - hooks_.on_prefailed(report.fault_code); + hooks_.on_prefailed(id); } } @@ -467,7 +543,8 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { return outcome; } -ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_correlation_auto_clear) { +ClearOutcome FaultManagerCore::clear_locked(const std::string & code, const std::string & source_id, + bool skip_correlation_auto_clear) { const std::string & logger = config_.logger_name; ClearOutcome outcome; @@ -477,6 +554,17 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_ return outcome; } + // Which record. Resolved before anything changes, so an ambiguous unscoped call + // clears nothing at all rather than one arbitrary owner's record. + std::string resolve_error; + const auto target = resolve_target_locked(code, source_id, resolve_error); + if (!target) { + outcome.message = resolve_error; + logf(log_, LogLevel::kWarn, logger, "ClearFault rejected: %s", resolve_error.c_str()); + return outcome; + } + const FaultId & id = *target; + const auto connection_lost = [&](const FaultStorage::IgnorableConnectionException & e) { logf(log_, LogLevel::kWarn, logger, "Failed to connect with the fault storage server: %s", e.what()); }; @@ -484,21 +572,22 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_ // Storage first: a clear that the storage refuses must not change the correlation state. bool cleared = false; try { - cleared = storage_->clear_fault(code); + cleared = storage_->clear_fault(id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); outcome.message = storage_unavailable(e); return outcome; } - // Then the correlation engine (to get the auto-clear list). + // Then the correlation engine (to get the auto-clear list). Symptoms come back as + // records of the same owner, so a cascade never crosses an owner boundary. // `skip_correlation_auto_clear` lets the caller opt out of cascade-clearing - // correlated symptom fault codes. Per-entity DELETE routes set it to true + // correlated symptom faults. Per-entity DELETE routes set it to true // so they cannot reach across entity boundaries via the correlation graph. - std::vector auto_cleared_codes; + std::vector auto_cleared; if (correlation_ && !skip_correlation_auto_clear) { - auto clear_result = correlation_->process_clear(code); - auto_cleared_codes = clear_result.auto_cleared_codes; + auto clear_result = correlation_->process_clear(id); + auto_cleared = clear_result.auto_cleared_symptoms; } outcome.success = cleared; @@ -508,38 +597,41 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_ return outcome; } - // Evict cooldown tracking for the cleared fault and the auto-cleared symptoms - last_capture_times_.erase(code); - for (const auto & symptom_code : auto_cleared_codes) { - last_capture_times_.erase(symptom_code); + // Evict cooldown tracking for the cleared record and the auto-cleared symptoms + last_capture_times_.erase(id); + for (const auto & symptom_id : auto_cleared) { + last_capture_times_.erase(symptom_id); } // Auto-clear correlated symptoms - for (const auto & symptom_code : auto_cleared_codes) { + std::vector auto_cleared_codes; + auto_cleared_codes.reserve(auto_cleared.size()); + for (const auto & symptom_id : auto_cleared) { try { - storage_->clear_fault(symptom_code); + storage_->clear_fault(symptom_id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); outcome.message = storage_unavailable(e); return outcome; } + auto_cleared_codes.push_back(symptom_id.fault_code); if (audit_) { std::optional symptom; try { - symptom = storage_->get_fault(symptom_code); + symptom = storage_->get_fault(symptom_id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); outcome.message = storage_unavailable(e); return outcome; } if (symptom) { - audit_transition(kTransitionCleared, *symptom, "clear_service", clock_.now_ns()); + audit_transition(kTransitionCleared, *symptom, symptom_id.owner, clock_.now_ns()); } } - logf(log_, LogLevel::kDebug, logger, "Auto-cleared symptom: %s (root cause: %s)", symptom_code.c_str(), + logf(log_, LogLevel::kDebug, logger, "Auto-cleared symptom: %s (root cause: %s)", symptom_id.fault_code.c_str(), code.c_str()); - // Also cleanup rosbag for auto-cleared faults - hooks_.on_cleared(symptom_code); + // Also cleanup rosbag for auto-cleared records + hooks_.on_cleared(symptom_id); } outcome.auto_cleared_codes = auto_cleared_codes; @@ -549,16 +641,16 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_ outcome.message = "Fault cleared: " + code + " (auto-cleared " + std::to_string(auto_cleared_codes.size()) + " symptoms)"; } - logf(log_, LogLevel::kInfo, logger, "Fault cleared: %s (auto-cleared %zu symptoms)", code.c_str(), - auto_cleared_codes.size()); + logf(log_, LogLevel::kInfo, logger, "Fault cleared: %s (source=%s, auto-cleared %zu symptoms)", code.c_str(), + id.owner.c_str(), auto_cleared_codes.size()); - // Cleanup rosbag for the main fault (auto_cleanup handled inside RosbagCapture) - hooks_.on_cleared(code); + // Cleanup rosbag for the cleared record (auto_cleanup handled inside RosbagCapture) + hooks_.on_cleared(id); - // Publish the cleared event - get the cleared fault to include in the event + // Publish the cleared event - get the cleared record to include in the event std::optional fault; try { - fault = storage_->get_fault(code); + fault = storage_->get_fault(id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); outcome.message = storage_unavailable(e); @@ -566,7 +658,7 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_ } if (fault) { publish_event(fd::fault_event_type::kCleared, *fault, auto_cleared_codes); - audit_transition(kTransitionCleared, *fault, "clear_service", clock_.now_ns()); + audit_transition(kTransitionCleared, *fault, id.owner, clock_.now_ns()); } return outcome; } @@ -587,30 +679,26 @@ ListOutcome FaultManagerCore::list_locked(const ListQuery & query) { outcome.muted_count = correlation_->get_muted_count(); outcome.cluster_count = correlation_->get_cluster_count(); - auto muted_faults = correlation_->get_muted_faults(); - if (query.include_muted) { + const auto muted_faults = correlation_->get_muted_faults(); outcome.muted_faults.reserve(muted_faults.size()); for (const auto & muted : muted_faults) { MutedFault info; info.fault_code = muted.fault_code; + info.source_id = muted.owner; info.root_cause_code = muted.root_cause_code; info.rule_id = muted.rule_id; info.delay_ms = muted.delay_ms; outcome.muted_faults.push_back(info); } } else { - // Build a set of muted codes for O(1) lookup. - std::unordered_set muted_codes; - for (const auto & muted : muted_faults) { - muted_codes.insert(muted.fault_code); - } - - // Remove all faults whose code is muted, in one pass. + // Hide the muted RECORDS, never every record of a muted code. A root cause mutes the symptoms of + // its own owner, so another owner's record of that same code was never suppressed and hiding it + // would drop a live fault from the default view. auto & faults = outcome.faults; faults.erase(std::remove_if(faults.begin(), faults.end(), - [&muted_codes](const fd::FaultRecord & fault) { - return muted_codes.count(fault.fault_code) > 0; + [this](const fd::FaultRecord & fault) { + return correlation_->is_muted(FaultId{fault.fault_code, owner_of(fault)}); }), faults.end()); } @@ -642,7 +730,7 @@ ListOutcome FaultManagerCore::list_locked(const ListQuery & query) { return outcome; } -FaultDetail FaultManagerCore::get_locked(const std::string & code, +FaultDetail FaultManagerCore::get_locked(const std::string & code, const std::string & source_id, std::vector> & rosbag_paths) { const std::string & logger = config_.logger_name; FaultDetail detail; @@ -662,9 +750,18 @@ FaultDetail FaultManagerCore::get_locked(const std::string & code, return detail; }; + // Which record + std::string resolve_error; + const auto target = resolve_target_locked(code, source_id, resolve_error); + if (!target) { + detail.error_message = resolve_error; + return detail; + } + const FaultId & id = *target; + std::optional fault; try { - fault = storage_->get_fault(code); + fault = storage_->get_fault(id); } catch (const FaultStorage::IgnorableConnectionException & e) { return unavailable(e); } @@ -681,7 +778,7 @@ FaultDetail FaultManagerCore::get_locked(const std::string & code, // Get freeze frame snapshots from storage std::vector stored_snapshots = {}; try { - stored_snapshots = storage_->get_snapshots(code); + stored_snapshots = storage_->get_snapshots(id); } catch (const FaultStorage::IgnorableConnectionException & e) { return unavailable(e); } @@ -706,7 +803,7 @@ FaultDetail FaultManagerCore::get_locked(const std::string & code, if (stored_snapshots.empty() || storage_->retains_snapshots_on_clear()) { std::optional frame; try { - frame = storage_->get_freeze_frame(code); + frame = storage_->get_freeze_frame(id); } catch (const FaultStorage::IgnorableConnectionException & e) { return unavailable(e); } @@ -726,7 +823,7 @@ FaultDetail FaultManagerCore::get_locked(const std::string & code, // this list is the only place the fault itself advertises them. std::vector> paths; try { - for (const auto & rosbag_info : storage_->get_rosbag_files(code)) { + for (const auto & rosbag_info : storage_->get_rosbag_files(id)) { SnapshotEntry rosbag_snapshot; rosbag_snapshot.type = kSnapshotRosbag; rosbag_snapshot.name = "rosbag_" + rosbag_info.recording_id; @@ -767,27 +864,16 @@ DebounceConfig FaultManagerCore::resolve_config(const std::string & source_id, c return config; } -void FaultManagerCore::warn_on_conflicting_debounce_policy(const std::string & fault_code, - const std::string & source_id, - const DebounceConfig & resolved) { - auto [it, inserted] = - debounce_policy_witness_.try_emplace(fault_code, DebouncePolicyWitness{resolved, source_id, false}); - if (inserted || it->second.warned || debounce_policy_equal(it->second.config, resolved)) { +void FaultManagerCore::warn_severity_mismatch(const fd::FaultReport & report, uint8_t stored_severity) { + if (!severity_mismatch_logged_.emplace(report.fault_code, report.source_id, report.severity).second) { return; } - - // One counter, two policies: the report that arrives decides the transition, so the other - // source's debounce policy is bypassed for as long as both report this code. - it->second.warned = true; logf(log_, LogLevel::kWarn, config_.logger_name, - "Fault code '%s' is debounced two ways: '%s' resolves confirmation=%d healing_enabled=%s healing=%d, " - "'%s' resolves confirmation=%d healing_enabled=%s healing=%d. The debounce counter belongs to the fault " - "code, so whichever source reports decides the transition and the other policy is bypassed. Pin " - "confirmation_threshold, healing_enabled and healing_threshold for this code in " - "fault_thresholds.config_file.", - fault_code.c_str(), it->second.source_id.c_str(), it->second.config.confirmation_threshold, - it->second.config.healing_enabled ? "true" : "false", it->second.config.healing_threshold, source_id.c_str(), - resolved.confirmation_threshold, resolved.healing_enabled ? "true" : "false", resolved.healing_threshold); + "Fault '%s' reported with severity %s by '%s', but its record is stored as %s. A record has one " + "severity and keeps the highest its source ever reported. Check the reporter's configuration. Logged " + "once per code, source and severity until restart.", + report.fault_code.c_str(), severity_name(report.severity), report.source_id.c_str(), + severity_name(stored_severity)); } void FaultManagerCore::audit_transition(const char * transition, const fd::FaultRecord & fault, diff --git a/src/ros2_medkit_fault_manager/src/core/fault_manager_startup.cpp b/src/ros2_medkit_fault_manager/src/core/fault_manager_startup.cpp index a2f73b250..1c014431e 100644 --- a/src/ros2_medkit_fault_manager/src/core/fault_manager_startup.cpp +++ b/src/ros2_medkit_fault_manager/src/core/fault_manager_startup.cpp @@ -332,8 +332,8 @@ void reclassify_stale_healed(FaultStorage & storage, const FaultCodeThresholdRes if (!reclassified.empty()) { if (audit_enabled) { const int64_t reclassified_at_ns = wall_clock_ns(); - for (const auto & fault_code : reclassified) { - auto fault = storage.get_fault(fault_code); + for (const auto & id : reclassified) { + auto fault = storage.get_fault(id); if (fault) { audit(*fault, reclassified_at_ns); } diff --git a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp index 41bd0b347..f407f219a 100644 --- a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp +++ b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp @@ -24,10 +24,13 @@ CorrelationEngine::CorrelationEngine(const CorrelationConfig & config) : config_(config), matcher_(std::make_unique(config.patterns)) { } -ProcessFaultResult CorrelationEngine::process_fault(const std::string & fault_code, const std::string & severity, +ProcessFaultResult CorrelationEngine::process_fault(const FaultId & id, const std::string & severity, std::chrono::steady_clock::time_point timestamp) { std::lock_guard lock(mutex_); + // Rules match on the CODE, and every relation formed below is between records of the + // same owner. + const std::string & fault_code = id.fault_code; ProcessFaultResult result; // First, clean up expired entries @@ -42,8 +45,8 @@ ProcessFaultResult CorrelationEngine::process_fault(const std::string & fault_co }), pending_root_causes_.end()); - // Check if this fault is a symptom of an existing root cause - auto symptom_result = try_as_symptom(fault_code, timestamp); + // Check if this record is a symptom of an existing root cause of the same owner + auto symptom_result = try_as_symptom(id, timestamp); if (symptom_result) { return *symptom_result; } @@ -59,14 +62,14 @@ ProcessFaultResult CorrelationEngine::process_fault(const std::string & fault_co if (rule.id == *root_cause_rule) { // Add to pending root causes PendingRootCause prc; - prc.fault_code = fault_code; + prc.fault_id = id; prc.rule_id = rule.id; prc.timestamp = timestamp; prc.window_ms = rule.window_ms; pending_root_causes_.push_back(prc); // Initialize symptom list - root_to_symptoms_[fault_code] = {}; + root_to_symptoms_[id] = {}; break; } } @@ -74,8 +77,8 @@ ProcessFaultResult CorrelationEngine::process_fault(const std::string & fault_co return result; } - // Check if this fault matches an auto-cluster rule - auto cluster_result = try_auto_cluster(fault_code, severity, timestamp); + // Check if this record matches an auto-cluster rule + auto cluster_result = try_auto_cluster(id, severity, timestamp); if (cluster_result) { return *cluster_result; } @@ -84,20 +87,22 @@ ProcessFaultResult CorrelationEngine::process_fault(const std::string & fault_co return result; } -ProcessClearResult CorrelationEngine::process_clear(const std::string & fault_code) { +ProcessClearResult CorrelationEngine::process_clear(const FaultId & id) { std::lock_guard lock(mutex_); + const std::string & fault_code = id.fault_code; ProcessClearResult result; - // Check if this is a root cause with symptoms - auto it = root_to_symptoms_.find(fault_code); + // Check if this record is a root cause with symptoms. The lookup is by record, so + // clearing owner X's root cause never reaches owner Y's symptoms. + auto it = root_to_symptoms_.find(id); if (it != root_to_symptoms_.end()) { // Find the rule to check auto_clear_with_root for (const auto & prc : pending_root_causes_) { - if (prc.fault_code == fault_code) { + if (prc.fault_id == id) { for (const auto & rule : config_.rules) { if (rule.id == prc.rule_id && rule.auto_clear_with_root) { - result.auto_cleared_codes = it->second; + result.auto_cleared_symptoms = it->second; break; } } @@ -106,21 +111,21 @@ ProcessClearResult CorrelationEngine::process_clear(const std::string & fault_co } // Also check finalized root causes (not in pending anymore) - if (result.auto_cleared_codes.empty()) { + if (result.auto_cleared_symptoms.empty()) { for (const auto & rule : config_.rules) { if (rule.mode == CorrelationMode::HIERARCHICAL && rule.auto_clear_with_root) { // Check if fault matches this rule's root cause if (matcher_->matches_any(fault_code, rule.root_cause_codes)) { - result.auto_cleared_codes = it->second; + result.auto_cleared_symptoms = it->second; break; } } } } - // Clean up muted faults - for (const auto & symptom_code : it->second) { - muted_faults_.erase(symptom_code); + // Clean up muted records + for (const auto & symptom_id : it->second) { + muted_faults_.erase(symptom_id); } // Remove from root_to_symptoms @@ -129,13 +134,13 @@ ProcessClearResult CorrelationEngine::process_clear(const std::string & fault_co // Remove from pending root causes pending_root_causes_.erase(std::remove_if(pending_root_causes_.begin(), pending_root_causes_.end(), - [&fault_code](const PendingRootCause & prc) { - return prc.fault_code == fault_code; + [&id](const PendingRootCause & prc) { + return prc.fault_id == id; }), pending_root_causes_.end()); - // Check if this fault is part of a cluster - auto cluster_it = fault_to_cluster_.find(fault_code); + // Check if this record is part of a cluster + auto cluster_it = fault_to_cluster_.find(id); if (cluster_it != fault_to_cluster_.end()) { const std::string cluster_id = cluster_it->second; @@ -184,8 +189,8 @@ ProcessClearResult CorrelationEngine::process_clear(const std::string & fault_co fault_to_cluster_.erase(cluster_it); } - // Remove from muted faults if it was a symptom - muted_faults_.erase(fault_code); + // Remove from muted records if it was a symptom + muted_faults_.erase(id); return result; } @@ -196,8 +201,12 @@ std::vector CorrelationEngine::get_muted_faults() const { std::vector result; result.reserve(muted_faults_.size()); - for (const auto & [code, data] : muted_faults_) { - result.push_back(data); + for (const auto & [muted_id, data] : muted_faults_) { + // Owner taken from the key rather than from the stored copy, so the entry always + // names the record it is filed under. + MutedFaultData entry = data; + entry.owner = muted_id.owner; + result.push_back(entry); } return result; @@ -208,9 +217,9 @@ uint32_t CorrelationEngine::get_muted_count() const { return static_cast(muted_faults_.size()); } -bool CorrelationEngine::is_muted(const std::string & fault_code) const { +bool CorrelationEngine::is_muted(const FaultId & id) const { std::lock_guard lock(mutex_); - return muted_faults_.find(fault_code) != muted_faults_.end(); + return muted_faults_.find(id) != muted_faults_.end(); } std::vector CorrelationEngine::get_clusters() const { @@ -247,26 +256,26 @@ void CorrelationEngine::cleanup_expired() { pending_root_causes_.end()); // Clean up expired pending clusters - std::vector expired_pending; - for (const auto & [rule_id, pending] : pending_clusters_) { + std::vector> expired_pending; + for (const auto & [key, pending] : pending_clusters_) { // Find rule to get window_ms for (const auto & rule : config_.rules) { - if (rule.id == rule_id) { + if (rule.id == key.first) { auto elapsed = std::chrono::duration_cast(now - pending.steady_first_at).count(); if (elapsed > static_cast(rule.window_ms)) { - expired_pending.push_back(rule_id); + expired_pending.push_back(key); } break; } } } - for (const auto & rule_id : expired_pending) { - auto it = pending_clusters_.find(rule_id); + for (const auto & key : expired_pending) { + auto it = pending_clusters_.find(key); if (it != pending_clusters_.end()) { if (active_clusters_.find(it->second.data.cluster_id) == active_clusters_.end()) { for (const auto & fault_code : it->second.data.fault_codes) { - fault_to_cluster_.erase(fault_code); + fault_to_cluster_.erase(FaultId{fault_code, it->second.owner}); } } pending_clusters_.erase(it); @@ -288,9 +297,15 @@ std::optional CorrelationEngine::try_as_root_cause(const std::strin return std::nullopt; } -std::optional CorrelationEngine::try_as_symptom(const std::string & fault_code, +std::optional CorrelationEngine::try_as_symptom(const FaultId & id, std::chrono::steady_clock::time_point timestamp) { + const std::string & fault_code = id.fault_code; for (const auto & prc : pending_root_causes_) { + // A root cause only explains its OWN reporter's faults. Without this the first + // owner to report a root-cause code would mute every other owner's symptom. + if (prc.fault_id.owner != id.owner) { + continue; + } // Find the rule for (const auto & rule : config_.rules) { if (rule.id != prc.rule_id || rule.mode != CorrelationMode::HIERARCHICAL) { @@ -324,23 +339,23 @@ std::optional CorrelationEngine::try_as_symptom(const std::s // This fault is a symptom! ProcessFaultResult result; result.should_mute = rule.mute_symptoms; - result.root_cause_code = prc.fault_code; + result.root_cause_code = prc.fault_id.fault_code; result.rule_id = rule.id; result.delay_ms = static_cast(elapsed); // Track the symptom (avoid duplicates) - auto & symptoms = root_to_symptoms_[prc.fault_code]; - if (std::find(symptoms.begin(), symptoms.end(), fault_code) == symptoms.end()) { - symptoms.push_back(fault_code); + auto & symptoms = root_to_symptoms_[prc.fault_id]; + if (std::find(symptoms.begin(), symptoms.end(), id) == symptoms.end()) { + symptoms.push_back(id); } if (rule.mute_symptoms) { MutedFaultData muted; muted.fault_code = fault_code; - muted.root_cause_code = prc.fault_code; + muted.root_cause_code = prc.fault_id.fault_code; muted.rule_id = rule.id; muted.delay_ms = result.delay_ms; - muted_faults_[fault_code] = muted; + muted_faults_[id] = muted; } return result; @@ -350,9 +365,9 @@ std::optional CorrelationEngine::try_as_symptom(const std::s return std::nullopt; } -std::optional CorrelationEngine::try_auto_cluster(const std::string & fault_code, - const std::string & severity, +std::optional CorrelationEngine::try_auto_cluster(const FaultId & id, const std::string & severity, std::chrono::steady_clock::time_point timestamp) { + const std::string & fault_code = id.fault_code; for (const auto & rule : config_.rules) { if (rule.mode != CorrelationMode::AUTO_CLUSTER) { continue; @@ -373,8 +388,11 @@ std::optional CorrelationEngine::try_auto_cluster(const std: auto now_system = std::chrono::system_clock::now(); - // Check if we have a pending cluster for this rule - auto pending_it = pending_clusters_.find(rule.id); + // Check if we have a pending cluster for this rule AND this owner. One rule forms + // one cluster per owner, so a burst on owner A never counts owner B's faults + // towards min_count and never mutes them as non-representative members. + const auto pending_key = std::make_pair(rule.id, id.owner); + auto pending_it = pending_clusters_.find(pending_key); if (pending_it != pending_clusters_.end()) { // Check if within time window using steady_clock timestamp auto elapsed = @@ -384,7 +402,7 @@ std::optional CorrelationEngine::try_auto_cluster(const std: // Window expired, start new cluster if (active_clusters_.find(pending_it->second.data.cluster_id) == active_clusters_.end()) { for (const auto & old_code : pending_it->second.data.fault_codes) { - fault_to_cluster_.erase(old_code); + fault_to_cluster_.erase(FaultId{old_code, id.owner}); } } pending_clusters_.erase(pending_it); @@ -393,7 +411,7 @@ std::optional CorrelationEngine::try_auto_cluster(const std: } // Keep faults that already belong to an active cluster for this rule in that cluster - auto existing_cluster_it = fault_to_cluster_.find(fault_code); + auto existing_cluster_it = fault_to_cluster_.find(id); if (existing_cluster_it != fault_to_cluster_.end()) { auto active_it = active_clusters_.find(existing_cluster_it->second); if (active_it != active_clusters_.end() && active_it->second.rule_id == rule.id) { @@ -418,6 +436,7 @@ std::optional CorrelationEngine::try_auto_cluster(const std: if (pending_it == pending_clusters_.end()) { // Start new pending cluster PendingCluster pending; + pending.owner = id.owner; pending.steady_first_at = timestamp; pending.data.cluster_id = generate_cluster_id(rule.id); pending.data.rule_id = rule.id; @@ -430,8 +449,8 @@ std::optional CorrelationEngine::try_auto_cluster(const std: pending.data.first_at = now_system; pending.data.last_at = now_system; - pending_clusters_[rule.id] = pending; - fault_to_cluster_[fault_code] = pending.data.cluster_id; + pending_clusters_[pending_key] = pending; + fault_to_cluster_[id] = pending.data.cluster_id; // Not enough faults yet for a cluster ProcessFaultResult result; @@ -460,7 +479,7 @@ std::optional CorrelationEngine::try_auto_cluster(const std: cluster.fault_codes.push_back(fault_code); pending.fault_severities[fault_code] = severity; cluster.last_at = now_system; - fault_to_cluster_[fault_code] = cluster.cluster_id; + fault_to_cluster_[id] = cluster.cluster_id; // Update representative based on rule's representative selection bool update_representative = false; diff --git a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp index 3ea975413..a83aeee96 100644 --- a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp @@ -117,17 +117,17 @@ FaultManagerNode::FaultManagerNode(const rclcpp::NodeOptions & options) : Node(" event_publisher_ = create_publisher("~/events", rclcpp::QoS(100).reliable()); event_sink_ = std::make_unique(event_publisher_); capture_hooks_ = std::make_unique( - [this](const std::string & code) { - return enqueue_capture(code); + [this](const FaultId & id) { + return enqueue_capture(id); }, - [this](const std::string & code) { + [this](const FaultId & id) { if (rosbag_capture_) { - rosbag_capture_->on_fault_prefailed(code); + rosbag_capture_->on_fault_prefailed(id); } }, - [this](const std::string & code) { + [this](const FaultId & id) { if (rosbag_capture_) { - rosbag_capture_->on_fault_cleared(code); + rosbag_capture_->on_fault_cleared(id); } }); core_ = std::make_unique(config_, std::move(parts), *event_sink_, *capture_hooks_, @@ -136,11 +136,13 @@ FaultManagerNode::FaultManagerNode(const rclcpp::NodeOptions & options) : Node(" // Runs after the override files are loaded, so a code with its own healing_enabled: true keeps // its HEALED status. Each flipped fault is audited, or the audit log's verify() would not see it. - core::reclassify_stale_healed(*storage_, core_->fault_code_resolver(), core_->global_debounce(), - core_->audit_log() != nullptr, log_sink_, config_.logger_name, - [this](const fd::FaultRecord & fault, int64_t reclassified_at_ns) { - audit_transition(kTransitionCleared, fault, "startup_reclassify", reclassified_at_ns); - }); + core::reclassify_stale_healed( + *storage_, core_->fault_code_resolver(), core_->global_debounce(), core_->audit_log() != nullptr, log_sink_, + config_.logger_name, [this](const fd::FaultRecord & fault, int64_t reclassified_at_ns) { + audit_transition(kTransitionCleared, fault, + fault.reporting_sources.empty() ? std::string() : fault.reporting_sources.front(), + reclassified_at_ns); + }); // Create service servers report_fault_srv_ = create_service( @@ -231,13 +233,13 @@ FaultManagerNode::FaultManagerNode(const rclcpp::NodeOptions & options) : Node(" capture_pool_ = std::make_unique( static_cast(capture_pool_size_), static_cast(capture_queue_depth_), capture_queue_full_policy_, log_sink_, - [snap, bag, rosbag_mutex](const std::string & fault_code) { + [snap, bag, rosbag_mutex](const FaultId & id) { if (snap) { - snap->capture(fault_code); + snap->capture(id); } if (bag) { std::lock_guard bag_lock(*rosbag_mutex); - bag->on_fault_confirmed(fault_code); + bag->on_fault_confirmed(id); } }, config_.logger_name); @@ -302,11 +304,11 @@ FaultManagerNode::~FaultManagerNode() { } } -core::CaptureResult FaultManagerNode::enqueue_capture(const std::string & fault_code) { +core::CaptureResult FaultManagerNode::enqueue_capture(const FaultId & id) { if (!capture_pool_) { return {}; } - const EnqueueOutcome outcome = capture_pool_->enqueue(fault_code); + const EnqueueOutcome outcome = capture_pool_->enqueue(id); // RCLCPP_WARN_THROTTLE needs a non-const Clock lvalue (Humble/Lyrical compat); mirror // rosbag_capture.cpp's local-copy pattern. The counter is cast to unsigned long long for %llu // to avoid -Wuseless-cast (uint64_t == unsigned long on LP64). @@ -318,19 +320,19 @@ core::CaptureResult FaultManagerNode::enqueue_capture(const std::string & fault_ RCLCPP_WARN_THROTTLE(get_logger(), throttle_clock, 2000, "Capture queue full (drop_oldest): evicted pending '%s' for '%s' " "(pool=%d, queue=%d, total_dropped=%llu)", - outcome.evicted_code ? outcome.evicted_code->c_str() : "?", fault_code.c_str(), + outcome.evicted_id ? outcome.evicted_id->fault_code.c_str() : "?", id.fault_code.c_str(), capture_pool_size_, capture_queue_depth_, static_cast(capture_pool_->dropped_captures())); - return {core::CaptureEnqueue::kEvictedOldest, outcome.evicted_code.value_or(std::string{})}; + return {core::CaptureEnqueue::kEvictedOldest, outcome.evicted_id.value_or(FaultId{})}; case EnqueueResult::kDroppedNewest: RCLCPP_WARN_THROTTLE(get_logger(), throttle_clock, 2000, "Capture queue full (reject_newest): dropped capture for '%s' " "(pool=%d, queue=%d, total_dropped=%llu)", - fault_code.c_str(), capture_pool_size_, capture_queue_depth_, + id.fault_code.c_str(), capture_pool_size_, capture_queue_depth_, static_cast(capture_pool_->dropped_captures())); return {core::CaptureEnqueue::kDroppedNewest, {}}; case EnqueueResult::kRejectedShuttingDown: - RCLCPP_DEBUG(get_logger(), "Capture pool shutting down; skipped capture for '%s'", fault_code.c_str()); + RCLCPP_DEBUG(get_logger(), "Capture pool shutting down; skipped capture for '%s'", id.fault_code.c_str()); return {core::CaptureEnqueue::kShuttingDown, {}}; } return {}; @@ -369,6 +371,7 @@ void FaultManagerNode::handle_list_faults( for (const auto & muted : outcome.muted_faults) { ros2_medkit_msgs::msg::MutedFaultInfo info; info.fault_code = muted.fault_code; + info.source_id = muted.source_id; info.root_cause_code = muted.root_cause_code; info.rule_id = muted.rule_id; info.delay_ms = muted.delay_ms; @@ -394,7 +397,7 @@ void FaultManagerNode::handle_list_faults( void FaultManagerNode::handle_clear_fault( const std::shared_ptr & request, const std::shared_ptr & response) { - auto outcome = core_->clear(request->fault_code, request->skip_correlation_auto_clear); + auto outcome = core_->clear(request->fault_code, request->source_id, request->skip_correlation_auto_clear); response->success = outcome.success; response->message = std::move(outcome.message); response->auto_cleared_codes = std::move(outcome.auto_cleared_codes); @@ -402,7 +405,7 @@ void FaultManagerNode::handle_clear_fault( void FaultManagerNode::handle_get_fault(const std::shared_ptr & request, const std::shared_ptr & response) { - const auto detail = core_->get(request->fault_code); + const auto detail = core_->get(request->fault_code, request->source_id); response->success = detail.success; response->error_message = detail.error_message; if (detail.fault.fault_code.empty()) { @@ -644,26 +647,19 @@ void FaultManagerNode::handle_get_snapshots( return; } - // Check if fault exists - std::optional fault; - try { - fault = storage_->get_fault(request->fault_code); - } catch (const FaultStorage::IgnorableConnectionException & e) { - RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); + // Which record + std::string resolve_error; + const auto target = core_->resolve_target(request->fault_code, request->source_id, resolve_error); + if (!target) { response->success = false; - response->error_message = storage_unavailable(e); - return; - } - if (!fault) { - response->success = false; - response->error_message = "Fault not found: " + request->fault_code; + response->error_message = resolve_error; return; } // Get snapshots from storage, newest capture set first. std::vector snapshots = {}; try { - snapshots = storage_->get_snapshots(request->fault_code, request->topic); + snapshots = storage_->get_snapshots(*target, request->topic); } catch (const FaultStorage::IgnorableConnectionException & e) { RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); response->success = false; @@ -733,7 +729,7 @@ void FaultManagerNode::handle_get_snapshots( // No download_url: rosbags are addressed per entity by the gateway. std::optional rosbag_info; try { - rosbag_info = storage_->get_rosbag_file(request->fault_code); + rosbag_info = storage_->get_rosbag_file(*target); } catch (const FaultStorage::IgnorableConnectionException & e) { RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); response->success = false; @@ -760,9 +756,11 @@ void FaultManagerNode::handle_get_snapshots( void FaultManagerNode::handle_get_rosbag(const std::shared_ptr & request, const std::shared_ptr & response) { - // Two ways in. recording_id addresses one specific bag and is tried first; - // fault_code keeps working and means "the newest recording of this fault", which - // is what it has always meant back when a fault could only have one. + // Two ways in. recording_id addresses one specific bag and is tried first, and + // source_id does not scope it: a recording is named directly and is shared by every + // record of its burst. fault_code (with source_id) keeps working and means "the + // newest recording of this record", which is what it has always meant back when a + // fault could only have one. // // A caller that cannot tell the two apart - the gateway, holding one URL segment // that may be either - sets both to the same string and relies on this fallthrough. @@ -771,6 +769,17 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptr attached_codes; std::string subject; + // fault_codes answers "which faults does this recording cover", so it is a set of + // codes. One row per RECORD means a burst of several owners of one code produces + // several rows pointing at the same bag, and pushing the code once per row would + // repeat it. First-seen order is kept, so the codes still read in the order the + // recording collected them. + const auto attach_code = [&attached_codes](const std::string & code) { + if (std::find(attached_codes.begin(), attached_codes.end(), code) == attached_codes.end()) { + attached_codes.push_back(code); + } + }; + if (!request->recording_id.empty()) { // Validation stays a hard failure: a traversal-shaped id is malformed on // either reading, and falling back would quietly re-admit it as a fault code. @@ -794,7 +803,7 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptrrecording_id; rosbag_info = rows.front(); for (const auto & row : rows) { - attached_codes.push_back(row.fault_code); + attach_code(row.fault_code); } } else if (request->fault_code.empty()) { response->success = false; @@ -812,24 +821,17 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptrfault_code; - // Check if fault exists - std::optional fault; - try { - fault = storage_->get_fault(request->fault_code); - } catch (const FaultStorage::IgnorableConnectionException & e) { - RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); - response->success = false; - response->error_message = storage_unavailable(e); - return; - } - if (!fault) { + // Which record + std::string resolve_error; + const auto target = core_->resolve_target(request->fault_code, request->source_id, resolve_error); + if (!target) { response->success = false; - response->error_message = "Fault not found: " + request->fault_code; + response->error_message = resolve_error; return; } try { - rosbag_info = storage_->get_rosbag_file(request->fault_code); + rosbag_info = storage_->get_rosbag_file(*target); } catch (const FaultStorage::IgnorableConnectionException & e) { RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); response->success = false; @@ -845,7 +847,7 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptrget_rosbag_files_by_recording(rosbag_info->recording_id)) { - attached_codes.push_back(row.fault_code); + attach_code(row.fault_code); } } catch (const FaultStorage::IgnorableConnectionException & e) { RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); diff --git a/src/ros2_medkit_fault_manager/src/fault_storage.cpp b/src/ros2_medkit_fault_manager/src/fault_storage.cpp index ceddfff19..11f81f2c5 100644 --- a/src/ros2_medkit_fault_manager/src/fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_storage.cpp @@ -84,17 +84,21 @@ bool sanitize_debounce_config(DebounceConfig & config) { } fd::FaultRecord FaultState::to_record() const { - fd::FaultRecord record; - record.fault_code = fault_code; - record.severity = severity; - record.description = description; - record.first_occurred_ns = first_occurred_ns; - record.last_occurred_ns = last_occurred_ns; - record.last_passed_ns = last_passed_ns; - record.occurrence_count = occurrence_count; - record.status = status; - record.reporting_sources.assign(reporting_sources.begin(), reporting_sources.end()); - return record; + fd::FaultRecord msg; + msg.fault_code = fault_code; + msg.severity = severity; + msg.description = description; + msg.first_occurred_ns = first_occurred_ns; + msg.last_occurred_ns = last_occurred_ns; + msg.last_passed_ns = last_passed_ns; + msg.occurrence_count = occurrence_count; + msg.status = status; + + // The owner, as a one-element list: the field names the source that owns the record, + // and the services that act on a single record take that value back as their source_id. + msg.reporting_sources.assign(1, owner); + + return msg; } void InMemoryFaultStorage::set_debounce_config(const DebounceConfig & config) { @@ -120,15 +124,19 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd const bool is_failed = (kind == fd::ReportKind::kFailed); - auto it = faults_.find(fault_code); + const FaultId id{fault_code, source_id}; + + auto it = faults_.find(id); if (it == faults_.end()) { - // New fault - only create entry for FAILED events + // New record - only create one for FAILED events. A source that has never reported + // this code owns no record yet, so a PASSED from it has nothing to heal. if (!is_failed) { - return false; // PASSED event for non-existent fault is ignored + return false; // PASSED event for non-existent record is ignored } FaultState state; state.fault_code = fault_code; + state.owner = source_id; state.severity = severity; state.description = description; state.first_occurred_ns = timestamp_ns; @@ -136,7 +144,6 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd state.last_failed_ns = timestamp_ns; state.occurrence_count = 1; state.debounce_counter = -1; // First FAILED event - state.reporting_sources.insert(source_id); // CRITICAL severity bypasses debounce and confirms immediately if (config.critical_immediate_confirm && severity == fd::severity::kCritical) { @@ -150,11 +157,11 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd record_near_miss(state, config, severity, source_id, timestamp_ns); } - faults_.emplace(fault_code, std::move(state)); + faults_.emplace(id, std::move(state)); return true; } - // Existing fault - update + // Existing record - update auto & state = it->second; // CLEARED faults can be reactivated by FAILED events @@ -169,7 +176,6 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd state.first_occurred_ns = timestamp_ns; state.last_failed_ns = timestamp_ns; state.last_occurred_ns = timestamp_ns; - state.reporting_sources.insert(source_id); if (state.occurrence_count < std::numeric_limits::max()) { ++state.occurrence_count; } @@ -179,8 +185,8 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd if (!description.empty()) { state.description = description; } - // Check for immediate CRITICAL confirmation - if (config.critical_immediate_confirm && severity == fd::severity::kCritical) { + // Immediate confirmation reads the stored severity: a CRITICAL record confirms on any FAILED report. + if (config.critical_immediate_confirm && state.severity == fd::severity::kCritical) { state.status = fd::fault_status::kConfirmed; } else { update_status(state, config); @@ -192,10 +198,11 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd } // Bring a counter left outside this config's band back into range before applying the report. - // Per-entity threshold overrides mean two sources of the same fault code can be evaluated - // against different bands, so a stored value clamped to one source's ceiling can sit above - // another's. The SQLite backend clamps on read for the same reason, and the two backends have to - // agree on the counter they record and on the status it produces. + // A record's counter belongs to one source, but that source's resolved band can change under + // it: an entity-threshold file reloaded with different values, or a global config that moved + // between runs, leaves a stored value clamped to the old ceiling sitting above the new one. + // The SQLite backend clamps on read for the same reason, and the two backends have to agree on + // the counter they record and on the status it produces. state.debounce_counter = clamp_debounce_counter(state.debounce_counter, config); if (is_failed) { @@ -214,10 +221,8 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd // run the counter off to INT32_MIN and delay later healing). state.debounce_counter = clamp_debounce_counter(state.debounce_counter - 1, config); - // Add source if not already present - state.reporting_sources.insert(source_id); - - // Update severity if higher + // Update severity if higher. Per record: a CRITICAL from one owner does not escalate + // another owner's record of the same code. if (severity > state.severity) { state.severity = severity; } @@ -227,8 +232,8 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd state.description = description; } - // Check for immediate confirmation of CRITICAL - if (config.critical_immediate_confirm && severity == fd::severity::kCritical) { + // Immediate confirmation reads the stored severity, so it also overrides the HEALED latch. + if (config.critical_immediate_confirm && state.severity == fd::severity::kCritical) { state.debounce_counter = std::min(state.debounce_counter, -1); state.status = fd::fault_status::kConfirmed; return false; @@ -278,7 +283,7 @@ std::vector InMemoryFaultStorage::list_faults(bool filter_by_se std::vector result; result.reserve(faults_.size()); - for (const auto & [code, state] : faults_) { + for (const auto & [id, state] : faults_) { // Filter by status if (status_filter.find(state.status) == status_filter.end()) { continue; @@ -295,10 +300,10 @@ std::vector InMemoryFaultStorage::list_faults(bool filter_by_se return result; } -std::optional InMemoryFaultStorage::get_fault(const std::string & fault_code) const { +std::optional InMemoryFaultStorage::get_fault(const FaultId & id) const { std::lock_guard lock(mutex_); - auto it = faults_.find(fault_code); + auto it = faults_.find(id); if (it == faults_.end()) { return std::nullopt; } @@ -306,21 +311,38 @@ std::optional InMemoryFaultStorage::get_fault(const std::string return it->second.to_record(); } -bool InMemoryFaultStorage::clear_fault(const std::string & fault_code) { +std::vector InMemoryFaultStorage::get_faults_by_code(const std::string & fault_code) const { + std::lock_guard lock(mutex_); + + // FaultId orders by code first, so the records of one code are one contiguous range + // and this is a range scan. Owner order inside it is the map's, which is what makes + // an ambiguity message list the owners the same way every time. + std::vector result; + for (auto it = faults_.lower_bound(FaultId{fault_code, ""}); it != faults_.end(); ++it) { + if (it->first.fault_code != fault_code) { + break; + } + result.push_back(it->second.to_record()); + } + return result; +} + +bool InMemoryFaultStorage::clear_fault(const FaultId & id) { std::lock_guard lock(mutex_); - auto it = faults_.find(fault_code); + auto it = faults_.find(id); if (it == faults_.end()) { return false; } - // Acknowledging a fault drops its value snapshots, unless a history was asked + // Acknowledging a record drops its value snapshots, unless a history was asked // for: with recordings retained past a clear, deleting the readings that go with - // them leaves a fault holding bags whose matching values are gone. + // them leaves a record holding bags whose matching values are gone. Scoped to the + // record, so another owner's readings for the same code stay where they are. if (!retain_snapshots_on_clear_) { snapshots_.erase(std::remove_if(snapshots_.begin(), snapshots_.end(), - [&fault_code](const SnapshotData & s) { - return s.fault_code == fault_code; + [&id](const SnapshotData & s) { + return s.fault_code == id.fault_code && s.owner == id.owner; }), snapshots_.end()); } @@ -334,27 +356,27 @@ size_t InMemoryFaultStorage::size() const { return faults_.size(); } -bool InMemoryFaultStorage::contains(const std::string & fault_code) const { +bool InMemoryFaultStorage::contains(const FaultId & id) const { std::lock_guard lock(mutex_); - return faults_.find(fault_code) != faults_.end(); + return faults_.find(id) != faults_.end(); } -std::vector InMemoryFaultStorage::check_time_based_confirmation(int64_t now_ns) { +std::vector InMemoryFaultStorage::check_time_based_confirmation(int64_t now_ns) { std::lock_guard lock(mutex_); - std::vector confirmed; + std::vector confirmed; if (config_.auto_confirm_after_sec <= 0.0) { return confirmed; // Time-based confirmation disabled } const int64_t threshold_ns = std::llround(config_.auto_confirm_after_sec * 1e9); - for (auto & [code, state] : faults_) { + for (auto & [id, state] : faults_) { if (state.status == fd::fault_status::kPrefailed) { const int64_t age_ns = now_ns - state.last_failed_ns; if (age_ns >= threshold_ns) { state.status = fd::fault_status::kConfirmed; - confirmed.push_back(code); + confirmed.push_back(id); } } } @@ -387,7 +409,7 @@ void InMemoryFaultStorage::store_snapshots(const std::vector & sna } std::lock_guard lock(mutex_); - const std::string fault_code = snapshots.front().fault_code; + const FaultId id{snapshots.front().fault_code, snapshots.front().owner}; // Built beside the live vector and swapped in, the shape store_rosbag_files // uses: a capture is all-or-nothing, so a throw partway must not leave half of @@ -396,15 +418,15 @@ void InMemoryFaultStorage::store_snapshots(const std::vector & sna updated.insert(updated.end(), snapshots.begin(), snapshots.end()); if (max_snapshots_per_fault_ > 0) { - // Evict whole capture sets, oldest first, until this fault fits. + // Evict whole capture sets, oldest first, until this record fits. // // The old rule counted rows and rejected the NEW row once full, so a capture // that straddled the cap was stored in part: some topics present, the rest // silently absent, indistinguishable from "that topic was not publishing". // Keep-newest also stops this cap from opposing the rosbag one. - const auto rows_for_fault = [&updated, &fault_code]() { - return static_cast(std::count_if(updated.begin(), updated.end(), [&fault_code](const SnapshotData & s) { - return s.fault_code == fault_code; + const auto rows_for_fault = [&updated, &id]() { + return static_cast(std::count_if(updated.begin(), updated.end(), [&id](const SnapshotData & s) { + return s.fault_code == id.fault_code && s.owner == id.owner; })); }; @@ -415,7 +437,7 @@ void InMemoryFaultStorage::store_snapshots(const std::vector & sna int64_t newest = 0; bool have_newest = false; for (const auto & s : updated) { - if (s.fault_code == fault_code && (!have_newest || s.capture_id > newest)) { + if (s.fault_code == id.fault_code && s.owner == id.owner && (!have_newest || s.capture_id > newest)) { newest = s.capture_id; have_newest = true; } @@ -425,7 +447,8 @@ void InMemoryFaultStorage::store_snapshots(const std::vector & sna bool found = false; int64_t oldest = 0; for (const auto & s : updated) { - if (s.fault_code == fault_code && s.capture_id != newest && (!found || s.capture_id < oldest)) { + if (s.fault_code == id.fault_code && s.owner == id.owner && s.capture_id != newest && + (!found || s.capture_id < oldest)) { oldest = s.capture_id; found = true; } @@ -434,8 +457,9 @@ void InMemoryFaultStorage::store_snapshots(const std::vector & sna break; // only the newest capture is left and it is over the cap on its own } updated.erase(std::remove_if(updated.begin(), updated.end(), - [&fault_code, oldest](const SnapshotData & s) { - return s.fault_code == fault_code && s.capture_id == oldest; + [&id, oldest](const SnapshotData & s) { + return s.fault_code == id.fault_code && s.owner == id.owner && + s.capture_id == oldest; }), updated.end()); } @@ -444,13 +468,13 @@ void InMemoryFaultStorage::store_snapshots(const std::vector & sna snapshots_.swap(updated); } -std::vector InMemoryFaultStorage::get_snapshots(const std::string & fault_code, +std::vector InMemoryFaultStorage::get_snapshots(const FaultId & id, const std::string & topic_filter) const { std::lock_guard lock(mutex_); std::vector result; for (const auto & snapshot : snapshots_) { - if (snapshot.fault_code == fault_code) { + if (snapshot.fault_code == id.fault_code && snapshot.owner == id.owner) { if (topic_filter.empty() || snapshot.topic == topic_filter) { result.push_back(snapshot); } @@ -477,12 +501,12 @@ int64_t InMemoryFaultStorage::get_max_capture_id() const { void InMemoryFaultStorage::store_freeze_frame(const FreezeFrameData & frame) { std::lock_guard lock(mutex_); - freeze_frames_[frame.fault_code] = frame; + freeze_frames_[FaultId{frame.fault_code, frame.owner}] = frame; } -std::optional InMemoryFaultStorage::get_freeze_frame(const std::string & fault_code) const { +std::optional InMemoryFaultStorage::get_freeze_frame(const FaultId & id) const { std::lock_guard lock(mutex_); - auto it = freeze_frames_.find(fault_code); + auto it = freeze_frames_.find(id); if (it == freeze_frames_.end()) { return std::nullopt; } @@ -491,7 +515,7 @@ std::optional InMemoryFaultStorage::get_freeze_frame(const std: void InMemoryFaultStorage::record_near_miss(const FaultState & state, const DebounceConfig & config, uint8_t severity, const std::string & source_id, int64_t timestamp_ns) { - auto & series = near_misses_[state.fault_code]; + auto & series = near_misses_[FaultId{state.fault_code, state.owner}]; NearMissRecord record; record.fault_code = state.fault_code; @@ -523,8 +547,8 @@ size_t InMemoryFaultStorage::set_max_near_misses_per_fault(size_t max_count) { // stay over the new one until the next near miss for that code happens to arrive. using DiffType = std::vector::difference_type; size_t evicted = 0; - for (auto & [code, series] : near_misses_) { - (void)code; + for (auto & [id, series] : near_misses_) { + (void)id; if (series.size() > max_count) { const size_t excess = series.size() - max_count; series.erase(series.begin(), series.begin() + static_cast(excess)); @@ -534,9 +558,9 @@ size_t InMemoryFaultStorage::set_max_near_misses_per_fault(size_t max_count) { return evicted; } -std::vector InMemoryFaultStorage::get_near_misses(const std::string & fault_code) const { +std::vector InMemoryFaultStorage::get_near_misses(const FaultId & id) const { std::lock_guard lock(mutex_); - auto it = near_misses_.find(fault_code); + auto it = near_misses_.find(id); if (it == near_misses_.end()) { return {}; } @@ -576,11 +600,11 @@ void InMemoryFaultStorage::store_rosbag_files(const std::vector row.recording_id = rosbag_recording_id(row.file_path); } - // Upsert on the (fault, recording) LINK - the same grain as the SQLite unique + // Upsert on the (record, recording) LINK - the same grain as the SQLite unique // index. Re-storing the same link refreshes it; a link to a different recording // appends, which is the feature. auto it = std::find_if(updated.begin(), updated.end(), [&row](const RosbagRow & r) { - return r.info.fault_code == row.fault_code && r.info.file_path == row.file_path; + return r.info.fault_code == row.fault_code && r.info.owner == row.owner && r.info.file_path == row.file_path; }); if (it != updated.end()) { it->info = row; @@ -592,12 +616,12 @@ void InMemoryFaultStorage::store_rosbag_files(const std::vector continue; } - // Keep the newest N recordings of this fault, oldest evicted first - the same + // Keep the newest N recordings of this record, oldest evicted first - the same // direction as evict_bags_over_quota, so the two eviction owners never need a // tiebreak. At N = 1 this is the pre-#620 behaviour exactly. std::vector mine; for (size_t i = 0; i < updated.size(); ++i) { - if (updated[i].info.fault_code == row.fault_code) { + if (updated[i].info.fault_code == row.fault_code && updated[i].info.owner == row.owner) { mine.push_back(i); } } @@ -623,7 +647,7 @@ void InMemoryFaultStorage::store_rosbag_files(const std::vector // Unlinked only once every row is in, the way the SQLite backend unlinks after its // COMMIT: a throw above must leave the old rows pointing at bags that still exist. // Referencing is decided on the finished batch, not on the state before it - two - // faults of one burst can share the bag being evicted, and checking row by row + // records of one burst can share the bag being evicted, and checking row by row // beforehand would find it still held by a sibling that a later iteration then // evicts, leaking the directory. for (const auto & path : evicted) { @@ -636,13 +660,13 @@ void InMemoryFaultStorage::store_rosbag_files(const std::vector } } -std::optional InMemoryFaultStorage::get_rosbag_file(const std::string & fault_code) const { +std::optional InMemoryFaultStorage::get_rosbag_file(const FaultId & id) const { std::lock_guard lock(mutex_); // Newest, matching the SQLite backend's ORDER BY created_at_ns DESC, id DESC. const RosbagRow * best = nullptr; for (const auto & row : rosbag_files_) { - if (row.info.fault_code != fault_code) { + if (row.info.fault_code != id.fault_code || row.info.owner != id.owner) { continue; } if (best == nullptr || std::tie(best->info.created_at_ns, best->seq) < std::tie(row.info.created_at_ns, row.seq)) { @@ -655,12 +679,12 @@ std::optional InMemoryFaultStorage::get_rosbag_file(const std::s return best->info; } -std::vector InMemoryFaultStorage::get_rosbag_files(const std::string & fault_code) const { +std::vector InMemoryFaultStorage::get_rosbag_files(const FaultId & id) const { std::lock_guard lock(mutex_); std::vector mine; for (const auto & row : rosbag_files_) { - if (row.info.fault_code == fault_code) { + if (row.info.fault_code == id.fault_code && row.info.owner == id.owner) { mine.push_back(&row); } } @@ -687,20 +711,20 @@ InMemoryFaultStorage::get_rosbag_files_by_recording(const std::string & recordin } } std::sort(result.begin(), result.end(), [](const RosbagFileInfo & a, const RosbagFileInfo & b) { - return a.fault_code < b.fault_code; + return std::tie(a.fault_code, a.owner) < std::tie(b.fault_code, b.owner); }); return result; } -bool InMemoryFaultStorage::delete_rosbag_file(const std::string & fault_code) { +bool InMemoryFaultStorage::delete_rosbag_file(const FaultId & id) { std::lock_guard lock(mutex_); - // ALL recordings of this fault. Used by auto_cleanup on clear, where dropping the - // fault's whole black-box history is the intent. + // ALL recordings of this record. Used by auto_cleanup on clear, where dropping the + // record's whole black-box history is the intent. std::set touched; const size_t before = rosbag_files_.size(); for (auto it = rosbag_files_.begin(); it != rosbag_files_.end();) { - if (it->info.fault_code == fault_code) { + if (it->info.fault_code == id.fault_code && it->info.owner == id.owner) { touched.insert(it->info.file_path); it = rosbag_files_.erase(it); } else { @@ -713,7 +737,7 @@ bool InMemoryFaultStorage::delete_rosbag_file(const std::string & fault_code) { for (const auto & path : touched) { if (path_referenced(path)) { - continue; // a sibling fault of the burst still holds it + continue; // a sibling record of the burst still holds it } std::error_code ec; std::filesystem::remove_all(path, ec); @@ -761,7 +785,7 @@ bool InMemoryFaultStorage::path_referenced(const std::string & file_path) const size_t InMemoryFaultStorage::get_total_rosbag_storage_bytes() const { std::lock_guard lock(mutex_); - // Sum per bag, not per fault: one recording can back a burst of correlated + // Sum per bag, not per record: one recording can back a burst of correlated // faults, and double-counting it would evict bags that still fit the quota. // Rows sharing a path can disagree on size while a recording is being // finalised, so take the largest - matching the SQLite backend's MAX() and @@ -807,13 +831,13 @@ std::vector InMemoryFaultStorage::list_rosbags_for_entity(const std::vector rows; for (const auto & row : rosbag_files_) { - // Check if any of the fault's reporting sources contain this entity - auto fault_it = faults_.find(row.info.fault_code); - if (fault_it == faults_.end()) { + // The row already carries its record's owner, so the entity match is on the row. + // The faults_ lookup stays as the existence check it always was: a row whose + // record is gone is not served. + if (faults_.find(FaultId{row.info.fault_code, row.info.owner}) == faults_.end()) { continue; } - const auto & fault_state = fault_it->second; - if (fault_state.reporting_sources.find(entity_fqn) != fault_state.reporting_sources.end()) { + if (row.info.owner == entity_fqn) { rows.push_back(&row); } } @@ -836,20 +860,20 @@ std::vector InMemoryFaultStorage::get_all_faults() const { std::vector result; result.reserve(faults_.size()); - for (const auto & [code, state] : faults_) { + for (const auto & [id, state] : faults_) { result.push_back(state.to_record()); } return result; } -std::vector InMemoryFaultStorage::reclassify_healed_as_cleared(const KeepHealedFn & keep_healed) { +std::vector InMemoryFaultStorage::reclassify_healed_as_cleared(const KeepHealedFn & keep_healed) { std::lock_guard lock(mutex_); - std::vector reclassified; - for (auto & [code, state] : faults_) { - if (state.status == fd::fault_status::kHealed && !(keep_healed && keep_healed(code))) { + std::vector reclassified; + for (auto & [id, state] : faults_) { + if (state.status == fd::fault_status::kHealed && !(keep_healed && keep_healed(id.fault_code))) { state.status = fd::fault_status::kCleared; - reclassified.push_back(code); + reclassified.push_back(id); } } @@ -857,10 +881,10 @@ std::vector InMemoryFaultStorage::reclassify_healed_as_cleared(cons // them here, and a backend that kept them would answer a snapshot query differently for the // same sequence of calls. The near-miss series is retained, as it is on clear_fault. if (!retain_snapshots_on_clear_ && !reclassified.empty()) { - const std::set affected(reclassified.begin(), reclassified.end()); + const std::set affected(reclassified.begin(), reclassified.end()); snapshots_.erase(std::remove_if(snapshots_.begin(), snapshots_.end(), [&affected](const SnapshotData & s) { - return affected.count(s.fault_code) > 0; + return affected.count(FaultId{s.fault_code, s.owner}) > 0; }), snapshots_.end()); } diff --git a/src/ros2_medkit_fault_manager/src/rosbag_capture.cpp b/src/ros2_medkit_fault_manager/src/rosbag_capture.cpp index fb3be1d36..7a0fd3030 100644 --- a/src/ros2_medkit_fault_manager/src/rosbag_capture.cpp +++ b/src/ros2_medkit_fault_manager/src/rosbag_capture.cpp @@ -422,54 +422,54 @@ void RosbagCapture::store_worker_loop() { } } -void RosbagCapture::on_fault_prefailed(const std::string & fault_code) { +void RosbagCapture::on_fault_prefailed(const FaultId & id) { if (!config_.enabled) { return; } // Start buffer if lazy_start and not already running if (config_.lazy_start && !running_.load()) { - RCLCPP_INFO(node_->get_logger(), "RosbagCapture starting on PREFAILED for fault '%s'", fault_code.c_str()); + RCLCPP_INFO(node_->get_logger(), "RosbagCapture starting on PREFAILED for fault '%s'", id.fault_code.c_str()); start(); } } -bool RosbagCapture::is_current_recording_primary(const std::string & fault_code) const { +bool RosbagCapture::is_current_recording_primary(const FaultId & id) const { std::lock_guard lock(post_fault_timer_mutex_); - return recording_post_fault_.load() && fault_code == current_fault_code_; + return recording_post_fault_.load() && id == current_fault_id_; } -bool RosbagCapture::attach_to_active_recording(const std::string & fault_code, - const std::set & entity_topics) { +bool RosbagCapture::attach_to_active_recording(const FaultId & id, const std::set & entity_topics) { std::lock_guard lock(post_fault_timer_mutex_); if (!recording_post_fault_.load()) { return false; } - if (fault_code == current_fault_code_) { + if (id == current_fault_id_) { return true; } // Cap the attachment set: a flapping detector must not grow it without bound // between two post-roll windows. Past the cap the burst is already recorded, // only the per-fault lookup key is missing. - if (attached_fault_codes_.size() >= kMaxAttachedFaults) { + if (attached_fault_ids_.size() >= kMaxAttachedFaults) { RCLCPP_WARN(node_->get_logger(), "Post-fault recording already covers %zu faults, not attaching '%s'", - attached_fault_codes_.size(), fault_code.c_str()); + attached_fault_ids_.size(), id.fault_code.c_str()); // Widened anyway. What the cap withholds is the lookup key, not the data - and in // entity mode returning here withheld the data too, because the recording never // learned this fault's topics and should_capture_topic() then dropped every one of // its messages. The bag is the burst's black box either way. - widen_capture_filter_for(fault_code, entity_topics); + widen_capture_filter_for(id, entity_topics); return true; } - if (attached_fault_codes_.insert(fault_code).second) { + if (attached_fault_ids_.insert(id).second) { RCLCPP_INFO(node_->get_logger(), "Fault '%s' confirmed during post-fault recording, attaching it to bag for '%s'", - fault_code.c_str(), current_fault_code_.empty() ? "" : current_fault_code_.c_str()); - widen_capture_filter_for(fault_code, entity_topics); + id.fault_code.c_str(), + current_fault_id_.fault_code.empty() ? "" : current_fault_id_.fault_code.c_str()); + widen_capture_filter_for(id, entity_topics); } return true; } -void RosbagCapture::widen_capture_filter_for(const std::string & fault_code, const std::set & topics) { +void RosbagCapture::widen_capture_filter_for(const FaultId & id, const std::set & topics) { // Entity mode scoped the in-flight writes to the first fault's topics; an // attached fault needs its own topics in the bag from the attach onwards, or // its row would serve a recording with none of its data. Union them in, and @@ -490,14 +490,15 @@ void RosbagCapture::widen_capture_filter_for(const std::string & fault_code, con if (topics.empty()) { RCLCPP_WARN(node_->get_logger(), "Entity scope unresolved for attached fault '%s'; widening the recording to all topics", - fault_code.c_str()); + id.fault_code.c_str()); active_capture_topics_.clear(); return; } active_capture_topics_.insert(topics.begin(), topics.end()); } -void RosbagCapture::on_fault_confirmed(const std::string & fault_code) { +void RosbagCapture::on_fault_confirmed(const FaultId & id) { + const std::string & fault_code = id.fault_code; if (!config_.enabled || !running_.load()) { return; } @@ -509,15 +510,15 @@ void RosbagCapture::on_fault_confirmed(const std::string & fault_code) { // guard stays true, messages keep entering the bag, and the recording overruns // duration_after_sec by however long the store stalls. std::set entity_topics; - if (config_.topics == "entity" && !is_current_recording_primary(fault_code)) { - entity_topics = compute_entity_topics(fault_code); + if (config_.topics == "entity" && !is_current_recording_primary(id)) { + entity_topics = compute_entity_topics(id); } // Faults arrive in bursts from one root cause, so a confirmation landing inside // an in-flight post-roll is exactly the correlated fault whose black box matters // most. It shares the recording window, so attach it to the running bag instead // of leaving it with no recording at all. - if (attach_to_active_recording(fault_code, entity_topics)) { + if (attach_to_active_recording(id, entity_topics)) { return; } @@ -575,9 +576,9 @@ void RosbagCapture::on_fault_confirmed(const std::string & fault_code) { // confirmation either attaches to this bag or starts its own - never both. { std::lock_guard lock(post_fault_timer_mutex_); - current_fault_code_ = fault_code; + current_fault_id_ = id; current_bag_path_ = bag_path; - attached_fault_codes_.clear(); + attached_fault_ids_.clear(); recording_post_fault_.store(true); arm_post_fault_timer(); } @@ -619,6 +620,7 @@ void RosbagCapture::on_fault_confirmed(const std::string & fault_code) { RosbagFileInfo info; info.fault_code = fault_code; + info.owner = id.owner; info.recording_id = rosbag_recording_id(bag_path); info.file_path = bag_path; info.format = config_.format; @@ -642,7 +644,8 @@ void RosbagCapture::on_fault_confirmed(const std::string & fault_code) { } } -void RosbagCapture::on_fault_cleared(const std::string & fault_code) { +void RosbagCapture::on_fault_cleared(const FaultId & id) { + const std::string & fault_code = id.fault_code; if (!config_.enabled || !config_.auto_cleanup) { return; } @@ -666,18 +669,18 @@ void RosbagCapture::on_fault_cleared(const std::string & fault_code) { { std::lock_guard lock(post_fault_timer_mutex_); if (recording_post_fault_.load()) { - attached_fault_codes_.erase(fault_code); - if (fault_code == current_fault_code_) { - current_fault_code_.clear(); + attached_fault_ids_.erase(id); + if (id == current_fault_id_) { + current_fault_id_ = FaultId{}; } } } - // Delete the bag file for this fault. Called from the clear service, where a throw terminates the process. + // Delete the bag rows of this record. Called from the clear service, where a throw terminates the process. // Under clear_rows_mutex_: a row the store worker is inserting lands before this delete, never after. try { std::lock_guard rows_lock(clear_rows_mutex_); - if (storage_->delete_rosbag_file(fault_code)) { + if (storage_->delete_rosbag_file(id)) { RCLCPP_INFO(node_->get_logger(), "Auto-cleanup: deleted bag file for fault '%s'", fault_code.c_str()); } } catch (const std::exception & e) { @@ -1022,20 +1025,20 @@ rclcpp::QoS RosbagCapture::resolve_topic_qos(const std::string & topic) const { return qos; } -std::set RosbagCapture::compute_entity_topics(const std::string & fault_code) { +std::set RosbagCapture::compute_entity_topics(const FaultId & id) { std::set topics; // The whole resolution is guarded: this runs on a detached capture thread with - // no outer catch, and storage_->get_fault() can throw on the sqlite backend - // (e.g. SQLITE_BUSY). A throw here would terminate the process - the exact crash - // the crash-safety work removed - so any failure degrades to "write everything". + // no outer catch, and the store can throw on the sqlite backend (e.g. SQLITE_BUSY). + // A throw here would terminate the process - the exact crash the crash-safety work + // removed - so any failure degrades to "write everything". try { - auto fault = storage_->get_fault(fault_code); - if (fault && !fault->reporting_sources.empty()) { - // reporting_sources hold the reporting node's FQN (e.g. "/planner_server"). - // Split each into (name, namespace) to match against topic endpoints. + if (storage_->contains(id)) { + // The record's owner is the reporting node's FQN (e.g. "/planner_server"). + // Split it into (name, namespace) to match against topic endpoints. std::set> wanted; - for (const auto & source : fault->reporting_sources) { + { + const std::string & source = id.owner; std::string ns = "/"; std::string name = source; const auto slash = source.rfind('/'); @@ -1086,7 +1089,7 @@ std::set RosbagCapture::compute_entity_topics(const std::string & f } } catch (const std::exception & e) { RCLCPP_WARN(node_->get_logger(), "Entity scope resolution failed for fault '%s' (%s); writing full buffer", - fault_code.c_str(), e.what()); + id.fault_code.c_str(), e.what()); topics.clear(); } @@ -1094,13 +1097,14 @@ std::set RosbagCapture::compute_entity_topics(const std::string & f } void RosbagCapture::resolve_entity_topics(const std::string & fault_code, std::set topics) { + // By code only for the log line. The caller already resolved the topics from the record. // The topics arrive resolved, for the same reason widen_capture_filter_for() takes // them resolved: computing them reads the fault store and walks the ROS graph, and // the caller does that once, before it touches any lock. if (config_.topics != "entity") { topics.clear(); } else if (!topics.empty()) { - RCLCPP_INFO(node_->get_logger(), "Entity scope for fault '%s': %zu topic(s) from the faulting node(s) + /tf", + RCLCPP_INFO(node_->get_logger(), "Entity scope for fault '%s': %zu topic(s) from the owning node + /tf", fault_code.c_str(), topics.size()); } else { RCLCPP_WARN(node_->get_logger(), @@ -1536,7 +1540,7 @@ void RosbagCapture::write_manifest(const std::string & bag_path, int64_t created } } -bool RosbagCapture::row_still_wanted(const std::string & code) { +bool RosbagCapture::row_still_wanted(const FaultId & id) { // Same rule as on_fault_cleared: without auto_cleanup, or with a history // configured, a cleared fault keeps its black box. Checking only auto_cleanup // here let an acknowledgement arriving during the post-roll drop every row and @@ -1548,10 +1552,10 @@ bool RosbagCapture::row_still_wanted(const std::string & code) { // the row: the bag stays referenced. std::optional fault; try { - fault = storage_->get_fault(code); + fault = storage_->get_fault(id); } catch (const std::exception & e) { RCLCPP_WARN(node_->get_logger(), "Could not read fault '%s' before storing its bag, keeping its row: %s", - code.c_str(), e.what()); + id.fault_code.c_str(), e.what()); return true; } // Absent is not cleared. A caller driving the capture directly, or a fault the @@ -1575,7 +1579,7 @@ bool RosbagCapture::store_rows_or_discard_bag(std::vector rows, std::lock_guard lock(clear_rows_mutex_); rows.erase(std::remove_if(rows.begin(), rows.end(), [this](const RosbagFileInfo & row) { - return !row_still_wanted(row.fault_code); + return !row_still_wanted(FaultId{row.fault_code, row.owner}); }), rows.end()); if (!rows.empty()) { @@ -1675,8 +1679,8 @@ void RosbagCapture::finalize_post_fault_recording() { // a recording that is already being finalised. Testing the guard under the same // lock also settles a timer firing concurrently with stop() - the loser sees the // recording already claimed and returns. - std::set attached; - std::string fault_code; + std::set attached; + FaultId fault_id; std::string bag_path; int64_t started_at_ns = 0; double recording_span_sec = 0.0; @@ -1692,8 +1696,8 @@ void RosbagCapture::finalize_post_fault_recording() { post_fault_timer_->cancel(); } recording_post_fault_.store(false); - attached.swap(attached_fault_codes_); - fault_code.swap(current_fault_code_); + attached.swap(attached_fault_ids_); + std::swap(fault_id, current_fault_id_); bag_path.swap(current_bag_path_); // Taken here, under the lock that clears the guard: a confirmation racing this // finalise opens its own recording the moment the guard drops, and would @@ -1762,8 +1766,8 @@ void RosbagCapture::finalize_post_fault_recording() { // is never listed before its manifest exists. ClosedRecording recording; recording.bag_path = std::move(bag_path); - recording.fault_code = std::move(fault_code); - recording.attached_fault_codes = std::move(attached); + recording.fault_id = std::move(fault_id); + recording.attached_ids = std::move(attached); recording.duration_sec = recording_span_sec; recording.created_at_ns = get_wall_clock_ns(); enqueue_closed_recording(std::move(recording)); @@ -1785,23 +1789,26 @@ void RosbagCapture::store_closed_recording(const ClosedRecording & recording) { info.duration_sec = recording.duration_sec; info.size_bytes = bag_size; info.created_at_ns = recording.created_at_ns; - // One recording, one row per fault it covers: the correlated faults that - // confirmed inside this window each need their own lookup key to serve it. Which + // One recording, one row per RECORD it covers: the correlated faults that + // confirmed inside this window each need their own lookup key to serve it, and + // two owners of one code in the same burst each get theirs. Which // of them still get one is decided in store_rows_or_discard_bag(). std::vector rows; - if (!recording.fault_code.empty()) { - info.fault_code = recording.fault_code; + if (!recording.fault_id.fault_code.empty()) { + info.fault_code = recording.fault_id.fault_code; + info.owner = recording.fault_id.owner; rows.push_back(info); } - for (const auto & code : recording.attached_fault_codes) { - info.fault_code = code; + for (const auto & attached_id : recording.attached_ids) { + info.fault_code = attached_id.fault_code; + info.owner = attached_id.owner; rows.push_back(info); } if (store_rows_or_discard_bag(std::move(rows), bag_path)) { RCLCPP_INFO(node_->get_logger(), "Bag file completed: %s (%.2f MB, %.1fs, %zu attached fault(s))", bag_path.c_str(), static_cast(bag_size) / (1024.0 * 1024.0), info.duration_sec, - recording.attached_fault_codes.size()); + recording.attached_ids.size()); } } catch (const std::exception & e) { RCLCPP_WARN(node_->get_logger(), "Failed to store recording '%s': %s", recording.bag_path.c_str(), e.what()); diff --git a/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp b/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp index 95de9275e..6ae8925b7 100644 --- a/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp +++ b/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp @@ -146,12 +146,15 @@ bool SnapshotCapture::seed_capture_seq() { return true; } -void SnapshotCapture::capture(const std::string & fault_code) { +void SnapshotCapture::capture(const FaultId & id) { + const std::string & fault_code = id.fault_code; if (!config_.enabled) { RCLCPP_DEBUG(node_->get_logger(), "Snapshot capture disabled, skipping for fault '%s'", fault_code.c_str()); return; } + // Resolution is by CODE: fault_specific and patterns are config about what a code + // means, not about who reported it. bool explicit_match = false; auto topics = resolve_topics(fault_code, explicit_match); // Zero-config fallback: nothing in the explicit config even mentioned this @@ -159,7 +162,7 @@ void SnapshotCapture::capture(const std::string & fault_code) { // per-fault opt-out (explicit_match == true) and must NOT fall through here. bool entity_scoped = false; if (topics.empty() && !explicit_match && config_.entity_default) { - topics = resolve_entity_topics(fault_code); + topics = resolve_entity_topics(id); entity_scoped = !topics.empty(); } if (topics.empty()) { @@ -193,9 +196,9 @@ void SnapshotCapture::capture(const std::string & fault_code) { // Entity-default topics are not known at construction, so the background // cache never holds them - always sample those on demand. if (config_.background_capture && !entity_scoped) { - success = capture_topic_from_cache(fault_code, topic, freeze_frame, rows); + success = capture_topic_from_cache(id, topic, freeze_frame, rows); } else { - success = capture_topic_on_demand(fault_code, topic, freeze_frame, rows); + success = capture_topic_on_demand(id, topic, freeze_frame, rows); } if (success) { @@ -221,7 +224,7 @@ void SnapshotCapture::capture(const std::string & fault_code) { // asked to retain snapshots there is no such promise, so the batch stands. // Absent is not cleared: a caller driving the capture directly still stores. // Mirrors rosbag_capture's wants_a_row. - const auto fault = storage_->get_fault(fault_code); + const auto fault = storage_->get_fault(id); const bool acknowledged_mid_capture = fault.has_value() && fault->status == fd::fault_status::kCleared; if (acknowledged_mid_capture && !storage_->retains_snapshots_on_clear()) { RCLCPP_DEBUG(node_->get_logger(), "Dropping capture for '%s' - acknowledged while it ran", fault_code.c_str()); @@ -229,7 +232,7 @@ void SnapshotCapture::capture(const std::string & fault_code) { storage_->store_snapshots(rows); } - // Persist the compact freeze-frame keyed by fault_code (retained across clear_fault). + // Persist the compact freeze-frame keyed by the RECORD (retained across clear_fault). // Only reached for faults with a configured capture set (unconfigured codes returned // early above and get no row). If nothing was publishing at confirmation time the row // holds an empty object, recording that a configured capture ran but sampled nothing. @@ -244,7 +247,7 @@ void SnapshotCapture::capture(const std::string & fault_code) { fault_code.c_str()); return; } - auto existing = storage_->get_freeze_frame(fault_code); + auto existing = storage_->get_freeze_frame(id); if (existing.has_value() && existing->data != "{}") { RCLCPP_WARN(node_->get_logger(), "Nothing captured for fault '%s'; keeping previously retained freeze-frame", fault_code.c_str()); @@ -254,6 +257,7 @@ void SnapshotCapture::capture(const std::string & fault_code) { FreezeFrameData frame; frame.fault_code = fault_code; + frame.owner = id.owner; frame.data = freeze_frame.dump(); frame.captured_at_ns = get_wall_clock_ns(); storage_->store_freeze_frame(frame); @@ -287,7 +291,8 @@ std::vector SnapshotCapture::resolve_topics(const std::string & fau return {}; } -std::vector SnapshotCapture::resolve_entity_topics(const std::string & fault_code) const { +std::vector SnapshotCapture::resolve_entity_topics(const FaultId & id) const { + const std::string & fault_code = id.fault_code; // Bound the on-demand sweep: each silent-but-published topic costs up to // timeout_sec, so a node with many declared topics must not stall a capture // pool worker indefinitely. @@ -298,8 +303,7 @@ std::vector SnapshotCapture::resolve_entity_topics(const std::strin // pool worker with no outer catch, and storage_->get_fault() can throw on // the sqlite backend. Any failure degrades to "no entity-default capture". try { - auto fault = storage_->get_fault(fault_code); - if (!fault || fault->reporting_sources.empty()) { + if (!storage_->contains(id)) { return {}; } @@ -308,16 +312,19 @@ std::vector SnapshotCapture::resolve_entity_topics(const std::strin // as (name, ns="/") could match an unrelated root-namespace node of the // same name and freeze-frame its topics. Their zero-config frames come // from the gateway's EntityFreezeFrameCapture instead. + // + // One owner, so one (name, namespace) pair: the record names exactly the + // source whose own data belongs in its frame. std::set> wanted; - for (const auto & source : fault->reporting_sources) { - if (source.size() < 2 || source.front() != '/') { - continue; - } - const auto slash = source.rfind('/'); - const std::string name = source.substr(slash + 1); - const std::string ns = (slash == 0) ? "/" : source.substr(0, slash); - if (!name.empty()) { - wanted.emplace(name, ns); + { + const std::string & source = id.owner; + if (source.size() >= 2 && source.front() == '/') { + const auto slash = source.rfind('/'); + const std::string name = source.substr(slash + 1); + const std::string ns = (slash == 0) ? "/" : source.substr(0, slash); + if (!name.empty()) { + wanted.emplace(name, ns); + } } } if (wanted.empty()) { @@ -357,13 +364,13 @@ std::vector SnapshotCapture::resolve_entity_topics(const std::strin } if (!topics.empty()) { - RCLCPP_INFO(node_->get_logger(), "Entity-default capture for fault '%s': %zu topic(s) from the reporting node(s)", + RCLCPP_INFO(node_->get_logger(), "Entity-default capture for fault '%s': %zu topic(s) from the owning node", fault_code.c_str(), topics.size()); } return {topics.begin(), topics.end()}; } -bool SnapshotCapture::capture_topic_on_demand(const std::string & fault_code, const std::string & topic, +bool SnapshotCapture::capture_topic_on_demand(const FaultId & id, const std::string & topic, nlohmann::json & freeze_frame, std::vector & rows) { // Get topic type std::string msg_type = get_topic_type(topic); @@ -468,7 +475,8 @@ bool SnapshotCapture::capture_topic_on_demand(const std::string & fault_code, co // Store snapshot (use wall clock time, not sim time, for proper timestamps) SnapshotData snapshot; - snapshot.fault_code = fault_code; + snapshot.fault_code = id.fault_code; + snapshot.owner = id.owner; snapshot.topic = topic; snapshot.message_type = msg_type; snapshot.data = json_data.dump(); @@ -479,7 +487,8 @@ bool SnapshotCapture::capture_topic_on_demand(const std::string & fault_code, co // Record the value into the compact freeze-frame dict under the topic key. freeze_frame[topic] = std::move(json_data); - RCLCPP_DEBUG(node_->get_logger(), "Captured snapshot from '%s' for fault '%s'", topic.c_str(), fault_code.c_str()); + RCLCPP_DEBUG(node_->get_logger(), "Captured snapshot from '%s' for fault '%s'", topic.c_str(), + id.fault_code.c_str()); return true; } catch (const ros2_medkit_serialization::TypeNotFoundError & e) { @@ -493,8 +502,9 @@ bool SnapshotCapture::capture_topic_on_demand(const std::string & fault_code, co return false; } -bool SnapshotCapture::capture_topic_from_cache(const std::string & fault_code, const std::string & topic, +bool SnapshotCapture::capture_topic_from_cache(const FaultId & id, const std::string & topic, nlohmann::json & freeze_frame, std::vector & rows) { + const std::string & fault_code = id.fault_code; std::lock_guard lock(cache_mutex_); auto it = message_cache_.find(topic); @@ -507,7 +517,8 @@ bool SnapshotCapture::capture_topic_from_cache(const std::string & fault_code, c // Store snapshot from cache SnapshotData snapshot; - snapshot.fault_code = fault_code; + snapshot.fault_code = id.fault_code; + snapshot.owner = id.owner; snapshot.topic = topic; snapshot.message_type = cached.message_type; snapshot.data = cached.data; diff --git a/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp b/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp index 29b2a23b1..3016d3467 100644 --- a/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp @@ -14,12 +14,19 @@ #include "ros2_medkit_fault_manager/sqlite_fault_storage.hpp" +#include +#include #include +#include +#include #include #include #include #include #include +#include +#include +#include namespace ros2_medkit_fault_manager { @@ -103,6 +110,109 @@ class SqliteStatement { sqlite3_stmt * stmt_{nullptr}; }; +/// First reporting source of a LEGACY `reporting_sources` value, recovered without +/// requiring the text to be valid JSON. +/// +/// Earlier builds wrote this column with an escape set that covered only the quote, +/// the backslash and \b \f \n \r \t, so a source_id carrying any other control byte +/// was stored as text no JSON parser accepts. Asking SQLite's json_extract to read it +/// raises "malformed JSON", which inside the migration means the rebuild rolls back +/// and the node refuses to start again for as long as the row exists. The recovery +/// therefore reads the text itself and treats JSON as a shape it recognises, not as a +/// precondition. +/// +/// A record always ends up addressable. Where nothing can be read the answer is the +/// synthetic owner `legacy`, never the empty string: an empty owner means one thing +/// only, a child row not yet assigned to a record, and a fault carrying it would be +/// unreachable through source_id and would keep the child backfill looking for work. +/// +/// The mapping, exhaustively: +/// "" -> "legacy" (nothing recorded) +/// "sensor_a" -> "sensor_a" (bare word, never a JSON array) +/// "[]" -> "legacy" (array with no source) +/// "[\"a<0x01>b\"]" -> "a<0x01>b" (control byte kept verbatim) +/// "[...]", "{...}" -> "legacy" (neither an array nor a bare word) +/// anything unterminated -> what was read before the text ran out +std::string legacy_first_source(const std::string & raw) { + std::size_t pos = 0; + while (pos < raw.size() && std::isspace(static_cast(raw[pos])) != 0) { + ++pos; + } + if (pos >= raw.size()) { + return kLegacyOwner; + } + + if (raw[pos] != '[') { + // Not an array. Only a bare word is recoverable here, meaning text carrying no + // JSON punctuation at all: an older writer, or a hand edit, put the source in the + // column unquoted. Anything else (a byte-order mark in front of the array, an + // object wrapper, a quoted scalar) is structure this function does not claim to + // read, and returning it whole would mint an owner out of punctuation. + const bool bare_word = + raw.find_first_of("[]{}\",:", pos) == std::string::npos && static_cast(raw[pos]) < 0x80; + return bare_word ? raw.substr(pos) : std::string(kLegacyOwner); + } + + ++pos; // past '[' + while (pos < raw.size() && std::isspace(static_cast(raw[pos])) != 0) { + ++pos; + } + if (pos >= raw.size() || raw[pos] != '"') { + return kLegacyOwner; // "[]", or a first element that is not a string + } + + ++pos; // past the opening quote + std::string out; + while (pos < raw.size() && raw[pos] != '"') { + if (raw[pos] != '\\' || pos + 1 >= raw.size()) { + out.push_back(raw[pos]); // every other byte is itself, control bytes included + ++pos; + continue; + } + const char escaped = raw[pos + 1]; + pos += 2; + switch (escaped) { + case 'b': + out.push_back('\b'); + break; + case 'f': + out.push_back('\f'); + break; + case 'n': + out.push_back('\n'); + break; + case 'r': + out.push_back('\r'); + break; + case 't': + out.push_back('\t'); + break; + case 'u': { + // Only the \u00XX range this writer ever emits is decoded. Anything wider is + // left as written rather than guessed at, because a wrong transcoding would + // change an identity that other rows are keyed by. + if (pos + 4 <= raw.size() && raw.compare(pos, 2, "00") == 0) { + const std::string digits = raw.substr(pos + 2, 2); + char * end = nullptr; + const auto value = std::strtol(digits.c_str(), &end, 16); + if (end != nullptr && *end == '\0') { + out.push_back(static_cast(value)); + pos += 4; + break; + } + } + out.push_back('\\'); + out.push_back('u'); + break; + } + default: + out.push_back(escaped); // covers \" and \\, and keeps anything unexpected + break; + } + } + return out.empty() ? std::string(kLegacyOwner) : out; +} + } // namespace SqliteFaultStorage::SqliteFaultStorage(const std::string & db_path, core::LogSink & log_sink) @@ -150,9 +260,17 @@ DebounceConfig SqliteFaultStorage::get_debounce_config() const { } void SqliteFaultStorage::initialize_schema() { + // A row is one fault RECORD, identified by (fault_code, owner). Uniqueness is the + // index below, never a column constraint, per the house rule spelled out at the + // rosbag_files table: a column constraint cannot be dropped with ALTER TABLE and + // forces a full rebuild the day the identity changes again. + // + // reporting_sources is kept as the serialized form of owner so a reader of the raw + // table still sees the field it always saw. owner is the authoritative one, and + // every write below derives the JSON from it, so the two cannot drift. const char * create_faults_table_sql = R"( CREATE TABLE IF NOT EXISTS faults ( - fault_code TEXT PRIMARY KEY, + fault_code TEXT NOT NULL, severity INTEGER NOT NULL, description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, @@ -163,7 +281,8 @@ void SqliteFaultStorage::initialize_schema() { debounce_counter INTEGER NOT NULL DEFAULT 0, last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, - confirmed_at_ns INTEGER NOT NULL DEFAULT 0 + confirmed_at_ns INTEGER NOT NULL DEFAULT 0, + owner TEXT NOT NULL ); )"; @@ -214,6 +333,7 @@ void SqliteFaultStorage::initialize_schema() { CREATE TABLE IF NOT EXISTS snapshots ( id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', topic TEXT NOT NULL, message_type TEXT NOT NULL, data TEXT NOT NULL, @@ -230,11 +350,13 @@ void SqliteFaultStorage::initialize_schema() { } // Create freeze_frames table: one compact JSON dict of captured topic values per fault - // code. Unlike snapshots, freeze frames are keyed by fault_code and are NOT removed on - // clear_fault, so the confirmed-state record is retained after acknowledgement. + // RECORD. Unlike snapshots, freeze frames are keyed by the record identity and are NOT + // removed on clear_fault, so the confirmed-state record is retained after acknowledgement. + // Uniqueness through the index, not a column constraint, per the house rule. const char * create_freeze_frames_table_sql = R"( CREATE TABLE IF NOT EXISTS freeze_frames ( - fault_code TEXT PRIMARY KEY, + fault_code TEXT NOT NULL, + owner TEXT NOT NULL, data TEXT NOT NULL, captured_at_ns INTEGER NOT NULL ); @@ -272,8 +394,8 @@ void SqliteFaultStorage::initialize_schema() { // Create near_misses table: append-only series of FAILED reports that moved the debounce // counter without confirming the fault. One row per qualifying report, never updated in // place, and NOT removed on clear_fault - acknowledging a fault cycle must not erase how - // often that code approached confirmation. Bounded per fault code by the caller-supplied - // limit, evicting the oldest rows first. + // often that record approached confirmation. Bounded per record (fault_code, source_id) by + // the caller-supplied limit, evicting the oldest rows first. const char * create_near_misses_table_sql = R"( CREATE TABLE IF NOT EXISTS near_misses ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -315,8 +437,8 @@ void SqliteFaultStorage::initialize_schema() { } } - // Create rosbag_files table. One row = one LINK (a fault claiming a recording): - // several faults of a burst link to one bag, and one fault links to several bags + // Create rosbag_files table. One row = one LINK (a fault record claiming a recording): + // several records of a burst link to one bag, and one record links to several bags // over time. Bytes belong to file_path, not to the row. // // House rule, learned the hard way here: uniqueness is expressed with @@ -328,6 +450,7 @@ void SqliteFaultStorage::initialize_schema() { CREATE TABLE IF NOT EXISTS rosbag_files ( id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', recording_id TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL, format TEXT NOT NULL, @@ -348,6 +471,28 @@ void SqliteFaultStorage::initialize_schema() { migrate_rosbag_files_drop_unique(); migrate_rosbag_files_add_recording_id(); + // After the rosbag rebuilds, so it sees a rosbag_files whose column set is settled: + // the rebuild above copies a fixed column list and would drop an owner column added + // before it. + migrate_faults_add_owner(); + + // The record identity, expressed as an index rather than a PRIMARY KEY so the next + // identity change is a DROP INDEX instead of another table rebuild. + if (sqlite3_exec(db_, "CREATE UNIQUE INDEX IF NOT EXISTS idx_faults_code_owner ON faults(fault_code, owner)", nullptr, + nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + throw std::runtime_error("Failed to create faults unique index: " + error); + } + + if (sqlite3_exec(db_, + "CREATE UNIQUE INDEX IF NOT EXISTS idx_freeze_frames_code_owner ON freeze_frames(fault_code, owner)", + nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + throw std::runtime_error("Failed to create freeze_frames unique index: " + error); + } + // Indexes last, so they serve a fresh table and a rebuilt one alike. // // idx_rosbag_files_path is capability, not tuning: path_referenced() scans on @@ -356,7 +501,7 @@ void SqliteFaultStorage::initialize_schema() { const char * create_rosbag_files_indexes_sql = R"( CREATE INDEX IF NOT EXISTS idx_rosbag_files_fault_code ON rosbag_files(fault_code); CREATE INDEX IF NOT EXISTS idx_rosbag_files_created_at ON rosbag_files(created_at_ns); - CREATE INDEX IF NOT EXISTS idx_rosbag_files_fault_created ON rosbag_files(fault_code, created_at_ns, id); + CREATE INDEX IF NOT EXISTS idx_rosbag_files_fault_created ON rosbag_files(fault_code, owner, created_at_ns, id); CREATE INDEX IF NOT EXISTS idx_rosbag_files_recording ON rosbag_files(recording_id); CREATE INDEX IF NOT EXISTS idx_rosbag_files_path ON rosbag_files(file_path); )"; @@ -376,7 +521,7 @@ void SqliteFaultStorage::initialize_schema() { // rows this index would reject. if (sqlite3_exec(db_, "DELETE FROM rosbag_files WHERE id NOT IN " - "(SELECT MAX(id) FROM rosbag_files GROUP BY fault_code, file_path)", + "(SELECT MAX(id) FROM rosbag_files GROUP BY fault_code, owner, file_path)", nullptr, nullptr, &err_msg) != SQLITE_OK) { std::string error = err_msg ? err_msg : "Unknown error"; sqlite3_free(err_msg); @@ -385,7 +530,7 @@ void SqliteFaultStorage::initialize_schema() { if (sqlite3_exec(db_, "CREATE UNIQUE INDEX IF NOT EXISTS idx_rosbag_files_fault_path " - "ON rosbag_files(fault_code, file_path)", + "ON rosbag_files(fault_code, owner, file_path)", nullptr, nullptr, &err_msg) != SQLITE_OK) { std::string error = err_msg ? err_msg : "Unknown error"; sqlite3_free(err_msg); @@ -463,6 +608,298 @@ void SqliteFaultStorage::migrate_rosbag_files_drop_unique() { } } +bool SqliteFaultStorage::table_has_owner(const char * table) const { + SqliteStatement info(db_, (std::string("PRAGMA table_info(") + table + ")").c_str()); + while (info.step() == SQLITE_ROW) { + if (info.column_text(1) == "owner") { + return true; + } + } + return false; +} + +bool SqliteFaultStorage::has_assignable_child_rows() const { + for (const char * table : {"freeze_frames", "snapshots", "rosbag_files"}) { + if (!table_has_owner(table)) { + continue; // the column arrives with the migration, which then fills it + } + // The same condition the backfill's UPDATE applies, so "is there work" and "what + // the work does" cannot disagree. Asking only for an empty owner would call a row + // that no open can ever assign (its code has several owners, or none) pending + // work forever, and every open would re-enter the write transaction and repeat + // the warning that says the row cannot be assigned. + const std::string sql = std::string("SELECT 1 FROM ") + table + " t WHERE t.owner = '' AND (SELECT COUNT(" + + "DISTINCT f.owner) FROM faults f WHERE f.fault_code = t.fault_code) = 1 LIMIT 1"; + SqliteStatement stmt(db_, sql.c_str()); + if (stmt.step() == SQLITE_ROW) { + return true; + } + } + return false; +} + +bool SqliteFaultStorage::table_exists(const char * table) const { + SqliteStatement stmt(db_, "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?"); + stmt.bind_text(1, table); + return stmt.step() == SQLITE_ROW; +} + +void SqliteFaultStorage::drop_scratch_table(const char * table) { + // Announced rather than done quietly. Debris under these names means an earlier + // rebuild did not finish, or someone repaired the database by hand, and an operator + // reading the log after a surprising restart should find out that this open threw + // work away. The names are reserved for this procedure, so nothing else loses data. + if (table_exists(table)) { + core::logf(log_sink_, core::LogLevel::kWarn, "sqlite_fault_storage", + "Dropping leftover migration scratch table '%s' from an unfinished earlier rebuild", table); + } + char * err_msg = nullptr; + const std::string sql = std::string("DROP TABLE IF EXISTS ") + table; + if (sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + throw std::runtime_error(std::string("fault owner migration failed (drop leftover ") + table + "): " + error); + } +} + +void SqliteFaultStorage::copy_legacy_fault_rows() { + struct LegacyFault { + std::string fault_code; + std::string description; + std::string status; + std::string owner; + int severity{0}; + int64_t first_occurred_ns{0}; + int64_t last_occurred_ns{0}; + int64_t occurrence_count{0}; + int64_t debounce_counter{0}; + int64_t last_failed_ns{0}; + int64_t last_passed_ns{0}; + int64_t confirmed_at_ns{0}; + }; + + // Read the whole table before writing a row of it. The write target is a different + // table, but the old one is dropped immediately afterwards, and holding a read cursor + // across that is a shape worth not having. + std::vector rows; + { + SqliteStatement select(db_, + "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " + "occurrence_count, status, reporting_sources, debounce_counter, last_failed_ns, " + "last_passed_ns, confirmed_at_ns FROM faults"); + while (select.step() == SQLITE_ROW) { + LegacyFault row; + row.fault_code = select.column_text(0); + row.severity = select.column_int(1); + row.description = select.column_text(2); + row.first_occurred_ns = select.column_int64(3); + row.last_occurred_ns = select.column_int64(4); + row.occurrence_count = select.column_int64(5); + row.status = select.column_text(6); + row.owner = legacy_first_source(select.column_text(7)); + row.debounce_counter = select.column_int64(8); + row.last_failed_ns = select.column_int64(9); + row.last_passed_ns = select.column_int64(10); + row.confirmed_at_ns = select.column_int64(11); + rows.push_back(std::move(row)); + } + } + + SqliteStatement insert(db_, + "INSERT INTO faults_new " + "(fault_code, severity, description, first_occurred_ns, last_occurred_ns, occurrence_count, " + " status, reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, " + " confirmed_at_ns, owner) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + + for (const auto & row : rows) { + if (row.owner == kLegacyOwner) { + // Kept, not dropped. The fault state is real and an operator can still see it, + // it simply belongs to no source that this database ever recorded readably. It + // gets a name rather than an empty owner so it stays addressable by source_id. + core::logf(log_sink_, core::LogLevel::kWarn, "sqlite_fault_storage", + "Fault '%s' carried no readable reporting source, migrated under owner '%s'", row.fault_code.c_str(), + kLegacyOwner); + } + insert.reset(); + insert.bind_text(1, row.fault_code); + insert.bind_int(2, row.severity); + insert.bind_text(3, row.description); + insert.bind_int64(4, row.first_occurred_ns); + insert.bind_int64(5, row.last_occurred_ns); + insert.bind_int64(6, row.occurrence_count); + insert.bind_text(7, row.status); + // reporting_sources is rewritten to the one owner at the same time, so the column + // and owner agree from the first open after the migration, exactly as every write + // keeps them agreeing afterwards. Through serialize_json_array, so the result is + // valid JSON even when the value that came in was not. + insert.bind_text(8, serialize_json_array({row.owner})); + insert.bind_int64(9, row.debounce_counter); + insert.bind_int64(10, row.last_failed_ns); + insert.bind_int64(11, row.last_passed_ns); + // A row that confirmed stays CONFIRMED or HEALED until a clear. A pending row with a value + // carries it from the occurrence before a reactivation, which earlier releases did not reset. + const bool pending = row.status == fd::fault_status::kPrefailed || row.status == fd::fault_status::kPrepassed; + insert.bind_int64(12, pending ? 0 : row.confirmed_at_ns); + insert.bind_text(13, row.owner); + if (insert.step() != SQLITE_DONE) { + throw std::runtime_error(std::string("fault owner migration failed (copy fault rows): ") + sqlite3_errmsg(db_)); + } + } +} + +void SqliteFaultStorage::backfill_child_owners() { + char * err_msg = nullptr; + for (const char * table : {"freeze_frames", "snapshots", "rosbag_files"}) { + if (!table_has_owner(table)) { + continue; + } + + // Correlated on the child row's own fault_code, and applied only where that code + // has exactly ONE owner. The uncorrelated form this replaces picked whichever row + // SQLite reached first, which on a code with two owners handed one owner's + // snapshots and recordings to the other. + const std::string sql = std::string("UPDATE ") + table + " SET owner = (SELECT f.owner FROM faults f WHERE " + + "f.fault_code = " + table + ".fault_code) WHERE owner = '' AND (SELECT COUNT(DISTINCT " + + "f.owner) FROM faults f WHERE f.fault_code = " + table + ".fault_code) = 1"; + if (sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + err_msg = nullptr; + throw std::runtime_error(std::string("fault owner migration failed (backfill ") + table + ".owner): " + error); + } + + // What is left is either evidence for a code no fault row carries any more, or for + // a code several owners share. Neither can be assigned without guessing, so it is + // named instead. The row stays: no (code, owner) read returns it to an owner, and + // deleting an operator's evidence to tidy a column is the worse trade. + SqliteStatement leftover( + db_, + (std::string("SELECT fault_code, COUNT(*) FROM ") + table + " WHERE owner = '' GROUP BY fault_code").c_str()); + while (leftover.step() == SQLITE_ROW) { + core::logf(log_sink_, core::LogLevel::kWarn, "sqlite_fault_storage", + "%s: %lld row(s) for fault '%s' keep an empty owner, the code has no single owner to " + "assign them to", + table, static_cast(leftover.column_int64(1)), leftover.column_text(0).c_str()); + } + } +} + +void SqliteFaultStorage::migrate_faults_add_owner() { + // Probed PER TABLE, not once for the whole migration. A database can hold a legacy + // faults table next to a snapshots table this release just created with owner + // already on it: the four tables come from four independent CREATE TABLE IF NOT + // EXISTS statements, so they do not move as a unit. One gate on faults alone either + // re-ALTERs a column that is already there or skips one that is missing, and both + // throw on the very next open. + const bool faults_needs_owner = !table_has_owner("faults"); + const bool freeze_frames_needs_owner = !table_has_owner("freeze_frames"); + const bool snapshots_needs_owner = !table_has_owner("snapshots"); + const bool rosbag_files_needs_owner = !table_has_owner("rosbag_files"); + const bool schema_work = + faults_needs_owner || freeze_frames_needs_owner || snapshots_needs_owner || rosbag_files_needs_owner; + + // The child backfill is not tied to the ALTER that adds the column, so its own + // condition is asked separately: an open interrupted between the faults rebuild and + // the backfill, or a child table that gained the column while faults was still + // legacy, leaves rows no owner can see. They are healed on the next open. + if (!schema_work && !has_assignable_child_rows()) { + return; // fresh database, or fully migrated - safe to re-run on every open + } + + char * err_msg = nullptr; + const auto exec = [this, &err_msg](const char * sql, const char * what) { + if (sqlite3_exec(db_, sql, nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + err_msg = nullptr; + throw std::runtime_error(std::string("fault owner migration failed (") + what + "): " + error); + } + }; + + exec("BEGIN IMMEDIATE", "begin"); + try { + if (faults_needs_owner) { + // faults: SQLite cannot change a PRIMARY KEY in place and CREATE TABLE IF NOT + // EXISTS is a no-op on an existing database, so the identity change is the + // documented table-rebuild procedure. + // + // The DROP first because a rebuild that was interrupted by a crash, or a manual + // recovery with an external tool, can leave faults_new behind. Without the DROP + // that leftover makes CREATE TABLE fail on every open and the schema stays + // legacy forever. It is safe because faults_new is this procedure's scratch + // table and nothing else ever reads it. + drop_scratch_table("faults_new"); + exec( + "CREATE TABLE faults_new (" + " fault_code TEXT NOT NULL," + " severity INTEGER NOT NULL," + " description TEXT NOT NULL," + " first_occurred_ns INTEGER NOT NULL," + " last_occurred_ns INTEGER NOT NULL," + " occurrence_count INTEGER NOT NULL," + " status TEXT NOT NULL," + " reporting_sources TEXT NOT NULL," + " debounce_counter INTEGER NOT NULL DEFAULT 0," + " last_failed_ns INTEGER NOT NULL DEFAULT 0," + " last_passed_ns INTEGER NOT NULL DEFAULT 0," + " confirmed_at_ns INTEGER NOT NULL DEFAULT 0," + " owner TEXT NOT NULL)", + "create new faults table"); + + copy_legacy_fault_rows(); + + exec("DROP TABLE faults", "drop old faults table"); + exec("ALTER TABLE faults_new RENAME TO faults", "rename faults"); + } + + if (freeze_frames_needs_owner) { + // freeze_frames: same problem, same procedure, same reason for the DROP. + // The owner is left empty here and assigned by the shared child backfill below, + // so one rule decides who a frame belongs to no matter which open produced it. + drop_scratch_table("freeze_frames_new"); + exec( + "CREATE TABLE freeze_frames_new (" + " fault_code TEXT NOT NULL," + " owner TEXT NOT NULL," + " data TEXT NOT NULL," + " captured_at_ns INTEGER NOT NULL)", + "create new freeze_frames table"); + exec( + "INSERT INTO freeze_frames_new (fault_code, owner, data, captured_at_ns) " + "SELECT fault_code, '', data, captured_at_ns FROM freeze_frames", + "copy freeze frames"); + exec("DROP TABLE freeze_frames", "drop old freeze_frames table"); + exec("ALTER TABLE freeze_frames_new RENAME TO freeze_frames", "rename freeze_frames"); + } + + // snapshots and rosbag_files only gain a column: their identity is a rowid, not + // the fault code, so ALTER TABLE is enough. The column arrives empty and the + // shared backfill fills it. + if (snapshots_needs_owner) { + exec("ALTER TABLE snapshots ADD COLUMN owner TEXT NOT NULL DEFAULT ''", "add snapshots.owner"); + } + + if (rosbag_files_needs_owner) { + exec("ALTER TABLE rosbag_files ADD COLUMN owner TEXT NOT NULL DEFAULT ''", "add rosbag_files.owner"); + + // The rosbag indexes that carry fault_code have to widen with it. CREATE INDEX IF + // NOT EXISTS in initialize_schema() is a no-op on an index that already exists, so + // without these drops the unique index would keep its two-column shape and refuse + // two owners of one code linking the same bag. + exec("DROP INDEX IF EXISTS idx_rosbag_files_fault_path", "drop rosbag unique index"); + exec("DROP INDEX IF EXISTS idx_rosbag_files_fault_created", "drop rosbag fault_created index"); + } + + backfill_child_owners(); + + exec("COMMIT", "commit"); + } catch (...) { + sqlite3_exec(db_, "ROLLBACK", nullptr, nullptr, nullptr); + throw; + } +} + void SqliteFaultStorage::migrate_rosbag_files_add_recording_id() { bool has_recording_id = false; { @@ -529,97 +966,6 @@ void SqliteFaultStorage::migrate_rosbag_files_add_recording_id() { } } -std::vector SqliteFaultStorage::parse_json_array(const std::string & json_str) const { - std::vector result; - - // Simple JSON array parser for ["a", "b", "c"] format - if (json_str.size() < 2 || json_str.front() != '[' || json_str.back() != ']') { - if (!json_str.empty()) { - core::logf(log_sink_, core::LogLevel::kWarn, "sqlite_fault_storage", "Malformed JSON array in database: '%s'", - json_str.c_str()); - } - return result; - } - - std::string content = json_str.substr(1, json_str.size() - 2); - if (content.empty()) { - return result; - } - - size_t pos = 0; - while (pos < content.size()) { - // Skip whitespace - while (pos < content.size() && std::isspace(static_cast(content[pos]))) { - ++pos; - } - if (pos >= content.size()) { - break; - } - - // Expect opening quote - if (content[pos] != '"') { - break; - } - ++pos; - - // Find closing quote (handle escape sequences) - std::string value; - while (pos < content.size() && content[pos] != '"') { - if (content[pos] == '\\' && pos + 1 < content.size()) { - ++pos; - char escaped = content[pos]; - switch (escaped) { - case '"': - value.push_back('"'); - break; - case '\\': - value.push_back('\\'); - break; - case '/': - value.push_back('/'); - break; - case 'b': - value.push_back('\b'); - break; - case 'f': - value.push_back('\f'); - break; - case 'n': - value.push_back('\n'); - break; - case 'r': - value.push_back('\r'); - break; - case 't': - value.push_back('\t'); - break; - default: - // Unknown escape sequence: preserve character as-is - value.push_back(escaped); - break; - } - ++pos; - continue; - } - value.push_back(content[pos]); - ++pos; - } - - if (pos < content.size()) { - ++pos; // Skip closing quote - } - - result.push_back(value); - - // Skip whitespace and comma - while (pos < content.size() && (std::isspace(static_cast(content[pos])) || content[pos] == ',')) { - ++pos; - } - } - - return result; -} - std::string SqliteFaultStorage::serialize_json_array(const std::vector & vec) { std::ostringstream oss; oss << '['; @@ -653,7 +999,19 @@ std::string SqliteFaultStorage::serialize_json_array(const std::vector(c) < 0x20) { + char escape[7]; + std::snprintf(escape, sizeof(escape), "\\u%04x", static_cast(static_cast(c))); + oss << escape; + } else { + oss << c; + } break; } } @@ -699,21 +1057,22 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod const DebounceConfig & config) { const bool is_failed = (kind == fd::ReportKind::kFailed); - // Check if fault exists + // Check if the record exists. The report is addressed to (fault_code, source_id): + // another source's record of the same code is a different row and is not touched. SqliteStatement check_stmt(db_, - "SELECT severity, occurrence_count, reporting_sources, status, debounce_counter, " - "confirmed_at_ns, first_occurred_ns FROM faults WHERE fault_code = ?"); + "SELECT severity, occurrence_count, status, debounce_counter, " + "confirmed_at_ns, first_occurred_ns FROM faults WHERE fault_code = ? AND owner = ?"); check_stmt.bind_text(1, fault_code); + check_stmt.bind_text(2, source_id); if (check_stmt.step() == SQLITE_ROW) { - // Fault exists - update it + // Record exists - update it int existing_severity = check_stmt.column_int(0); int64_t existing_count = check_stmt.column_int64(1); - std::string sources_json = check_stmt.column_text(2); - std::string current_status = check_stmt.column_text(3); - int32_t debounce_counter = check_stmt.column_int(4); - int64_t confirmed_at_ns = check_stmt.column_int64(5); - int64_t first_occurred_ns = check_stmt.column_int64(6); + std::string current_status = check_stmt.column_text(2); + int32_t debounce_counter = check_stmt.column_int(3); + int64_t confirmed_at_ns = check_stmt.column_int64(4); + int64_t first_occurred_ns = check_stmt.column_int64(5); // Bring a runaway counter persisted by an older build (the bug this fixes) back into range on // first touch; this also keeps the +1/-1 below overflow-safe. The counter is local to this call. @@ -729,21 +1088,18 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod // FAILED event reactivates - reset debounce counter to 0 so FAILED branch // decrements it to -1, then reuse the existing FAILED logic below. Also // reset first_occurred: this is a new outage cycle, not a continuation - // of the one that just cleared. + // of the one that just cleared. The new occurrence has no confirmation + // time until it confirms. debounce_counter = 0; first_occurred_ns = timestamp_ns; + confirmed_at_ns = 0; is_reactivation = true; } if (is_failed) { // FAILED event - // Parse existing sources and add new one - std::vector sources = parse_json_array(sources_json); - std::set sources_set(sources.begin(), sources.end()); - sources_set.insert(source_id); - sources.assign(sources_set.begin(), sources_set.end()); - - // Escalate severity if new severity is higher + // Escalate severity if new severity is higher. Per record: a CRITICAL from one + // owner does not escalate another owner's record of the same code. int new_severity = std::max(existing_severity, static_cast(severity)); // Increment count with saturation - only on a genuine new occurrence (reactivation @@ -757,9 +1113,10 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod // Decrement towards confirmation, clamped to the thresholds. debounce_counter = clamp_debounce_counter(debounce_counter - 1, config); - // CRITICAL bypasses debounce; otherwise the shared state machine decides (with hysteresis). + // A CRITICAL record bypasses debounce and the HEALED latch. The stored severity decides, not the + // report's. Otherwise the shared state machine decides (with hysteresis). std::string new_status; - if (config.critical_immediate_confirm && severity == fd::severity::kCritical) { + if (config.critical_immediate_confirm && new_severity == fd::severity::kCritical) { debounce_counter = std::min(debounce_counter, -1); new_status = fd::fault_status::kConfirmed; } else { @@ -777,36 +1134,36 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod SqliteStatement update_stmt( db_, description.empty() ? "UPDATE faults SET severity = ?, last_occurred_ns = ?, last_failed_ns = ?, " "occurrence_count = ?, " - "reporting_sources = ?, status = ?, debounce_counter = ?, confirmed_at_ns = ?, " - "first_occurred_ns = ? WHERE fault_code = ?" + "status = ?, debounce_counter = ?, confirmed_at_ns = ?, " + "first_occurred_ns = ? WHERE fault_code = ? AND owner = ?" : "UPDATE faults SET severity = ?, description = ?, last_occurred_ns = ?, " "last_failed_ns = ?, " - "occurrence_count = ?, reporting_sources = ?, status = ?, debounce_counter = ?, " - "confirmed_at_ns = ?, first_occurred_ns = ? WHERE fault_code = ?"); + "occurrence_count = ?, status = ?, debounce_counter = ?, " + "confirmed_at_ns = ?, first_occurred_ns = ? WHERE fault_code = ? AND owner = ?"); if (description.empty()) { update_stmt.bind_int(1, new_severity); update_stmt.bind_int64(2, timestamp_ns); update_stmt.bind_int64(3, timestamp_ns); update_stmt.bind_int64(4, new_count); - update_stmt.bind_text(5, serialize_json_array(sources)); - update_stmt.bind_text(6, new_status); - update_stmt.bind_int(7, debounce_counter); - update_stmt.bind_int64(8, confirmed_at_ns); - update_stmt.bind_int64(9, first_occurred_ns); - update_stmt.bind_text(10, fault_code); + update_stmt.bind_text(5, new_status); + update_stmt.bind_int(6, debounce_counter); + update_stmt.bind_int64(7, confirmed_at_ns); + update_stmt.bind_int64(8, first_occurred_ns); + update_stmt.bind_text(9, fault_code); + update_stmt.bind_text(10, source_id); } else { update_stmt.bind_int(1, new_severity); update_stmt.bind_text(2, description); update_stmt.bind_int64(3, timestamp_ns); update_stmt.bind_int64(4, timestamp_ns); update_stmt.bind_int64(5, new_count); - update_stmt.bind_text(6, serialize_json_array(sources)); - update_stmt.bind_text(7, new_status); - update_stmt.bind_int(8, debounce_counter); - update_stmt.bind_int64(9, confirmed_at_ns); - update_stmt.bind_int64(10, first_occurred_ns); - update_stmt.bind_text(11, fault_code); + update_stmt.bind_text(6, new_status); + update_stmt.bind_int(7, debounce_counter); + update_stmt.bind_int64(8, confirmed_at_ns); + update_stmt.bind_int64(9, first_occurred_ns); + update_stmt.bind_text(10, fault_code); + update_stmt.bind_text(11, source_id); } if (update_stmt.step() != SQLITE_DONE) { @@ -826,11 +1183,13 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod // ENDING, not occurring. Bumping it made a long-stale CONFIRMED fault look // freshly active. The PASSED instant is recorded in last_passed_ns. SqliteStatement update_stmt( - db_, "UPDATE faults SET last_passed_ns = ?, status = ?, debounce_counter = ? WHERE fault_code = ?"); + db_, + "UPDATE faults SET last_passed_ns = ?, status = ?, debounce_counter = ? WHERE fault_code = ? AND owner = ?"); update_stmt.bind_int64(1, timestamp_ns); update_stmt.bind_text(2, new_status); update_stmt.bind_int(3, debounce_counter); update_stmt.bind_text(4, fault_code); + update_stmt.bind_text(5, source_id); if (update_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to update fault: ") + sqlite3_errmsg(db_)); @@ -840,9 +1199,10 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod return is_reactivation; // Reactivation treated as new occurrence for event publishing } - // New fault - only create for FAILED events + // New record - only create one for FAILED events. A source that has never reported + // this code owns no record yet, so a PASSED from it has nothing to heal. if (!is_failed) { - return false; // PASSED event for non-existent fault is ignored + return false; // PASSED event for non-existent record is ignored } // Determine initial status based on debounce logic (shared with the in-memory backend). @@ -854,12 +1214,12 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod initial_status = compute_debounce_status(initial_counter, "", config); } - // New fault - insert with debounce_counter = -1 + // New record - insert with debounce_counter = -1 SqliteStatement insert_stmt(db_, "INSERT INTO faults (fault_code, severity, description, first_occurred_ns, " "last_occurred_ns, occurrence_count, status, reporting_sources, " - "debounce_counter, last_failed_ns, last_passed_ns, confirmed_at_ns) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + "debounce_counter, last_failed_ns, last_passed_ns, confirmed_at_ns, owner) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); const bool confirmed_now = initial_status == fd::fault_status::kConfirmed; insert_stmt.bind_text(1, fault_code); @@ -874,6 +1234,7 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod insert_stmt.bind_int64(10, timestamp_ns); // last_failed_ns insert_stmt.bind_int64(11, 0); // last_passed_ns (never passed) insert_stmt.bind_int64(12, confirmed_now ? timestamp_ns : 0); + insert_stmt.bind_text(13, source_id); if (insert_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to insert fault: ") + sqlite3_errmsg(db_)); @@ -909,7 +1270,7 @@ std::vector SqliteFaultStorage::list_faults(bool filter_by_seve // Build query std::string sql = "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, last_passed_ns FROM faults WHERE status IN ("; + "occurrence_count, status, owner, last_passed_ns FROM faults WHERE status IN ("; for (size_t i = 0; i < status_filter.size(); ++i) { if (i > 0) { sql += ", "; @@ -946,7 +1307,9 @@ std::vector SqliteFaultStorage::list_faults(bool filter_by_seve fault.occurrence_count = static_cast(stmt.column_int64(5)); fault.status = stmt.column_text(6); - fault.reporting_sources = parse_json_array(stmt.column_text(7)); + // The owner, as a one-element list: the column is the record identity, and the + // services that act on a single record take that value back as their source_id. + fault.reporting_sources.assign(1, stmt.column_text(7)); fault.last_passed_ns = stmt.column_int64(8); result.push_back(fault); @@ -955,58 +1318,85 @@ std::vector SqliteFaultStorage::list_faults(bool filter_by_seve return result; } -std::optional SqliteFaultStorage::get_fault(const std::string & fault_code) const { +namespace { + +/// Shared projection so every fault read decodes the same column order. +fd::FaultRecord read_fault_row(SqliteStatement & stmt) { + fd::FaultRecord fault; + fault.fault_code = stmt.column_text(0); + fault.severity = static_cast(stmt.column_int(1)); + fault.description = stmt.column_text(2); + fault.first_occurred_ns = stmt.column_int64(3); + fault.last_occurred_ns = stmt.column_int64(4); + fault.occurrence_count = static_cast(stmt.column_int64(5)); + fault.status = stmt.column_text(6); + fault.reporting_sources.assign(1, stmt.column_text(7)); + fault.last_passed_ns = stmt.column_int64(8); + return fault; +} + +constexpr const char * kFaultColumns = + "fault_code, severity, description, first_occurred_ns, last_occurred_ns, " + "occurrence_count, status, owner, last_passed_ns"; + +} // namespace + +std::optional SqliteFaultStorage::get_fault(const FaultId & id) const { std::lock_guard lock(mutex_); - SqliteStatement stmt(db_, - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, last_passed_ns FROM faults WHERE fault_code = ?"); - stmt.bind_text(1, fault_code); + SqliteStatement stmt( + db_, (std::string("SELECT ") + kFaultColumns + " FROM faults WHERE fault_code = ? AND owner = ?").c_str()); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); if (stmt.step() != SQLITE_ROW) { return std::nullopt; } - fd::FaultRecord fault; - fault.fault_code = stmt.column_text(0); - fault.severity = static_cast(stmt.column_int(1)); - fault.description = stmt.column_text(2); + return read_fault_row(stmt); +} - int64_t first_ns = stmt.column_int64(3); - int64_t last_ns = stmt.column_int64(4); - fault.first_occurred_ns = first_ns; - fault.last_occurred_ns = last_ns; +std::vector SqliteFaultStorage::get_faults_by_code(const std::string & fault_code) const { + std::lock_guard lock(mutex_); - fault.occurrence_count = static_cast(stmt.column_int64(5)); - fault.status = stmt.column_text(6); - fault.reporting_sources = parse_json_array(stmt.column_text(7)); - fault.last_passed_ns = stmt.column_int64(8); + // Ordered by owner so an ambiguity message lists the owners the same way every time, + // and so the two backends answer in the same order. + SqliteStatement stmt( + db_, (std::string("SELECT ") + kFaultColumns + " FROM faults WHERE fault_code = ? ORDER BY owner ASC").c_str()); + stmt.bind_text(1, fault_code); - return fault; + std::vector result; + while (stmt.step() == SQLITE_ROW) { + result.push_back(read_fault_row(stmt)); + } + return result; } -bool SqliteFaultStorage::clear_fault(const std::string & fault_code) { +bool SqliteFaultStorage::clear_fault(const FaultId & id) { std::lock_guard lock(mutex_); - // The near_misses rows for this code are deliberately left alone. Clearing acknowledges one + // The near_misses rows of this record are deliberately left alone. Clearing acknowledges one // fault cycle; the record of how often the code approached confirmation spans cycles and // cannot be reconstructed once deleted. - // Delete associated snapshots when fault is cleared - // Acknowledging a fault drops its value snapshots, unless a history was asked + // Delete the record's snapshots when it is cleared. + // Acknowledging a record drops its value snapshots, unless a history was asked // for: with recordings retained past a clear, deleting the readings that go with - // them leaves a fault holding bags whose matching values are gone. + // them leaves a record holding bags whose matching values are gone. Scoped to the + // owner, so another owner's readings for the same code stay where they are. if (!retain_snapshots_on_clear_) { - SqliteStatement delete_snapshots(db_, "DELETE FROM snapshots WHERE fault_code = ?"); - delete_snapshots.bind_text(1, fault_code); + SqliteStatement delete_snapshots(db_, "DELETE FROM snapshots WHERE fault_code = ? AND owner = ?"); + delete_snapshots.bind_text(1, id.fault_code); + delete_snapshots.bind_text(2, id.owner); if (delete_snapshots.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to delete snapshots: ") + sqlite3_errmsg(db_)); } } - SqliteStatement stmt(db_, "UPDATE faults SET status = ? WHERE fault_code = ?"); + SqliteStatement stmt(db_, "UPDATE faults SET status = ? WHERE fault_code = ? AND owner = ?"); stmt.bind_text(1, fd::fault_status::kCleared); - stmt.bind_text(2, fault_code); + stmt.bind_text(2, id.fault_code); + stmt.bind_text(3, id.owner); if (stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to clear fault: ") + sqlite3_errmsg(db_)); @@ -1015,20 +1405,22 @@ bool SqliteFaultStorage::clear_fault(const std::string & fault_code) { return sqlite3_changes(db_) > 0; } -std::vector SqliteFaultStorage::reclassify_healed_as_cleared(const KeepHealedFn & keep_healed) { +std::vector SqliteFaultStorage::reclassify_healed_as_cleared(const KeepHealedFn & keep_healed) { std::lock_guard lock(mutex_); - // Collect the codes that will flip first so the caller can audit each one. Codes the + // Collect the records that will flip first so the caller can audit each one. Codes the // caller keeps HEALED are left out. Everything runs under the same lock, so the - // returned list matches the rows actually reclassified below. - std::vector reclassified; + // returned list matches the rows actually reclassified below. Identities, not codes: + // one code can hold a HEALED record for one owner and an untouched CONFIRMED one for + // another. + std::vector reclassified; { - SqliteStatement select_stmt(db_, "SELECT fault_code FROM faults WHERE status = ?"); + SqliteStatement select_stmt(db_, "SELECT fault_code, owner FROM faults WHERE status = ?"); select_stmt.bind_text(1, fd::fault_status::kHealed); while (select_stmt.step() == SQLITE_ROW) { - std::string code = select_stmt.column_text(0); - if (!(keep_healed && keep_healed(code))) { - reclassified.push_back(std::move(code)); + FaultId id{select_stmt.column_text(0), select_stmt.column_text(1)}; + if (!(keep_healed && keep_healed(id.fault_code))) { + reclassified.push_back(std::move(id)); } } } @@ -1037,26 +1429,28 @@ std::vector SqliteFaultStorage::reclassify_healed_as_cleared(const return reclassified; } - // One transaction, so a failure part way does not leave some codes flipped. + // One transaction, so a failure part way does not leave some records flipped. exec_or_throw("BEGIN IMMEDIATE"); try { - for (const auto & code : reclassified) { + for (const auto & id : reclassified) { // Drop snapshots for the affected faults so a reclassified row matches CLEARED semantics. // clear_fault is not the only place that takes a fault's readings, so retain_snapshots_on_clear_ // has to reach here too: otherwise the setting holds until the next restart and then the // reclassification deletes exactly what it was set to keep. if (!retain_snapshots_on_clear_) { - SqliteStatement del(db_, "DELETE FROM snapshots WHERE fault_code = ?"); - del.bind_text(1, code); + SqliteStatement del(db_, "DELETE FROM snapshots WHERE fault_code = ? AND owner = ?"); + del.bind_text(1, id.fault_code); + del.bind_text(2, id.owner); if (del.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to delete snapshots: ") + sqlite3_errmsg(db_)); } } - SqliteStatement stmt(db_, "UPDATE faults SET status = ? WHERE fault_code = ? AND status = ?"); + SqliteStatement stmt(db_, "UPDATE faults SET status = ? WHERE fault_code = ? AND owner = ? AND status = ?"); stmt.bind_text(1, fd::fault_status::kCleared); - stmt.bind_text(2, code); - stmt.bind_text(3, fd::fault_status::kHealed); + stmt.bind_text(2, id.fault_code); + stmt.bind_text(3, id.owner); + stmt.bind_text(4, fd::fault_status::kHealed); if (stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to reclassify HEALED faults: ") + sqlite3_errmsg(db_)); } @@ -1081,35 +1475,37 @@ size_t SqliteFaultStorage::size() const { return static_cast(stmt.column_int64(0)); } -bool SqliteFaultStorage::contains(const std::string & fault_code) const { +bool SqliteFaultStorage::contains(const FaultId & id) const { std::lock_guard lock(mutex_); - SqliteStatement stmt(db_, "SELECT 1 FROM faults WHERE fault_code = ? LIMIT 1"); - stmt.bind_text(1, fault_code); + SqliteStatement stmt(db_, "SELECT 1 FROM faults WHERE fault_code = ? AND owner = ? LIMIT 1"); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); return stmt.step() == SQLITE_ROW; } -std::vector SqliteFaultStorage::check_time_based_confirmation(int64_t now_ns) { +std::vector SqliteFaultStorage::check_time_based_confirmation(int64_t now_ns) { std::lock_guard lock(mutex_); - std::vector confirmed; + std::vector confirmed; if (config_.auto_confirm_after_sec <= 0.0) { return confirmed; // Time-based confirmation disabled } + int64_t threshold_ns = std::llround(config_.auto_confirm_after_sec * 1e9); int64_t cutoff_ns = now_ns - threshold_ns; - // Collect the codes that will flip first so the caller can audit each one. The + // Collect the records that will flip first so the caller can audit each one. The // SELECT predicate mirrors the UPDATE exactly, and both run under the same lock, // so the returned list matches the rows actually confirmed below. { SqliteStatement select_stmt( - db_, "SELECT fault_code FROM faults WHERE status = ? AND last_failed_ns <= ? AND last_failed_ns > 0"); + db_, "SELECT fault_code, owner FROM faults WHERE status = ? AND last_failed_ns <= ? AND last_failed_ns > 0"); select_stmt.bind_text(1, fd::fault_status::kPrefailed); select_stmt.bind_int64(2, cutoff_ns); while (select_stmt.step() == SQLITE_ROW) { - confirmed.push_back(select_stmt.column_text(0)); + confirmed.push_back(FaultId{select_stmt.column_text(0), select_stmt.column_text(1)}); } } @@ -1157,24 +1553,26 @@ void SqliteFaultStorage::store_snapshots(const std::vector & snaps } std::lock_guard lock(mutex_); - const std::string & fault_code = snapshots.front().fault_code; + const FaultId id{snapshots.front().fault_code, snapshots.front().owner}; // One transaction for the whole capture: a capture is all-or-nothing, and the // old row-at-a-time path could leave a confirmation's values half stored. exec_or_throw("BEGIN IMMEDIATE"); try { { - SqliteStatement stmt(db_, - "INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns, capture_id) " - "VALUES (?, ?, ?, ?, ?, ?)"); + SqliteStatement stmt( + db_, + "INSERT INTO snapshots (fault_code, owner, topic, message_type, data, captured_at_ns, capture_id) " + "VALUES (?, ?, ?, ?, ?, ?, ?)"); for (const auto & snapshot : snapshots) { stmt.reset(); stmt.bind_text(1, snapshot.fault_code); - stmt.bind_text(2, snapshot.topic); - stmt.bind_text(3, snapshot.message_type); - stmt.bind_text(4, snapshot.data); - stmt.bind_int64(5, snapshot.captured_at_ns); - stmt.bind_int64(6, snapshot.capture_id); + stmt.bind_text(2, snapshot.owner); + stmt.bind_text(3, snapshot.topic); + stmt.bind_text(4, snapshot.message_type); + stmt.bind_text(5, snapshot.data); + stmt.bind_int64(6, snapshot.captured_at_ns); + stmt.bind_int64(7, snapshot.capture_id); if (stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to store snapshot: ") + sqlite3_errmsg(db_)); } @@ -1182,36 +1580,39 @@ void SqliteFaultStorage::store_snapshots(const std::vector & snaps } if (max_snapshots_per_fault_ > 0) { - // Trim whole capture sets, oldest first, until the fault fits. The old rule + // Trim whole capture sets, oldest first, until the record fits. The old rule // counted rows and rejected the NEW row once full, so a capture straddling // the cap was stored in part - some topics present, the rest silently gone, // indistinguishable from "that topic was not publishing". Keep-newest also // stops this cap from opposing the rosbag one. // // The newest capture is never trimmed: if it alone exceeds the cap, the cap - // is smaller than this fault's topic count and tearing it would be the very + // is smaller than this record's topic count and tearing it would be the very // thing being fixed. - SqliteStatement newest(db_, "SELECT MAX(capture_id) FROM snapshots WHERE fault_code = ?"); - newest.bind_text(1, fault_code); + SqliteStatement newest(db_, "SELECT MAX(capture_id) FROM snapshots WHERE fault_code = ? AND owner = ?"); + newest.bind_text(1, id.fault_code); + newest.bind_text(2, id.owner); int64_t newest_capture = 0; if (newest.step() == SQLITE_ROW) { newest_capture = newest.column_int64(0); } SqliteStatement trim(db_, - "DELETE FROM snapshots WHERE fault_code = ?1 AND capture_id = " - "(SELECT MIN(capture_id) FROM snapshots WHERE fault_code = ?1) " - "AND capture_id <> ?2"); - SqliteStatement count(db_, "SELECT COUNT(*) FROM snapshots WHERE fault_code = ?"); + "DELETE FROM snapshots WHERE fault_code = ?1 AND owner = ?2 AND capture_id = " + "(SELECT MIN(capture_id) FROM snapshots WHERE fault_code = ?1 AND owner = ?2) " + "AND capture_id <> ?3"); + SqliteStatement count(db_, "SELECT COUNT(*) FROM snapshots WHERE fault_code = ? AND owner = ?"); while (true) { count.reset(); - count.bind_text(1, fault_code); + count.bind_text(1, id.fault_code); + count.bind_text(2, id.owner); if (count.step() != SQLITE_ROW || static_cast(count.column_int64(0)) <= max_snapshots_per_fault_) { break; } trim.reset(); - trim.bind_text(1, fault_code); - trim.bind_int64(2, newest_capture); + trim.bind_text(1, id.fault_code); + trim.bind_text(2, id.owner); + trim.bind_int64(3, newest_capture); if (trim.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to trim snapshots: ") + sqlite3_errmsg(db_)); } @@ -1228,15 +1629,15 @@ void SqliteFaultStorage::store_snapshots(const std::vector & snaps } } -std::vector SqliteFaultStorage::get_snapshots(const std::string & fault_code, +std::vector SqliteFaultStorage::get_snapshots(const FaultId & id, const std::string & topic_filter) const { std::lock_guard lock(mutex_); std::vector result; std::string sql = - "SELECT fault_code, topic, message_type, data, captured_at_ns, capture_id FROM snapshots WHERE fault_code " - "= ?"; + "SELECT fault_code, owner, topic, message_type, data, captured_at_ns, capture_id FROM snapshots " + "WHERE fault_code = ? AND owner = ?"; if (!topic_filter.empty()) { sql += " AND topic = ?"; } @@ -1246,19 +1647,21 @@ std::vector SqliteFaultStorage::get_snapshots(const std::string & sql += " ORDER BY capture_id DESC, captured_at_ns DESC"; SqliteStatement stmt(db_, sql.c_str()); - stmt.bind_text(1, fault_code); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); if (!topic_filter.empty()) { - stmt.bind_text(2, topic_filter); + stmt.bind_text(3, topic_filter); } while (stmt.step() == SQLITE_ROW) { SnapshotData snapshot; snapshot.fault_code = stmt.column_text(0); - snapshot.topic = stmt.column_text(1); - snapshot.message_type = stmt.column_text(2); - snapshot.data = stmt.column_text(3); - snapshot.captured_at_ns = stmt.column_int64(4); - snapshot.capture_id = stmt.column_int64(5); + snapshot.owner = stmt.column_text(1); + snapshot.topic = stmt.column_text(2); + snapshot.message_type = stmt.column_text(3); + snapshot.data = stmt.column_text(4); + snapshot.captured_at_ns = stmt.column_int64(5); + snapshot.capture_id = stmt.column_int64(6); result.push_back(snapshot); } @@ -1281,24 +1684,29 @@ int64_t SqliteFaultStorage::get_max_capture_id() const { void SqliteFaultStorage::store_freeze_frame(const FreezeFrameData & frame) { std::lock_guard lock(mutex_); - // Keyed by fault_code (PRIMARY KEY): a re-confirm replaces the previous frame. + // Keyed by (fault_code, owner) through idx_freeze_frames_code_owner: a re-confirm of + // the same record replaces its frame, and another owner's frame for the same code is + // a different row. SqliteStatement stmt(db_, - "INSERT OR REPLACE INTO freeze_frames (fault_code, data, captured_at_ns) " - "VALUES (?, ?, ?)"); + "INSERT OR REPLACE INTO freeze_frames (fault_code, owner, data, captured_at_ns) " + "VALUES (?, ?, ?, ?)"); stmt.bind_text(1, frame.fault_code); - stmt.bind_text(2, frame.data); - stmt.bind_int64(3, frame.captured_at_ns); + stmt.bind_text(2, frame.owner); + stmt.bind_text(3, frame.data); + stmt.bind_int64(4, frame.captured_at_ns); if (stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to store freeze frame: ") + sqlite3_errmsg(db_)); } } -std::optional SqliteFaultStorage::get_freeze_frame(const std::string & fault_code) const { +std::optional SqliteFaultStorage::get_freeze_frame(const FaultId & id) const { std::lock_guard lock(mutex_); - SqliteStatement stmt(db_, "SELECT fault_code, data, captured_at_ns FROM freeze_frames WHERE fault_code = ?"); - stmt.bind_text(1, fault_code); + SqliteStatement stmt( + db_, "SELECT fault_code, owner, data, captured_at_ns FROM freeze_frames WHERE fault_code = ? AND owner = ?"); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); if (stmt.step() != SQLITE_ROW) { return std::nullopt; @@ -1306,8 +1714,9 @@ std::optional SqliteFaultStorage::get_freeze_frame(const std::s FreezeFrameData frame; frame.fault_code = stmt.column_text(0); - frame.data = stmt.column_text(1); - frame.captured_at_ns = stmt.column_int64(2); + frame.owner = stmt.column_text(1); + frame.data = stmt.column_text(2); + frame.captured_at_ns = stmt.column_int64(3); return frame; } @@ -1323,12 +1732,12 @@ size_t SqliteFaultStorage::set_max_near_misses_per_fault(size_t max_count) { } // Apply the bound to what is already in the database. Without this, a database that grew under - // a larger bound (or none) stays over the new bound until each fault code happens to record - // another near miss - and a code that never does keeps its rows for good. + // a larger bound (or none) stays over the new bound until each record happens to log + // another near miss - and a record that never does keeps its rows for good. SqliteStatement trim_stmt(db_, "DELETE FROM near_misses WHERE id IN (" "SELECT id FROM (SELECT id, ROW_NUMBER() OVER " - "(PARTITION BY fault_code ORDER BY id DESC) AS rn FROM near_misses) " + "(PARTITION BY fault_code, source_id ORDER BY id DESC) AS rn FROM near_misses) " "WHERE rn > ?)"); trim_stmt.bind_int64(1, static_cast(max_count)); @@ -1376,27 +1785,31 @@ void SqliteFaultStorage::record_near_miss_locked(const std::string & fault_code, // by timestamp would then drop the row that was just appended and make the two backends, which // append in arrival order, disagree on the same input. SqliteStatement trim_stmt(db_, - "DELETE FROM near_misses WHERE fault_code = ?1 AND id NOT IN " - "(SELECT id FROM near_misses WHERE fault_code = ?1 " - "ORDER BY id DESC LIMIT ?2)"); + "DELETE FROM near_misses WHERE fault_code = ?1 AND source_id = ?2 AND id NOT IN " + "(SELECT id FROM near_misses WHERE fault_code = ?1 AND source_id = ?2 " + "ORDER BY id DESC LIMIT ?3)"); trim_stmt.bind_text(1, fault_code); - trim_stmt.bind_int64(2, static_cast(max_near_misses_per_fault_)); + trim_stmt.bind_text(2, source_id); + trim_stmt.bind_int64(3, static_cast(max_near_misses_per_fault_)); if (trim_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to trim near-miss series: ") + sqlite3_errmsg(db_)); } } -std::vector SqliteFaultStorage::get_near_misses(const std::string & fault_code) const { +std::vector SqliteFaultStorage::get_near_misses(const FaultId & id) const { std::lock_guard lock(mutex_); std::vector result; + // source_id already IS the owner on this table, so the series only needs the extra + // predicate, not a new column. SqliteStatement stmt(db_, "SELECT fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, " - "severity, source_id, resulting_status FROM near_misses WHERE fault_code = ? " - "ORDER BY id ASC"); - stmt.bind_text(1, fault_code); + "severity, source_id, resulting_status FROM near_misses " + "WHERE fault_code = ? AND source_id = ? ORDER BY id ASC"); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); while (stmt.step() == SQLITE_ROW) { NearMissRecord record; @@ -1441,7 +1854,7 @@ void SqliteFaultStorage::store_rosbag_files(const std::vector & std::lock_guard lock(mutex_); // One transaction for the whole burst: a crash mid-store must not leave some - // faults of the shared recording without their lookup row. Evicted bags are + // records of the shared recording without their lookup row. Evicted bags are // unlinked only after COMMIT - a ROLLBACK resurrects the rows, which must keep // pointing at bags that still exist. std::vector evicted; @@ -1458,7 +1871,7 @@ void SqliteFaultStorage::store_rosbag_files(const std::vector & } // Referencing is re-checked on the committed state, not on the state each - // eviction saw: two faults of one burst can link the same bag, and a row-by-row + // eviction saw: two records of one burst can link the same bag, and a row-by-row // check inside the loop would find it still held by a sibling that a later // iteration then evicts, leaking the directory. std::set unique_paths(evicted.begin(), evicted.end()); @@ -1477,7 +1890,7 @@ std::vector SqliteFaultStorage::store_rosbag_file_locked(const Rosb row.recording_id = rosbag_recording_id(row.file_path); } - // Upserts on idx_rosbag_files_fault_path, i.e. on the (fault, recording) link. + // Upserts on idx_rosbag_files_fault_path, i.e. on the (record, recording) link. // Re-storing the SAME link refreshes it; a link to a DIFFERENT recording is a new // row now, which is the feature. Nothing is unlinked here - byte lifetime is the // cap's business below, and the caller's, after the commit. @@ -1489,46 +1902,48 @@ std::vector SqliteFaultStorage::store_rosbag_file_locked(const Rosb // two backends in a different order for a re-stored row inside a tie group. SqliteStatement stmt(db_, "INSERT INTO rosbag_files " - "(fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns) " - "VALUES (?, ?, ?, ?, ?, ?, ?) " - "ON CONFLICT(fault_code, file_path) DO UPDATE SET " + "(fault_code, owner, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT(fault_code, owner, file_path) DO UPDATE SET " "recording_id = excluded.recording_id, format = excluded.format, " "duration_sec = excluded.duration_sec, size_bytes = excluded.size_bytes, " "created_at_ns = excluded.created_at_ns"); stmt.bind_text(1, row.fault_code); - stmt.bind_text(2, row.recording_id); - stmt.bind_text(3, row.file_path); - stmt.bind_text(4, row.format); + stmt.bind_text(2, row.owner); + stmt.bind_text(3, row.recording_id); + stmt.bind_text(4, row.file_path); + stmt.bind_text(5, row.format); // Bind duration_sec as a double using sqlite3_bind_double directly - if (sqlite3_bind_double(stmt.get(), 5, row.duration_sec) != SQLITE_OK) { + if (sqlite3_bind_double(stmt.get(), 6, row.duration_sec) != SQLITE_OK) { throw std::runtime_error(std::string("Failed to bind duration_sec: ") + sqlite3_errmsg(db_)); } - stmt.bind_int64(6, static_cast(row.size_bytes)); - stmt.bind_int64(7, row.created_at_ns); + stmt.bind_int64(7, static_cast(row.size_bytes)); + stmt.bind_int64(8, row.created_at_ns); if (stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to store rosbag file: ") + sqlite3_errmsg(db_)); } if (max_rosbags_per_fault_ == 0) { - return {}; // unlimited per fault; only the global byte quota bounds this + return {}; // unlimited per record, only the global byte quota bounds this } - // Keep the newest N recordings of this fault. Oldest-first eviction, the same + // Keep the newest N recordings of this record. Oldest-first eviction, the same // direction as evict_bags_over_quota, so the two eviction owners never need a // tiebreak. At N = 1 this reproduces the pre-#620 behaviour exactly: the new // recording replaces the old and the old bag is unlinked. const char * const doomed_sql = - "FROM rosbag_files WHERE fault_code = ?1 AND id NOT IN " - "(SELECT id FROM rosbag_files WHERE fault_code = ?1 " - " ORDER BY created_at_ns DESC, id DESC LIMIT ?2)"; + "FROM rosbag_files WHERE fault_code = ?1 AND owner = ?2 AND id NOT IN " + "(SELECT id FROM rosbag_files WHERE fault_code = ?1 AND owner = ?2 " + " ORDER BY created_at_ns DESC, id DESC LIMIT ?3)"; std::vector evicted; { SqliteStatement select(db_, (std::string("SELECT DISTINCT file_path ") + doomed_sql).c_str()); select.bind_text(1, row.fault_code); - select.bind_int64(2, static_cast(max_rosbags_per_fault_)); + select.bind_text(2, row.owner); + select.bind_int64(3, static_cast(max_rosbags_per_fault_)); while (select.step() == SQLITE_ROW) { evicted.push_back(select.column_text(0)); } @@ -1539,7 +1954,8 @@ std::vector SqliteFaultStorage::store_rosbag_file_locked(const Rosb SqliteStatement del(db_, (std::string("DELETE ") + doomed_sql).c_str()); del.bind_text(1, row.fault_code); - del.bind_int64(2, static_cast(max_rosbags_per_fault_)); + del.bind_text(2, row.owner); + del.bind_int64(3, static_cast(max_rosbags_per_fault_)); if (del.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to trim rosbag rows: ") + sqlite3_errmsg(db_)); } @@ -1552,27 +1968,30 @@ namespace { RosbagFileInfo read_rosbag_row(SqliteStatement & stmt) { RosbagFileInfo info; info.fault_code = stmt.column_text(0); - info.recording_id = stmt.column_text(1); - info.file_path = stmt.column_text(2); - info.format = stmt.column_text(3); - info.duration_sec = sqlite3_column_double(stmt.get(), 4); - info.size_bytes = static_cast(stmt.column_int64(5)); - info.created_at_ns = stmt.column_int64(6); + info.owner = stmt.column_text(1); + info.recording_id = stmt.column_text(2); + info.file_path = stmt.column_text(3); + info.format = stmt.column_text(4); + info.duration_sec = sqlite3_column_double(stmt.get(), 5); + info.size_bytes = static_cast(stmt.column_int64(6)); + info.created_at_ns = stmt.column_int64(7); return info; } constexpr const char * kRosbagColumns = - "fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns"; + "fault_code, owner, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns"; } // namespace -std::vector SqliteFaultStorage::get_rosbag_files(const std::string & fault_code) const { +std::vector SqliteFaultStorage::get_rosbag_files(const FaultId & id) const { std::lock_guard lock(mutex_); - SqliteStatement stmt(db_, (std::string("SELECT ") + kRosbagColumns + - " FROM rosbag_files WHERE fault_code = ? ORDER BY created_at_ns DESC, id DESC") - .c_str()); - stmt.bind_text(1, fault_code); + SqliteStatement stmt(db_, + (std::string("SELECT ") + kRosbagColumns + + " FROM rosbag_files WHERE fault_code = ? AND owner = ? ORDER BY created_at_ns DESC, id DESC") + .c_str()); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); std::vector result; while (stmt.step() == SQLITE_ROW) { @@ -1585,7 +2004,7 @@ std::vector SqliteFaultStorage::get_rosbag_files_by_recording(co std::lock_guard lock(mutex_); SqliteStatement stmt(db_, (std::string("SELECT ") + kRosbagColumns + - " FROM rosbag_files WHERE recording_id = ? ORDER BY fault_code ASC") + " FROM rosbag_files WHERE recording_id = ? ORDER BY fault_code ASC, owner ASC") .c_str()); stmt.bind_text(1, recording_id); @@ -1644,46 +2063,39 @@ size_t SqliteFaultStorage::delete_rosbag_recording(const std::string & recording return removed; } -std::optional SqliteFaultStorage::get_rosbag_file(const std::string & fault_code) const { +std::optional SqliteFaultStorage::get_rosbag_file(const FaultId & id) const { std::lock_guard lock(mutex_); - // ORDER BY is load-bearing now that a fault can hold several recordings. Without + // ORDER BY is load-bearing now that a record can hold several recordings. Without // it SQLite may return any matching row, so the fault detail and the download // would serve an arbitrary recording - non-deterministically, which no test // catches reliably. id breaks the tie because a burst stamps one created_at_ns // across all its rows. - SqliteStatement stmt(db_, - "SELECT fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns " - "FROM rosbag_files WHERE fault_code = ? " - "ORDER BY created_at_ns DESC, id DESC LIMIT 1"); - stmt.bind_text(1, fault_code); + SqliteStatement stmt(db_, (std::string("SELECT ") + kRosbagColumns + + " FROM rosbag_files WHERE fault_code = ? AND owner = ? " + "ORDER BY created_at_ns DESC, id DESC LIMIT 1") + .c_str()); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); if (stmt.step() != SQLITE_ROW) { return std::nullopt; } - RosbagFileInfo info; - info.fault_code = stmt.column_text(0); - info.recording_id = stmt.column_text(1); - info.file_path = stmt.column_text(2); - info.format = stmt.column_text(3); - info.duration_sec = sqlite3_column_double(stmt.get(), 4); - info.size_bytes = static_cast(stmt.column_int64(5)); - info.created_at_ns = stmt.column_int64(6); - - return info; + return read_rosbag_row(stmt); } -bool SqliteFaultStorage::delete_rosbag_file(const std::string & fault_code) { +bool SqliteFaultStorage::delete_rosbag_file(const FaultId & id) { std::lock_guard lock(mutex_); - // Every path, not the first one: a fault holds as many recordings as its cap + // Every path, not the first one: a record holds as many recordings as its cap // allows, and stepping once would unlink one bag and leak the rest - rows gone, // directories left behind, uncounted by a quota that sums rows. std::set paths; { - SqliteStatement select_stmt(db_, "SELECT file_path FROM rosbag_files WHERE fault_code = ?"); - select_stmt.bind_text(1, fault_code); + SqliteStatement select_stmt(db_, "SELECT file_path FROM rosbag_files WHERE fault_code = ? AND owner = ?"); + select_stmt.bind_text(1, id.fault_code); + select_stmt.bind_text(2, id.owner); while (select_stmt.step() == SQLITE_ROW) { paths.insert(select_stmt.column_text(0)); } @@ -1694,8 +2106,9 @@ bool SqliteFaultStorage::delete_rosbag_file(const std::string & fault_code) { // done, that would leave a surviving row pointing at a bag that is gone - // unreadable for good, and still charged against the storage quota, which sums // rows. This way the worst case is an orphaned directory instead. - SqliteStatement delete_stmt(db_, "DELETE FROM rosbag_files WHERE fault_code = ?"); - delete_stmt.bind_text(1, fault_code); + SqliteStatement delete_stmt(db_, "DELETE FROM rosbag_files WHERE fault_code = ? AND owner = ?"); + delete_stmt.bind_text(1, id.fault_code); + delete_stmt.bind_text(2, id.owner); if (delete_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to delete rosbag file record: ") + sqlite3_errmsg(db_)); @@ -1703,7 +2116,7 @@ bool SqliteFaultStorage::delete_rosbag_file(const std::string & fault_code) { const bool deleted = sqlite3_changes(db_) > 0; - // Unlink only once no fault references the bag any more. These rows are already + // Unlink only once no row references the bag any more. These rows are already // gone, so path_referenced() sees exactly the siblings of a shared recording. for (const auto & path : paths) { if (!path_referenced(path)) { @@ -1716,8 +2129,8 @@ bool SqliteFaultStorage::delete_rosbag_file(const std::string & fault_code) { return deleted; } -size_t SqliteFaultStorage::delete_rosbag_files(const std::vector & fault_codes) { - if (fault_codes.empty()) { +size_t SqliteFaultStorage::delete_rosbag_files(const std::vector & ids) { + if (ids.empty()) { return 0; } std::lock_guard lock(mutex_); @@ -1729,18 +2142,20 @@ size_t SqliteFaultStorage::delete_rosbag_files(const std::vector & size_t deleted = 0; exec_or_throw("BEGIN IMMEDIATE"); try { - for (const auto & code : fault_codes) { + for (const auto & id : ids) { { - // while, not if: one fault code can name several recordings now, and the + // while, not if: one record can name several recordings now, and the // sweep must be able to reclaim every one of their bags. - SqliteStatement select_stmt(db_, "SELECT file_path FROM rosbag_files WHERE fault_code = ?"); - select_stmt.bind_text(1, code); + SqliteStatement select_stmt(db_, "SELECT file_path FROM rosbag_files WHERE fault_code = ? AND owner = ?"); + select_stmt.bind_text(1, id.fault_code); + select_stmt.bind_text(2, id.owner); while (select_stmt.step() == SQLITE_ROW) { paths.insert(select_stmt.column_text(0)); } } - SqliteStatement delete_stmt(db_, "DELETE FROM rosbag_files WHERE fault_code = ?"); - delete_stmt.bind_text(1, code); + SqliteStatement delete_stmt(db_, "DELETE FROM rosbag_files WHERE fault_code = ? AND owner = ?"); + delete_stmt.bind_text(1, id.fault_code); + delete_stmt.bind_text(2, id.owner); if (delete_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to delete rosbag file record: ") + sqlite3_errmsg(db_)); } @@ -1773,7 +2188,7 @@ bool SqliteFaultStorage::path_referenced(const std::string & file_path) const { size_t SqliteFaultStorage::get_total_rosbag_storage_bytes() const { std::lock_guard lock(mutex_); - // Sum per bag, not per fault: one recording can back a burst of correlated + // Sum per bag, not per record: one recording can back a burst of correlated // faults, and double-counting it would evict bags that still fit the quota. SqliteStatement stmt( db_, @@ -1792,20 +2207,11 @@ std::vector SqliteFaultStorage::get_all_rosbag_files() const { std::vector result; - SqliteStatement stmt(db_, - "SELECT fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns " - "FROM rosbag_files ORDER BY created_at_ns ASC, id ASC"); + SqliteStatement stmt( + db_, (std::string("SELECT ") + kRosbagColumns + " FROM rosbag_files ORDER BY created_at_ns ASC, id ASC").c_str()); while (stmt.step() == SQLITE_ROW) { - RosbagFileInfo info; - info.fault_code = stmt.column_text(0); - info.recording_id = stmt.column_text(1); - info.file_path = stmt.column_text(2); - info.format = stmt.column_text(3); - info.duration_sec = sqlite3_column_double(stmt.get(), 4); - info.size_bytes = static_cast(stmt.column_int64(5)); - info.created_at_ns = stmt.column_int64(6); - result.push_back(info); + result.push_back(read_rosbag_row(stmt)); } return result; @@ -1816,29 +2222,21 @@ std::vector SqliteFaultStorage::list_rosbags_for_entity(const st std::vector result; - // Join rosbag_files with faults table and filter by reporting_sources containing entity_fqn. - // Use json_each() for proper JSON array querying instead of LIKE, which treats - // '_' as a single-char wildcard and would produce false positives on ROS names. + // The row carries its record's owner, so the entity match is an equality on the row + // and the json_each() scan over reporting_sources is gone. The join to faults stays + // as the existence check it always was: a row whose record is gone is not served. SqliteStatement stmt(db_, - "SELECT r.fault_code, r.recording_id, r.file_path, r.format, r.duration_sec, r.size_bytes, " - "r.created_at_ns " + "SELECT r.fault_code, r.owner, r.recording_id, r.file_path, r.format, r.duration_sec, " + "r.size_bytes, r.created_at_ns " "FROM rosbag_files r " - "JOIN faults f ON r.fault_code = f.fault_code " - "JOIN json_each(f.reporting_sources) j ON j.value = ? " + "JOIN faults f ON r.fault_code = f.fault_code AND r.owner = f.owner " + "WHERE r.owner = ? " "ORDER BY r.created_at_ns DESC, r.id DESC"); stmt.bind_text(1, entity_fqn); while (stmt.step() == SQLITE_ROW) { - RosbagFileInfo info; - info.fault_code = stmt.column_text(0); - info.recording_id = stmt.column_text(1); - info.file_path = stmt.column_text(2); - info.format = stmt.column_text(3); - info.duration_sec = sqlite3_column_double(stmt.get(), 4); - info.size_bytes = static_cast(stmt.column_int64(5)); - info.created_at_ns = stmt.column_int64(6); - result.push_back(info); + result.push_back(read_rosbag_row(stmt)); } return result; @@ -1847,28 +2245,11 @@ std::vector SqliteFaultStorage::list_rosbags_for_entity(const st std::vector SqliteFaultStorage::get_all_faults() const { std::lock_guard lock(mutex_); - SqliteStatement stmt(db_, - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, last_passed_ns FROM faults"); + SqliteStatement stmt(db_, (std::string("SELECT ") + kFaultColumns + " FROM faults").c_str()); std::vector result; while (stmt.step() == SQLITE_ROW) { - fd::FaultRecord fault; - fault.fault_code = stmt.column_text(0); - fault.severity = static_cast(stmt.column_int(1)); - fault.description = stmt.column_text(2); - - int64_t first_ns = stmt.column_int64(3); - int64_t last_ns = stmt.column_int64(4); - fault.first_occurred_ns = first_ns; - fault.last_occurred_ns = last_ns; - - fault.occurrence_count = static_cast(stmt.column_int64(5)); - fault.status = stmt.column_text(6); - fault.reporting_sources = parse_json_array(stmt.column_text(7)); - fault.last_passed_ns = stmt.column_int64(8); - - result.push_back(fault); + result.push_back(read_fault_row(stmt)); } return result; diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_code_thresholds_integration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_code_thresholds_integration.test.py index 85206fab3..6506aeac5 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_code_thresholds_integration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_code_thresholds_integration.test.py @@ -17,11 +17,10 @@ Integration tests for per-fault_code debounce thresholds. The node is launched with both layers configured, because the point of the -fault-code layer is what it does to the entity layer underneath it: the debounce -counter belongs to the fault code while an entity override is chosen by the -reporting source, so without this layer two entities reporting one code debounce -it two ways. The last case here covers the warning that says so when no -fault-code override settles it. +fault-code layer is what it does to the entity layer underneath it: an entity +override is chosen by the reporting source, so without this layer two entities +reporting one code debounce it two ways. Each owner's record keeps its own +counter, and the fault-code override sets the band it is judged against. """ import os @@ -37,27 +36,11 @@ from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import GetFault, ReportFault -# The node's own wording, so a reworded warning fails here rather than going quiet. -CONFLICT_WARNING = 'is debounced two ways' - # Entities whose debounce policies differ, from test_entity_thresholds.yaml. LIDAR = '/sensors/lidar/front' MOTOR = '/powertrain/motor/left' -def _output_text(proc_output, process): - """ - Return everything the process has written so far, as one string. - - Concatenated with NO separator: proc_output yields raw stream chunks, not - lines, so joining on a newline would splice one into the middle of a log - line and break a substring match on a message that is plainly there. - """ - return ''.join( - output.text.decode(errors='replace') for output in proc_output[process] - ) - - def generate_test_description(): """Launch fault_manager with both the entity and fault-code layers.""" pkg_share = get_package_share_directory('ros2_medkit_fault_manager') @@ -142,10 +125,11 @@ def _report(self, fault_code, source_id): resp = self._call(self.report_client, req) self.assertTrue(resp.accepted) - def _status(self, fault_code): - """Get a fault's current status.""" + def _status(self, fault_code, source_id): + """Get the current status of the record one source owns.""" req = GetFault.Request() req.fault_code = fault_code + req.source_id = source_id resp = self._call(self.get_client, req) self.assertTrue(resp.success, resp.error_message) return resp.fault.status @@ -160,131 +144,63 @@ def test_01_fault_code_beats_the_entity_that_reports_it(self): top of the entity one. """ self._report('SHARED.OVERHEAT', LIDAR) - self.assertEqual(self._status('SHARED.OVERHEAT'), Fault.STATUS_PREFAILED) + self.assertEqual(self._status('SHARED.OVERHEAT', LIDAR), Fault.STATUS_PREFAILED) self._report('SHARED.OVERHEAT', LIDAR) - self.assertEqual(self._status('SHARED.OVERHEAT'), Fault.STATUS_PREFAILED) + self.assertEqual(self._status('SHARED.OVERHEAT', LIDAR), Fault.STATUS_PREFAILED) self._report('SHARED.OVERHEAT', LIDAR) - self.assertEqual(self._status('SHARED.OVERHEAT'), Fault.STATUS_CONFIRMED) + self.assertEqual(self._status('SHARED.OVERHEAT', LIDAR), Fault.STATUS_CONFIRMED) # @verifies REQ_INTEROP_107 - def test_02_one_code_debounces_alike_from_two_entities(self): + def test_02_one_code_debounces_alike_for_each_entitys_record(self): """ - Confirm SHARED.JAM on its own fourth event, whoever reported it. + Confirm each owner's SHARED.JAM record on its own fourth event. - Alternating the two sources is the problem case: under the - entity layer alone lidar's -1 would confirm this on the second event, - bypassing the motor's policy. + The two sources own two records on their own counters. Under the entity + layer alone lidar's -1 would confirm its record on the first event and + the motor's -5 would take five. """ - self._report('SHARED.JAM', MOTOR) - self.assertEqual(self._status('SHARED.JAM'), Fault.STATUS_PREFAILED) - - self._report('SHARED.JAM', LIDAR) - self.assertEqual(self._status('SHARED.JAM'), Fault.STATUS_PREFAILED) + for source in (MOTOR, LIDAR): + for _ in range(3): + self._report('SHARED.JAM', source) + self.assertEqual( + self._status('SHARED.JAM', source), Fault.STATUS_PREFAILED) - self._report('SHARED.JAM', MOTOR) - self.assertEqual(self._status('SHARED.JAM'), Fault.STATUS_PREFAILED) - - self._report('SHARED.JAM', LIDAR) - self.assertEqual(self._status('SHARED.JAM'), Fault.STATUS_CONFIRMED) + self._report('SHARED.JAM', source) + self.assertEqual( + self._status('SHARED.JAM', source), Fault.STATUS_CONFIRMED) # @verifies REQ_INTEROP_107 def test_03_a_code_with_no_override_keeps_the_entity_policy(self): """The new layer is opt-in: an unlisted code still follows its entity.""" self._report('MOTOR.ONLY', MOTOR) - self.assertEqual(self._status('MOTOR.ONLY'), Fault.STATUS_PREFAILED) + self.assertEqual(self._status('MOTOR.ONLY', MOTOR), Fault.STATUS_PREFAILED) self._report('LIDAR.ONLY', LIDAR) - self.assertEqual(self._status('LIDAR.ONLY'), Fault.STATUS_CONFIRMED) + self.assertEqual(self._status('LIDAR.ONLY', LIDAR), Fault.STATUS_CONFIRMED) # @verifies REQ_INTEROP_107 - def test_04_a_half_pinned_code_is_still_reported(self, proc_output, - fault_manager_node): + def test_04_a_half_pinned_code_confirms_on_the_pinned_threshold(self): """ Report SHARED.PARTIAL, which pins confirmation but not healing. - The override settles the direction it names and no more: the two sources - heal this code under thresholds 1 and 5. Warning on the whole policy - rather than on the presence of an override is what keeps that visible. + Both owners' records confirm on their own second event, whatever + confirmation threshold their entity carries. """ - self._report('SHARED.PARTIAL', LIDAR) - self.assertEqual(self._status('SHARED.PARTIAL'), Fault.STATUS_PREFAILED) - - self._report('SHARED.PARTIAL', MOTOR) - self.assertEqual(self._status('SHARED.PARTIAL'), Fault.STATUS_CONFIRMED) - - proc_output.assertWaitFor( - f"Fault code 'SHARED.PARTIAL' {CONFLICT_WARNING}", - process=fault_manager_node, - timeout=10.0, - ) + for source in (LIDAR, MOTOR): + self._report('SHARED.PARTIAL', source) + self.assertEqual( + self._status('SHARED.PARTIAL', source), Fault.STATUS_PREFAILED) - def test_05_conflicting_policies_are_reported_once(self, proc_output, - fault_manager_node): - """ - Warn on two entities reporting one unlisted code, and warn once. - - Without the warning the bypass is invisible - the fault confirms under - whichever policy happened to report, and nothing in the log distinguishes - that from the policy the operator configured. - """ - self._report('CONFLICTING.CODE', LIDAR) - self._report('CONFLICTING.CODE', MOTOR) - - proc_output.assertWaitFor( - f"Fault code 'CONFLICTING.CODE' {CONFLICT_WARNING}", - process=fault_manager_node, - timeout=10.0, - ) - - # Report again. The node must not warn a second time; that is checked - # in TestShutdown, once the whole output has been read. - self._report('CONFLICTING.CODE', LIDAR) - self._report('CONFLICTING.CODE', MOTOR) + self._report('SHARED.PARTIAL', source) + self.assertEqual( + self._status('SHARED.PARTIAL', source), Fault.STATUS_CONFIRMED) @launch_testing.post_shutdown_test() class TestShutdown(unittest.TestCase): - """Verify fault_manager exits cleanly and stayed quiet where it should.""" - - def test_an_overridden_code_never_warns(self, proc_output, - fault_manager_node): - """ - Stay quiet about a code whose override settles it for both sources. - - SHARED.JAM had two sources and no conflict: a fault-code override - resolves the same for both, so there is nothing to warn about. Checked - after shutdown, when the whole output is in hand - asserting an absence - against a stream still being written proves nothing. - """ - text = _output_text(proc_output, fault_manager_node) - self.assertNotIn("Fault code 'SHARED.JAM'", text) - - # And a code only one source ever reports has nothing to conflict with. - # A witness compared against the global config instead of against the - # first report would warn here, on every ordinary single-reporter robot. - self.assertNotIn("Fault code 'SHARED.OVERHEAT'", text) - self.assertNotIn("Fault code 'MOTOR.ONLY'", text) - self.assertNotIn("Fault code 'LIDAR.ONLY'", text) - - def test_conflict_is_warned_once_with_both_policies(self, proc_output, - fault_manager_node): - """ - Warn once for CONFLICTING.CODE, naming both sources and both policies. - - Counted here, after shutdown: during the run proc_output may not have - read the last warning yet, so a count there can pass on two warnings. - """ - text = _output_text(proc_output, fault_manager_node) - self.assertEqual(text.count("Fault code 'CONFLICTING.CODE'"), 1) - # Lidar reported first, so it is named first. Values come from - # test_entity_thresholds.yaml; motor inherits healing_enabled=false. - self.assertIn( - f"'{LIDAR}' resolves confirmation=-1 healing_enabled=true healing=1, " - f"'{MOTOR}' resolves confirmation=-3 healing_enabled=false healing=5", - text, - ) + """Verify fault_manager exits cleanly.""" def test_exit_code(self, proc_info): """Check process exit code.""" diff --git a/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py index 1743b40be..d45916824 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py @@ -322,8 +322,8 @@ def test_09_clear_fault_empty_code_fails(self): self.assertIn('empty', response.message.lower()) print(f'Clear empty code response: {response.message}') - def test_10_report_updates_existing_fault(self): - """Test that reporting same fault_code updates existing fault.""" + def test_10_two_sources_are_two_records(self): + """Two sources reporting one fault_code own two independent records.""" fault_code = 'TEST_FAULT_UPDATE' # Report first time @@ -337,18 +337,17 @@ def test_10_report_updates_existing_fault(self): response1 = self._call_service(self.report_fault_client, request1) self.assertTrue(response1.accepted) - # Report second time from different source + # Report second time from a different source, at a higher severity request2 = ReportFault.Request() request2.fault_code = fault_code request2.event_type = ReportFault.Request.EVENT_FAILED - request2.severity = Fault.SEVERITY_ERROR # Higher severity + request2.severity = Fault.SEVERITY_ERROR request2.description = 'Second report' request2.source_id = '/node2' response2 = self._call_service(self.report_fault_client, request2) self.assertTrue(response2.accepted) - # Verify fault was updated get_request = ListFaults.Request() get_request.filter_by_severity = False get_request.severity = 0 @@ -356,34 +355,37 @@ def test_10_report_updates_existing_fault(self): get_response = self._call_service(self.list_faults_client, get_request) - updated_fault = None + by_owner = {} for f in get_response.faults: if f.fault_code == fault_code: - updated_fault = f - break - - self.assertIsNotNone(updated_fault) - self.assertEqual(updated_fault.severity, Fault.SEVERITY_ERROR) - # Edge-counting: a re-report of a still-active fault is the same - # continuous occurrence, so the count stays at 1 (it only increments - # on reactivation after CLEARED). - self.assertEqual(updated_fault.occurrence_count, 1) - self.assertEqual(len(updated_fault.reporting_sources), 2) - print(f'Updated fault: occurrence_count={updated_fault.occurrence_count}, ' - f'sources={updated_fault.reporting_sources}') + self.assertEqual(len(f.reporting_sources), 1) + by_owner[f.reporting_sources[0]] = f + + self.assertEqual(sorted(by_owner), ['/node1', '/node2']) + # Severity is the record's own maximum, so one source reporting ERROR does + # not escalate the other source's record. + self.assertEqual(by_owner['/node1'].severity, Fault.SEVERITY_WARN) + self.assertEqual(by_owner['/node2'].severity, Fault.SEVERITY_ERROR) + self.assertEqual(by_owner['/node1'].description, 'First report') + self.assertEqual(by_owner['/node2'].description, 'Second report') + # Each record counts its own occurrences. + self.assertEqual(by_owner['/node1'].occurrence_count, 1) + self.assertEqual(by_owner['/node2'].occurrence_count, 1) + print(f'Two records for {fault_code}: owners={sorted(by_owner)}') def test_11_confirmation_workflow(self): """Test fault auto-confirms after reaching threshold (default=-3).""" fault_code = 'TEST_FAULT_CONFIRM' - # Report fault 3 times (default threshold) + # Report 3 times from ONE source: the debounce counter belongs to that + # source's record, so depth is what one reporter has said. for i in range(3): request = ReportFault.Request() request.fault_code = fault_code request.event_type = ReportFault.Request.EVENT_FAILED request.severity = Fault.SEVERITY_ERROR request.description = f'Report {i + 1}' - request.source_id = f'/node{i + 1}' + request.source_id = '/node1' self._call_service(self.report_fault_client, request) # Query CONFIRMED faults (should include our fault now) @@ -444,19 +446,22 @@ def test_12_prefailed_excluded_from_default_query(self): print('PREFAILED query found the fault as expected') def test_13_multi_source_confirmation(self): - """Test fault confirms when multiple sources report same fault.""" + """Each source reporting one fault_code confirms its own record.""" fault_code = 'TEST_FAULT_MULTI_SRC' - # Report from 3 different sources + # Each of 3 sources reports enough times to confirm its OWN record. Under a + # shared counter three single reports confirmed one fault between them. Now + # each reporter has to reach its own threshold. sources = ['/sensor1', '/sensor2', '/sensor3'] for source in sources: - request = ReportFault.Request() - request.fault_code = fault_code - request.event_type = ReportFault.Request.EVENT_FAILED - request.severity = Fault.SEVERITY_ERROR - request.description = 'Multi-source fault' - request.source_id = source - self._call_service(self.report_fault_client, request) + for _ in range(3): # launch threshold is -3 + request = ReportFault.Request() + request.fault_code = fault_code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = 'Multi-source fault' + request.source_id = source + self._call_service(self.report_fault_client, request) # Query and verify get_request = ListFaults.Request() @@ -466,14 +471,17 @@ def test_13_multi_source_confirmation(self): response = self._call_service(self.list_faults_client, get_request) - fault = next((f for f in response.faults if f.fault_code == fault_code), None) - self.assertIsNotNone(fault) - self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) - self.assertEqual(len(fault.reporting_sources), 3) - # Edge-counting: three sources reporting the same active fault are one - # continuous occurrence; the count tracks raise edges, not reports. - self.assertEqual(fault.occurrence_count, 1) - print(f'Multi-source fault confirmed: sources={fault.reporting_sources}') + matching = [f for f in response.faults if f.fault_code == fault_code] + self.assertEqual(len(matching), 3) + owners = [] + for fault in matching: + self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(len(fault.reporting_sources), 1) + # Edge-counting is per record: one raise edge each. + self.assertEqual(fault.occurrence_count, 1) + owners.append(fault.reporting_sources[0]) + self.assertEqual(sorted(owners), sorted(sources)) + print(f'Three records confirmed: owners={sorted(owners)}') def test_14_critical_severity_immediate_confirmation(self): """Test CRITICAL severity bypasses debounce and confirms immediately.""" @@ -506,14 +514,14 @@ def test_15_passed_event_increments_counter(self): """Test PASSED events increment debounce counter towards healing.""" fault_code = 'TEST_FAULT_PASSED' - # Report 2 FAILED events (counter = -2) + # Report 2 FAILED events from one source (its counter = -2) for i in range(2): request = ReportFault.Request() request.fault_code = fault_code request.event_type = ReportFault.Request.EVENT_FAILED request.severity = Fault.SEVERITY_ERROR request.description = f'Failed report {i + 1}' - request.source_id = f'/node{i + 1}' + request.source_id = '/node1' self._call_service(self.report_fault_client, request) # Verify fault is PREFAILED @@ -534,7 +542,9 @@ def test_15_passed_event_increments_counter(self): request.event_type = ReportFault.Request.EVENT_PASSED request.severity = 0 # Ignored for PASSED request.description = '' # Ignored for PASSED - request.source_id = '/test_node' + # The same source: a PASSED moves the counter of the record its own + # source owns, and a source that owns no record has nothing to heal. + request.source_id = '/node1' self._call_service(self.report_fault_client, request) # Verify fault is PREPASSED (counter > 0) @@ -801,7 +811,9 @@ def test_22_correlation_symptoms_muted(self): request.event_type = ReportFault.Request.EVENT_FAILED request.severity = Fault.SEVERITY_CRITICAL request.description = f'Symptom: {symptom}' - request.source_id = '/motor_node' + # Same owner as the root cause: correlation relates records of one owner, + # so a root cause only explains faults its own reporter raised. + request.source_id = '/estop_node' response = self._call_service(self.report_fault_client, request) self.assertTrue(response.accepted) @@ -837,9 +849,10 @@ def test_23_correlation_auto_clear_with_root(self): When ESTOP_001 is cleared, all its correlated symptoms should also be cleared (auto_clear_with_root=true). """ - # Clear the root cause + # Clear the root cause record clear_request = ClearFault.Request() clear_request.fault_code = 'ESTOP_001' + clear_request.source_id = '/estop_node' clear_response = self._call_service(self.clear_fault_client, clear_request) @@ -866,6 +879,57 @@ def test_23_correlation_auto_clear_with_root(self): self.assertEqual(len(remaining_muted), 0) print('All symptoms auto-cleared with root cause') + def test_23b_auto_clear_stays_inside_one_owner(self): + """A root cause auto-clears only its own reporter's symptom record.""" + # Two owners raise the same root cause and the same symptom. + for owner in ['/cascade_a', '/cascade_b']: + for code in ['ESTOP_001', 'MOTOR_COMM_CASCADE']: + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_CRITICAL + request.description = f'{code} from {owner}' + request.source_id = owner + self.assertTrue(self._call_service(self.report_fault_client, request).accepted) + + clear_request = ClearFault.Request() + clear_request.fault_code = 'ESTOP_001' + clear_request.source_id = '/cascade_a' + clear_response = self._call_service(self.clear_fault_client, clear_request) + self.assertTrue(clear_response.success) + self.assertIn('MOTOR_COMM_CASCADE', clear_response.auto_cleared_codes) + + # B's identical symptom is untouched: it is muted by B's own root cause, + # which nobody cleared. + get_request = ListFaults.Request() + get_request.filter_by_severity = False + get_request.statuses = [Fault.STATUS_CONFIRMED] + get_request.include_muted = True + faults = self._call_service(self.list_faults_client, get_request).faults + + symptom_owners = sorted(f.reporting_sources[0] for f in faults + if f.fault_code == 'MOTOR_COMM_CASCADE') + self.assertEqual(symptom_owners, ['/cascade_b']) + + # include_muted left at its default on purpose: muting is per record, so the + # default query hides B's muted record alone and keeps A's cleared record, + # which carries the same code. + cleared_request = ListFaults.Request() + cleared_request.statuses = [Fault.STATUS_CLEARED] + cleared = [f for f in self._call_service(self.list_faults_client, cleared_request).faults + if f.fault_code == 'MOTOR_COMM_CASCADE'] + self.assertEqual([f.reporting_sources[0] for f in cleared], ['/cascade_a']) + + # The same default query still hides the record that IS muted: B's symptom is + # CONFIRMED under B's own root cause and must not appear. + default_request = ListFaults.Request() + default_request.statuses = [Fault.STATUS_CONFIRMED] + default_confirmed = [f for f in + self._call_service(self.list_faults_client, default_request).faults + if f.fault_code == 'MOTOR_COMM_CASCADE'] + self.assertEqual(default_confirmed, []) + print('Auto-clear cascade stayed inside one owner') + def test_24_correlation_auto_cluster(self): """ Test auto-cluster correlation: sensor errors are grouped. @@ -948,23 +1012,24 @@ def test_26_cleared_fault_can_be_reactivated(self): response = self._call_service(self.report_fault_client, request) self.assertTrue(response.accepted) - # Clear the fault + # Clear that owner's record clear_request = ClearFault.Request() clear_request.fault_code = fault_code + clear_request.source_id = '/node1' clear_response = self._call_service(self.clear_fault_client, clear_request) self.assertTrue(clear_response.success) - # Report again - should reactivate + # The same owner reports again - reactivates its own record request = ReportFault.Request() request.fault_code = fault_code request.event_type = ReportFault.Request.EVENT_FAILED request.severity = Fault.SEVERITY_CRITICAL request.description = 'Reactivated' - request.source_id = '/node2' + request.source_id = '/node1' response = self._call_service(self.report_fault_client, request) self.assertTrue(response.accepted) - # Verify fault is CONFIRMED again + # Verify the record is CONFIRMED again, on its second occurrence get_request = ListFaults.Request() get_request.statuses = [Fault.STATUS_CONFIRMED] get_response = self._call_service(self.list_faults_client, get_request) @@ -973,8 +1038,58 @@ def test_26_cleared_fault_can_be_reactivated(self): self.assertIsNotNone(fault) self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) self.assertEqual(fault.occurrence_count, 2) - self.assertEqual(len(fault.reporting_sources), 2) - print(f'Cleared fault reactivated: occurrence_count={fault.occurrence_count}') + self.assertEqual(fault.reporting_sources, ['/node1']) + print(f'Cleared record reactivated: occurrence_count={fault.occurrence_count}') + + def test_26b_scoped_clear_leaves_the_other_owners_record(self): + """A scoped clear takes one record, an unscoped clear of two is refused.""" + fault_code = 'TEST_SCOPED_CLEAR' + owners = ['/owner_a', '/owner_b'] + + for owner in owners: + request = ReportFault.Request() + request.fault_code = fault_code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_CRITICAL + request.description = 'Two owners' + request.source_id = owner + self.assertTrue(self._call_service(self.report_fault_client, request).accepted) + + # Unscoped: two records carry the code, so the call is refused outright. + unscoped = ClearFault.Request() + unscoped.fault_code = fault_code + unscoped_response = self._call_service(self.clear_fault_client, unscoped) + self.assertFalse(unscoped_response.success) + self.assertTrue(unscoped_response.message.startswith('ambiguous:'), + unscoped_response.message) + for owner in owners: + self.assertIn(owner, unscoped_response.message) + + # Nothing was cleared by the refusal. + confirmed = ListFaults.Request() + confirmed.statuses = [Fault.STATUS_CONFIRMED] + still = [f for f in self._call_service(self.list_faults_client, confirmed).faults + if f.fault_code == fault_code] + self.assertEqual(len(still), 2) + + # Scoped: exactly one record moves. + scoped = ClearFault.Request() + scoped.fault_code = fault_code + scoped.source_id = '/owner_a' + self.assertTrue(self._call_service(self.clear_fault_client, scoped).success) + + after = self._call_service(self.list_faults_client, confirmed).faults + remaining = [f for f in after if f.fault_code == fault_code] + self.assertEqual(len(remaining), 1) + self.assertEqual(remaining[0].reporting_sources, ['/owner_b']) + + cleared_request = ListFaults.Request() + cleared_request.statuses = [Fault.STATUS_CLEARED] + cleared = [f for f in self._call_service(self.list_faults_client, cleared_request).faults + if f.fault_code == fault_code] + self.assertEqual(len(cleared), 1) + self.assertEqual(cleared[0].reporting_sources, ['/owner_a']) + print('Scoped clear took one record and left the other CONFIRMED') def test_27_capture_pool_bounds_fault_storm(self, proc_output): """ diff --git a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_history.test.py b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_history.test.py index 7187b9dc1..ec6879b8a 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_history.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_history.test.py @@ -14,7 +14,7 @@ # limitations under the License. """ -End-to-end tests for keeping several recordings per fault code (Issue #620). +End-to-end tests for keeping several recordings per fault record (Issue #620). The other rosbag suites all run at the shipped default of one recording per fault, which is what every release before this one could store. This one raises @@ -74,7 +74,7 @@ def get_coverage_env(): def generate_test_description(): - """Launch a fault_manager keeping several recordings per fault code.""" + """Launch a fault_manager keeping several recordings per fault record.""" publisher_script = """ import rclpy from rclpy.node import Node @@ -212,7 +212,7 @@ def start_fault_manager_when_publisher_up(event): class TestRosbagHistory(unittest.TestCase): - """Several recordings per fault code, addressed by recording id.""" + """Several recordings per fault record, addressed by recording id.""" @classmethod def setUpClass(cls): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py b/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py new file mode 100644 index 000000000..6d2e9c1d9 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Severity contract of a fault record through a real fault manager on SQLite. + +The database starts as one written before records had an owner, so the first +start migrates it. The fault manager is then restarted twice on it. Every +record here has one owner. A record keeps the highest severity its owner ever +reported. A report on a CRITICAL record confirms it. A reactivated occurrence, +migrated or not, has no confirmation time until it confirms. A report whose +severity differs from the stored one is logged once per process. +""" + +import os +import shutil +import signal +import sqlite3 +import tempfile +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from rclpy.qos import QoSProfile, ReliabilityPolicy +from ros2_medkit_msgs.msg import Fault, FaultEvent +from ros2_medkit_msgs.srv import ClearFault, ListFaults, ReportFault + +DB_DIR = tempfile.mkdtemp(prefix='medkit_severity_') +DB_PATH = os.path.join(DB_DIR, 'faults.db') +EVENT_TOPIC = '/fault_manager/events' +SOURCE = '/robot/reporter' +ALL_STATUSES = [ + Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, + Fault.STATUS_HEALED, Fault.STATUS_CLEARED, +] +MISMATCH_LINE = "Fault 'SEV_CRIT' reported with severity WARN" +TIMEOUT_SEC = 30.0 +LEGACY_CONFIRMED_AT_NS = 1_000_000_000 + +# The faults table as releases before record identity created it: one row per code. +LEGACY_FAULTS_DDL = ( + 'CREATE TABLE faults (fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, ' + 'description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, ' + 'last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, ' + 'status TEXT NOT NULL, reporting_sources TEXT NOT NULL, ' + 'debounce_counter INTEGER NOT NULL DEFAULT 0, ' + 'last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, ' + 'confirmed_at_ns INTEGER NOT NULL DEFAULT 0)') + + +# Every legacy row carries LEGACY_CONFIRMED_AT_NS. The pending ones were left so by a reactivation. +LEGACY_ROWS = ( + ('LEGACY_CONF_AT', Fault.SEVERITY_ERROR, Fault.STATUS_CLEARED), + ('LEGACY_CRIT', Fault.SEVERITY_CRITICAL, Fault.STATUS_CLEARED), + ('LEGACY_PREFAILED', Fault.SEVERITY_ERROR, Fault.STATUS_PREFAILED), + ('LEGACY_PREPASSED', Fault.SEVERITY_WARN, Fault.STATUS_PREPASSED), + ('LEGACY_CONFIRMED', Fault.SEVERITY_ERROR, Fault.STATUS_CONFIRMED), + ('LEGACY_HEALED', Fault.SEVERITY_ERROR, Fault.STATUS_HEALED), +) + + +def seed_legacy_database(): + """Write the LEGACY_ROWS into a faults table that has no owner column.""" + with sqlite3.connect(DB_PATH) as db: + db.execute(LEGACY_FAULTS_DDL) + for code, severity, status in LEGACY_ROWS: + db.execute( + 'INSERT INTO faults VALUES (?, ?, ?, ?, ?, 1, ?, ?, -3, ?, 0, ?)', + (code, severity, 'legacy', 500_000_000, 500_000_000, status, + f'["{SOURCE}"]', 500_000_000, LEGACY_CONFIRMED_AT_NS)) + + +def generate_test_description(): + seed_legacy_database() + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'sqlite', + 'database_path': DB_PATH, + 'confirmation_threshold': -3, + 'healing_enabled': True, + 'healing_threshold': 1, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + # The test restarts the node by PID; launch brings it back on the same database. + respawn=True, + respawn_delay=1.0, + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestSeverityConsistency(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_severity_consistency_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + cls.events = [] + qos = QoSProfile(depth=100, reliability=ReliabilityPolicy.RELIABLE) + cls.event_sub = cls.node.create_subscription( + FaultEvent, EVENT_TOPIC, cls.events.append, qos) + assert cls.report_client.wait_for_service(timeout_sec=TIMEOUT_SEC) + assert cls.list_client.wait_for_service(timeout_sec=TIMEOUT_SEC) + assert cls.clear_client.wait_for_service(timeout_sec=TIMEOUT_SEC) + cls.primes = 0 + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request, timeout=10.0): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=timeout) + if not future.done(): + client.remove_pending_request(future) + return None + return future.result() + + def _report(self, code, severity, event_type=ReportFault.Request.EVENT_FAILED): + request = ReportFault.Request() + request.fault_code = code + request.event_type = event_type + request.severity = severity + request.description = f'{code} report' + request.source_id = SOURCE + response = self._call(self.report_client, request) + self.assertIsNotNone(response, 'report_fault timed out') + self.assertTrue(response.accepted) + + def _passed(self, code): + self._report(code, Fault.SEVERITY_INFO, ReportFault.Request.EVENT_PASSED) + + def _clear(self, code): + request = ClearFault.Request() + request.fault_code = code + request.source_id = SOURCE + response = self._call(self.clear_client, request) + self.assertIsNotNone(response, 'clear_fault timed out') + self.assertTrue(response.success) + + def _list(self, statuses=None, severity=None): + request = ListFaults.Request() + request.statuses = statuses if statuses is not None else ALL_STATUSES + if severity is not None: + request.filter_by_severity = True + request.severity = severity + response = self._call(self.list_client, request) + self.assertIsNotNone(response, 'list_faults timed out') + return {f.fault_code: f for f in response.faults} + + def _fault(self, code): + fault = self._list().get(code) + self.assertIsNotNone(fault, f'{code} is not stored') + return fault + + def _confirmed_at(self, code): + with sqlite3.connect(DB_PATH, timeout=10.0) as db: + row = db.execute( + 'SELECT confirmed_at_ns FROM faults WHERE fault_code = ? AND owner = ?', + (code, SOURCE)).fetchone() + self.assertIsNotNone(row, f'{code} has no row in the database') + return row[0] + + def _wait_for_event(self, code, event_type, occurrence): + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + for event in list(self.events): + if (event.fault.fault_code, event.event_type, event.fault.occurrence_count) == \ + (code, event_type, occurrence): + return event + rclpy.spin_once(self.node, timeout_sec=0.1) + return None + + def _prime_events(self): + """Report fresh CRITICAL codes until one event arrives, so the event topic is matched.""" + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + type(self).primes += 1 + code = f'PRIME_{self.primes}' + self._report(code, Fault.SEVERITY_CRITICAL) + settle = time.monotonic() + 1.0 + while time.monotonic() < settle: + if any(e.fault.fault_code == code for e in self.events): + return + rclpy.spin_once(self.node, timeout_sec=0.1) + self.fail('no fault event arrived from the fault manager') + + def _restart(self, fault_manager_node): + old_pid = fault_manager_node.process_details['pid'] + os.kill(old_pid, signal.SIGTERM) + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline and fault_manager_node.process_details['pid'] == old_pid: + time.sleep(0.2) + self.assertNotEqual( + fault_manager_node.process_details['pid'], old_pid, + 'the fault manager was not restarted') + + probe = ListFaults.Request() + probe.statuses = ALL_STATUSES + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + if self.list_client.wait_for_service(timeout_sec=1.0): + if self._call(self.list_client, probe, timeout=2.0) is not None: + self._prime_events() + return + self.fail('the restarted fault manager never answered list_faults') + + def _mismatch_lines(self, proc_output, expected): + """Count SEV_CRIT mismatch warnings once ``expected`` arrived, or at the timeout.""" + def count(): + text = ''.join(io.text.decode(errors='replace') for io in proc_output) + return text.count(MISMATCH_LINE) + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline and count() < expected: + time.sleep(0.2) + # Anything logged late would show here as more than expected. + time.sleep(0.5) + return count() + + def test_severity_contract_across_restarts(self, fault_manager_node, proc_output): + self._prime_events() + + # Process 1 migrated the legacy rows. Only a row that confirmed keeps its time. + for code, _, status in LEGACY_ROWS: + pending = status in (Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED) + with self.subTest(case='migrated row', code=code): + self.assertEqual(self._fault(code).status, status) + self.assertEqual(self._confirmed_at(code), + 0 if pending else LEGACY_CONFIRMED_AT_NS) + + # A cleared row's next FAILED starts a new occurrence. + self._report('LEGACY_CONF_AT', Fault.SEVERITY_ERROR) + self._report('LEGACY_CRIT', Fault.SEVERITY_WARN) + with self.subTest(case='migrated, reactivated, not yet confirmed'): + fault = self._fault('LEGACY_CONF_AT') + self.assertEqual(fault.status, Fault.STATUS_PREFAILED) + self.assertEqual(fault.occurrence_count, 2) + self.assertEqual(self._confirmed_at('LEGACY_CONF_AT'), 0) + with self.subTest(case='migrated CRITICAL, WARN'): + fault = self._fault('LEGACY_CRIT') + self.assertEqual(fault.severity, Fault.SEVERITY_CRITICAL) + self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) + self.assertGreater(self._confirmed_at('LEGACY_CRIT'), LEGACY_CONFIRMED_AT_NS) + + # Process 1: CRITICAL, cleared; CRITICAL, healed; ERROR confirmed, cleared. + self._report('SEV_CRIT', Fault.SEVERITY_CRITICAL) + self.assertEqual(self._fault('SEV_CRIT').status, Fault.STATUS_CONFIRMED) + self._clear('SEV_CRIT') + + self._report('HEAL_CRIT', Fault.SEVERITY_CRITICAL) + self._passed('HEAL_CRIT') + self._passed('HEAL_CRIT') + self.assertEqual(self._fault('HEAL_CRIT').status, Fault.STATUS_HEALED) + + for _ in range(3): + self._report('CONF_AT', Fault.SEVERITY_ERROR) + self.assertEqual(self._fault('CONF_AT').status, Fault.STATUS_CONFIRMED) + first_confirmed_at = self._confirmed_at('CONF_AT') + self.assertGreater(first_confirmed_at, 0) + self._clear('CONF_AT') + + self._restart(fault_manager_node) + + # Process 2: one WARN report on a CRITICAL code confirms it, after a clear or a heal. + self._report('SEV_CRIT', Fault.SEVERITY_WARN) + self._report('HEAL_CRIT', Fault.SEVERITY_WARN) + with self.subTest(case='CRITICAL, clear, restart, WARN'): + fault = self._fault('SEV_CRIT') + self.assertEqual(fault.severity, Fault.SEVERITY_CRITICAL) + self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(fault.occurrence_count, 2) + event = self._wait_for_event('SEV_CRIT', FaultEvent.EVENT_CONFIRMED, 2) + self.assertIsNotNone(event, 'no fault_confirmed event for occurrence 2') + self.assertEqual(event.fault.severity, Fault.SEVERITY_CRITICAL) + + with self.subTest(case='CRITICAL, heal, restart, WARN'): + fault = self._fault('HEAL_CRIT') + self.assertEqual(fault.severity, Fault.SEVERITY_CRITICAL) + self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(fault.occurrence_count, 1) + + with self.subTest(case='no pending CRITICAL'): + self.assertEqual( + self._list(statuses=[Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED], + severity=Fault.SEVERITY_CRITICAL), + {}) + + for _ in range(9): + self._report('SEV_CRIT', Fault.SEVERITY_WARN) + + self._report('CONF_AT', Fault.SEVERITY_ERROR) + with self.subTest(case='reactivated, not yet confirmed'): + self.assertEqual(self._fault('CONF_AT').status, Fault.STATUS_PREFAILED) + self.assertEqual(self._confirmed_at('CONF_AT'), 0) + + with self.subTest(case='mismatch logged once in this process'): + self.assertEqual(self._mismatch_lines(proc_output, 1), 1) + + self._restart(fault_manager_node) + + # Process 3: the reactivated occurrences still have no confirmation time, then confirm. + with self.subTest(case='migrated rows after two restarts'): + self.assertEqual(self._confirmed_at('LEGACY_PREPASSED'), 0) + self.assertEqual(self._confirmed_at('LEGACY_CONFIRMED'), LEGACY_CONFIRMED_AT_NS) + self.assertEqual(self._confirmed_at('LEGACY_HEALED'), LEGACY_CONFIRMED_AT_NS) + with self.subTest(case='reactivated, restarted, not yet confirmed'): + self.assertEqual(self._confirmed_at('CONF_AT'), 0) + self.assertEqual(self._confirmed_at('LEGACY_CONF_AT'), 0) + self._report('LEGACY_CONF_AT', Fault.SEVERITY_ERROR) + self._report('LEGACY_CONF_AT', Fault.SEVERITY_ERROR) + self.assertEqual(self._fault('LEGACY_CONF_AT').status, Fault.STATUS_CONFIRMED) + with self.subTest(case='migrated record confirms its new occurrence'): + self.assertGreater(self._confirmed_at('LEGACY_CONF_AT'), LEGACY_CONFIRMED_AT_NS) + self._report('CONF_AT', Fault.SEVERITY_ERROR) + self._report('CONF_AT', Fault.SEVERITY_ERROR) + fault = self._fault('CONF_AT') + self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) + started = fault.first_occurred + occurrence_start_ns = started.sec * 1_000_000_000 + started.nanosec + with self.subTest(case='confirmation time belongs to this occurrence'): + self.assertGreater(self._confirmed_at('CONF_AT'), occurrence_start_ns) + self.assertGreater(occurrence_start_ns, first_confirmed_at) + + for _ in range(10): + self._report('SEV_CRIT', Fault.SEVERITY_WARN) + with self.subTest(case='the bound restarts with the process'): + self.assertEqual(self._mismatch_lines(proc_output, 2), 2) + + +@launch_testing.post_shutdown_test() +class TestSeverityConsistencyShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + shutil.rmtree(DB_DIR, ignore_errors=True) + launch_testing.asserts.assertExitCodes(proc_info) diff --git a/src/ros2_medkit_fault_manager/test/test_capture_thread_pool.cpp b/src/ros2_medkit_fault_manager/test/test_capture_thread_pool.cpp index defba8395..8fce2c335 100644 --- a/src/ros2_medkit_fault_manager/test/test_capture_thread_pool.cpp +++ b/src/ros2_medkit_fault_manager/test/test_capture_thread_pool.cpp @@ -33,6 +33,7 @@ using ros2_medkit_fault_manager::CaptureThreadPool; using ros2_medkit_fault_manager::EnqueueResult; +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::QueueFullPolicy; namespace { @@ -42,13 +43,21 @@ ros2_medkit_fault_manager::ros2::RosLogSink & test_logger() { return sink; } +// The pool queues fault RECORDS. These tests are about queue mechanics, so every +// job here belongs to one owner and the code alone tells the jobs apart. +constexpr const char * kOwner = "/reporter"; + +FaultId job(const std::string & code) { + return FaultId{code, kOwner}; +} + // Capture_fn that records every invocation and blocks each job until released, // so tests can deterministically hold workers busy and inspect concurrency. class GatedCallback { public: - std::function fn() { - return [this](const std::string & code) { - run(code); + std::function fn() { + return [this](const FaultId & id) { + run(id.fault_code); }; } @@ -132,7 +141,7 @@ TEST_F(CaptureThreadPoolTest, BoundsAndReachesPoolSize) { GatedCallback gc; CaptureThreadPool pool(2, 8, QueueFullPolicy::kRejectNewest, test_logger(), gc.fn()); for (int i = 0; i < 6; ++i) { - pool.enqueue("j" + std::to_string(i)); + pool.enqueue(job("j" + std::to_string(i))); } gc.wait_until_active(2); // prove 2 ran concurrently EXPECT_EQ(gc.max_active(), 2u); // never more than pool_size @@ -144,8 +153,8 @@ TEST_F(CaptureThreadPoolTest, BoundsAndReachesPoolSize) { TEST_F(CaptureThreadPoolTest, PoolSizeOneSerializes) { GatedCallback gc; CaptureThreadPool pool(1, 8, QueueFullPolicy::kRejectNewest, test_logger(), gc.fn()); - pool.enqueue("a"); - pool.enqueue("b"); + pool.enqueue(job("a")); + pool.enqueue(job("b")); gc.wait_until_active(1); EXPECT_EQ(gc.max_active(), 1u); gc.release(); @@ -159,13 +168,13 @@ TEST_F(CaptureThreadPoolTest, PoolSizeOneSerializes) { TEST_F(CaptureThreadPoolTest, ZeroSizesClampToOne) { GatedCallback gc; CaptureThreadPool pool(0, 0, QueueFullPolicy::kDropOldest, test_logger(), gc.fn()); - EXPECT_EQ(pool.enqueue("a").result, EnqueueResult::kAccepted); - gc.wait_until_active(1); // a worker exists (pool_size clamped to 1) - EXPECT_EQ(pool.enqueue("b").result, EnqueueResult::kAccepted); // queue_depth clamped to 1 - auto outcome = pool.enqueue("c"); // full -> evict oldest, no UB + EXPECT_EQ(pool.enqueue(job("a")).result, EnqueueResult::kAccepted); + gc.wait_until_active(1); // a worker exists (pool_size clamped to 1) + EXPECT_EQ(pool.enqueue(job("b")).result, EnqueueResult::kAccepted); // queue_depth clamped to 1 + auto outcome = pool.enqueue(job("c")); // full -> evict oldest, no UB EXPECT_EQ(outcome.result, EnqueueResult::kEvictedOldest); - ASSERT_TRUE(outcome.evicted_code.has_value()); - EXPECT_EQ(*outcome.evicted_code, "b"); + ASSERT_TRUE(outcome.evicted_id.has_value()); + EXPECT_EQ(outcome.evicted_id->fault_code, "b"); gc.release(); pool.shutdown(); } @@ -173,14 +182,14 @@ TEST_F(CaptureThreadPoolTest, ZeroSizesClampToOne) { TEST_F(CaptureThreadPoolTest, RejectNewestDropsExcessExactly) { GatedCallback gc; CaptureThreadPool pool(2, 8, QueueFullPolicy::kRejectNewest, test_logger(), gc.fn()); - pool.enqueue("blk0"); - pool.enqueue("blk1"); + pool.enqueue(job("blk0")); + pool.enqueue(job("blk1")); gc.wait_until_active(2); // both workers busy; queue empties for (int i = 0; i < 8; ++i) { // fill pending queue (queue_depth) - EXPECT_EQ(pool.enqueue("f" + std::to_string(i)).result, EnqueueResult::kAccepted); + EXPECT_EQ(pool.enqueue(job("f" + std::to_string(i))).result, EnqueueResult::kAccepted); } for (int i = 0; i < 5; ++i) { // overflow - EXPECT_EQ(pool.enqueue("x" + std::to_string(i)).result, EnqueueResult::kDroppedNewest); + EXPECT_EQ(pool.enqueue(job("x" + std::to_string(i))).result, EnqueueResult::kDroppedNewest); } EXPECT_EQ(pool.dropped_captures(), 5u); gc.release(); @@ -193,17 +202,17 @@ TEST_F(CaptureThreadPoolTest, RejectNewestDropsExcessExactly) { TEST_F(CaptureThreadPoolTest, DropOldestEvictsOldestPending) { GatedCallback gc; CaptureThreadPool pool(2, 8, QueueFullPolicy::kDropOldest, test_logger(), gc.fn()); - pool.enqueue("blk0"); - pool.enqueue("blk1"); + pool.enqueue(job("blk0")); + pool.enqueue(job("blk1")); gc.wait_until_active(2); for (int i = 0; i < 8; ++i) { - EXPECT_EQ(pool.enqueue("f" + std::to_string(i)).result, EnqueueResult::kAccepted); + EXPECT_EQ(pool.enqueue(job("f" + std::to_string(i))).result, EnqueueResult::kAccepted); } for (int i = 0; i < 5; ++i) { // each overflow evicts current oldest f0..f4 - auto outcome = pool.enqueue("x" + std::to_string(i)); + auto outcome = pool.enqueue(job("x" + std::to_string(i))); EXPECT_EQ(outcome.result, EnqueueResult::kEvictedOldest); - ASSERT_TRUE(outcome.evicted_code.has_value()); - EXPECT_EQ(*outcome.evicted_code, "f" + std::to_string(i)); + ASSERT_TRUE(outcome.evicted_id.has_value()); + EXPECT_EQ(outcome.evicted_id->fault_code, "f" + std::to_string(i)); } EXPECT_EQ(pool.dropped_captures(), 5u); gc.release(); @@ -221,10 +230,10 @@ TEST_F(CaptureThreadPoolTest, DropOldestEvictsOldestPending) { TEST_F(CaptureThreadPoolTest, FifoOrder) { GatedCallback gc; CaptureThreadPool pool(1, 8, QueueFullPolicy::kRejectNewest, test_logger(), gc.fn()); - pool.enqueue("A"); + pool.enqueue(job("A")); gc.wait_until_active(1); - pool.enqueue("B"); - pool.enqueue("C"); + pool.enqueue(job("B")); + pool.enqueue(job("C")); gc.release(); gc.wait_until_started(3); // A, B, C must all run before shutdown discards queue pool.shutdown(); @@ -236,8 +245,8 @@ TEST_F(CaptureThreadPoolTest, CallbackThrowKeepsWorkerAlive) { std::mutex m; std::condition_variable cv; int ran = 0; - auto fn = [&](const std::string & code) { - if (code == "boom") { + auto fn = [&](const FaultId & id) { + if (id.fault_code == "boom") { throw std::runtime_error("boom"); } { @@ -247,9 +256,9 @@ TEST_F(CaptureThreadPoolTest, CallbackThrowKeepsWorkerAlive) { cv.notify_all(); }; CaptureThreadPool pool(1, 8, QueueFullPolicy::kRejectNewest, test_logger(), fn); - pool.enqueue("boom"); - pool.enqueue("ok1"); - pool.enqueue("ok2"); + pool.enqueue(job("boom")); + pool.enqueue(job("ok1")); + pool.enqueue(job("ok2")); { std::unique_lock lock(m); cv.wait(lock, [&] { @@ -261,9 +270,9 @@ TEST_F(CaptureThreadPoolTest, CallbackThrowKeepsWorkerAlive) { } TEST_F(CaptureThreadPoolTest, EnqueueAfterShutdownRejected) { - CaptureThreadPool pool(1, 8, QueueFullPolicy::kRejectNewest, test_logger(), [](const std::string &) {}); + CaptureThreadPool pool(1, 8, QueueFullPolicy::kRejectNewest, test_logger(), [](const FaultId &) {}); pool.shutdown(); - auto outcome = pool.enqueue("x"); + auto outcome = pool.enqueue(job("x")); EXPECT_EQ(outcome.result, EnqueueResult::kRejectedShuttingDown); EXPECT_EQ(pool.dropped_captures(), 0u); } @@ -271,9 +280,9 @@ TEST_F(CaptureThreadPoolTest, EnqueueAfterShutdownRejected) { TEST_F(CaptureThreadPoolTest, ShutdownDiscardsPendingCompletesInFlight) { GatedCallback gc; CaptureThreadPool pool(1, 8, QueueFullPolicy::kRejectNewest, test_logger(), gc.fn()); - EXPECT_EQ(pool.enqueue("inflight").result, EnqueueResult::kAccepted); + EXPECT_EQ(pool.enqueue(job("inflight")).result, EnqueueResult::kAccepted); gc.wait_until_active(1); - EXPECT_EQ(pool.enqueue("pending").result, EnqueueResult::kAccepted); + EXPECT_EQ(pool.enqueue(job("pending")).result, EnqueueResult::kAccepted); EXPECT_EQ(pool.pending_size(), 1u); std::thread shut([&] { @@ -291,7 +300,7 @@ TEST_F(CaptureThreadPoolTest, ShutdownDiscardsPendingCompletesInFlight) { } TEST_F(CaptureThreadPoolTest, ShutdownIsIdempotent) { - CaptureThreadPool pool(2, 8, QueueFullPolicy::kRejectNewest, test_logger(), [](const std::string &) {}); + CaptureThreadPool pool(2, 8, QueueFullPolicy::kRejectNewest, test_logger(), [](const FaultId &) {}); pool.shutdown(); pool.shutdown(); // must not throw, hang, or double-join SUCCEED(); @@ -305,14 +314,14 @@ TEST_F(CaptureThreadPoolTest, ShutdownIsIdempotent) { TEST_F(CaptureThreadPoolTest, ConcurrentEnqueueDuringShutdownIsClean) { std::atomic ran{0}; std::atomic accepted{0}; - CaptureThreadPool pool(4, 64, QueueFullPolicy::kRejectNewest, test_logger(), [&ran](const std::string &) { + CaptureThreadPool pool(4, 64, QueueFullPolicy::kRejectNewest, test_logger(), [&ran](const FaultId &) { ran.fetch_add(1, std::memory_order_relaxed); }); std::atomic saw_shutdown{false}; std::thread producer([&] { for (int i = 0; i < 100000; ++i) { - const EnqueueResult r = pool.enqueue("j" + std::to_string(i)).result; + const EnqueueResult r = pool.enqueue(job("j" + std::to_string(i))).result; if (r == EnqueueResult::kAccepted) { accepted.fetch_add(1, std::memory_order_relaxed); } else if (r == EnqueueResult::kRejectedShuttingDown) { @@ -329,7 +338,7 @@ TEST_F(CaptureThreadPoolTest, ConcurrentEnqueueDuringShutdownIsClean) { // The producer must have crossed the shutdown transition (else the race was not // exercised), and every post-shutdown enqueue must reject cleanly. EXPECT_TRUE(saw_shutdown.load()); - EXPECT_EQ(pool.enqueue("after").result, EnqueueResult::kRejectedShuttingDown); + EXPECT_EQ(pool.enqueue(job("after")).result, EnqueueResult::kRejectedShuttingDown); // shutdown() discards pending work, so some accepted jobs may not run - but no // job may run that was never accepted. EXPECT_LE(ran.load(), accepted.load()); @@ -341,7 +350,7 @@ TEST_F(CaptureThreadPoolTest, ReleasesCaptureFnAfterShutdown) { auto sentinel = std::make_shared(42); { std::weak_ptr weak = sentinel; - CaptureThreadPool pool(2, 8, QueueFullPolicy::kRejectNewest, test_logger(), [held = sentinel](const std::string &) { + CaptureThreadPool pool(2, 8, QueueFullPolicy::kRejectNewest, test_logger(), [held = sentinel](const FaultId &) { (void)held; }); sentinel.reset(); // pool's capture_fn now holds the only strong ref diff --git a/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp b/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp index 3d803825b..b23fdb858 100644 --- a/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp +++ b/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp @@ -14,7 +14,9 @@ #include +#include #include +#include #include #include "ros2_medkit_fault_manager/correlation/config_parser.hpp" @@ -23,6 +25,27 @@ using namespace ros2_medkit_fault_manager::correlation; using namespace std::chrono_literals; +using ros2_medkit_fault_manager::FaultId; + +namespace { + +// The engine relates fault RECORDS, and every relation it forms is between records +// of the same owner. Most of this suite exercises the rule vocabulary, which is +// keyed by code, so those tests report everything from one owner. The tests that +// name a second owner are the ones checking that a relation does not cross owners. +constexpr const char * kOwner = "/reporter_a"; +constexpr const char * kOtherOwner = "/reporter_b"; + +FaultId rec(const std::string & code) { + return FaultId{code, kOwner}; +} + +FaultId other(const std::string & code) { + return FaultId{code, kOtherOwner}; +} + +} // namespace + class CorrelationEngineTest : public ::testing::Test { protected: void SetUp() override { @@ -114,7 +137,7 @@ TEST_F(CorrelationEngineTest, RootCauseRecognized) { auto config = create_hierarchical_config(); CorrelationEngine engine(config); - auto result = engine.process_fault("ESTOP_001", "CRITICAL"); + auto result = engine.process_fault(rec("ESTOP_001"), "CRITICAL"); EXPECT_FALSE(result.should_mute); EXPECT_TRUE(result.is_root_cause); @@ -126,11 +149,11 @@ TEST_F(CorrelationEngineTest, SymptomMuted) { CorrelationEngine engine(config); // First, report root cause - auto root_result = engine.process_fault("ESTOP_001", "CRITICAL"); + auto root_result = engine.process_fault(rec("ESTOP_001"), "CRITICAL"); EXPECT_TRUE(root_result.is_root_cause); // Then report symptom - auto symptom_result = engine.process_fault("MOTOR_COMM_FL", "ERROR"); + auto symptom_result = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); EXPECT_TRUE(symptom_result.should_mute); EXPECT_FALSE(symptom_result.is_root_cause); @@ -151,10 +174,10 @@ TEST_F(CorrelationEngineTest, MultipleSymptomsMuted) { auto config = create_hierarchical_config(); CorrelationEngine engine(config); - engine.process_fault("ESTOP_001", "CRITICAL"); - engine.process_fault("MOTOR_COMM_FL", "ERROR"); - engine.process_fault("MOTOR_COMM_FR", "ERROR"); - engine.process_fault("DRIVE_FAULT", "ERROR"); + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); + engine.process_fault(rec("MOTOR_COMM_FR"), "ERROR"); + engine.process_fault(rec("DRIVE_FAULT"), "ERROR"); EXPECT_EQ(3u, engine.get_muted_count()); } @@ -164,7 +187,7 @@ TEST_F(CorrelationEngineTest, SymptomBeforeRootCauseNotCorrelated) { CorrelationEngine engine(config); // Report symptom BEFORE root cause - auto symptom_result = engine.process_fault("MOTOR_COMM_FL", "ERROR"); + auto symptom_result = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); // Should NOT be muted (no root cause yet) EXPECT_FALSE(symptom_result.should_mute); @@ -177,11 +200,11 @@ TEST_F(CorrelationEngineTest, SymptomAfterWindowNotCorrelated) { CorrelationEngine engine(config); auto start = std::chrono::steady_clock::now(); - engine.process_fault("ESTOP_001", "CRITICAL", start); + engine.process_fault(rec("ESTOP_001"), "CRITICAL", start); // Report symptom AFTER window (window is 1000ms) auto after_window = start + 1500ms; - auto symptom_result = engine.process_fault("MOTOR_COMM_FL", "ERROR", after_window); + auto symptom_result = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", after_window); // Should NOT be muted (outside window) EXPECT_FALSE(symptom_result.should_mute); @@ -192,31 +215,66 @@ TEST_F(CorrelationEngineTest, ClearRootCauseClearsSymptoms) { auto config = create_hierarchical_config(); CorrelationEngine engine(config); - engine.process_fault("ESTOP_001", "CRITICAL"); - engine.process_fault("MOTOR_COMM_FL", "ERROR"); - engine.process_fault("MOTOR_COMM_FR", "ERROR"); + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); + engine.process_fault(rec("MOTOR_COMM_FR"), "ERROR"); EXPECT_EQ(2u, engine.get_muted_count()); // Clear root cause - auto clear_result = engine.process_clear("ESTOP_001"); + auto clear_result = engine.process_clear(rec("ESTOP_001")); - EXPECT_EQ(2u, clear_result.auto_cleared_codes.size()); - EXPECT_NE(std::find(clear_result.auto_cleared_codes.begin(), clear_result.auto_cleared_codes.end(), "MOTOR_COMM_FL"), - clear_result.auto_cleared_codes.end()); - EXPECT_NE(std::find(clear_result.auto_cleared_codes.begin(), clear_result.auto_cleared_codes.end(), "MOTOR_COMM_FR"), - clear_result.auto_cleared_codes.end()); + EXPECT_EQ(2u, clear_result.auto_cleared_symptoms.size()); + const auto & symptoms = clear_result.auto_cleared_symptoms; + EXPECT_NE(std::find(symptoms.begin(), symptoms.end(), rec("MOTOR_COMM_FL")), symptoms.end()); + EXPECT_NE(std::find(symptoms.begin(), symptoms.end(), rec("MOTOR_COMM_FR")), symptoms.end()); - // Muted faults should be cleared + // Muted records should be cleared EXPECT_EQ(0u, engine.get_muted_count()); } +// A root cause explains its own reporter's faults, not everyone's. Without the owner +// on the relation, the first reporter of a root-cause code would mute and auto-clear +// every other reporter's identical symptom. +TEST_F(CorrelationEngineTest, RootCauseDoesNotMuteAnotherOwnersSymptom) { + auto config = create_hierarchical_config(); + CorrelationEngine engine(config); + + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + + const auto same_owner = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); + const auto other_owner = engine.process_fault(other("MOTOR_COMM_FL"), "ERROR"); + + EXPECT_TRUE(same_owner.should_mute); + EXPECT_FALSE(other_owner.should_mute) << "a root cause of one owner must not mute another owner's record"; + EXPECT_TRUE(engine.is_muted(rec("MOTOR_COMM_FL"))); + EXPECT_FALSE(engine.is_muted(other("MOTOR_COMM_FL"))); + EXPECT_EQ(1u, engine.get_muted_count()); +} + +// The auto-clear cascade is the same relation seen from the clear side. +TEST_F(CorrelationEngineTest, ClearRootCauseLeavesAnotherOwnersSymptomAlone) { + auto config = create_hierarchical_config(); + CorrelationEngine engine(config); + + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + engine.process_fault(other("ESTOP_001"), "CRITICAL"); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); + engine.process_fault(other("MOTOR_COMM_FL"), "ERROR"); + + const auto clear_result = engine.process_clear(rec("ESTOP_001")); + + ASSERT_EQ(1u, clear_result.auto_cleared_symptoms.size()); + EXPECT_EQ(rec("MOTOR_COMM_FL"), clear_result.auto_cleared_symptoms.front()); + EXPECT_TRUE(engine.is_muted(other("MOTOR_COMM_FL"))) << "the other owner's symptom stays muted by its own root cause"; +} + TEST_F(CorrelationEngineTest, UnrelatedFaultNotCorrelated) { auto config = create_hierarchical_config(); CorrelationEngine engine(config); - engine.process_fault("ESTOP_001", "CRITICAL"); - auto result = engine.process_fault("SENSOR_TIMEOUT", "ERROR"); + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + auto result = engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR"); // SENSOR_TIMEOUT doesn't match motor_errors or drive_faults patterns EXPECT_FALSE(result.should_mute); @@ -242,19 +300,19 @@ TEST_F(CorrelationEngineTest, InlineSymptomCodesMuted) { CorrelationEngine engine(config); // Report root cause - auto root_result = engine.process_fault("ESTOP_001", "CRITICAL"); + auto root_result = engine.process_fault(rec("ESTOP_001"), "CRITICAL"); EXPECT_TRUE(root_result.is_root_cause); // Report symptoms matching inline codes - auto motor_result = engine.process_fault("MOTOR_COMM_FL", "ERROR"); + auto motor_result = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); EXPECT_TRUE(motor_result.should_mute); EXPECT_EQ("ESTOP_001", motor_result.root_cause_code); - auto drive_result = engine.process_fault("DRIVE_FAULT", "ERROR"); + auto drive_result = engine.process_fault(rec("DRIVE_FAULT"), "ERROR"); EXPECT_TRUE(drive_result.should_mute); // Unrelated fault should not be muted - auto sensor_result = engine.process_fault("SENSOR_ERROR", "ERROR"); + auto sensor_result = engine.process_fault(rec("SENSOR_ERROR"), "ERROR"); EXPECT_FALSE(sensor_result.should_mute); EXPECT_EQ(2u, engine.get_muted_count()); @@ -282,13 +340,13 @@ TEST_F(CorrelationEngineTest, MixedPatternAndInlineSymptoms) { auto config = parse_config_string(yaml); CorrelationEngine engine(config); - engine.process_fault("ESTOP_001", "CRITICAL"); + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); // Both pattern-matched and inline-matched faults should be muted - auto sensor_result = engine.process_fault("SENSOR_TIMEOUT", "ERROR"); + auto sensor_result = engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR"); EXPECT_TRUE(sensor_result.should_mute); - auto motor_result = engine.process_fault("MOTOR_FAULT", "ERROR"); + auto motor_result = engine.process_fault(rec("MOTOR_FAULT"), "ERROR"); EXPECT_TRUE(motor_result.should_mute); EXPECT_EQ(2u, engine.get_muted_count()); @@ -303,16 +361,16 @@ TEST_F(CorrelationEngineTest, AutoClusterTriggersAtMinCount) { CorrelationEngine engine(config); // Report faults - need 3 for cluster - auto r1 = engine.process_fault("MOTOR_COMM_FL", "ERROR"); + auto r1 = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); EXPECT_FALSE(r1.should_mute); EXPECT_EQ(0u, engine.get_cluster_count()); - auto r2 = engine.process_fault("SENSOR_TIMEOUT", "ERROR"); + auto r2 = engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR"); EXPECT_FALSE(r2.should_mute); EXPECT_EQ(0u, engine.get_cluster_count()); // Third fault triggers cluster - auto r3 = engine.process_fault("DRIVE_COMM_ERROR", "WARNING"); + auto r3 = engine.process_fault(rec("DRIVE_COMM_ERROR"), "WARNING"); // Third fault should be muted (show_as_single=true) EXPECT_TRUE(r3.should_mute); EXPECT_EQ(1u, engine.get_cluster_count()); @@ -322,8 +380,8 @@ TEST_F(CorrelationEngineTest, AutoClusterNotTriggeredBelowMinCount) { auto config = create_auto_cluster_config(); CorrelationEngine engine(config); - engine.process_fault("MOTOR_COMM_FL", "ERROR"); - engine.process_fault("SENSOR_TIMEOUT", "ERROR"); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR"); // Only 2 faults, need 3 EXPECT_EQ(0u, engine.get_cluster_count()); @@ -333,9 +391,9 @@ TEST_F(CorrelationEngineTest, AutoClusterHighestSeverityRepresentative) { auto config = create_auto_cluster_config(); CorrelationEngine engine(config); - engine.process_fault("MOTOR_COMM_FL", "WARNING"); - engine.process_fault("SENSOR_TIMEOUT", "CRITICAL"); // Higher severity - engine.process_fault("DRIVE_COMM_ERROR", "ERROR"); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING"); + engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL"); // Higher severity + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR"); auto clusters = engine.get_clusters(); ASSERT_EQ(1u, clusters.size()); @@ -361,8 +419,8 @@ TEST_F(CorrelationEngineTest, AutoClusterFirstRepresentative) { auto config = parse_config_string(yaml); CorrelationEngine engine(config); - engine.process_fault("FIRST_ERROR", "WARNING"); - engine.process_fault("SECOND_ERROR", "CRITICAL"); + engine.process_fault(rec("FIRST_ERROR"), "WARNING"); + engine.process_fault(rec("SECOND_ERROR"), "CRITICAL"); auto clusters = engine.get_clusters(); ASSERT_EQ(1u, clusters.size()); @@ -375,11 +433,11 @@ TEST_F(CorrelationEngineTest, AutoClusterWindowExpires) { auto start = std::chrono::steady_clock::now(); - engine.process_fault("MOTOR_COMM_FL", "ERROR", start); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", start + 100ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", start); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", start + 100ms); // Third fault after window (500ms) - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", start + 600ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", start + 600ms); // Cluster should have been reset, so still not at min_count // Actually the third fault starts a new pending cluster @@ -394,10 +452,10 @@ TEST_F(CorrelationEngineTest, CleanupExpiredRemovesPendingRootCauses) { auto past = std::chrono::steady_clock::now() - std::chrono::milliseconds(2000); // Report root cause with old timestamp - engine.process_fault("ESTOP_001", "CRITICAL", past); + engine.process_fault(rec("ESTOP_001"), "CRITICAL", past); // Symptom reported NOW should NOT be correlated (root cause expired) - auto result = engine.process_fault("MOTOR_COMM_FL", "ERROR"); + auto result = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); EXPECT_FALSE(result.should_mute); // Not muted - root cause window expired EXPECT_EQ(0u, engine.get_muted_count()); @@ -405,7 +463,7 @@ TEST_F(CorrelationEngineTest, CleanupExpiredRemovesPendingRootCauses) { engine.cleanup_expired(); // Another symptom should also not be correlated - auto result2 = engine.process_fault("MOTOR_TIMEOUT_RR", "ERROR"); + auto result2 = engine.process_fault(rec("MOTOR_TIMEOUT_RR"), "ERROR"); EXPECT_FALSE(result2.should_mute); EXPECT_EQ(0u, engine.get_muted_count()); } @@ -416,8 +474,8 @@ TEST_F(CorrelationEngineTest, CleanupExpiredRemovesPendingClusters) { // Create pending cluster with old timestamp auto past = std::chrono::steady_clock::now() - std::chrono::milliseconds(1000); - engine.process_fault("MOTOR_COMM_FL", "ERROR", past); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", past + 10ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", past); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", past + 10ms); // Cluster should not be active (only 2 faults, need 3) EXPECT_EQ(0u, engine.get_cluster_count()); @@ -426,7 +484,7 @@ TEST_F(CorrelationEngineTest, CleanupExpiredRemovesPendingClusters) { engine.cleanup_expired(); // New fault should start fresh pending cluster, not join expired one - auto result = engine.process_fault("DRIVE_COMM_ERROR", "ERROR"); + auto result = engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR"); EXPECT_FALSE(result.should_mute); // First in new cluster EXPECT_EQ(0u, engine.get_cluster_count()); // Still not enough } @@ -437,9 +495,9 @@ TEST_F(CorrelationEngineTest, CleanupExpiredDoesNotBreakActiveCluster) { // Create an active cluster with old timestamps (past the 500ms window) auto past = std::chrono::steady_clock::now() - std::chrono::milliseconds(1000); - engine.process_fault("MOTOR_COMM_FL", "ERROR", past); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", past + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "WARNING", past + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", past); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", past + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "WARNING", past + 20ms); // Cluster should be active (3 faults >= min_count=3) EXPECT_EQ(1u, engine.get_cluster_count()); @@ -452,7 +510,7 @@ TEST_F(CorrelationEngineTest, CleanupExpiredDoesNotBreakActiveCluster) { EXPECT_EQ(1u, engine.get_cluster_count()); // process_clear must still find the active cluster via fault_to_cluster_ - engine.process_clear("MOTOR_COMM_FL"); + engine.process_clear(rec("MOTOR_COMM_FL")); // Cluster should still exist with 2 remaining fault codes and updated representative EXPECT_EQ(1u, engine.get_cluster_count()); @@ -472,14 +530,14 @@ TEST_F(CorrelationEngineTest, HierarchicalAndClusterCoexist) { CorrelationEngine engine(config); // Hierarchical: ESTOP_001 -> MOTOR_* - engine.process_fault("ESTOP_001", "CRITICAL"); - auto motor_result = engine.process_fault("MOTOR_COMM_FL", "ERROR"); + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + auto motor_result = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); EXPECT_TRUE(motor_result.should_mute); EXPECT_EQ(1u, engine.get_muted_count()); // Auto-cluster: SENSOR_* (need 2) - engine.process_fault("SENSOR_LIDAR", "ERROR"); - engine.process_fault("SENSOR_IMU", "ERROR"); + engine.process_fault(rec("SENSOR_LIDAR"), "ERROR"); + engine.process_fault(rec("SENSOR_IMU"), "ERROR"); EXPECT_EQ(1u, engine.get_cluster_count()); // Both should coexist @@ -495,9 +553,9 @@ TEST_F(CorrelationEngineTest, DuplicateFaultCode) { auto config = create_hierarchical_config(); CorrelationEngine engine(config); - engine.process_fault("ESTOP_001", "CRITICAL"); - engine.process_fault("MOTOR_COMM_FL", "ERROR"); - engine.process_fault("MOTOR_COMM_FL", "ERROR"); // Duplicate + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); // Duplicate // Should still only have 1 muted fault (no duplicate) // Note: current implementation allows duplicates in symptoms list @@ -521,9 +579,9 @@ TEST_F(CorrelationEngineTest, ClearNonExistentFault) { CorrelationEngine engine(config); // Clear fault that was never reported - auto result = engine.process_clear("NONEXISTENT"); + auto result = engine.process_clear(rec("NONEXISTENT")); - EXPECT_TRUE(result.auto_cleared_codes.empty()); + EXPECT_TRUE(result.auto_cleared_symptoms.empty()); } TEST_F(CorrelationEngineTest, AutoClusterRetroactiveMuting) { @@ -550,18 +608,18 @@ TEST_F(CorrelationEngineTest, AutoClusterRetroactiveMuting) { auto t0 = std::chrono::steady_clock::now(); // Fault #1 - representative (FIRST policy) - auto result1 = engine.process_fault("SENSOR_001", "ERROR", t0); + auto result1 = engine.process_fault(rec("SENSOR_001"), "ERROR", t0); EXPECT_FALSE(result1.should_mute); // First fault is representative EXPECT_FALSE(result1.cluster_id.empty()); EXPECT_TRUE(result1.retroactive_mute_codes.empty()); // Cluster not active yet // Fault #2 - not muted because cluster not active - auto result2 = engine.process_fault("SENSOR_002", "ERROR", t0 + std::chrono::milliseconds(10)); + auto result2 = engine.process_fault(rec("SENSOR_002"), "ERROR", t0 + std::chrono::milliseconds(10)); EXPECT_FALSE(result2.should_mute); // Cluster still not active EXPECT_TRUE(result2.retroactive_mute_codes.empty()); // Fault #3 - triggers cluster activation (min_count=3) - auto result3 = engine.process_fault("SENSOR_003", "ERROR", t0 + std::chrono::milliseconds(20)); + auto result3 = engine.process_fault(rec("SENSOR_003"), "ERROR", t0 + std::chrono::milliseconds(20)); EXPECT_TRUE(result3.should_mute); // #3 is muted (not representative) EXPECT_EQ(1u, result3.retroactive_mute_codes.size()); // #2 should be retroactively muted EXPECT_EQ("SENSOR_002", result3.retroactive_mute_codes[0]); @@ -584,16 +642,16 @@ TEST_F(CorrelationEngineTest, ClearFaultRemovesFromPendingCluster) { auto t0 = std::chrono::steady_clock::now(); // Add 2 faults (below min_count=3), creating a pending cluster - engine.process_fault("MOTOR_COMM_FL", "ERROR", t0); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", t0 + 10ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", t0 + 10ms); EXPECT_EQ(0u, engine.get_cluster_count()); // Still pending // Clear one of the faults - engine.process_clear("MOTOR_COMM_FL"); + engine.process_clear(rec("MOTOR_COMM_FL")); // Now add a third fault - should NOT activate the cluster because // the cleared fault was removed from pending, so only 2 faults total - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", t0 + 20ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", t0 + 20ms); EXPECT_EQ(0u, engine.get_cluster_count()); // Still not enough } @@ -620,15 +678,15 @@ TEST_F(CorrelationEngineTest, ClearRepresentativeReassignsPendingCluster) { auto t0 = std::chrono::steady_clock::now(); // Add 2 faults, creating a pending cluster - engine.process_fault("FIRST_ERROR", "CRITICAL", t0); - engine.process_fault("SECOND_ERROR", "ERROR", t0 + 10ms); + engine.process_fault(rec("FIRST_ERROR"), "CRITICAL", t0); + engine.process_fault(rec("SECOND_ERROR"), "ERROR", t0 + 10ms); // Clear the representative (first fault) - engine.process_clear("FIRST_ERROR"); + engine.process_clear(rec("FIRST_ERROR")); // Add 2 more faults to reach min_count (SECOND_ERROR + 2 new = 3) - engine.process_fault("THIRD_ERROR", "ERROR", t0 + 20ms); - engine.process_fault("FOURTH_ERROR", "ERROR", t0 + 30ms); + engine.process_fault(rec("THIRD_ERROR"), "ERROR", t0 + 20ms); + engine.process_fault(rec("FOURTH_ERROR"), "ERROR", t0 + 30ms); EXPECT_EQ(1u, engine.get_cluster_count()); auto clusters = engine.get_clusters(); @@ -645,16 +703,16 @@ TEST_F(CorrelationEngineTest, ClearAllFaultsRemovesPendingCluster) { auto t0 = std::chrono::steady_clock::now(); // Add 2 faults - engine.process_fault("MOTOR_COMM_FL", "ERROR", t0); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", t0 + 10ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", t0 + 10ms); // Clear both - engine.process_clear("MOTOR_COMM_FL"); - engine.process_clear("SENSOR_TIMEOUT"); + engine.process_clear(rec("MOTOR_COMM_FL")); + engine.process_clear(rec("SENSOR_TIMEOUT")); // Adding 2 new faults should start a fresh pending cluster, not join old one - engine.process_fault("DRIVE_COMM_NEW", "ERROR", t0 + 100ms); - engine.process_fault("MOTOR_COMM_NEW", "ERROR", t0 + 110ms); + engine.process_fault(rec("DRIVE_COMM_NEW"), "ERROR", t0 + 100ms); + engine.process_fault(rec("MOTOR_COMM_NEW"), "ERROR", t0 + 110ms); EXPECT_EQ(0u, engine.get_cluster_count()); // Still 2 faults, below min_count=3 } @@ -684,9 +742,9 @@ TEST_F(CorrelationEngineTest, ClearMostRecentRepresentativeReassignsActiveCluste auto t0 = std::chrono::steady_clock::now(); - engine.process_fault("FIRST_ERROR", "WARNING", t0); - engine.process_fault("SECOND_ERROR", "ERROR", t0 + 10ms); - engine.process_fault("THIRD_ERROR", "CRITICAL", t0 + 20ms); + engine.process_fault(rec("FIRST_ERROR"), "WARNING", t0); + engine.process_fault(rec("SECOND_ERROR"), "ERROR", t0 + 10ms); + engine.process_fault(rec("THIRD_ERROR"), "CRITICAL", t0 + 20ms); EXPECT_EQ(1u, engine.get_cluster_count()); auto clusters = engine.get_clusters(); @@ -694,7 +752,7 @@ TEST_F(CorrelationEngineTest, ClearMostRecentRepresentativeReassignsActiveCluste EXPECT_EQ("THIRD_ERROR", clusters[0].representative_code); EXPECT_EQ("CRITICAL", clusters[0].representative_severity); - engine.process_clear("THIRD_ERROR"); + engine.process_clear(rec("THIRD_ERROR")); clusters = engine.get_clusters(); ASSERT_EQ(1u, clusters.size()); @@ -713,9 +771,9 @@ TEST_F(CorrelationEngineTest, ClearHighestSeverityRepresentativeReassigns) { auto t0 = std::chrono::steady_clock::now(); // WARNING, CRITICAL (rep), ERROR - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "CRITICAL", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", t0 + 20ms); // SENSOR_TIMEOUT is representative (highest severity) auto clusters = engine.get_clusters(); @@ -723,7 +781,7 @@ TEST_F(CorrelationEngineTest, ClearHighestSeverityRepresentativeReassigns) { EXPECT_EQ("SENSOR_TIMEOUT", clusters[0].representative_code); // Clear the representative - engine.process_clear("SENSOR_TIMEOUT"); + engine.process_clear(rec("SENSOR_TIMEOUT")); // Remaining: WARNING, ERROR -> ERROR should become new representative clusters = engine.get_clusters(); @@ -742,8 +800,8 @@ TEST_F(CorrelationEngineTest, CleanupExpiredRemovesFaultToClusterEntries) { // Create pending cluster with old timestamp auto past = std::chrono::steady_clock::now() - std::chrono::milliseconds(1000); - engine.process_fault("MOTOR_COMM_FL", "ERROR", past); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", past + 10ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", past); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", past + 10ms); EXPECT_EQ(0u, engine.get_cluster_count()); // Cleanup removes expired pending cluster @@ -753,9 +811,9 @@ TEST_F(CorrelationEngineTest, CleanupExpiredRemovesFaultToClusterEntries) { // If fault_to_cluster_ was NOT cleaned, these would try to join // a non-existent cluster instead of creating a new one. auto t1 = std::chrono::steady_clock::now(); - engine.process_fault("MOTOR_COMM_FL", "ERROR", t1); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", t1 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", t1 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", t1); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", t1 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", t1 + 20ms); // Should form a NEW cluster with all 3 faults EXPECT_EQ(1u, engine.get_cluster_count()); @@ -771,18 +829,18 @@ TEST_F(CorrelationEngineTest, ClearRepresentativeAfterNewWindowReplacesPendingRe auto t0 = std::chrono::steady_clock::now(); // Activate first cluster: SENSOR_TIMEOUT (CRITICAL) is representative - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "CRITICAL", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", t0 + 20ms); ASSERT_EQ(1u, engine.get_cluster_count()); // After window_ms (500ms), a new fault starts a fresh pending cluster for the same rule, // replacing pending_clusters_["comm_storm"] - engine.process_fault("OTHER_COMM_ERROR", "WARNING", t0 + 600ms); + engine.process_fault(rec("OTHER_COMM_ERROR"), "WARNING", t0 + 600ms); // Clearing the representative of the first active cluster must still reassign // its representative to the next-highest remaining fault (DRIVE_COMM_ERROR) - engine.process_clear("SENSOR_TIMEOUT"); + engine.process_clear(rec("SENSOR_TIMEOUT")); auto clusters = engine.get_clusters(); ASSERT_EQ(1u, clusters.size()); @@ -798,20 +856,20 @@ TEST_F(CorrelationEngineTest, ReReportRepresentativeAfterWindowAndClearReassigns auto t0 = std::chrono::steady_clock::now(); // Activate cluster: SENSOR_TIMEOUT (CRITICAL) is representative - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "CRITICAL", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", t0 + 20ms); ASSERT_EQ(1u, engine.get_cluster_count()); const auto active_id = engine.get_clusters()[0].cluster_id; // Re-report the representative after window_ms (500ms); it must stay in the active cluster // rather than starting a new pending cluster and overwriting fault_to_cluster_ - auto rereport = engine.process_fault("SENSOR_TIMEOUT", "CRITICAL", t0 + 600ms); + auto rereport = engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL", t0 + 600ms); EXPECT_EQ(active_id, rereport.cluster_id); EXPECT_FALSE(rereport.should_mute); // Clearing the representative must still update the active cluster - engine.process_clear("SENSOR_TIMEOUT"); + engine.process_clear(rec("SENSOR_TIMEOUT")); auto clusters = engine.get_clusters(); ASSERT_EQ(1u, clusters.size()); @@ -827,18 +885,18 @@ TEST_F(CorrelationEngineTest, ClearBelowMinCountInWindowThenNewFaultJoinsUpdates auto t0 = std::chrono::steady_clock::now(); // Activate cluster with 3 faults: SENSOR_TIMEOUT (CRITICAL) is representative - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "CRITICAL", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "WARNING", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "WARNING", t0 + 20ms); ASSERT_EQ(1u, engine.get_cluster_count()); // Clear 2 members inside the window so the cluster holds only 1 code (< min_count) - engine.process_clear("MOTOR_COMM_FL"); - engine.process_clear("DRIVE_COMM_ERROR"); + engine.process_clear(rec("MOTOR_COMM_FL")); + engine.process_clear(rec("DRIVE_COMM_ERROR")); ASSERT_EQ(1u, engine.get_cluster_count()); // A new code joins inside the window while cluster size (2) is still below min_count (3) - auto join_res = engine.process_fault("NEW_COMM_ERROR", "ERROR", t0 + 50ms); + auto join_res = engine.process_fault(rec("NEW_COMM_ERROR"), "ERROR", t0 + 50ms); EXPECT_TRUE(join_res.should_mute); auto clusters = engine.get_clusters(); @@ -846,7 +904,7 @@ TEST_F(CorrelationEngineTest, ClearBelowMinCountInWindowThenNewFaultJoinsUpdates EXPECT_EQ(2u, clusters[0].fault_codes.size()); // Clearing the original representative must reassign to the newly joined code - engine.process_clear("SENSOR_TIMEOUT"); + engine.process_clear(rec("SENSOR_TIMEOUT")); clusters = engine.get_clusters(); ASSERT_EQ(1u, clusters.size()); @@ -860,12 +918,12 @@ TEST_F(CorrelationEngineTest, ReReportActiveMemberWithHigherSeverityInWindowBeco CorrelationEngine engine(config); auto t0 = std::chrono::steady_clock::now(); - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "WARNING", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "WARNING", t0 + 20ms); ASSERT_EQ(1u, engine.get_cluster_count()); - auto res = engine.process_fault("MOTOR_COMM_FL", "CRITICAL", t0 + 30ms); + auto res = engine.process_fault(rec("MOTOR_COMM_FL"), "CRITICAL", t0 + 30ms); EXPECT_FALSE(res.should_mute); auto clusters = engine.get_clusters(); @@ -879,12 +937,12 @@ TEST_F(CorrelationEngineTest, ReReportActiveMemberWithHigherSeverityAfterWindowB CorrelationEngine engine(config); auto t0 = std::chrono::steady_clock::now(); - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "WARNING", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "WARNING", t0 + 20ms); ASSERT_EQ(1u, engine.get_cluster_count()); - auto res = engine.process_fault("MOTOR_COMM_FL", "CRITICAL", t0 + 600ms); + auto res = engine.process_fault(rec("MOTOR_COMM_FL"), "CRITICAL", t0 + 600ms); EXPECT_FALSE(res.should_mute); auto clusters = engine.get_clusters(); @@ -898,12 +956,12 @@ TEST_F(CorrelationEngineTest, ReReportRepresentativeWithLowerSeverityHandsOverRe CorrelationEngine engine(config); auto t0 = std::chrono::steady_clock::now(); - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "CRITICAL", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", t0 + 20ms); ASSERT_EQ(1u, engine.get_cluster_count()); - auto res = engine.process_fault("SENSOR_TIMEOUT", "WARNING", t0 + 600ms); + auto res = engine.process_fault(rec("SENSOR_TIMEOUT"), "WARNING", t0 + 600ms); EXPECT_TRUE(res.should_mute); auto clusters = engine.get_clusters(); diff --git a/src/ros2_medkit_fault_manager/test/test_entity_thresholds.cpp b/src/ros2_medkit_fault_manager/test/test_entity_thresholds.cpp index 36fe66355..1e96a1071 100644 --- a/src/ros2_medkit_fault_manager/test/test_entity_thresholds.cpp +++ b/src/ros2_medkit_fault_manager/test/test_entity_thresholds.cpp @@ -289,7 +289,7 @@ TEST_F(PerEntityStorageTest, LidarFaultConfirmsImmediately) { storage_.report_fault_event("LIDAR_FAIL", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Lidar failure", "/sensors/lidar", ts.nanoseconds(), lidar_config_); - auto fault = storage_.get_fault("LIDAR_FAIL"); + auto fault = storage_.get_fault({"LIDAR_FAIL", "/sensors/lidar"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -302,7 +302,7 @@ TEST_F(PerEntityStorageTest, MotorFaultNeedsDebouncing) { storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor hot", "/powertrain/motor_left", ts.nanoseconds(), motor_config_); - auto fault = storage_.get_fault("MOTOR_OVERHEAT"); + auto fault = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor_left"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); @@ -311,13 +311,13 @@ TEST_F(PerEntityStorageTest, MotorFaultNeedsDebouncing) { storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", "/powertrain/motor_left", clock_.now().nanoseconds(), motor_config_); } - fault = storage_.get_fault("MOTOR_OVERHEAT"); + fault = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor_left"}); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Event 5: counter = -5, meets threshold -> CONFIRMED storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", "/powertrain/motor_left", clock_.now().nanoseconds(), motor_config_); - fault = storage_.get_fault("MOTOR_OVERHEAT"); + fault = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor_left"}); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -327,27 +327,38 @@ TEST_F(PerEntityStorageTest, UnknownEntityUsesGlobalConfig) { storage_.report_fault_event("UNKNOWN_FAULT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Unknown", "/some/unknown/entity", ts.nanoseconds(), global_); - auto fault = storage_.get_fault("UNKNOWN_FAULT"); + auto fault = storage_.get_fault({"UNKNOWN_FAULT", "/some/unknown/entity"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Global threshold=-1 -> immediate } // @verifies REQ_INTEROP_095 -TEST_F(PerEntityStorageTest, DifferentEntitiesSameFaultCode) { +// Two entities reporting one code are two records, each evaluated against its own +// entity's band. This used to be one row with one shared counter, so the lidar's +// single report inherited the motor's decrement and confirmed a fault the motor's +// own policy said was still debouncing. +TEST_F(PerEntityStorageTest, DifferentEntitiesSameFaultCodeDebounceIndependently) { auto ts = clock_.now(); - // Motor reports with motor config (debounced) + // Motor reports with motor config (debounced, threshold -5) storage_.report_fault_event("OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheat", "/powertrain/motor_left", ts.nanoseconds(), motor_config_); - auto fault = storage_.get_fault("OVERHEAT"); - EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Motor needs 5 events - // Lidar reports same fault_code with lidar config (threshold=-1) - // Counter is now -2, lidar threshold is -1, -2 <= -1 -> CONFIRMED + // Lidar reports the same fault_code with lidar config (threshold -1) storage_.report_fault_event("OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Lidar overheat", "/sensors/lidar", clock_.now().nanoseconds(), lidar_config_); - fault = storage_.get_fault("OVERHEAT"); - EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Lidar config confirms at -1, counter is -2 + + auto motor = storage_.get_fault({"OVERHEAT", "/powertrain/motor_left"}); + ASSERT_TRUE(motor.has_value()); + EXPECT_EQ(motor->status, Fault::STATUS_PREFAILED) << "the motor's own record is one report into a band of five"; + EXPECT_EQ(motor->description, "Motor overheat"); + + auto lidar = storage_.get_fault({"OVERHEAT", "/sensors/lidar"}); + ASSERT_TRUE(lidar.has_value()); + EXPECT_EQ(lidar->status, Fault::STATUS_CONFIRMED) << "the lidar's own record confirms on its first report"; + EXPECT_EQ(lidar->description, "Lidar overheat"); + + EXPECT_EQ(storage_.size(), 2u); } // @verifies REQ_INTEROP_095 @@ -357,7 +368,7 @@ TEST_F(PerEntityStorageTest, LidarHealingWithThreshold1) { // Confirm fault storage_.report_fault_event("LIDAR_FAIL", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Lidar failure", "/sensors/lidar", ts.nanoseconds(), lidar_config_); - auto fault = storage_.get_fault("LIDAR_FAIL"); + auto fault = storage_.get_fault({"LIDAR_FAIL", "/sensors/lidar"}); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // One PASSED event should heal (healing_threshold=1 for lidar) @@ -366,7 +377,7 @@ TEST_F(PerEntityStorageTest, LidarHealingWithThreshold1) { clock_.now().nanoseconds(), lidar_config_); storage_.report_fault_event("LIDAR_FAIL", fd::ReportKind::kPassed, 0, "", "/sensors/lidar", clock_.now().nanoseconds(), lidar_config_); - fault = storage_.get_fault("LIDAR_FAIL"); + fault = storage_.get_fault({"LIDAR_FAIL", "/sensors/lidar"}); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); } @@ -377,7 +388,7 @@ TEST_F(PerEntityStorageTest, BackwardCompatibleWithoutResolver) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", ts.nanoseconds(), global_); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Global threshold=-1 } diff --git a/src/ros2_medkit_fault_manager/test/test_fault_code_thresholds.cpp b/src/ros2_medkit_fault_manager/test/test_fault_code_thresholds.cpp index a32295959..a031abc9c 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_code_thresholds.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_code_thresholds.cpp @@ -33,6 +33,7 @@ using ros2_medkit_fault_manager::EntityDebounceOverride; using ros2_medkit_fault_manager::EntityThresholdResolver; using ros2_medkit_fault_manager::FaultCodeDebounceOverride; using ros2_medkit_fault_manager::FaultCodeThresholdResolver; +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::InMemoryFaultStorage; using ros2_medkit_msgs::msg::Fault; using ros2_medkit_msgs::srv::ReportFault; @@ -456,27 +457,32 @@ class FaultCodeStorageTest : public ::testing::Test { }; // @verifies REQ_INTEROP_107 -TEST_F(FaultCodeStorageTest, LidarNoLongerConfirmsTheMotorsCodeEarly) { - // Without the fault-code override, lidar's threshold of -1 would confirm this - // on the second event (the counter is shared). With it, both sources debounce - // at -3. - report("OVERHEAT", "/powertrain/motor/left"); - EXPECT_EQ(storage_.get_fault("OVERHEAT")->status, Fault::STATUS_PREFAILED); - - report("OVERHEAT", "/sensors/lidar/front"); - EXPECT_EQ(storage_.get_fault("OVERHEAT")->status, Fault::STATUS_PREFAILED); - - report("OVERHEAT", "/sensors/lidar/front"); - EXPECT_EQ(storage_.get_fault("OVERHEAT")->status, Fault::STATUS_CONFIRMED); +TEST_F(FaultCodeStorageTest, TheCodeOverrideGovernsEachOwnersRecord) { + // Without the fault-code override, lidar's threshold of -1 would confirm its record on + // its first event. With it, the record of each owner debounces at -3, on its own counter. + const FaultId motor{"OVERHEAT", "/powertrain/motor/left"}; + const FaultId lidar{"OVERHEAT", "/sensors/lidar/front"}; + + report(motor.fault_code, motor.owner); + EXPECT_EQ(storage_.get_fault(motor)->status, Fault::STATUS_PREFAILED); + + report(lidar.fault_code, lidar.owner); + EXPECT_EQ(storage_.get_fault(lidar)->status, Fault::STATUS_PREFAILED); + report(lidar.fault_code, lidar.owner); + EXPECT_EQ(storage_.get_fault(lidar)->status, Fault::STATUS_PREFAILED); + report(lidar.fault_code, lidar.owner); + EXPECT_EQ(storage_.get_fault(lidar)->status, Fault::STATUS_CONFIRMED); + + EXPECT_EQ(storage_.get_fault(motor)->status, Fault::STATUS_PREFAILED) << "the motor's own counter is at 1 of 3"; } // @verifies REQ_INTEROP_107 TEST_F(FaultCodeStorageTest, ACodeWithoutAnOverrideKeepsEntityBehaviour) { report("MOTOR_STALL", "/powertrain/motor/left"); - EXPECT_EQ(storage_.get_fault("MOTOR_STALL")->status, Fault::STATUS_PREFAILED); + EXPECT_EQ(storage_.get_fault({"MOTOR_STALL", "/powertrain/motor/left"})->status, Fault::STATUS_PREFAILED); report("LIDAR_BLOCKED", "/sensors/lidar/front"); - EXPECT_EQ(storage_.get_fault("LIDAR_BLOCKED")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_.get_fault({"LIDAR_BLOCKED", "/sensors/lidar/front"})->status, Fault::STATUS_CONFIRMED); } // The startup pass flips HEALED to CLEARED only for codes whose healing is off. @@ -488,10 +494,10 @@ TEST_F(FaultCodeStorageTest, ReclassifyKeepsCodesThatStillHeal) { for (const std::string code : {"KEEPS.HEALING", "STOPPED.HEALING"}) { storage_.report_fault_event(code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", "/src", clock_.now().nanoseconds(), healing); - for (int i = 0; i < 3 && storage_.get_fault(code)->status != Fault::STATUS_HEALED; ++i) { + for (int i = 0; i < 3 && storage_.get_fault({code, "/src"})->status != Fault::STATUS_HEALED; ++i) { storage_.report_fault_event(code, fd::ReportKind::kPassed, 0, "", "/src", clock_.now().nanoseconds(), healing); } - ASSERT_EQ(storage_.get_fault(code)->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_.get_fault({code, "/src"})->status, Fault::STATUS_HEALED); } auto reclassified = storage_.reclassify_healed_as_cleared([](const std::string & code) { @@ -499,7 +505,7 @@ TEST_F(FaultCodeStorageTest, ReclassifyKeepsCodesThatStillHeal) { }); ASSERT_EQ(reclassified.size(), 1u); - EXPECT_EQ(reclassified[0], "STOPPED.HEALING"); - EXPECT_EQ(storage_.get_fault("KEEPS.HEALING")->status, Fault::STATUS_HEALED); - EXPECT_EQ(storage_.get_fault("STOPPED.HEALING")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(reclassified[0], (FaultId{"STOPPED.HEALING", "/src"})); + EXPECT_EQ(storage_.get_fault({"KEEPS.HEALING", "/src"})->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_.get_fault({"STOPPED.HEALING", "/src"})->status, Fault::STATUS_CLEARED); } diff --git a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp index 7050e1d16..e76cdd547 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp @@ -15,15 +15,21 @@ #include #include +#include +#include #include #include +#include +#include #include #include #include +#include #include #include #include #include +#include #include #include @@ -32,6 +38,7 @@ #include #include "rclcpp/rclcpp.hpp" +#include "rcutils/logging.h" #include "ros2_medkit_fault_manager/fault_audit_log.hpp" #include "ros2_medkit_fault_manager/fault_manager_node.hpp" #include "ros2_medkit_fault_manager/fault_storage.hpp" @@ -44,6 +51,7 @@ #include "ros2_medkit_msgs/srv/get_fault.hpp" #include "ros2_medkit_msgs/srv/get_rosbag.hpp" #include "ros2_medkit_msgs/srv/get_snapshots.hpp" +#include "ros2_medkit_msgs/srv/list_faults.hpp" #include "ros2_medkit_msgs/srv/list_faults_for_entity.hpp" #include "ros2_medkit_msgs/srv/list_rosbags.hpp" #include "ros2_medkit_msgs/srv/report_fault.hpp" @@ -61,6 +69,7 @@ using ros2_medkit_msgs::srv::ClearFault; using ros2_medkit_msgs::srv::GetFault; using ros2_medkit_msgs::srv::GetRosbag; using ros2_medkit_msgs::srv::GetSnapshots; +using ros2_medkit_msgs::srv::ListFaults; using ros2_medkit_msgs::srv::ListFaultsForEntity; using ros2_medkit_msgs::srv::ListRosbags; using ros2_medkit_msgs::srv::ReportFault; @@ -97,7 +106,7 @@ TEST_F(FaultStorageTest, ReportNewFaultEvent) { EXPECT_TRUE(is_new); EXPECT_EQ(storage_.size(), 1u); - EXPECT_TRUE(storage_.contains("MOTOR_OVERHEAT")); + EXPECT_TRUE(storage_.contains({"MOTOR_OVERHEAT", "/powertrain/motor"})); } TEST_F(FaultStorageTest, PassedEventForNonExistentFaultIgnored) { @@ -121,18 +130,88 @@ TEST_F(FaultStorageTest, ReportExistingFaultEventUpdates) { bool is_new = storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Second report", - "/powertrain/motor2", timestamp2.nanoseconds(), default_config()); + "/powertrain/motor1", timestamp2.nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_.size(), 1u); - auto fault = storage_.get_fault("MOTOR_OVERHEAT"); + auto fault = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor1"}); ASSERT_TRUE(fault.has_value()); // Still the same continuous occurrence (not CLEARED in between): occurrence_count - // does not bump on every report, only severity/sources/description update. + // does not bump on every report, only severity/description update. EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->severity, Fault::SEVERITY_ERROR); // Updated to higher severity - EXPECT_EQ(fault->reporting_sources.size(), 2u); + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources.front(), "/powertrain/motor1"); +} + +// Two sources reporting one code are two records. This used to be one row whose +// counter, status, occurrence count, severity and timestamps were all shared, so one +// source's report moved another source's fault. +TEST_F(FaultStorageTest, TwoSourcesReportingOneCodeAreTwoRecords) { + DebounceConfig config; + config.confirmation_threshold = -2; + + const rclcpp::Time a_time(1000, 0, RCL_SYSTEM_TIME); + const rclcpp::Time b_time(2000, 0, RCL_SYSTEM_TIME); + + // A reports twice and reaches its threshold. B reports once and does not. + EXPECT_TRUE(storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "from a", + "/powertrain/motor1", a_time.nanoseconds(), config)); + storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "from a", + "/powertrain/motor1", a_time.nanoseconds(), config); + EXPECT_TRUE(storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "from b", + "/powertrain/motor2", b_time.nanoseconds(), config)) + << "a source that owns no record yet gets a new one, not an update"; + + EXPECT_EQ(storage_.size(), 2u); + + auto a = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor1"}); + auto b = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor2"}); + ASSERT_TRUE(a.has_value()); + ASSERT_TRUE(b.has_value()); + + EXPECT_EQ(a->status, Fault::STATUS_CONFIRMED) << "A reached its own threshold"; + EXPECT_EQ(b->status, Fault::STATUS_PREFAILED) << "B is one report into its own band"; + EXPECT_EQ(a->severity, Fault::SEVERITY_WARN) << "B's ERROR must not escalate A's record"; + EXPECT_EQ(b->severity, Fault::SEVERITY_ERROR); + EXPECT_EQ(a->description, "from a"); + EXPECT_EQ(b->description, "from b"); + EXPECT_EQ(a->occurrence_count, 1u); + EXPECT_EQ(b->occurrence_count, 1u); + EXPECT_EQ(a->first_occurred_ns, a_time.nanoseconds()); + EXPECT_EQ(b->first_occurred_ns, b_time.nanoseconds()); + ASSERT_EQ(a->reporting_sources.size(), 1u); + EXPECT_EQ(a->reporting_sources.front(), "/powertrain/motor1"); + ASSERT_EQ(b->reporting_sources.size(), 1u); + EXPECT_EQ(b->reporting_sources.front(), "/powertrain/motor2"); +} + +// The resolution step behind an unscoped service call. +TEST_F(FaultStorageTest, GetFaultsByCodeReturnsEveryOwnerAndGetFaultExactlyOne) { + rclcpp::Clock clock; + storage_.report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_b", + clock.now().nanoseconds(), default_config()); + storage_.report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_a", + clock.now().nanoseconds(), default_config()); + storage_.report_fault_event("OTHER", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_a", + clock.now().nanoseconds(), default_config()); + + const auto shared = storage_.get_faults_by_code("SHARED"); + ASSERT_EQ(shared.size(), 2u); + // Ordered by owner, so an ambiguity message names them the same way every time. + ASSERT_EQ(shared[0].reporting_sources.size(), 1u); + ASSERT_EQ(shared[1].reporting_sources.size(), 1u); + EXPECT_EQ(shared[0].reporting_sources.front(), "/owner_a"); + EXPECT_EQ(shared[1].reporting_sources.front(), "/owner_b"); + + EXPECT_EQ(storage_.get_faults_by_code("OTHER").size(), 1u); + EXPECT_TRUE(storage_.get_faults_by_code("NEVER_REPORTED").empty()); + + auto one = storage_.get_fault({"SHARED", "/owner_a"}); + ASSERT_TRUE(one.has_value()); + EXPECT_EQ(one->reporting_sources.front(), "/owner_a"); + EXPECT_FALSE(storage_.get_fault({"SHARED", "/nobody"}).has_value()); } TEST_F(FaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCount) { @@ -145,7 +224,7 @@ TEST_F(FaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCount) { "/tank", clock.now().nanoseconds(), default_config()); } - auto fault = storage_.get_fault("TANK_OVERFILL"); + auto fault = storage_.get_fault({"TANK_OVERFILL", "/tank"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -207,16 +286,16 @@ TEST_F(FaultStorageTest, ClearFault) { storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", timestamp.nanoseconds(), default_config()); - bool cleared = storage_.clear_fault("MOTOR_OVERHEAT"); + bool cleared = storage_.clear_fault({"MOTOR_OVERHEAT", "/node1"}); EXPECT_TRUE(cleared); - auto fault = storage_.get_fault("MOTOR_OVERHEAT"); + auto fault = storage_.get_fault({"MOTOR_OVERHEAT", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); } TEST_F(FaultStorageTest, ClearNonExistentFault) { - bool cleared = storage_.clear_fault("NON_EXISTENT"); + bool cleared = storage_.clear_fault({"NON_EXISTENT", "/node1"}); EXPECT_FALSE(cleared); } @@ -226,7 +305,7 @@ TEST_F(FaultStorageTest, GetClearedFaults) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", timestamp.nanoseconds(), default_config()); - storage_.clear_fault("FAULT_1"); + storage_.clear_fault({"FAULT_1", "/node1"}); // Query cleared faults auto faults = storage_.list_faults(false, 0, {Fault::STATUS_CLEARED}); @@ -283,12 +362,14 @@ TEST_F(FaultStorageTest, FaultStaysPrefailedAboveThreshold) { config.confirmation_threshold = -3; storage_.set_debounce_config(config); + // Both reports from one source: the counter belongs to that source's record, so + // debounce depth is what one reporter has said, not what several have said between them. storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Still debouncing towards confirmation, same occurrence EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); @@ -302,14 +383,12 @@ TEST_F(FaultStorageTest, FaultConfirmsAtThreshold) { config.confirmation_threshold = -3; storage_.set_debounce_config(config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), config); + for (int i = 0; i < 3; ++i) { + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", + clock.now().nanoseconds(), config); + } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Debounce build-up is still one occurrence EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -318,13 +397,13 @@ TEST_F(FaultStorageTest, FaultConfirmsAtThreshold) { TEST_F(FaultStorageTest, ConfirmedFaultStaysConfirmed) { rclcpp::Clock clock; - // Report fault 4 times + // One source reports 4 times: occurrence_count counts outages, not reports. for (int i = 0; i < 4; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", - "/node" + std::to_string(i), clock.now().nanoseconds(), default_config()); + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", + clock.now().nanoseconds(), default_config()); } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Same occurrence throughout, not one per report EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -333,18 +412,21 @@ TEST_F(FaultStorageTest, ConfirmedFaultStaysConfirmed) { TEST_F(FaultStorageTest, MultiSourceConfirmsFault) { rclcpp::Clock clock; - // Report same fault from 3 different sources - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/sensor1", - clock.now().nanoseconds(), default_config()); - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/sensor2", - clock.now().nanoseconds(), default_config()); - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/sensor3", - clock.now().nanoseconds(), default_config()); + // Report the same fault code from 3 different sources + for (const auto * source : {"/sensor1", "/sensor2", "/sensor3"}) { + storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", source, + clock.now().nanoseconds(), default_config()); + } - auto fault = storage_.get_fault("MOTOR_OVERHEAT"); - ASSERT_TRUE(fault.has_value()); - EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); - EXPECT_EQ(fault->reporting_sources.size(), 3u); + // Three records, each confirmed on its own report and naming only its own source. + EXPECT_EQ(storage_.size(), 3u); + for (const auto * source : {"/sensor1", "/sensor2", "/sensor3"}) { + auto fault = storage_.get_fault({"MOTOR_OVERHEAT", source}); + ASSERT_TRUE(fault.has_value()) << source; + EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources.front(), source); + } } TEST_F(FaultStorageTest, SameSourceMultipleReportsConfirms) { @@ -358,7 +440,7 @@ TEST_F(FaultStorageTest, SameSourceMultipleReportsConfirms) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), default_config()); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Repeated reports from one source, one occurrence EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -375,7 +457,7 @@ TEST_F(FaultStorageTest, ImmediateConfirmationWithThresholdZero) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -388,7 +470,7 @@ TEST_F(FaultStorageTest, CriticalSeverityBypassesDebounce) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "Critical test", "/node1", clock.now().nanoseconds(), default_config()); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -405,7 +487,7 @@ TEST_F(FaultStorageTest, CriticalBypassCanBeDisabled) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "Critical test", "/node1", clock.now().nanoseconds(), config); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); } @@ -419,32 +501,46 @@ TEST_F(FaultStorageTest, ClearedFaultCanBeReactivated) { "/node1", first_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->first_occurred_ns, first_ts.nanoseconds()); - // Clear the fault - storage_.clear_fault("FAULT_1"); - fault = storage_.get_fault("FAULT_1"); + // Clear the record + storage_.clear_fault({"FAULT_1", "/node1"}); + fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); - // Report again after a gap - should reactivate as a new cycle + // The same owner reports again after a gap - reactivates its own record as a new cycle auto second_ts = rclcpp::Time(first_ts.nanoseconds() + 1'000'000'000LL); // +1s is_new = storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Reactivated", - "/node2", second_ts.nanoseconds(), default_config()); + "/node1", second_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); // Should return true like a new fault - fault = storage_.get_fault("FAULT_1"); + fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Should be reconfirmed EXPECT_EQ(fault->occurrence_count, 2u); // Should increment - EXPECT_EQ(fault->reporting_sources.size(), 2u); // Both sources - EXPECT_EQ(fault->description, "Reactivated"); // Updated description + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources.front(), "/node1"); + EXPECT_EQ(fault->description, "Reactivated"); // Updated description // #25: first_occurred must reflect the new cycle, not the outage that already cleared. EXPECT_EQ(fault->first_occurred_ns, second_ts.nanoseconds()); + + // A different source reporting the same code after the clear opens its OWN record + // rather than reactivating another owner's. It used to reactivate the shared row, + // which is what made occurrence_count and reporting_sources grow across owners. + const rclcpp::Time third_ts(first_ts.nanoseconds() + 2'000'000'000LL, RCL_SYSTEM_TIME); + EXPECT_TRUE(storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "From the other source", "/node2", third_ts.nanoseconds(), default_config())); + auto other = storage_.get_fault({"FAULT_1", "/node2"}); + ASSERT_TRUE(other.has_value()); + EXPECT_EQ(other->occurrence_count, 1u) << "a first report is one occurrence, whatever the code has seen before"; + ASSERT_EQ(other->reporting_sources.size(), 1u); + EXPECT_EQ(other->reporting_sources.front(), "/node2"); + EXPECT_EQ(storage_.size(), 2u); } TEST_F(FaultStorageTest, PassedEventForClearedFaultIgnored) { @@ -455,14 +551,14 @@ TEST_F(FaultStorageTest, PassedEventForClearedFaultIgnored) { clock.now().nanoseconds(), default_config()); // Clear the fault - storage_.clear_fault("FAULT_1"); + storage_.clear_fault({"FAULT_1", "/node1"}); // PASSED event should be ignored for CLEARED fault bool result = storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), default_config()); EXPECT_FALSE(result); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); // Should stay cleared } @@ -475,36 +571,34 @@ TEST_F(FaultStorageTest, ClearedFaultReactivationRestartsDebounce) { config.confirmation_threshold = -3; storage_.set_debounce_config(config); - // Report 3 times to confirm - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), config); + // One source reports 3 times to confirm its own record + for (int i = 0; i < 3; ++i) { + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", + clock.now().nanoseconds(), config); + } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); - // Clear the fault - storage_.clear_fault("FAULT_1"); + // Clear the record + storage_.clear_fault({"FAULT_1", "/node1"}); // Reactivate - should start in PREFAILED with counter=-1 - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node4", + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - fault = storage_.get_fault("FAULT_1"); + fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Not yet confirmed, needs 2 more FAILED // Report 2 more times to re-confirm - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node5", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node6", - clock.now().nanoseconds(), config); + for (int i = 0; i < 2; ++i) { + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", + clock.now().nanoseconds(), config); + } - fault = storage_.get_fault("FAULT_1"); + fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Now confirmed } @@ -517,22 +611,85 @@ TEST_F(FaultStorageTest, PassedEventIncrementsCounter) { config.confirmation_threshold = -3; config.healing_threshold = 3; // explicit upper clamp (don't depend on the default) - // Report 2 FAILED events (counter = -2, PREFAILED) - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); + // Report 2 FAILED events from one source (its counter = -2, PREFAILED) + for (int i = 0; i < 2; ++i) { + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", + clock.now().nanoseconds(), config); + } // Report 3 PASSED events (counter = -2 + 3 = +1) for (int i = 0; i < 3; ++i) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREPASSED); // Counter > 0 } +// A PASSED moves only its own source's counter. It used to move the one counter the +// code had, so one reporter recovering walked another reporter's fault back from +// confirmation, and with healing on it healed it outright. +TEST_F(FaultStorageTest, PassedFromOneSourceMovesOnlyThatSourcesRecord) { + rclcpp::Clock clock; + DebounceConfig config; + config.confirmation_threshold = -1; + config.healing_enabled = true; + config.healing_threshold = 1; + + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node_a", + clock.now().nanoseconds(), config); + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node_b", + clock.now().nanoseconds(), config); + + // Two PASSED from A only: -1 -> 0 -> +1, which is A's healing threshold. + for (int i = 0; i < 2; ++i) { + storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node_a", clock.now().nanoseconds(), + config); + } + + auto a = storage_.get_fault({"FAULT_1", "/node_a"}); + auto b = storage_.get_fault({"FAULT_1", "/node_b"}); + ASSERT_TRUE(a.has_value()); + ASSERT_TRUE(b.has_value()); + EXPECT_EQ(a->status, Fault::STATUS_HEALED); + EXPECT_EQ(b->status, Fault::STATUS_CONFIRMED) << "B never reported PASSED, so B's fault is still active"; + EXPECT_EQ(b->last_passed_ns, 0) << "A's recovery must not stamp B's record"; +} + +// A PASSED from a source that owns no record for this code has nothing to heal. +TEST_F(FaultStorageTest, PassedFromASourceWithNoRecordIsIgnored) { + rclcpp::Clock clock; + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node_a", + clock.now().nanoseconds(), default_config()); + + const bool is_new = storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node_b", + clock.now().nanoseconds(), default_config()); + + EXPECT_FALSE(is_new); + EXPECT_EQ(storage_.size(), 1u) << "a PASSED must never create a record"; + EXPECT_FALSE(storage_.get_fault({"FAULT_1", "/node_b"}).has_value()); + auto owner_a = storage_.get_fault({"FAULT_1", "/node_a"}); + ASSERT_TRUE(owner_a.has_value()); + EXPECT_EQ(owner_a->status, Fault::STATUS_CONFIRMED); +} + +// Severity is per record: a CRITICAL from one source does not escalate another's. +TEST_F(FaultStorageTest, CriticalFromOneSourceDoesNotEscalateAnothersRecord) { + rclcpp::Clock clock; + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "warn", "/node_b", + clock.now().nanoseconds(), default_config()); + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "critical", "/node_a", + clock.now().nanoseconds(), default_config()); + + auto a = storage_.get_fault({"FAULT_1", "/node_a"}); + auto b = storage_.get_fault({"FAULT_1", "/node_b"}); + ASSERT_TRUE(a.has_value()); + ASSERT_TRUE(b.has_value()); + EXPECT_EQ(a->severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(b->severity, Fault::SEVERITY_WARN) << "severity is the record's own maximum, not the code's"; +} + TEST_F(FaultStorageTest, PassedEventDoesNotAdvanceLastOccurred) { // Observed on a live box: a plugin sent one PASSED on reconnect, healing was // disabled so the fault stayed CONFIRMED (by design), but last_occurred jumped @@ -546,7 +703,7 @@ TEST_F(FaultStorageTest, PassedEventDoesNotAdvanceLastOccurred) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", passed_at.nanoseconds(), default_config()); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // healing disabled: latched, by design EXPECT_EQ(fault->last_occurred_ns, failed_at.nanoseconds()); @@ -568,7 +725,7 @@ TEST_F(FaultStorageTest, HealingDisabledByDefault) { default_config()); } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // stays confirmed, not silently downgraded } @@ -589,7 +746,7 @@ TEST_F(FaultStorageTest, HealingWhenEnabled) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); } @@ -611,7 +768,7 @@ TEST_F(FaultStorageTest, TimeBasedConfirmationDisabledByDefault) { EXPECT_TRUE(confirmed.empty()); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); } @@ -636,9 +793,9 @@ TEST_F(FaultStorageTest, TimeBasedConfirmationWhenEnabled) { auto after_timeout = rclcpp::Time(now.nanoseconds() + static_cast(15e9)); auto confirmed = storage_.check_time_based_confirmation(after_timeout.nanoseconds()); ASSERT_EQ(confirmed.size(), 1u); - EXPECT_EQ(confirmed[0], "FAULT_1"); + EXPECT_EQ(confirmed[0], (ros2_medkit_fault_manager::FaultId{"FAULT_1", "/node1"})); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -656,7 +813,7 @@ TEST_F(FaultStorageTest, ConfirmedFaultCanHealWithPassedEvents) { "/node" + std::to_string(i), clock.now().nanoseconds(), config); } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -665,7 +822,7 @@ TEST_F(FaultStorageTest, ConfirmedFaultCanHealWithPassedEvents) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); } - fault = storage_.get_fault("FAULT_1"); + fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); } @@ -686,17 +843,17 @@ TEST_F(FaultStorageTest, HealedFaultCanRecurWithFailedEvents) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); // Report 3 FAILED events - fault should recur and confirm (counter = +3 - 3 = 0, then -3) for (int i = 0; i < 6; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Recurrence", "/node2", + storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Recurrence", "/node1", clock.now().nanoseconds(), config); } - fault = storage_.get_fault("FAULT_1"); + fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -713,24 +870,25 @@ TEST_F(FaultStorageTest, HeartbeatHealClampedAndStatusLatched) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_.get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_.get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); // Long heal heartbeat: counter clamps at healing_threshold, then heals. for (int i = 0; i < 100; ++i) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); } - EXPECT_EQ(storage_.get_fault("FAULT_1")->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_.get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_HEALED); // Re-confirmation is bounded and the healed status latches until the counter // walks all the way back down to the confirmation threshold. for (int i = 0; i < 3; ++i) { storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("FAULT_1")->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); + EXPECT_EQ(storage_.get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_HEALED) + << "latch broke after " << (i + 1); } storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_.get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); } // Latch holds in both directions, same as the SQLite backend: CONFIRMED with a positive counter @@ -744,14 +902,14 @@ TEST_F(FaultStorageTest, ConfirmedLatchSurvivesFailedAtPositiveCounter) { storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); } // HEALED with a negative counter stays HEALED on PASSED. @@ -767,14 +925,14 @@ TEST_F(FaultStorageTest, HealedLatchSurvivesPassedAtNegativeCounter) { for (int i = 0; i < 4; ++i) { storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); for (int i = 0; i < 5; ++i) { // +3 -> -2, latched HEALED storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); } storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } TEST_F(FaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePassed) { @@ -790,19 +948,19 @@ TEST_F(FaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePassed) { for (int i = 0; i < 4; ++i) { // -1 -> +3 -> HEALED storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); // CRITICAL confirms immediately from HEALED; counter must not stay at +2 where one PASSED heals it. storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { // -1 -> +2, still CONFIRMED storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); } storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } TEST_F(FaultStorageTest, ReclassifyHealedAsCleared) { @@ -816,12 +974,12 @@ TEST_F(FaultStorageTest, ReclassifyHealedAsCleared) { for (int i = 0; i < 4; ++i) { storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); const auto reclassified = storage_.reclassify_healed_as_cleared(); ASSERT_EQ(reclassified.size(), 1u); - EXPECT_EQ(reclassified[0], "F"); - EXPECT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(reclassified[0], (ros2_medkit_fault_manager::FaultId{"F", "/n"})); + EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CLEARED); EXPECT_TRUE(storage_.reclassify_healed_as_cleared().empty()); } @@ -864,9 +1022,9 @@ TEST_F(FaultStorageTest, HealingThresholdZeroHealsOnSinglePassed) { storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } TEST(DebounceHelpers, ComputeDebounceStatusLatches) { @@ -1022,7 +1180,7 @@ TEST(FaultManagerNodeParameterTest, AppliesNearMissRetentionBound) { drive_near_misses(node->get_storage_for_test(), 10); - EXPECT_EQ(node->get_storage().get_near_misses("PUMP_PRESSURE_LOW").size(), 4u); + EXPECT_EQ(node->get_storage().get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 4u); } TEST(FaultManagerNodeParameterTest, NearMissRetentionDefaultsToBounded) { @@ -1032,8 +1190,8 @@ TEST(FaultManagerNodeParameterTest, NearMissRetentionDefaultsToBounded) { drive_near_misses(node->get_storage_for_test(), 205); - // The documented default is 200 per fault code, and it must be in force without configuration. - EXPECT_EQ(node->get_storage().get_near_misses("PUMP_PRESSURE_LOW").size(), 200u); + // The documented default is 200 per fault record, and it must be in force without configuration. + EXPECT_EQ(node->get_storage().get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 200u); } TEST(FaultManagerNodeParameterTest, NegativeNearMissBoundFallsBackToDefault) { @@ -1047,7 +1205,7 @@ TEST(FaultManagerNodeParameterTest, NegativeNearMissBoundFallsBackToDefault) { drive_near_misses(node->get_storage_for_test(), 205); - EXPECT_EQ(node->get_storage().get_near_misses("PUMP_PRESSURE_LOW").size(), 200u); + EXPECT_EQ(node->get_storage().get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 200u); } TEST(FaultManagerNodeParameterTest, ZeroNearMissBoundIsUnlimited) { @@ -1060,7 +1218,7 @@ TEST(FaultManagerNodeParameterTest, ZeroNearMissBoundIsUnlimited) { drive_near_misses(node->get_storage_for_test(), 205); - EXPECT_EQ(node->get_storage().get_near_misses("PUMP_PRESSURE_LOW").size(), 205u); + EXPECT_EQ(node->get_storage().get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 205u); } TEST(FaultManagerNodeParameterTest, ParsesDropOldestPolicy) { @@ -1092,8 +1250,10 @@ class FaultEventPublishingTest : public ::testing::Test { } void SetUp() override { - // Unique namespace per test iteration avoids DDS topic collisions - std::string ns = "/test_events_" + std::to_string(test_counter_.fetch_add(1)); + // Unique namespace per test iteration avoids DDS topic collisions. Kept as a + // member so a subclass can add a client of its own on the same namespace. + ns_ = "/test_events_" + std::to_string(test_counter_.fetch_add(1)); + const std::string & ns = ns_; // Create fault manager node with immediate confirmation rclcpp::NodeOptions fm_options; @@ -1207,20 +1367,31 @@ class FaultEventPublishingTest : public ::testing::Test { return future.get()->accepted; } - bool call_clear_fault(const std::string & fault_code) { + /// @param source_id owner to scope the call to. Empty leaves the call unscoped. + bool call_clear_fault(const std::string & fault_code, const std::string & source_id = "") { + const auto response = clear_fault_response(fault_code, source_id); + return response.has_value() && response->success; + } + + /// The whole response, for the tests that check the refusal message rather than the flag. + std::optional clear_fault_response(const std::string & fault_code, + const std::string & source_id = "") { auto request = std::make_shared(); request->fault_code = fault_code; + request->source_id = source_id; auto future = clear_fault_client_->async_send_request(request); if (!spin_until_future_ready(future)) { - return false; + return std::nullopt; } - return future.get()->success; + return *future.get(); } - std::optional call_get_fault(const std::string & fault_code) { + /// @param source_id owner to scope the call to. Empty leaves the call unscoped. + std::optional call_get_fault(const std::string & fault_code, const std::string & source_id = "") { auto request = std::make_shared(); request->fault_code = fault_code; + request->source_id = source_id; auto future = get_fault_client_->async_send_request(request); if (!spin_until_future_ready(future)) { @@ -1240,6 +1411,7 @@ class FaultEventPublishingTest : public ::testing::Test { return *future.get(); } + std::string ns_; std::shared_ptr fault_manager_; std::shared_ptr test_node_; rclcpp::Subscription::SharedPtr event_subscription_; @@ -1277,17 +1449,46 @@ TEST_F(FaultEventPublishingTest, UpdateExistingFaultPublishesUpdatedEvent) { // Clear received events received_events_.clear(); - // Report same fault again - should trigger EVENT_UPDATED - ASSERT_TRUE(call_report_fault("TEST_FAULT_2", Fault::SEVERITY_ERROR, "/test_node2")); + // The SAME owner reports again - its own record is updated, so EVENT_UPDATED. + ASSERT_TRUE(call_report_fault("TEST_FAULT_2", Fault::SEVERITY_ERROR, "/test_node1")); ASSERT_TRUE(spin_until([this]() { return !received_events_.empty(); })); - // Verify EVENT_UPDATED was published (severity/sources changed; still one occurrence) + // Verify EVENT_UPDATED was published (severity changed, still one occurrence) ASSERT_EQ(received_events_.size(), 1u); EXPECT_EQ(received_events_[0].event_type, FaultEvent::EVENT_UPDATED); EXPECT_EQ(received_events_[0].fault.fault_code, "TEST_FAULT_2"); EXPECT_EQ(received_events_[0].fault.occurrence_count, 1u); + ASSERT_EQ(received_events_[0].fault.reporting_sources.size(), 1u); + EXPECT_EQ(received_events_[0].fault.reporting_sources.front(), "/test_node1"); +} + +// A second source reporting a code someone else already reported opens its OWN record, +// so the event is a confirmation of a new record rather than an update of a shared one. +TEST_F(FaultEventPublishingTest, ASecondSourceConfirmsItsOwnRecord) { + ASSERT_TRUE(call_report_fault("TWO_OWNERS", Fault::SEVERITY_WARN, "/test_node1")); + ASSERT_TRUE(spin_until([this]() { + return !received_events_.empty(); + })); + received_events_.clear(); + + ASSERT_TRUE(call_report_fault("TWO_OWNERS", Fault::SEVERITY_ERROR, "/test_node2")); + ASSERT_TRUE(spin_until([this]() { + return !received_events_.empty(); + })); + + ASSERT_EQ(received_events_.size(), 1u); + EXPECT_EQ(received_events_[0].event_type, FaultEvent::EVENT_CONFIRMED); + ASSERT_EQ(received_events_[0].fault.reporting_sources.size(), 1u); + EXPECT_EQ(received_events_[0].fault.reporting_sources.front(), "/test_node2"); + EXPECT_EQ(received_events_[0].fault.severity, Fault::SEVERITY_ERROR); + + // The first owner's record is untouched, severity included. + auto first = call_get_fault("TWO_OWNERS", "/test_node1"); + ASSERT_TRUE(first.has_value()); + ASSERT_TRUE(first->success); + EXPECT_EQ(first->fault.severity, Fault::SEVERITY_WARN); } TEST_F(FaultEventPublishingTest, ClearFaultPublishesClearedEvent) { @@ -1358,6 +1559,339 @@ TEST_F(HealingFaultEventPublishingTest, HealPublishesClearedEventSoStreamConsume EXPECT_EQ(event->fault.status, Fault::STATUS_HEALED); } +/// Records every message logged through rcutils while an instance exists, then forwards it. +class LogCapture { + public: + struct Line { + int severity; + std::string text; + }; + + LogCapture() : previous_(rcutils_logging_get_output_handler()) { + std::lock_guard lock(mutex()); + active() = this; + rcutils_logging_set_output_handler(&LogCapture::handle); + } + + ~LogCapture() { + std::lock_guard lock(mutex()); + rcutils_logging_set_output_handler(previous_); + active() = nullptr; + } + + LogCapture(const LogCapture &) = delete; + LogCapture & operator=(const LogCapture &) = delete; + LogCapture(LogCapture &&) = delete; + LogCapture & operator=(LogCapture &&) = delete; + + /// WARN lines whose text contains @p needle. + std::vector warnings_with(const std::string & needle) const { + std::lock_guard lock(mutex()); + std::vector out; + for (const auto & line : lines_) { + if (line.severity == RCUTILS_LOG_SEVERITY_WARN && line.text.find(needle) != std::string::npos) { + out.push_back(line.text); + } + } + return out; + } + + private: + static std::mutex & mutex() { + static std::mutex m; + return m; + } + + static LogCapture *& active() { + static LogCapture * instance = nullptr; + return instance; + } + + static void handle(const rcutils_log_location_t * location, int severity, const char * name, + rcutils_time_point_value_t timestamp, const char * format, va_list * args) { + rcutils_logging_output_handler_t forward = nullptr; + { + std::lock_guard lock(mutex()); + LogCapture * self = active(); + if (self != nullptr) { + va_list copy; + va_copy(copy, *args); + std::array buffer{}; + // The format comes from the logging call site. There is no literal to pass. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" + std::vsnprintf(buffer.data(), buffer.size(), format, copy); +#pragma GCC diagnostic pop + va_end(copy); + self->lines_.push_back(Line{severity, buffer.data()}); + forward = self->previous_; + } + } + if (forward != nullptr) { + forward(location, severity, name, timestamp, format, args); + } + } + + rcutils_logging_output_handler_t previous_; + std::vector lines_; +}; + +/// Severity consistency of a record through the node, on both storage backends: events and audit +/// rows carry the stored severity, near-miss rows the reported one, and a differing severity is logged. +class SeverityConsistencyNodeTest : public FaultEventPublishingTest, public ::testing::WithParamInterface { + protected: + SeverityConsistencyNodeTest() { + std::random_device rd; + std::mt19937_64 gen(rd()); + std::uniform_int_distribution dist; + dir_ = std::filesystem::temp_directory_path() / ("test_severity_node_" + std::to_string(dist(gen))); + std::filesystem::create_directories(dir_); + } + + ~SeverityConsistencyNodeTest() override { + std::error_code ec; + std::filesystem::remove_all(dir_, ec); + } + + SeverityConsistencyNodeTest(const SeverityConsistencyNodeTest &) = delete; + SeverityConsistencyNodeTest & operator=(const SeverityConsistencyNodeTest &) = delete; + SeverityConsistencyNodeTest(SeverityConsistencyNodeTest &&) = delete; + SeverityConsistencyNodeTest & operator=(SeverityConsistencyNodeTest &&) = delete; + + std::vector fault_manager_overrides() override { + return { + rclcpp::Parameter("storage_type", GetParam()), + rclcpp::Parameter("database_path", (dir_ / "faults.db").string()), + rclcpp::Parameter("confirmation_threshold", -3), + rclcpp::Parameter("audit_log.enabled", true), + rclcpp::Parameter("audit_log.database_path", audit_path()), + }; + } + + std::string audit_path() const { + return (dir_ / "audit.db").string(); + } + + bool report(const std::string & code, uint8_t severity, const std::string & source) { + return call_report_fault(code, severity, source); + } + + /// The event of @p type for @p code in occurrence @p occurrence, once it arrives. + std::optional wait_for_event(const std::string & code, const std::string & type, uint32_t occurrence) { + std::optional found; + spin_until([&]() { + for (const auto & e : received_events_) { + if (e.fault.fault_code == code && e.event_type == type && e.fault.occurrence_count == occurrence) { + found = e; + return true; + } + } + return false; + }); + return found; + } + + /// @param source_id owner of the record. Empty resolves the code's only record. + Fault get(const std::string & code, const std::string & source_id = "") { + auto response = call_get_fault(code, source_id); + EXPECT_TRUE(response.has_value() && response->success) << code << " " << source_id; + return response ? response->fault : Fault{}; + } + + /// (transition, severity) of every audit row for @p code, oldest first. + std::vector> audit_rows(const std::string & code) const { + ros2_medkit_fault_manager::FaultAuditLog audit(audit_path()); + std::vector> rows; + for (const auto & record : audit.read()) { + if (record.event.fault_code == code) { + rows.emplace_back(record.event.transition, record.event.severity); + } + } + return rows; + } + + std::filesystem::path dir_; +}; + +INSTANTIATE_TEST_SUITE_P(Backends, SeverityConsistencyNodeTest, ::testing::Values("memory", "sqlite")); + +TEST_P(SeverityConsistencyNodeTest, EventsCarryTheSeverityGetShows) { + ASSERT_TRUE(report("EVT_SEV", Fault::SEVERITY_CRITICAL, "/a")); + auto confirmed = wait_for_event("EVT_SEV", FaultEvent::EVENT_CONFIRMED, 1); + ASSERT_TRUE(confirmed.has_value()); + EXPECT_EQ(confirmed->fault.severity, get("EVT_SEV").severity); + + ASSERT_TRUE(call_clear_fault("EVT_SEV")); + auto cleared = wait_for_event("EVT_SEV", FaultEvent::EVENT_CLEARED, 1); + ASSERT_TRUE(cleared.has_value()); + EXPECT_EQ(cleared->fault.severity, get("EVT_SEV").severity); + + // A WARN report on a CRITICAL code starts occurrence 2 and confirms it at once. + ASSERT_TRUE(report("EVT_SEV", Fault::SEVERITY_WARN, "/a")); + auto reconfirmed = wait_for_event("EVT_SEV", FaultEvent::EVENT_CONFIRMED, 2); + ASSERT_TRUE(reconfirmed.has_value()) << "no fault_confirmed event for occurrence 2"; + auto shown = get("EVT_SEV"); + EXPECT_EQ(shown.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(shown.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(reconfirmed->fault.severity, shown.severity); + EXPECT_EQ(reconfirmed->fault.status, shown.status); + + // A lower report inside the occurrence updates the fault and keeps the code's severity. + received_events_.clear(); + ASSERT_TRUE(report("EVT_SEV", Fault::SEVERITY_ERROR, "/a")); + auto updated = wait_for_event("EVT_SEV", FaultEvent::EVENT_UPDATED, 2); + ASSERT_TRUE(updated.has_value()); + EXPECT_EQ(updated->fault.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(updated->fault.severity, get("EVT_SEV").severity); +} + +TEST_P(SeverityConsistencyNodeTest, AuditRowsCarryTheStoredSeverity) { + using Rows = std::vector>; + const int warn = Fault::SEVERITY_WARN; + const int error = Fault::SEVERITY_ERROR; + + // Debounced: WARN opens the occurrence, ERROR escalates it, WARN confirms it. + ASSERT_TRUE(report("AUDIT_SEV", Fault::SEVERITY_WARN, "/a")); + ASSERT_TRUE(report("AUDIT_SEV", Fault::SEVERITY_ERROR, "/a")); + ASSERT_TRUE(report("AUDIT_SEV", Fault::SEVERITY_WARN, "/a")); + auto confirmed = wait_for_event("AUDIT_SEV", FaultEvent::EVENT_CONFIRMED, 1); + ASSERT_TRUE(confirmed.has_value()); + EXPECT_EQ(confirmed->fault.severity, error); + EXPECT_EQ(get("AUDIT_SEV").severity, error); + EXPECT_EQ(audit_rows("AUDIT_SEV"), (Rows{{"occurred", warn}, {"confirmed", error}})) + << "each row records the stored severity at its transition"; + + // ERROR confirmed and cleared, then WARN reports open and confirm occurrence 2. + for (int i = 0; i < 3; ++i) { + ASSERT_TRUE(report("AUDIT_REACT", Fault::SEVERITY_ERROR, "/a")); + } + ASSERT_TRUE(call_clear_fault("AUDIT_REACT")); + for (int i = 0; i < 3; ++i) { + ASSERT_TRUE(report("AUDIT_REACT", Fault::SEVERITY_WARN, "/a")); + } + ASSERT_TRUE(wait_for_event("AUDIT_REACT", FaultEvent::EVENT_CONFIRMED, 2).has_value()); + EXPECT_EQ(get("AUDIT_REACT").severity, error); + EXPECT_EQ( + audit_rows("AUDIT_REACT"), + (Rows{{"occurred", error}, {"confirmed", error}, {"cleared", error}, {"occurred", error}, {"confirmed", error}})); + + // The near-miss rows of the same reports keep what was reported. + std::vector near_miss_severities; + for (const auto & record : fault_manager_->get_storage().get_near_misses({"AUDIT_REACT", "/a"})) { + near_miss_severities.push_back(record.severity); + } + EXPECT_EQ(near_miss_severities, (std::vector{error, error, warn, warn})); +} + +/// Global confirmation threshold -1. Sources under /robot/slow have their own threshold of -3. +class SeverityEntityThresholdNodeTest : public SeverityConsistencyNodeTest { + protected: + SeverityEntityThresholdNodeTest() { + std::ofstream(dir_ / "thresholds.yaml") << "/robot/slow:\n confirmation_threshold: -3\n"; + } + + std::vector fault_manager_overrides() override { + return { + rclcpp::Parameter("storage_type", GetParam()), + rclcpp::Parameter("database_path", (dir_ / "faults.db").string()), + rclcpp::Parameter("confirmation_threshold", -1), + rclcpp::Parameter("entity_thresholds.config_file", (dir_ / "thresholds.yaml").string()), + }; + } +}; + +INSTANTIATE_TEST_SUITE_P(Backends, SeverityEntityThresholdNodeTest, ::testing::Values("memory", "sqlite")); + +TEST_P(SeverityEntityThresholdNodeTest, CriticalCodeConfirmsUnderAnEntityThreshold) { + const std::string slow = "/robot/slow/sensor"; + + // The override is in effect: one WARN report from the entity leaves a new code pending. + ASSERT_TRUE(report("ENTITY_WARN", Fault::SEVERITY_WARN, slow)); + EXPECT_EQ(get("ENTITY_WARN").status, Fault::STATUS_PREFAILED); + + // CRITICAL, clear, WARN, all from the entity. + ASSERT_TRUE(report("ENTITY_CRIT", Fault::SEVERITY_CRITICAL, slow)); + ASSERT_TRUE(wait_for_event("ENTITY_CRIT", FaultEvent::EVENT_CONFIRMED, 1).has_value()); + ASSERT_TRUE(call_clear_fault("ENTITY_CRIT")); + ASSERT_TRUE(report("ENTITY_CRIT", Fault::SEVERITY_WARN, slow)); + auto shown = get("ENTITY_CRIT"); + EXPECT_EQ(shown.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(shown.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(shown.occurrence_count, 2u); + auto event = wait_for_event("ENTITY_CRIT", FaultEvent::EVENT_CONFIRMED, 2); + ASSERT_TRUE(event.has_value()) << "no fault_confirmed event for occurrence 2"; + EXPECT_EQ(event->fault.severity, Fault::SEVERITY_CRITICAL); + + // A CRITICAL record of a source on the global threshold does not confirm the entity's record. + ASSERT_TRUE(report("ENTITY_MIXED", Fault::SEVERITY_CRITICAL, "/robot/fast/sensor")); + ASSERT_TRUE(report("ENTITY_MIXED", Fault::SEVERITY_WARN, slow)); + shown = get("ENTITY_MIXED", slow); + EXPECT_EQ(shown.status, Fault::STATUS_PREFAILED); + EXPECT_EQ(shown.severity, Fault::SEVERITY_WARN); + EXPECT_EQ(get("ENTITY_MIXED", "/robot/fast/sensor").status, Fault::STATUS_CONFIRMED); +} + +TEST_P(SeverityConsistencyNodeTest, SeverityMismatchIsLoggedOncePerCodeOwnerAndSeverity) { + LogCapture capture; + auto logged = [&capture]() { + return capture.warnings_with("'MISMATCH'"); + }; + + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_ERROR, "/a")); + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_ERROR, "/a")); + EXPECT_TRUE(logged().empty()) << "a new code and a matching report are not mismatches"; + + for (int i = 0; i < 50; ++i) { + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_WARN, "/a")); + } + auto lines = logged(); + ASSERT_EQ(lines.size(), 1u) << "fifty identical mismatches log one line"; + EXPECT_NE(lines[0].find("WARN"), std::string::npos) << lines[0]; + EXPECT_NE(lines[0].find("ERROR"), std::string::npos) << lines[0]; + EXPECT_NE(lines[0].find("'/a'"), std::string::npos) << lines[0]; + + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_WARN, "/b")); + EXPECT_EQ(logged().size(), 1u) << "a report that creates /b's record has nothing to differ from"; + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_ERROR, "/b")); + lines = logged(); + ASSERT_EQ(lines.size(), 2u) << "another owner is named on its own"; + EXPECT_NE(lines[1].find("'/b'"), std::string::npos) << lines[1]; + EXPECT_NE(lines[1].find("stored as WARN"), std::string::npos) << "compared with /b's own record: " << lines[1]; + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_INFO, "/a")); + EXPECT_EQ(logged().size(), 3u) << "another reported severity is named on its own"; + ASSERT_TRUE(call_report_passed("MISMATCH", "/a")); + EXPECT_EQ(logged().size(), 3u) << "a PASSED report carries no severity"; + + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_CRITICAL, "/a")); + EXPECT_EQ(logged().size(), 4u) << "escalation is a mismatch too"; + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_WARN, "/a")); + ASSERT_TRUE(call_clear_fault("MISMATCH", "/a")); + for (int i = 0; i < 20; ++i) { + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_WARN, "/a")); + } + EXPECT_EQ(logged().size(), 4u) << "the bound holds across the stored severity and across occurrences"; + + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_ERROR, "/a")); + lines = logged(); + ASSERT_EQ(lines.size(), 5u); + EXPECT_NE(lines[4].find("CRITICAL"), std::string::npos) << lines[4]; +} + +TEST_P(SeverityConsistencyNodeTest, SeverityMismatchIsLoggedPerCode) { + LogCapture capture; + ASSERT_TRUE(report("MISMATCH_A", Fault::SEVERITY_ERROR, "/a")); + ASSERT_TRUE(report("MISMATCH_B", Fault::SEVERITY_ERROR, "/a")); + + ASSERT_TRUE(report("MISMATCH_A", Fault::SEVERITY_WARN, "/a")); + ASSERT_EQ(capture.warnings_with("'MISMATCH_A'").size(), 1u); + + ASSERT_TRUE(report("MISMATCH_B", Fault::SEVERITY_WARN, "/a")); + const auto lines = capture.warnings_with("'MISMATCH_B'"); + ASSERT_EQ(lines.size(), 1u) << "the same owner and severity on another code is a mismatch of its own"; + EXPECT_NE(lines[0].find("'/a'"), std::string::npos) << lines[0]; + EXPECT_NE(lines[0].find("stored as ERROR"), std::string::npos) << lines[0]; +} + TEST_F(FaultEventPublishingTest, ClearNonExistentFaultNoEvent) { // Clear non-existent fault - should not publish event ASSERT_FALSE(call_clear_fault("NON_EXISTENT_FAULT")); @@ -1481,16 +2015,17 @@ TEST_F(FaultEventPublishingTest, GetFaultReturnsEnvironmentData) { } TEST_F(FaultEventPublishingTest, GetFaultReturnsExtendedDataRecords) { - // Report fault twice to have first and last occurrence timestamps differ + // Report twice from one owner so first and last occurrence are both stamped ASSERT_TRUE(call_report_fault("EDR_TEST", Fault::SEVERITY_ERROR, "/node1")); ASSERT_TRUE(spin_until([this]() { return !received_events_.empty(); })); - ASSERT_TRUE(call_report_fault("EDR_TEST", Fault::SEVERITY_ERROR, "/node2")); + ASSERT_TRUE(call_report_fault("EDR_TEST", Fault::SEVERITY_ERROR, "/node1")); ASSERT_TRUE(spin_until([this]() { return received_events_.size() >= 2; })); + // One record carries the code, so the unscoped call resolves to it. auto response = call_get_fault("EDR_TEST"); ASSERT_TRUE(response.has_value()); EXPECT_TRUE(response->success); @@ -1501,6 +2036,410 @@ TEST_F(FaultEventPublishingTest, GetFaultReturnsExtendedDataRecords) { EXPECT_NE(edr.last_occurrence_ns, 0); } +// --- Record resolution on the single-record services --- + +TEST_F(FaultEventPublishingTest, ScopedClearClearsOneRecordAndPublishesItsOwner) { + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_a")); + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_b")); + ASSERT_TRUE(spin_until([this]() { + return received_events_.size() >= 2; + })); + received_events_.clear(); + + ASSERT_TRUE(call_clear_fault("SHARED_CODE", "/owner_a")); + ASSERT_TRUE(spin_until([this]() { + return !received_events_.empty(); + })); + + ASSERT_EQ(received_events_.size(), 1u) << "one record was cleared, so one event"; + EXPECT_EQ(received_events_[0].event_type, FaultEvent::EVENT_CLEARED); + EXPECT_EQ(received_events_[0].fault.status, Fault::STATUS_CLEARED); + ASSERT_EQ(received_events_[0].fault.reporting_sources.size(), 1u); + EXPECT_EQ(received_events_[0].fault.reporting_sources.front(), "/owner_a"); + + auto other = call_get_fault("SHARED_CODE", "/owner_b"); + ASSERT_TRUE(other.has_value()); + ASSERT_TRUE(other->success); + EXPECT_EQ(other->fault.status, Fault::STATUS_CONFIRMED) << "clearing one owner must not clear another"; +} + +TEST_F(FaultEventPublishingTest, UnscopedClearOfTwoRecordsIsRefusedAndClearsNothing) { + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_a")); + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_b")); + ASSERT_TRUE(spin_until([this]() { + return received_events_.size() >= 2; + })); + received_events_.clear(); + + const auto response = clear_fault_response("SHARED_CODE"); + ASSERT_TRUE(response.has_value()); + EXPECT_FALSE(response->success); + EXPECT_EQ(response->message.rfind("ambiguous:", 0), 0u) << "got: " << response->message; + EXPECT_NE(response->message.find("/owner_a"), std::string::npos) << "the owners must be in the message"; + EXPECT_NE(response->message.find("/owner_b"), std::string::npos); + // The message is prose an operator reads, so it is two sentences, not a semicolon + // splice. The instruction has to survive whatever a client does to the rest of it. + EXPECT_NE(response->message.find("Set source_id to pick one."), std::string::npos) << "got: " << response->message; + EXPECT_EQ(response->message.find(';'), std::string::npos) << "got: " << response->message; + + // Nothing was cleared, and nothing was published. + spin_for(std::chrono::milliseconds(200)); + EXPECT_TRUE(received_events_.empty()); + for (const char * owner : {"/owner_a", "/owner_b"}) { + auto still = call_get_fault("SHARED_CODE", owner); + ASSERT_TRUE(still.has_value()); + ASSERT_TRUE(still->success) << owner; + EXPECT_EQ(still->fault.status, Fault::STATUS_CONFIRMED) << owner; + } +} + +TEST_F(FaultEventPublishingTest, UnscopedClearOfOneRecordSucceeds) { + ASSERT_TRUE(call_report_fault("ONLY_ONE", Fault::SEVERITY_ERROR, "/owner_a")); + ASSERT_TRUE(spin_until([this]() { + return !received_events_.empty(); + })); + + EXPECT_TRUE(call_clear_fault("ONLY_ONE")); + + auto response = call_get_fault("ONLY_ONE", "/owner_a"); + ASSERT_TRUE(response.has_value()); + ASSERT_TRUE(response->success); + EXPECT_EQ(response->fault.status, Fault::STATUS_CLEARED); +} + +TEST_F(FaultEventPublishingTest, GetFaultResolvesTheSameWayAsClear) { + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_WARN, "/owner_a")); + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_CRITICAL, "/owner_b")); + ASSERT_TRUE(call_report_fault("ONLY_ONE", Fault::SEVERITY_ERROR, "/owner_a")); + ASSERT_TRUE(spin_until([this]() { + return received_events_.size() >= 3; + })); + + // Scoped: exactly that owner's record. + auto scoped = call_get_fault("SHARED_CODE", "/owner_a"); + ASSERT_TRUE(scoped.has_value()); + ASSERT_TRUE(scoped->success); + EXPECT_EQ(scoped->fault.severity, Fault::SEVERITY_WARN); + ASSERT_EQ(scoped->fault.reporting_sources.size(), 1u); + EXPECT_EQ(scoped->fault.reporting_sources.front(), "/owner_a"); + + // Unscoped with two records: refused, not an arbitrary pick. + auto ambiguous = call_get_fault("SHARED_CODE"); + ASSERT_TRUE(ambiguous.has_value()); + EXPECT_FALSE(ambiguous->success); + EXPECT_EQ(ambiguous->error_message.rfind("ambiguous:", 0), 0u) << "got: " << ambiguous->error_message; + + // Unscoped with one record: that record. + auto single = call_get_fault("ONLY_ONE"); + ASSERT_TRUE(single.has_value()); + ASSERT_TRUE(single->success); + EXPECT_EQ(single->fault.severity, Fault::SEVERITY_ERROR); +} + +/// Gives the node a hierarchical rule that mutes symptoms, which is the only way two +/// owners can hold the same symptom code with exactly one of them muted. +class MutedFaultListingTest : public FaultEventPublishingTest { + protected: + std::vector fault_manager_overrides() override { + // Written here rather than in SetUp() because the base SetUp() asks for the + // overrides before it builds the node, and the node reads the file at construction. + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dist; + config_path_ = + std::filesystem::temp_directory_path() / ("test_muted_listing_" + std::to_string(dist(gen)) + ".yaml"); + std::ofstream out(config_path_); + out << R"( +correlation: + enabled: true + default_window_ms: 30000 + patterns: + symptom_codes: + codes: ["SYMPTOM_*"] + rules: + - id: root_cascade + name: "Root Cascade" + mode: hierarchical + root_cause: + codes: ["ROOT_CAUSE"] + symptoms: + - pattern: symptom_codes + window_ms: 30000 + mute_symptoms: true + auto_clear_with_root: true +)"; + out.close(); + + auto overrides = FaultEventPublishingTest::fault_manager_overrides(); + overrides.emplace_back("correlation.config_file", config_path_.string()); + return overrides; + } + + void SetUp() override { + FaultEventPublishingTest::SetUp(); + list_faults_client_ = test_node_->create_client(ns_ + "/fault_manager/list_faults"); + ASSERT_TRUE(list_faults_client_->wait_for_service(std::chrono::seconds(5))); + } + + void TearDown() override { + list_faults_client_.reset(); + FaultEventPublishingTest::TearDown(); + std::filesystem::remove(config_path_); + } + + std::optional call_list_faults(bool include_muted) { + auto request = std::make_shared(); + request->include_muted = include_muted; + request->statuses = {Fault::STATUS_CONFIRMED}; + + auto future = list_faults_client_->async_send_request(request); + if (!spin_until_future_ready(future)) { + return std::nullopt; + } + return *future.get(); + } + + /// Owners of every listed record carrying @p fault_code, sorted so the order the + /// backend happens to return does not decide the assertion. + static std::vector owners_of(const ListFaults::Response & response, const std::string & fault_code) { + std::vector owners; + for (const auto & fault : response.faults) { + if (fault.fault_code == fault_code && !fault.reporting_sources.empty()) { + owners.push_back(fault.reporting_sources.front()); + } + } + std::sort(owners.begin(), owners.end()); + return owners; + } + + std::filesystem::path config_path_; + rclcpp::Client::SharedPtr list_faults_client_; +}; + +// Muting belongs to a RECORD, so the default listing hides the muted record alone. +// Owner A's symptom is suppressed by A's own root cause. Owner B reported the same +// code with no root cause of its own, so B's record was never muted and a listing +// that drops it hides a live CRITICAL fault from the default view. +TEST_F(MutedFaultListingTest, DefaultListingHidesTheMutedRecordNotTheCode) { + ASSERT_TRUE(call_report_fault("ROOT_CAUSE", Fault::SEVERITY_CRITICAL, "/owner_a")); + ASSERT_TRUE(call_report_fault("SYMPTOM_SPEED", Fault::SEVERITY_CRITICAL, "/owner_a")); + ASSERT_TRUE(call_report_fault("SYMPTOM_SPEED", Fault::SEVERITY_CRITICAL, "/owner_b")); + + auto visible = call_list_faults(false); + ASSERT_TRUE(visible.has_value()); + EXPECT_EQ(visible->muted_count, 1u) << "only A's symptom record is muted"; + EXPECT_EQ(owners_of(*visible, "SYMPTOM_SPEED"), std::vector{"/owner_b"}); + // The root cause itself is never muted, so the listing is not simply empty. + EXPECT_EQ(owners_of(*visible, "ROOT_CAUSE"), std::vector{"/owner_a"}); + + auto everything = call_list_faults(true); + ASSERT_TRUE(everything.has_value()); + EXPECT_EQ(owners_of(*everything, "SYMPTOM_SPEED"), (std::vector{"/owner_a", "/owner_b"})); + // One entry per muted RECORD, naming both halves of its identity. + ASSERT_EQ(everything->muted_faults.size(), 1u); + EXPECT_EQ(everything->muted_faults.front().fault_code, "SYMPTOM_SPEED"); + EXPECT_EQ(everything->muted_faults.front().root_cause_code, "ROOT_CAUSE"); + EXPECT_EQ(everything->muted_faults.front().source_id, "/owner_a"); +} + +// Two owners muted on one code are two entries of that code. Without the owner on the +// wire they are indistinguishable, and a reader cannot tell one muted record from the +// other, nor address either of them. +TEST_F(MutedFaultListingTest, MutedEntriesOfOneCodeAreToldApartByTheirOwner) { + for (const char * owner : {"/owner_a", "/owner_b"}) { + ASSERT_TRUE(call_report_fault("ROOT_CAUSE", Fault::SEVERITY_CRITICAL, owner)); + ASSERT_TRUE(call_report_fault("SYMPTOM_SPEED", Fault::SEVERITY_CRITICAL, owner)); + } + + auto everything = call_list_faults(true); + ASSERT_TRUE(everything.has_value()); + EXPECT_EQ(everything->muted_count, 2u); + + std::vector muted_owners; + for (const auto & muted : everything->muted_faults) { + EXPECT_EQ(muted.fault_code, "SYMPTOM_SPEED"); + EXPECT_EQ(muted.root_cause_code, "ROOT_CAUSE"); + muted_owners.push_back(muted.source_id); + } + std::sort(muted_owners.begin(), muted_owners.end()); + EXPECT_EQ(muted_owners, (std::vector{"/owner_a", "/owner_b"})); + + // With both muted the default listing shows neither, which is what makes the owner + // on the entry the only way to tell the two suppressed records apart. + auto visible = call_list_faults(false); + ASSERT_TRUE(visible.has_value()); + EXPECT_TRUE(owners_of(*visible, "SYMPTOM_SPEED").empty()); +} + +/// A node backed by a database written before the owner column existed, so the record +/// the migration produced can be exercised over the wire the way a client reaches it. +class LegacyOwnedRecordTest : public FaultEventPublishingTest { + protected: + std::vector fault_manager_overrides() override { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dist; + db_path_ = std::filesystem::temp_directory_path() / ("test_legacy_owner_" + std::to_string(dist(gen)) + ".db"); + + // One legacy row whose reporting_sources records nothing readable. The migration + // has to give it a name, because a record nobody can address is a record an + // operator cannot clear. + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path_.string().c_str(), &raw), SQLITE_OK); + EXPECT_EQ(sqlite3_exec(raw, + "CREATE TABLE faults (fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, " + "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " + "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " + "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " + "debounce_counter INTEGER NOT NULL DEFAULT 0, " + "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " + "confirmed_at_ns INTEGER NOT NULL DEFAULT 0);" + "INSERT INTO faults VALUES ('SHARED_CODE', 2, 'from an older release', 1, 1, 1, " + "'CONFIRMED', '', -1, 1, 0, 0);", + nullptr, nullptr, nullptr), + SQLITE_OK); + sqlite3_close(raw); + + return { + rclcpp::Parameter("storage_type", "sqlite"), + rclcpp::Parameter("database_path", db_path_.string()), + rclcpp::Parameter("confirmation_threshold", -1), + }; + } + + void TearDown() override { + FaultEventPublishingTest::TearDown(); + std::filesystem::remove(db_path_); + std::filesystem::remove(db_path_.string() + "-wal"); + std::filesystem::remove(db_path_.string() + "-shm"); + } + + std::filesystem::path db_path_; +}; + +// The migrated record is addressable like any other. A live source reporting the same +// code creates a second record beside it, and from there the ordinary rules apply: the +// unscoped call is refused and names both owners, and a scoped call reaches exactly one. +// An empty owner would fail all of this, because no source_id could ever name it. +TEST_F(LegacyOwnedRecordTest, AMigratedRecordIsAddressableBesideALiveSource) { + const std::string legacy = ros2_medkit_fault_manager::kLegacyOwner; + + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_CRITICAL, "/live_source")); + ASSERT_TRUE(spin_until([this]() { + return !received_events_.empty(); + })); + + // Scoped read reaches the migrated record, not the live one. + auto migrated = call_get_fault("SHARED_CODE", legacy); + ASSERT_TRUE(migrated.has_value()); + ASSERT_TRUE(migrated->success) << migrated->error_message; + ASSERT_EQ(migrated->fault.reporting_sources.size(), 1u); + EXPECT_EQ(migrated->fault.reporting_sources.front(), legacy); + EXPECT_EQ(migrated->fault.description, "from an older release"); + + // Unscoped is ambiguous, and the refusal names the migrated owner rather than + // leaving a blank where an owner should be. + const auto refused = clear_fault_response("SHARED_CODE"); + ASSERT_TRUE(refused.has_value()); + EXPECT_FALSE(refused->success); + EXPECT_EQ(refused->message.rfind("ambiguous:", 0), 0u) << "got: " << refused->message; + EXPECT_NE(refused->message.find(legacy), std::string::npos) << "got: " << refused->message; + EXPECT_NE(refused->message.find("/live_source"), std::string::npos) << "got: " << refused->message; + EXPECT_EQ(refused->message.find("()"), std::string::npos) + << "an empty owner would show as a blank entry: " << refused->message; + + // Scoped clear takes the migrated record alone. + ASSERT_TRUE(call_clear_fault("SHARED_CODE", legacy)); + auto live = call_get_fault("SHARED_CODE", "/live_source"); + ASSERT_TRUE(live.has_value()); + ASSERT_TRUE(live->success) << live->error_message; + EXPECT_EQ(live->fault.status, Fault::STATUS_CONFIRMED) << "the live source's record must be untouched"; +} + +/// GetRosbag over the wire, with the links written straight into the node's store: the +/// recording itself comes from a background capture this suite does not run, and what +/// is under test is how the handler reports the links, not how they got there. +class GetRosbagAttachedCodesTest : public FaultEventPublishingTest { + protected: + void SetUp() override { + FaultEventPublishingTest::SetUp(); + get_rosbag_client_ = test_node_->create_client(ns_ + "/fault_manager/get_rosbag"); + ASSERT_TRUE(get_rosbag_client_->wait_for_service(std::chrono::seconds(5))); + + // The handler serves a link only while its bag is on disk, so the directory has to + // exist for the response to be an answer rather than a reap. + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dist; + bag_path_ = std::filesystem::temp_directory_path() / ("test_get_rosbag_" + std::to_string(dist(gen))); + std::filesystem::create_directories(bag_path_); + } + + void TearDown() override { + get_rosbag_client_.reset(); + FaultEventPublishingTest::TearDown(); + std::error_code ec; + std::filesystem::remove_all(bag_path_, ec); + } + + /// Link one record to the single shared recording, the way a burst does. + void link_recording(const std::string & fault_code, const std::string & owner) { + ros2_medkit_fault_manager::RosbagFileInfo info; + info.fault_code = fault_code; + info.owner = owner; + info.file_path = bag_path_.string(); + info.format = "mcap"; + info.duration_sec = 5.0; + info.size_bytes = 100; + info.created_at_ns = 1000; + fault_manager_->get_storage_for_test().store_rosbag_file(info); + } + + std::optional call_get_rosbag(const std::string & fault_code, const std::string & source_id) { + auto request = std::make_shared(); + request->fault_code = fault_code; + request->source_id = source_id; + + auto future = get_rosbag_client_->async_send_request(request); + if (!spin_until_future_ready(future)) { + return std::nullopt; + } + return *future.get(); + } + + std::filesystem::path bag_path_; + rclcpp::Client::SharedPtr get_rosbag_client_; +}; + +// fault_codes answers which faults a recording covers, so it is a set of codes. Two +// owners of one code are two links to the one bag, and repeating the code would make +// the caller authorize the same download twice over. +TEST_F(GetRosbagAttachedCodesTest, ACodeSharedByTwoOwnersIsReportedOnce) { + ASSERT_TRUE(call_report_fault("BURST_CODE", Fault::SEVERITY_CRITICAL, "/owner_a")); + ASSERT_TRUE(call_report_fault("BURST_CODE", Fault::SEVERITY_CRITICAL, "/owner_b")); + link_recording("BURST_CODE", "/owner_a"); + link_recording("BURST_CODE", "/owner_b"); + + auto response = call_get_rosbag("BURST_CODE", "/owner_a"); + ASSERT_TRUE(response.has_value()); + ASSERT_TRUE(response->success) << response->error_message; + EXPECT_EQ(response->fault_codes, std::vector{"BURST_CODE"}); +} + +// The control on the same harness: deduplicating must not collapse the set itself. +// A burst of two different codes on one bag still reports both, in first-seen order. +TEST_F(GetRosbagAttachedCodesTest, EveryDistinctCodeOfTheBurstIsReported) { + ASSERT_TRUE(call_report_fault("BURST_FIRST", Fault::SEVERITY_CRITICAL, "/owner_a")); + ASSERT_TRUE(call_report_fault("BURST_SECOND", Fault::SEVERITY_CRITICAL, "/owner_a")); + link_recording("BURST_FIRST", "/owner_a"); + link_recording("BURST_SECOND", "/owner_a"); + + auto response = call_get_rosbag("BURST_FIRST", "/owner_a"); + ASSERT_TRUE(response.has_value()); + ASSERT_TRUE(response->success) << response->error_message; + EXPECT_EQ(response->fault_codes, (std::vector{"BURST_FIRST", "BURST_SECOND"})); +} + // @verifies REQ_INTEROP_012 TEST_F(FaultEventPublishingTest, ListFaultsForEntitySuccess) { // Report faults from different sources @@ -1580,7 +2519,7 @@ TEST_F(FreezeFrameRetentionTest, GetFaultServesRetainedFreezeFrameAfterClear) { // Capture runs asynchronously on the pool; wait for a non-empty freeze-frame. ASSERT_TRUE(spin_until( [this]() { - auto frame = fault_manager_->get_storage().get_freeze_frame("FF_FAULT"); + auto frame = fault_manager_->get_storage().get_freeze_frame({"FF_FAULT", "/test_node"}); return frame.has_value() && frame->data != "{}"; }, std::chrono::milliseconds(10000))); @@ -1647,6 +2586,7 @@ struct ReportedBag { ros2_medkit_fault_manager::RosbagFileInfo row_for(const std::string & fault_code) const { ros2_medkit_fault_manager::RosbagFileInfo info; info.fault_code = fault_code; + info.owner = "/test_node"; info.file_path = dir.string(); info.recording_id = ros2_medkit_fault_manager::rosbag_recording_id(info.file_path); info.format = "sqlite3"; @@ -1694,7 +2634,7 @@ TEST_F(FaultEventPublishingTest, GetFaultReportsARecordingsServedBytesNotItsFoot EXPECT_NE(rosbag_snapshot->size_bytes, bag.footprint) << "the directory total is the quota's figure, not the API's"; // The row itself is untouched: the quota still sees the whole recording. - auto row = fault_manager_->get_storage().get_rosbag_file("SERVED_BYTES_FAULT"); + auto row = fault_manager_->get_storage().get_rosbag_file({"SERVED_BYTES_FAULT", "/test_node"}); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->size_bytes, bag.footprint) << "reporting must not have rewritten what the quota counts"; } @@ -1831,13 +2771,13 @@ TEST_F(UnlimitedSnapshotRetentionTest, RetentionAppliesWithNoPerFaultCap) { ASSERT_TRUE(spin_until( [this]() { - return !fault_manager_->get_storage().get_snapshots("UNCAPPED_FAULT").empty(); + return !fault_manager_->get_storage().get_snapshots({"UNCAPPED_FAULT", "/test_node"}).empty(); }, std::chrono::milliseconds(10000))); ASSERT_TRUE(call_clear_fault("UNCAPPED_FAULT")); - EXPECT_FALSE(fault_manager_->get_storage().get_snapshots("UNCAPPED_FAULT").empty()) + EXPECT_FALSE(fault_manager_->get_storage().get_snapshots({"UNCAPPED_FAULT", "/test_node"}).empty()) << "acknowledgement deleted readings the operator asked to keep"; } @@ -1886,6 +2826,7 @@ TEST(InMemorySnapshotRetentionTest, ClearKeepsSnapshotsWhenEvidenceIsRetained) { ros2_medkit_fault_manager::SnapshotData row; row.fault_code = "KEEP"; + row.owner = "/n"; row.topic = "/t"; row.message_type = "std_msgs/msg/String"; row.data = "{}"; @@ -1893,11 +2834,11 @@ TEST(InMemorySnapshotRetentionTest, ClearKeepsSnapshotsWhenEvidenceIsRetained) { row.capture_id = 1; storage.store_snapshots({row}); - ASSERT_TRUE(storage.clear_fault("KEEP")); + ASSERT_TRUE(storage.clear_fault({"KEEP", "/n"})); // Same rule as the SQLite side: with a history configured, acknowledging must // not leave recordings whose matching readings were deleted. - EXPECT_EQ(storage.get_snapshots("KEEP").size(), 1u); + EXPECT_EQ(storage.get_snapshots({"KEEP", "/n"}).size(), 1u); } TEST(InMemorySnapshotRetentionTest, ClearStillDropsSnapshotsByDefault) { @@ -1908,14 +2849,15 @@ TEST(InMemorySnapshotRetentionTest, ClearStillDropsSnapshotsByDefault) { ros2_medkit_fault_manager::SnapshotData row; row.fault_code = "DROP"; + row.owner = "/n"; row.topic = "/t"; row.message_type = "std_msgs/msg/String"; row.data = "{}"; row.capture_id = 1; storage.store_snapshots({row}); - ASSERT_TRUE(storage.clear_fault("DROP")); - EXPECT_TRUE(storage.get_snapshots("DROP").empty()) << "the default must stay what it always was"; + ASSERT_TRUE(storage.clear_fault({"DROP", "/n"})); + EXPECT_TRUE(storage.get_snapshots({"DROP", "/n"}).empty()) << "the default must stay what it always was"; } TEST(InMemorySnapshotLimitTest, KeepsTheNewestCaptureWholeAndDropsTheOldest) { @@ -1927,6 +2869,7 @@ TEST(InMemorySnapshotLimitTest, KeepsTheNewestCaptureWholeAndDropsTheOldest) { for (const char * topic : {"/a", "/b"}) { ros2_medkit_fault_manager::SnapshotData row; row.fault_code = "TEST"; + row.owner = "/n"; row.topic = topic; row.message_type = "std_msgs/msg/String"; row.data = "{}"; @@ -1943,7 +2886,7 @@ TEST(InMemorySnapshotLimitTest, KeepsTheNewestCaptureWholeAndDropsTheOldest) { // Whole captures in, whole capture out. Counting rows and rejecting the new one // stored capture 3 in part: one topic present, the rest silently absent. - auto result = storage.get_snapshots("TEST"); + auto result = storage.get_snapshots({"TEST", "/n"}); ASSERT_EQ(result.size(), 4u); for (const auto & s : result) { EXPECT_NE(s.capture_id, 1) << "the oldest capture should have gone whole"; @@ -1956,6 +2899,7 @@ TEST(InMemorySnapshotLimitTest, UnlimitedWhenZero) { ros2_medkit_fault_manager::SnapshotData snap; snap.fault_code = "TEST"; + snap.owner = "/n"; snap.topic = "/test"; snap.message_type = "std_msgs/msg/String"; snap.data = "{}"; @@ -1965,7 +2909,7 @@ TEST(InMemorySnapshotLimitTest, UnlimitedWhenZero) { storage.store_snapshot(snap); } - auto result = storage.get_snapshots("TEST"); + auto result = storage.get_snapshots({"TEST", "/n"}); EXPECT_EQ(result.size(), 20u); } @@ -1976,11 +2920,12 @@ TEST(InMemoryFreezeFrameTest, StoreAndRetrieve) { ros2_medkit_fault_manager::FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc"; frame.data = R"({"/plc/pressure":{"data":8.4}})"; frame.captured_at_ns = 1000; storage.store_freeze_frame(frame); - auto retrieved = storage.get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage.get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, frame.data); EXPECT_EQ(retrieved->captured_at_ns, 1000); @@ -1988,7 +2933,7 @@ TEST(InMemoryFreezeFrameTest, StoreAndRetrieve) { TEST(InMemoryFreezeFrameTest, AbsentForUnknownFault) { InMemoryFaultStorage storage; - EXPECT_FALSE(storage.get_freeze_frame("NEVER_CAPTURED").has_value()); + EXPECT_FALSE(storage.get_freeze_frame({"NEVER_CAPTURED", "/plc"}).has_value()); } TEST(InMemoryFreezeFrameTest, ReplacedOnRecapture) { @@ -1996,6 +2941,7 @@ TEST(InMemoryFreezeFrameTest, ReplacedOnRecapture) { ros2_medkit_fault_manager::FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc"; frame.data = R"({"/plc/pressure":{"data":8.4}})"; frame.captured_at_ns = 1000; storage.store_freeze_frame(frame); @@ -2004,7 +2950,7 @@ TEST(InMemoryFreezeFrameTest, ReplacedOnRecapture) { frame.captured_at_ns = 2000; storage.store_freeze_frame(frame); - auto retrieved = storage.get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage.get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, R"({"/plc/pressure":{"data":9.9}})"); EXPECT_EQ(retrieved->captured_at_ns, 2000); @@ -2019,6 +2965,7 @@ TEST(InMemoryFreezeFrameTest, SurvivesClearFault) { ros2_medkit_fault_manager::SnapshotData snap; snap.fault_code = "PLC_PRESSURE_HIGH"; + snap.owner = "/plc_node"; snap.topic = "/plc/pressure"; snap.message_type = "std_msgs/msg/Float64"; snap.data = R"({"data":8.4})"; @@ -2027,14 +2974,15 @@ TEST(InMemoryFreezeFrameTest, SurvivesClearFault) { ros2_medkit_fault_manager::FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc_node"; frame.data = R"({"/plc/pressure":{"data":8.4}})"; frame.captured_at_ns = 1000; storage.store_freeze_frame(frame); - ASSERT_TRUE(storage.clear_fault("PLC_PRESSURE_HIGH")); + ASSERT_TRUE(storage.clear_fault({"PLC_PRESSURE_HIGH", "/plc_node"})); - EXPECT_TRUE(storage.get_snapshots("PLC_PRESSURE_HIGH").empty()); - auto retrieved = storage.get_freeze_frame("PLC_PRESSURE_HIGH"); + EXPECT_TRUE(storage.get_snapshots({"PLC_PRESSURE_HIGH", "/plc_node"}).empty()); + auto retrieved = storage.get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc_node"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, frame.data); } @@ -2051,6 +2999,7 @@ TEST(InMemoryRosbagRestoreTest, RestoreWithNewPathUnlinksTheOldExclusiveBag) { ros2_medkit_fault_manager::RosbagFileInfo info; info.fault_code = "X"; + info.owner = "/n"; info.file_path = old_path; info.format = "mcap"; info.duration_sec = 5.0; @@ -2062,7 +3011,7 @@ TEST(InMemoryRosbagRestoreTest, RestoreWithNewPathUnlinksTheOldExclusiveBag) { storage.store_rosbag_file(info); EXPECT_FALSE(std::filesystem::exists(old_path)) << "nobody references the old bag, it must be unlinked"; - auto row = storage.get_rosbag_file("X"); + auto row = storage.get_rosbag_file({"X", "/n"}); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->file_path, old_path + "_new"); } @@ -2080,16 +3029,19 @@ TEST(InMemoryRosbagRestoreTest, RestoreWithNewPathKeepsTheBagASiblingStillRefere info.size_bytes = 100; info.created_at_ns = 1000; info.fault_code = "X"; + info.owner = "/n"; storage.store_rosbag_file(info); info.fault_code = "Y"; + info.owner = "/n"; storage.store_rosbag_file(info); info.fault_code = "X"; + info.owner = "/n"; info.file_path = shared_path + "_new"; storage.store_rosbag_file(info); EXPECT_TRUE(std::filesystem::exists(shared_path)) << "the sibling fault still owns the shared bag"; - auto sibling = storage.get_rosbag_file("Y"); + auto sibling = storage.get_rosbag_file({"Y", "/n"}); ASSERT_TRUE(sibling.has_value()); EXPECT_EQ(sibling->file_path, shared_path); @@ -2332,17 +3284,25 @@ TEST_F(FaultAuditIntegrationTest, TransitionsAppendVerifiableChain) { EXPECT_EQ(records[2].event.transition, ros2_medkit_fault_manager::kTransitionConfirmed); EXPECT_EQ(records[3].event.transition, ros2_medkit_fault_manager::kTransitionCleared); EXPECT_EQ(records[1].event.fault_code, "AUDIT_FAULT"); + // Every transition row names the record's owner, the clear included: the + // clear_service and auto_heal literals said which mechanism moved the row, not + // whose fault moved, and with two owners per code that is the question the log + // has to answer. + EXPECT_EQ(records[1].event.source_id, "/plc/pump"); EXPECT_EQ(records[2].event.source_id, "/plc/pump"); + EXPECT_EQ(records[3].event.source_id, "/plc/pump"); auto result = audit.verify(); EXPECT_TRUE(result.ok) << result.error; EXPECT_EQ(result.checked, 4); } -// Completeness: an auto-healed fault must record its END. One FAILED confirms the -// fault (occurred + confirmed); two PASSED drive the debounce counter to the -// healing threshold, which must append a distinct "healed" row (source auto_heal), -// and the full occurred -> confirmed -> healed chain must verify. +// Completeness: an auto-healed record must record its END. One FAILED confirms the +// record (occurred + confirmed), two PASSED drive the debounce counter to the +// healing threshold, which must append a distinct "healed" row. That row names the +// record's OWNER: the transition column already says the heal was automatic, so the +// source is free to answer the question the transition cannot, whose record ended. +// The full occurred -> confirmed -> healed chain must verify. TEST_F(FaultAuditIntegrationTest, AutoHealAppendsHealedRow) { auto send_report = [&](uint8_t event_type) { auto req = std::make_shared(); @@ -2381,7 +3341,7 @@ TEST_F(FaultAuditIntegrationTest, AutoHealAppendsHealedRow) { EXPECT_EQ(transitions[0], ros2_medkit_fault_manager::kTransitionOccurred); EXPECT_EQ(transitions[1], ros2_medkit_fault_manager::kTransitionConfirmed); EXPECT_EQ(transitions[2], ros2_medkit_fault_manager::kTransitionHealed); - EXPECT_EQ(heal_source, "auto_heal"); + EXPECT_EQ(heal_source, "/robot/sensor") << "every audit row names the record's owner, not the mechanism"; auto result = audit.verify(); EXPECT_TRUE(result.ok) << result.error; @@ -2502,7 +3462,7 @@ TEST(FaultAuditTimerTest, TimerConfirmationAppendsConfirmedAuditRow) { rclcpp::Clock clock(RCL_SYSTEM_TIME); node->get_storage_for_test().report_fault_event("AUTO_CONF_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "stuck", "/robot/src", clock.now().nanoseconds(), config); - ASSERT_EQ(node->get_storage().get_fault("AUTO_CONF_1")->status, Fault::STATUS_PREFAILED); + ASSERT_EQ(node->get_storage().get_fault({"AUTO_CONF_1", "/robot/src"})->status, Fault::STATUS_PREFAILED); // Spin until a confirmed audit row appears or the budget expires (the wall // timer fires once per second). @@ -2524,7 +3484,7 @@ TEST(FaultAuditTimerTest, TimerConfirmationAppendsConfirmedAuditRow) { } EXPECT_TRUE(saw_confirmed) << "timer-driven confirmation was not audited"; - EXPECT_EQ(node->get_storage().get_fault("AUTO_CONF_1")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(node->get_storage().get_fault({"AUTO_CONF_1", "/robot/src"})->status, Fault::STATUS_CONFIRMED); EXPECT_TRUE(audit->verify().ok); } @@ -2543,7 +3503,7 @@ void seed_healed_fault(const std::string & db_path, const std::string & fault_co seed.report_fault_event(fault_code, fd::ReportKind::kPassed, 0, "", "/robot/src", clock.now().nanoseconds(), config); } - ASSERT_EQ(seed.get_fault(fault_code)->status, Fault::STATUS_HEALED); + ASSERT_EQ(seed.get_fault({fault_code, "/robot/src"})->status, Fault::STATUS_HEALED); } std::filesystem::path make_temp_dir(const char * prefix) { @@ -2578,7 +3538,7 @@ TEST(FaultAuditStartupReclassifyTest, StartupReclassifyAppendsClearedRow) { }); auto node = std::make_shared(options); - EXPECT_EQ(node->get_storage().get_fault("STALE_HEALED")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(node->get_storage().get_fault({"STALE_HEALED", "/robot/src"})->status, Fault::STATUS_CLEARED); const auto * audit = node->get_audit_log_for_test(); ASSERT_NE(audit, nullptr); @@ -2586,7 +3546,7 @@ TEST(FaultAuditStartupReclassifyTest, StartupReclassifyAppendsClearedRow) { for (const auto & rec : audit->read()) { if (rec.event.fault_code == "STALE_HEALED") { EXPECT_EQ(rec.event.transition, ros2_medkit_fault_manager::kTransitionCleared); - EXPECT_EQ(rec.event.source_id, "startup_reclassify"); + EXPECT_EQ(rec.event.source_id, "/robot/src") << "the audit row names the record's owner"; EXPECT_EQ(rec.event.status, Fault::STATUS_CLEARED); ++cleared_rows; } @@ -2615,7 +3575,7 @@ TEST(FaultAuditStartupReclassifyTest, DisabledAuditStillReclassifies) { }); auto node = std::make_shared(options); - EXPECT_EQ(node->get_storage().get_fault("STALE_HEALED")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(node->get_storage().get_fault({"STALE_HEALED", "/robot/src"})->status, Fault::STATUS_CLEARED); EXPECT_EQ(node->get_audit_log_for_test(), nullptr); EXPECT_FALSE(std::filesystem::exists(dir / "fault_audit.db")); } @@ -2722,10 +3682,12 @@ TEST(FaultAuditFailClosedTest, FailClosedAbortsAndFlags) { // --- InMemoryFaultStorage snapshot retention tests --- -static void store_two_snapshots(InMemoryFaultStorage & storage, const std::string & fault_code) { +static void store_two_snapshots(InMemoryFaultStorage & storage, const std::string & fault_code, + const std::string & owner = "/test_node") { for (int i = 0; i < 2; ++i) { ros2_medkit_fault_manager::SnapshotData snapshot; snapshot.fault_code = fault_code; + snapshot.owner = owner; snapshot.topic = "/test/topic" + std::to_string(i); snapshot.message_type = "std_msgs/msg/String"; snapshot.data = R"({"data": "value"})"; @@ -2741,11 +3703,11 @@ TEST(InMemorySnapshotRetentionTest, RetainedOnClearWhenConfigured) { storage.report_fault_event("SNAPSHOT_RETAIN_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault with evidence", "/test_node", 1000, default_config()); store_two_snapshots(storage, "SNAPSHOT_RETAIN_TEST"); - ASSERT_EQ(storage.get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u); + ASSERT_EQ(storage.get_snapshots({"SNAPSHOT_RETAIN_TEST", "/test_node"}).size(), 2u); - ASSERT_TRUE(storage.clear_fault("SNAPSHOT_RETAIN_TEST")); + ASSERT_TRUE(storage.clear_fault({"SNAPSHOT_RETAIN_TEST", "/test_node"})); - EXPECT_EQ(storage.get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u); + EXPECT_EQ(storage.get_snapshots({"SNAPSHOT_RETAIN_TEST", "/test_node"}).size(), 2u); } TEST(InMemorySnapshotRetentionTest, DeletedOnClearByDefault) { @@ -2755,9 +3717,9 @@ TEST(InMemorySnapshotRetentionTest, DeletedOnClearByDefault) { "fault with evidence", "/test_node", 1000, default_config()); store_two_snapshots(storage, "SNAPSHOT_DROP_TEST"); - ASSERT_TRUE(storage.clear_fault("SNAPSHOT_DROP_TEST")); + ASSERT_TRUE(storage.clear_fault({"SNAPSHOT_DROP_TEST", "/test_node"})); - EXPECT_TRUE(storage.get_snapshots("SNAPSHOT_DROP_TEST").empty()); + EXPECT_TRUE(storage.get_snapshots({"SNAPSHOT_DROP_TEST", "/test_node"}).empty()); } TEST(InMemorySnapshotRetentionTest, HealedReclassificationDropsSnapshotsByDefault) { @@ -2775,13 +3737,13 @@ TEST(InMemorySnapshotRetentionTest, HealedReclassificationDropsSnapshotsByDefaul storage.report_fault_event("SNAPSHOT_DROP_TEST", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "/test_node", 1000 + i, config); } - auto healed = storage.get_fault("SNAPSHOT_DROP_TEST"); + auto healed = storage.get_fault({"SNAPSHOT_DROP_TEST", "/test_node"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; ASSERT_EQ(storage.reclassify_healed_as_cleared().size(), 1u); - EXPECT_TRUE(storage.get_snapshots("SNAPSHOT_DROP_TEST").empty()); + EXPECT_TRUE(storage.get_snapshots({"SNAPSHOT_DROP_TEST", "/test_node"}).empty()); } TEST(InMemorySnapshotRetentionTest, HealedReclassificationKeepsSnapshotsWhenConfigured) { @@ -2800,7 +3762,7 @@ TEST(InMemorySnapshotRetentionTest, HealedReclassificationKeepsSnapshotsWhenConf } ASSERT_EQ(storage.reclassify_healed_as_cleared().size(), 1u); - EXPECT_EQ(storage.get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u); + EXPECT_EQ(storage.get_snapshots({"SNAPSHOT_RETAIN_TEST", "/test_node"}).size(), 2u); } TEST(InMemorySnapshotRetentionTest, HealedReclassificationLeavesOtherFaultsSnapshotsAlone) { @@ -2823,8 +3785,9 @@ TEST(InMemorySnapshotRetentionTest, HealedReclassificationLeavesOtherFaultsSnaps ASSERT_EQ(storage.reclassify_healed_as_cleared().size(), 1u); - EXPECT_TRUE(storage.get_snapshots("HEALED_FAULT").empty()); - EXPECT_EQ(storage.get_snapshots("ACTIVE_FAULT").size(), 2u) << "reclassification touched an unrelated fault"; + EXPECT_TRUE(storage.get_snapshots({"HEALED_FAULT", "/test_node"}).empty()); + EXPECT_EQ(storage.get_snapshots({"ACTIVE_FAULT", "/test_node"}).size(), 2u) + << "reclassification touched an unrelated fault"; } TEST(InMemorySnapshotRetentionTest, RetentionIsPerFaultCodeNotGlobal) { @@ -2837,10 +3800,10 @@ TEST(InMemorySnapshotRetentionTest, RetentionIsPerFaultCodeNotGlobal) { store_two_snapshots(storage, code); } - ASSERT_TRUE(storage.clear_fault("FAULT_A")); + ASSERT_TRUE(storage.clear_fault({"FAULT_A", "/test_node"})); - EXPECT_EQ(storage.get_snapshots("FAULT_A").size(), 2u); - EXPECT_EQ(storage.get_snapshots("FAULT_B").size(), 2u); + EXPECT_EQ(storage.get_snapshots({"FAULT_A", "/test_node"}).size(), 2u); + EXPECT_EQ(storage.get_snapshots({"FAULT_B", "/test_node"}).size(), 2u); } // --- InMemoryFaultStorage near-miss series tests --- @@ -2870,14 +3833,14 @@ TEST(InMemoryNearMissTest, SeriesIsAppendedAndSurvivesClear) { "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); } - auto series = storage.get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u); EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[2].debounce_counter, -3); EXPECT_EQ(series[0].confirmation_threshold, -4); - ASSERT_TRUE(storage.clear_fault("PUMP_PRESSURE_LOW")); - EXPECT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + ASSERT_TRUE(storage.clear_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); + EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); } TEST(InMemoryNearMissTest, ConfirmingReportIsNotANearMiss) { @@ -2889,10 +3852,10 @@ TEST(InMemoryNearMissTest, ConfirmingReportIsNotANearMiss) { "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); } - auto fault = storage.get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage.get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); - EXPECT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); } TEST(InMemoryNearMissTest, SeriesBoundedKeepingNewest) { @@ -2906,7 +3869,7 @@ TEST(InMemoryNearMissTest, SeriesBoundedKeepingNewest) { "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); } - auto series = storage.get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u); EXPECT_EQ(series[0].debounce_counter, -3); EXPECT_EQ(series[2].debounce_counter, -5); @@ -2923,7 +3886,7 @@ TEST(InMemoryNearMissTest, PassedReportIsNotANearMiss) { storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", near_miss_time(2).nanoseconds(), config); - EXPECT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); + EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); } TEST(InMemoryNearMissTest, SeriesSurvivesHealedReclassification) { @@ -2938,14 +3901,14 @@ TEST(InMemoryNearMissTest, SeriesSurvivesHealedReclassification) { storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); } - auto healed = storage.get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage.get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; - ASSERT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + ASSERT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); ASSERT_EQ(storage.reclassify_healed_as_cleared().size(), 1u); - EXPECT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); } TEST(InMemoryNearMissTest, BoundIsPerFaultCode) { @@ -2961,8 +3924,8 @@ TEST(InMemoryNearMissTest, BoundIsPerFaultCode) { "/powertrain/motor", near_miss_time(i).nanoseconds(), config); } - EXPECT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); - EXPECT_EQ(storage.get_near_misses("MOTOR_OVERHEAT").size(), 2u); + EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); + EXPECT_EQ(storage.get_near_misses({"MOTOR_OVERHEAT", "/powertrain/motor"}).size(), 2u); } TEST(InMemoryNearMissTest, BoundZeroIsUnlimited) { @@ -2976,7 +3939,7 @@ TEST(InMemoryNearMissTest, BoundZeroIsUnlimited) { "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); } - EXPECT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 150u); + EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 150u); } TEST(InMemoryNearMissTest, ApplyingSmallerBoundTrimsExistingSeries) { @@ -2992,7 +3955,7 @@ TEST(InMemoryNearMissTest, ApplyingSmallerBoundTrimsExistingSeries) { storage.set_max_near_misses_per_fault(2); - auto series = storage.get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[0].debounce_counter, -4); EXPECT_EQ(series[1].debounce_counter, -5); @@ -3027,14 +3990,14 @@ TEST(InMemoryNearMissTest, EntriesCarryTheResultingStatus) { storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); } - auto healed = storage.get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage.get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", near_miss_time(4).nanoseconds(), config); - auto series = storage.get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[0].resulting_status, Fault::STATUS_PREFAILED); EXPECT_EQ(series[1].resulting_status, Fault::STATUS_HEALED) @@ -3042,10 +4005,11 @@ TEST(InMemoryNearMissTest, EntriesCarryTheResultingStatus) { } /// Run one identical report sequence against both backends and return their near-miss series. -/// Per-entity threshold overrides mean two reports for the SAME fault code can carry different -/// DebounceConfig values, which is the only way a stored counter ends up outside the band the next -/// report is evaluated against. -TEST(StorageBackendParityTest, MixedEntityThresholdsProduceTheSameSeries) { +/// A record's counter belongs to one source, but that source's resolved band can change under +/// it - an entity-threshold file reloaded with different values, or a global config that moved +/// between runs - which is how a stored counter ends up outside the band the next report is +/// evaluated against. Both backends clamp on read, and they have to agree on the result. +TEST(StorageBackendParityTest, AChangedBandProducesTheSameSeriesInBothBackends) { std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution dist; @@ -3057,34 +4021,36 @@ TEST(StorageBackendParityTest, MixedEntityThresholdsProduceTheSameSeries) { wide.healing_threshold = 6; wide.critical_immediate_confirm = false; - DebounceConfig narrow; // a second source, with a lower ceiling + DebounceConfig narrow; // the same source, re-resolved to a lower ceiling narrow.confirmation_threshold = -4; narrow.healing_threshold = 3; narrow.critical_immediate_confirm = false; + const ros2_medkit_fault_manager::FaultId id{"SHARED_CODE", "/one_source"}; + auto drive = [&](ros2_medkit_fault_manager::FaultStorage & storage) { - storage.report_fault_event("SHARED_CODE", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "dip", "/wide_source", - 1000, wide); + storage.report_fault_event("SHARED_CODE", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "dip", "/one_source", 1000, + wide); for (int i = 1; i <= 7; ++i) { - storage.report_fault_event("SHARED_CODE", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/wide_source", + storage.report_fault_event("SHARED_CODE", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/one_source", 1000 + i, wide); } - storage.report_fault_event("SHARED_CODE", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "dip", "/narrow_source", - 2000, narrow); + storage.report_fault_event("SHARED_CODE", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "dip", "/one_source", 2000, + narrow); }; InMemoryFaultStorage memory; drive(memory); - const auto memory_series = memory.get_near_misses("SHARED_CODE"); - const auto memory_fault = memory.get_fault("SHARED_CODE"); + const auto memory_series = memory.get_near_misses(id); + const auto memory_fault = memory.get_fault(id); std::vector sqlite_series; std::optional sqlite_fault; { ros2_medkit_fault_manager::SqliteFaultStorage sqlite(db_path.string(), test_log_sink()); drive(sqlite); - sqlite_series = sqlite.get_near_misses("SHARED_CODE"); - sqlite_fault = sqlite.get_fault("SHARED_CODE"); + sqlite_series = sqlite.get_near_misses(id); + sqlite_fault = sqlite.get_fault(id); } std::filesystem::remove(db_path); std::filesystem::remove(db_path.string() + "-wal"); @@ -3105,7 +4071,7 @@ TEST(StorageBackendParityTest, MixedEntityThresholdsProduceTheSameSeries) { TEST(InMemoryNearMissTest, EmptyForUnknownFault) { InMemoryFaultStorage storage; - EXPECT_TRUE(storage.get_near_misses("NEVER_REPORTED").empty()); + EXPECT_TRUE(storage.get_near_misses({"NEVER_REPORTED", "/node1"}).empty()); } // --- Snapshot read path with retention enabled --- @@ -3170,9 +4136,10 @@ class SnapshotReadPathTest : public ::testing::Test { } void store_snapshot(const std::string & fault_code, const std::string & topic, const std::string & data, - int64_t captured_at_ns) { + int64_t captured_at_ns, const std::string & owner = "/plc") { ros2_medkit_fault_manager::SnapshotData snapshot; snapshot.fault_code = fault_code; + snapshot.owner = owner; snapshot.topic = topic; snapshot.message_type = "std_msgs/msg/String"; snapshot.data = data; @@ -3216,6 +4183,7 @@ TEST_F(SnapshotReadPathTest, FreezeFrameStaysVisibleBehindRetainedSnapshots) { ros2_medkit_fault_manager::FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc"; frame.data = R"({"/plc/pressure":{"data":"latest confirmation"}})"; frame.captured_at_ns = 9000; storage.store_freeze_frame(frame); diff --git a/src/ros2_medkit_fault_manager/test/test_fault_manager_core_concurrency.cpp b/src/ros2_medkit_fault_manager/test/test_fault_manager_core_concurrency.cpp index 530adff39..4ad5fc721 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_manager_core_concurrency.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_manager_core_concurrency.cpp @@ -39,6 +39,9 @@ constexpr int kReportsPerThread = 500; constexpr int kCodes = 4; constexpr int kClearEvery = 50; const char * const kClearedCode = "CLEARED_CODE"; +/// Every load thread reports as one source, so the threads contend on the same records. +const char * const kLoadSource = "/load"; +const char * const kSeedSource = "/seed"; class NullLogSink : public core::LogSink { public: @@ -64,12 +67,12 @@ class CountingEventSink : public core::EventSink { class NoCaptureHooks : public core::CaptureHooks { public: - core::CaptureResult capture_now(const std::string & /*code*/) override { + core::CaptureResult capture_now(const ros2_medkit_fault_manager::FaultId & /*id*/) override { return {}; } - void on_prefailed(const std::string & /*code*/) override { + void on_prefailed(const ros2_medkit_fault_manager::FaultId & /*id*/) override { } - void on_cleared(const std::string & /*code*/) override { + void on_cleared(const ros2_medkit_fault_manager::FaultId & /*id*/) override { } }; @@ -92,8 +95,8 @@ class YieldingStorage : public ros2_medkit_fault_manager::InMemoryFaultStorage { return InMemoryFaultStorage::report_fault_event(fault_code, kind, severity, description, source_id, timestamp_ns, config); } - std::optional get_fault(const std::string & fault_code) const override { - auto result = InMemoryFaultStorage::get_fault(fault_code); + std::optional get_fault(const ros2_medkit_fault_manager::FaultId & id) const override { + auto result = InMemoryFaultStorage::get_fault(id); pause(); return result; } @@ -130,7 +133,7 @@ struct Fixture { fd::FaultReport seed; seed.fault_code = kClearedCode; seed.severity = fd::severity::kError; - seed.source_id = "/seed"; + seed.source_id = kSeedSource; EXPECT_TRUE(core.report(seed).accepted); } }; @@ -147,10 +150,10 @@ void run_thread_sequence(core::FaultManagerCore & core, int thread_index) { report.kind = fd::ReportKind::kFailed; report.severity = fd::severity::kError; report.description = "load"; - report.source_id = "/thread_" + std::to_string(thread_index); + report.source_id = kLoadSource; EXPECT_TRUE(core.report(report).accepted); if (i % kClearEvery == 0) { - EXPECT_TRUE(core.clear(kClearedCode, true).success); + EXPECT_TRUE(core.clear(kClearedCode, kSeedSource, true).success); } } } @@ -199,8 +202,8 @@ TEST(FaultManagerCoreConcurrency, ParallelReportsKeepCounts) { // Per code occurrence_count and status equal the replay's. for (int c = 0; c < kCodes; ++c) { const auto code = code_name(c); - const auto got = parallel.core.get(code); - const auto want = replay.core.get(code); + const auto got = parallel.core.get(code, kLoadSource); + const auto want = replay.core.get(code, kLoadSource); ASSERT_TRUE(got.success) << code; ASSERT_TRUE(want.success) << code; EXPECT_EQ(got.fault.occurrence_count, want.fault.occurrence_count) << code; diff --git a/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp b/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp new file mode 100644 index 000000000..c366a37b8 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp @@ -0,0 +1,893 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Severity contract of a fault record, run against both storage backends. +/// +/// A record (fault_code, owner) has one severity: the highest any FAILED report from its owner has +/// carried, within an occurrence and across occurrences. With critical_immediate_confirm, a report +/// on a record whose stored severity is CRITICAL confirms it, so no stored record is CRITICAL in +/// PREFAILED or PREPASSED. Another owner's record of the same code is untouched. + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rclcpp/rclcpp.hpp" +#include "ros2_medkit_fault_manager/fault_storage.hpp" +#include "ros2_medkit_fault_manager/sqlite_fault_storage.hpp" +#include "ros2_medkit_msgs/msg/fault.hpp" +#include "test_log_sink.hpp" + +namespace { + +using ros2_medkit_fault_manager::DebounceConfig; +using ros2_medkit_fault_manager::FaultId; +using ros2_medkit_fault_manager::FaultStorage; +using ros2_medkit_fault_manager::InMemoryFaultStorage; +using ros2_medkit_fault_manager::SqliteFaultStorage; +using ros2_medkit_msgs::msg::Fault; + +constexpr const char * kOwner = "/reporter"; + +constexpr int64_t kBaseNs = 1'700'000'000'000'000'000LL; + +int64_t at(int64_t second) { + return kBaseNs + second * 1'000'000'000LL; +} + +DebounceConfig confirm_at(int32_t confirmation_threshold) { + DebounceConfig config; + config.confirmation_threshold = confirmation_threshold; + return config; +} + +bool pending(const std::string & status) { + return status == Fault::STATUS_PREFAILED || status == Fault::STATUS_PREPASSED; +} + +std::filesystem::path unique_temp_dir() { + std::random_device rd; + std::mt19937_64 gen(rd()); + std::uniform_int_distribution dist; + return std::filesystem::temp_directory_path() / ("medkit_severity_" + std::to_string(dist(gen))); +} + +template +struct BackendFactory; + +template <> +struct BackendFactory { + static std::unique_ptr make(const std::filesystem::path & /*dir*/) { + return std::make_unique(); + } +}; + +template <> +struct BackendFactory { + static std::unique_ptr make(const std::filesystem::path & dir) { + return std::make_unique((dir / "faults.db").string(), test_log_sink()); + } +}; + +template +class FaultSeverityTest : public ::testing::Test { + protected: + void SetUp() override { + dir_ = unique_temp_dir(); + std::filesystem::create_directories(dir_); + storage_ = BackendFactory::make(dir_); + } + + void TearDown() override { + storage_.reset(); + std::error_code ec; + std::filesystem::remove_all(dir_, ec); + } + + bool failed(const std::string & code, uint8_t severity, int64_t second, const DebounceConfig & config, + const std::string & owner = kOwner) { + return storage_->report_fault_event(code, fd::ReportKind::kFailed, severity, "report", owner, at(second), config); + } + + void passed(const std::string & code, int64_t second, const DebounceConfig & config, + const std::string & owner = kOwner) { + storage_->report_fault_event(code, fd::ReportKind::kPassed, 0, "", owner, at(second), config); + } + + bool clear(const std::string & code, const std::string & owner = kOwner) { + return storage_->clear_fault(FaultId{code, owner}); + } + + fd::FaultRecord fault(const std::string & code, const std::string & owner = kOwner) { + auto f = storage_->get_fault(FaultId{code, owner}); + EXPECT_TRUE(f.has_value()) << code << " " << owner; + return f.value_or(fd::FaultRecord{}); + } + + /// Every stored CRITICAL record in a pending status. + std::vector pending_critical() { + return storage_->list_faults(true, Fault::SEVERITY_CRITICAL, {Fault::STATUS_PREFAILED, Fault::STATUS_PREPASSED}); + } + + std::filesystem::path dir_; + std::unique_ptr storage_; +}; + +using Backends = ::testing::Types; +TYPED_TEST_SUITE(FaultSeverityTest, Backends); + +TYPED_TEST(FaultSeverityTest, CriticalThenClearedWarnConfirmsAtEveryThreshold) { + for (const int32_t threshold : {-1, -2, -3}) { + const auto config = confirm_at(threshold); + ASSERT_TRUE(config.critical_immediate_confirm); + const std::string code = "CRIT_CLEAR_WARN_T" + std::to_string(-threshold); + SCOPED_TRACE(code); + + this->failed(code, Fault::SEVERITY_CRITICAL, 0, config); + ASSERT_EQ(this->fault(code).status, Fault::STATUS_CONFIRMED); + ASSERT_TRUE(this->clear(code)); + + EXPECT_TRUE(this->failed(code, Fault::SEVERITY_WARN, 1, config)) << "a report on a CLEARED record is an occurrence"; + const auto f = this->fault(code); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL) << "the record keeps the highest severity ever reported"; + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED) << "a CRITICAL record confirms on its first report"; + EXPECT_EQ(f.occurrence_count, 2u); + EXPECT_TRUE(this->pending_critical().empty()); + } +} + +TYPED_TEST(FaultSeverityTest, TimeBasedConfirmationFindsACriticalRecordConfirmedAtTheReport) { + auto config = confirm_at(-3); + config.auto_confirm_after_sec = 5.0; + this->storage_->set_debounce_config(config); + + this->failed("TIMED_CRIT", Fault::SEVERITY_CRITICAL, 0, config); + ASSERT_TRUE(this->clear("TIMED_CRIT")); + this->failed("TIMED_CRIT", Fault::SEVERITY_WARN, 1, config); + this->failed("TIMED_WARN", Fault::SEVERITY_WARN, 1, config); + EXPECT_EQ(this->fault("TIMED_CRIT").status, Fault::STATUS_CONFIRMED) << "confirmed by the report, not the sweep"; + EXPECT_EQ(this->fault("TIMED_WARN").status, Fault::STATUS_PREFAILED); + + EXPECT_TRUE(this->storage_->check_time_based_confirmation(at(3)).empty()) << "before the window ends"; + const auto swept = this->storage_->check_time_based_confirmation(at(10)); + ASSERT_EQ(swept.size(), 1u) << "the sweep confirms the pending record only"; + EXPECT_EQ(swept[0].fault_code, "TIMED_WARN"); + EXPECT_EQ(this->fault("TIMED_WARN").status, Fault::STATUS_CONFIRMED); + const auto f = this->fault("TIMED_CRIT"); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(f.occurrence_count, 2u); +} + +TYPED_TEST(FaultSeverityTest, AnotherOwnersCriticalDoesNotConfirmThisRecord) { + const auto config = confirm_at(-3); + + this->failed("SHARED_CRIT", Fault::SEVERITY_CRITICAL, 0, config, "/a"); + ASSERT_TRUE(this->clear("SHARED_CRIT", "/a")); + + EXPECT_TRUE(this->failed("SHARED_CRIT", Fault::SEVERITY_WARN, 1, config, "/b")) << "/b owns no record yet"; + auto b = this->fault("SHARED_CRIT", "/b"); + EXPECT_EQ(b.severity, Fault::SEVERITY_WARN) << "severity is per record"; + EXPECT_EQ(b.status, Fault::STATUS_PREFAILED); + + this->failed("SHARED_CRIT", Fault::SEVERITY_WARN, 2, config, "/a"); + const auto a = this->fault("SHARED_CRIT", "/a"); + EXPECT_EQ(a.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(a.status, Fault::STATUS_CONFIRMED); + b = this->fault("SHARED_CRIT", "/b"); + EXPECT_EQ(b.status, Fault::STATUS_PREFAILED) << "confirming /a leaves /b's record as it was"; + EXPECT_TRUE(this->pending_critical().empty()); +} + +TYPED_TEST(FaultSeverityTest, WarnCriticalWarnWithinAnOccurrence) { + const auto config = confirm_at(-3); + + this->failed("W_C_W", Fault::SEVERITY_WARN, 0, config); + EXPECT_EQ(this->fault("W_C_W").status, Fault::STATUS_PREFAILED); + EXPECT_EQ(this->fault("W_C_W").severity, Fault::SEVERITY_WARN); + + this->failed("W_C_W", Fault::SEVERITY_CRITICAL, 1, config); + EXPECT_EQ(this->fault("W_C_W").status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(this->fault("W_C_W").severity, Fault::SEVERITY_CRITICAL) << "escalation within an occurrence"; + + this->failed("W_C_W", Fault::SEVERITY_WARN, 2, config); + const auto f = this->fault("W_C_W"); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL) << "a lower report does not lower the record"; + EXPECT_EQ(f.occurrence_count, 1u); +} + +TYPED_TEST(FaultSeverityTest, WarnCriticalClearWarnConfirmsTheNewOccurrence) { + const auto config = confirm_at(-3); + + this->failed("W_C_CLR_W", Fault::SEVERITY_WARN, 0, config); + this->failed("W_C_CLR_W", Fault::SEVERITY_CRITICAL, 1, config); + ASSERT_TRUE(this->clear("W_C_CLR_W")); + + this->failed("W_C_CLR_W", Fault::SEVERITY_WARN, 2, config); + const auto f = this->fault("W_C_CLR_W"); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(f.occurrence_count, 2u); + EXPECT_TRUE(this->pending_critical().empty()); +} + +TYPED_TEST(FaultSeverityTest, HealedCriticalRecordConfirmsOnTheNextFailed) { + // Healing at 1: one FAILED confirms at -1, two PASSED reach 1 and heal. + DebounceConfig config = confirm_at(-3); + config.healing_enabled = true; + config.healing_threshold = 1; + + this->failed("HEAL_CRIT", Fault::SEVERITY_CRITICAL, 0, config); + this->passed("HEAL_CRIT", 1, config); + this->passed("HEAL_CRIT", 2, config); + ASSERT_EQ(this->fault("HEAL_CRIT").status, Fault::STATUS_HEALED); + + EXPECT_FALSE(this->failed("HEAL_CRIT", Fault::SEVERITY_WARN, 3, config)) << "HEALED is not an occurrence boundary"; + const auto f = this->fault("HEAL_CRIT"); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED) << "the latch does not hold a CRITICAL record back"; + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(f.occurrence_count, 1u); + EXPECT_EQ(f.first_occurred_ns, at(0)); +} + +TYPED_TEST(FaultSeverityTest, HealedWarnRecordStillWalksBackThroughTheLatch) { + // Control for the case above: a non-CRITICAL record keeps the latch. + DebounceConfig config = confirm_at(-3); + config.healing_enabled = true; + config.healing_threshold = 1; + + this->failed("HEAL_WARN", Fault::SEVERITY_WARN, 0, config); + this->failed("HEAL_WARN", Fault::SEVERITY_WARN, 1, config); + this->failed("HEAL_WARN", Fault::SEVERITY_WARN, 2, config); + ASSERT_EQ(this->fault("HEAL_WARN").status, Fault::STATUS_CONFIRMED); + for (int64_t s = 3; s < 7; ++s) { + this->passed("HEAL_WARN", s, config); + } + ASSERT_EQ(this->fault("HEAL_WARN").status, Fault::STATUS_HEALED); + + this->failed("HEAL_WARN", Fault::SEVERITY_WARN, 7, config); + EXPECT_EQ(this->fault("HEAL_WARN").status, Fault::STATUS_HEALED); +} + +TYPED_TEST(FaultSeverityTest, CriticalBypassOffLeavesTheDebounceInCharge) { + DebounceConfig config = confirm_at(-3); + config.critical_immediate_confirm = false; + + this->failed("NO_BYPASS", Fault::SEVERITY_CRITICAL, 0, config); + EXPECT_EQ(this->fault("NO_BYPASS").status, Fault::STATUS_PREFAILED); + this->failed("NO_BYPASS", Fault::SEVERITY_WARN, 1, config); + EXPECT_EQ(this->fault("NO_BYPASS").status, Fault::STATUS_PREFAILED); +} + +TYPED_TEST(FaultSeverityTest, CriticalBypassOffLeavesTheReactivationToTheDebounce) { + DebounceConfig config = confirm_at(-3); + config.critical_immediate_confirm = false; + + for (const int64_t second : {0, 1, 2}) { + this->failed("NO_BYPASS_REACT", Fault::SEVERITY_CRITICAL, second, config); + } + ASSERT_EQ(this->fault("NO_BYPASS_REACT").status, Fault::STATUS_CONFIRMED) << "confirmed by the debounce"; + ASSERT_TRUE(this->clear("NO_BYPASS_REACT")); + + this->failed("NO_BYPASS_REACT", Fault::SEVERITY_WARN, 3, config); + auto f = this->fault("NO_BYPASS_REACT"); + EXPECT_EQ(f.status, Fault::STATUS_PREFAILED) << "without the bypass a CRITICAL record reactivates like any other"; + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(f.occurrence_count, 2u); + + this->failed("NO_BYPASS_REACT", Fault::SEVERITY_WARN, 4, config); + EXPECT_EQ(this->fault("NO_BYPASS_REACT").status, Fault::STATUS_PREFAILED); + this->failed("NO_BYPASS_REACT", Fault::SEVERITY_WARN, 5, config); + EXPECT_EQ(this->fault("NO_BYPASS_REACT").status, Fault::STATUS_CONFIRMED); +} + +TYPED_TEST(FaultSeverityTest, SeverityNeverDropsWithinOrAcrossOccurrences) { + for (const int32_t threshold : {-1, -3}) { + const auto config = confirm_at(threshold); + const std::string code = "NO_DROP_T" + std::to_string(-threshold); + SCOPED_TRACE(code); + + this->failed(code, Fault::SEVERITY_ERROR, 0, config); + this->failed(code, Fault::SEVERITY_WARN, 1, config); + this->failed(code, Fault::SEVERITY_INFO, 2, config); + EXPECT_EQ(this->fault(code).severity, Fault::SEVERITY_ERROR); + + ASSERT_TRUE(this->clear(code)); + this->failed(code, Fault::SEVERITY_WARN, 3, config); + const auto f = this->fault(code); + EXPECT_EQ(f.severity, Fault::SEVERITY_ERROR); + EXPECT_EQ(f.occurrence_count, 2u); + EXPECT_TRUE( + this->storage_->list_faults(true, Fault::SEVERITY_WARN, {Fault::STATUS_PREFAILED, Fault::STATUS_CONFIRMED}) + .empty()) + << "the severity filter matches the stored severity"; + } +} + +TYPED_TEST(FaultSeverityTest, NearMissRowsKeepTheReportedSeverity) { + const auto config = confirm_at(-3); + + this->failed("NEAR", Fault::SEVERITY_ERROR, 0, config); + this->failed("NEAR", Fault::SEVERITY_WARN, 1, config); + ASSERT_EQ(this->fault("NEAR").severity, Fault::SEVERITY_ERROR); + + const auto series = this->storage_->get_near_misses(FaultId{"NEAR", kOwner}); + ASSERT_EQ(series.size(), 2u); + EXPECT_EQ(series[0].severity, Fault::SEVERITY_ERROR); + EXPECT_EQ(series[1].severity, Fault::SEVERITY_WARN) << "a near-miss row records its report"; +} + +// --------------------------------------------------------------------------- +// SQLite only: restart, the migration of a database without owners, and the confirmation time +// the in-memory backend does not store. +// --------------------------------------------------------------------------- + +/// The faults table as releases before record identity created it: one row per code. +constexpr const char * kLegacyFaultsDdl = + "CREATE TABLE faults (fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, " + "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " + "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " + "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " + "debounce_counter INTEGER NOT NULL DEFAULT 0, " + "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " + "confirmed_at_ns INTEGER NOT NULL DEFAULT 0);"; + +class SqliteSeverityTest : public ::testing::Test { + protected: + void SetUp() override { + dir_ = unique_temp_dir(); + std::filesystem::create_directories(dir_); + reopen(); + } + + void TearDown() override { + storage_.reset(); + std::error_code ec; + std::filesystem::remove_all(dir_, ec); + } + + /// Close the database and open it again, as a fault manager restart does. + void reopen() { + storage_.reset(); + storage_ = std::make_unique(db_path(), test_log_sink()); + } + + std::string db_path() const { + return (dir_ / "faults.db").string(); + } + + struct LegacyRow { + std::string code; + int severity; + std::string status; + int64_t confirmed_at_ns; + }; + + /// Replace the database with a legacy one holding these rows, then open it, which migrates it. + void open_legacy_database(const std::vector & rows) { + storage_.reset(); + std::error_code ec; + std::filesystem::remove(db_path(), ec); + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path().c_str(), &raw), SQLITE_OK); + const int64_t t1 = at(1); + std::ostringstream sql; + sql << kLegacyFaultsDdl; + for (const auto & row : rows) { + sql << "INSERT INTO faults VALUES ('" << row.code << "', " << row.severity << ", 'legacy', " << t1 << ", " << t1 + << ", 2, '" << row.status << "', '[\"" << kOwner << "\"]', -1, " << t1 << ", 0, " << row.confirmed_at_ns + << ");"; + } + EXPECT_EQ(sqlite3_exec(raw, sql.str().c_str(), nullptr, nullptr, nullptr), SQLITE_OK) << sqlite3_errmsg(raw); + sqlite3_close(raw); + reopen(); + } + + void open_legacy_database(const std::string & code, int severity, const std::string & status, + int64_t confirmed_at_ns) { + open_legacy_database({LegacyRow{code, severity, status, confirmed_at_ns}}); + } + + void failed(const std::string & code, uint8_t severity, int64_t second, const DebounceConfig & config) { + storage_->report_fault_event(code, fd::ReportKind::kFailed, severity, "report", kOwner, at(second), config); + } + + void passed(const std::string & code, int64_t second, const DebounceConfig & config) { + storage_->report_fault_event(code, fd::ReportKind::kPassed, 0, "", kOwner, at(second), config); + } + + bool clear(const std::string & code) { + return storage_->clear_fault(FaultId{code, kOwner}); + } + + fd::FaultRecord fault(const std::string & code) { + auto f = storage_->get_fault(FaultId{code, kOwner}); + EXPECT_TRUE(f.has_value()) << code; + return f.value_or(fd::FaultRecord{}); + } + + int64_t confirmed_at(const std::string & code) const { + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path().c_str(), &raw), SQLITE_OK); + sqlite3_stmt * stmt = nullptr; + EXPECT_EQ(sqlite3_prepare_v2(raw, "SELECT confirmed_at_ns FROM faults WHERE fault_code = ? AND owner = ?", -1, + &stmt, nullptr), + SQLITE_OK); + sqlite3_bind_text(stmt, 1, code.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, kOwner, -1, SQLITE_TRANSIENT); + int64_t value = -1; + if (sqlite3_step(stmt) == SQLITE_ROW) { + value = sqlite3_column_int64(stmt, 0); + } + sqlite3_finalize(stmt); + sqlite3_close(raw); + return value; + } + + std::filesystem::path dir_; + std::unique_ptr storage_; +}; + +TEST_F(SqliteSeverityTest, CriticalClearRestartWarnConfirms) { + const auto config = confirm_at(-3); + failed("CRIT_RESTART", Fault::SEVERITY_CRITICAL, 0, config); + ASSERT_TRUE(clear("CRIT_RESTART")); + + reopen(); + failed("CRIT_RESTART", Fault::SEVERITY_WARN, 1, config); + + const auto f = fault("CRIT_RESTART"); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(f.occurrence_count, 2u); + EXPECT_EQ(confirmed_at("CRIT_RESTART"), at(1)); +} + +TEST_F(SqliteSeverityTest, ReactivationHasNoConfirmationTimeUntilItConfirms) { + const auto config = confirm_at(-3); + failed("CONF_AT", Fault::SEVERITY_ERROR, 0, config); + failed("CONF_AT", Fault::SEVERITY_ERROR, 1, config); + failed("CONF_AT", Fault::SEVERITY_ERROR, 2, config); + ASSERT_EQ(fault("CONF_AT").status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(confirmed_at("CONF_AT"), at(2)); + ASSERT_TRUE(clear("CONF_AT")); + EXPECT_EQ(confirmed_at("CONF_AT"), at(2)) << "a CLEARED record keeps the time it confirmed"; + + failed("CONF_AT", Fault::SEVERITY_ERROR, 3, config); + ASSERT_EQ(fault("CONF_AT").status, Fault::STATUS_PREFAILED); + EXPECT_EQ(confirmed_at("CONF_AT"), 0) << "this occurrence has not confirmed"; + + failed("CONF_AT", Fault::SEVERITY_ERROR, 4, config); + EXPECT_EQ(confirmed_at("CONF_AT"), 0); + + failed("CONF_AT", Fault::SEVERITY_ERROR, 5, config); + ASSERT_EQ(fault("CONF_AT").status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(confirmed_at("CONF_AT"), at(5)); +} + +TEST_F(SqliteSeverityTest, ReactivationConfirmationTimeSurvivesRestarts) { + const auto config = confirm_at(-2); + failed("CONF_AT_RESTART", Fault::SEVERITY_ERROR, 0, config); + failed("CONF_AT_RESTART", Fault::SEVERITY_ERROR, 1, config); + ASSERT_EQ(confirmed_at("CONF_AT_RESTART"), at(1)); + ASSERT_TRUE(clear("CONF_AT_RESTART")); + + reopen(); + failed("CONF_AT_RESTART", Fault::SEVERITY_ERROR, 2, config); + ASSERT_EQ(fault("CONF_AT_RESTART").status, Fault::STATUS_PREFAILED); + EXPECT_EQ(confirmed_at("CONF_AT_RESTART"), 0); + + reopen(); + EXPECT_EQ(confirmed_at("CONF_AT_RESTART"), 0); + failed("CONF_AT_RESTART", Fault::SEVERITY_ERROR, 3, config); + ASSERT_EQ(fault("CONF_AT_RESTART").status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(confirmed_at("CONF_AT_RESTART"), at(3)); +} + +TEST_F(SqliteSeverityTest, ReactivationThatConfirmsAtOnceRecordsItsOwnTime) { + const auto config = confirm_at(-1); + failed("CONF_AT_NOW", Fault::SEVERITY_WARN, 0, config); + ASSERT_EQ(confirmed_at("CONF_AT_NOW"), at(0)); + ASSERT_TRUE(clear("CONF_AT_NOW")); + + failed("CONF_AT_NOW", Fault::SEVERITY_WARN, 1, config); + EXPECT_EQ(confirmed_at("CONF_AT_NOW"), at(1)); + + // A CRITICAL record confirms a debounced reactivation at once, and records that instant. + const auto debounced = confirm_at(-3); + failed("CONF_AT_CRIT", Fault::SEVERITY_CRITICAL, 2, debounced); + ASSERT_TRUE(clear("CONF_AT_CRIT")); + failed("CONF_AT_CRIT", Fault::SEVERITY_INFO, 3, debounced); + EXPECT_EQ(confirmed_at("CONF_AT_CRIT"), at(3)); +} + +TEST_F(SqliteSeverityTest, MigratedClearedRecordHasNoConfirmationTimeUntilItConfirms) { + const auto config = confirm_at(-3); + open_legacy_database("MIGRATED", Fault::SEVERITY_ERROR, Fault::STATUS_CLEARED, at(0)); + ASSERT_EQ(fault("MIGRATED").status, Fault::STATUS_CLEARED); + EXPECT_EQ(confirmed_at("MIGRATED"), at(0)) << "the migration keeps the cleared occurrence's time"; + + failed("MIGRATED", Fault::SEVERITY_ERROR, 2, config); + ASSERT_EQ(fault("MIGRATED").status, Fault::STATUS_PREFAILED); + EXPECT_EQ(fault("MIGRATED").occurrence_count, 3u); + EXPECT_EQ(confirmed_at("MIGRATED"), 0) << "this occurrence has not confirmed"; + + reopen(); + EXPECT_EQ(confirmed_at("MIGRATED"), 0); + failed("MIGRATED", Fault::SEVERITY_ERROR, 3, config); + failed("MIGRATED", Fault::SEVERITY_ERROR, 4, config); + ASSERT_EQ(fault("MIGRATED").status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(confirmed_at("MIGRATED"), at(4)); +} + +TEST_F(SqliteSeverityTest, MigratedClearedCriticalRecordConfirmsOnAWarn) { + const auto config = confirm_at(-3); + open_legacy_database("MIGRATED_CRIT", Fault::SEVERITY_CRITICAL, Fault::STATUS_CLEARED, at(0)); + + failed("MIGRATED_CRIT", Fault::SEVERITY_WARN, 2, config); + const auto f = fault("MIGRATED_CRIT"); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(confirmed_at("MIGRATED_CRIT"), at(2)); +} + +TEST_F(SqliteSeverityTest, PendingCriticalRowFromAnEarlierReleaseStaysUntilAReport) { + // The row an earlier release left after CRITICAL, clear, WARN at threshold -3: CRITICAL in + // PREFAILED, occurrence 2, still holding the first occurrence's confirmation time. + open_legacy_database("LEGACY", Fault::SEVERITY_CRITICAL, Fault::STATUS_PREFAILED, at(0)); + auto f = fault("LEGACY"); + EXPECT_EQ(f.status, Fault::STATUS_PREFAILED) << "the migration does not rewrite the status"; + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(confirmed_at("LEGACY"), 0) << "the migration drops the earlier occurrence's time"; + + reopen(); + f = fault("LEGACY"); + EXPECT_EQ(f.status, Fault::STATUS_PREFAILED); + EXPECT_EQ(f.occurrence_count, 2u); + EXPECT_EQ(confirmed_at("LEGACY"), 0); +} + +TEST_F(SqliteSeverityTest, MigrationKeepsAConfirmationTimeOnlyOnRowsThatConfirmed) { + // Under the latch a row that confirmed stays CONFIRMED or HEALED until a clear, so a pending row + // with a time carries it from the occurrence before a reactivation. + const int64_t earlier = at(0); + open_legacy_database({ + {"PENDING_FAILED", Fault::SEVERITY_ERROR, Fault::STATUS_PREFAILED, earlier}, + {"PENDING_PASSED", Fault::SEVERITY_WARN, Fault::STATUS_PREPASSED, earlier}, + {"PENDING_NEVER", Fault::SEVERITY_ERROR, Fault::STATUS_PREFAILED, 0}, + {"CONFIRMED", Fault::SEVERITY_ERROR, Fault::STATUS_CONFIRMED, earlier}, + {"HEALED", Fault::SEVERITY_ERROR, Fault::STATUS_HEALED, earlier}, + {"CLEARED", Fault::SEVERITY_ERROR, Fault::STATUS_CLEARED, earlier}, + }); + + const std::vector> expected{ + {"PENDING_FAILED", 0}, {"PENDING_PASSED", 0}, {"PENDING_NEVER", 0}, + {"CONFIRMED", earlier}, {"HEALED", earlier}, {"CLEARED", earlier}, + }; + for (int open = 0; open < 2; ++open) { + for (const auto & [code, confirmed] : expected) { + SCOPED_TRACE(code + " open " + std::to_string(open)); + EXPECT_EQ(confirmed_at(code), confirmed); + } + reopen(); + } + EXPECT_EQ(fault("PENDING_FAILED").status, Fault::STATUS_PREFAILED); + EXPECT_EQ(fault("PENDING_PASSED").status, Fault::STATUS_PREPASSED); + EXPECT_EQ(fault("CONFIRMED").status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(fault("HEALED").status, Fault::STATUS_HEALED); + EXPECT_EQ(fault("CLEARED").status, Fault::STATUS_CLEARED); +} + +TEST_F(SqliteSeverityTest, SweepRecordsTheConfirmationTimeOfAReactivatedOccurrence) { + auto config = confirm_at(-3); + config.auto_confirm_after_sec = 5.0; + storage_->set_debounce_config(config); + + for (const int64_t second : {0, 1, 2}) { + failed("SWEPT", Fault::SEVERITY_ERROR, second, config); + } + ASSERT_EQ(fault("SWEPT").status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(confirmed_at("SWEPT"), at(2)); + ASSERT_TRUE(clear("SWEPT")); + failed("SWEPT", Fault::SEVERITY_ERROR, 3, config); + ASSERT_EQ(fault("SWEPT").status, Fault::STATUS_PREFAILED); + EXPECT_EQ(confirmed_at("SWEPT"), 0) << "this occurrence has not confirmed"; + + reopen(); + storage_->set_debounce_config(config); + EXPECT_EQ(confirmed_at("SWEPT"), 0); + EXPECT_TRUE(storage_->check_time_based_confirmation(at(5)).empty()) << "before the window ends"; + ASSERT_EQ(storage_->check_time_based_confirmation(at(10)).size(), 1u); + EXPECT_EQ(fault("SWEPT").status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(confirmed_at("SWEPT"), at(10)) << "the sweep confirmed this occurrence"; +} + +TEST_F(SqliteSeverityTest, TimeBasedConfirmationKeepsTheConfirmationTimeOfTheReport) { + auto config = confirm_at(-3); + config.auto_confirm_after_sec = 5.0; + storage_->set_debounce_config(config); + + failed("TIMED", Fault::SEVERITY_CRITICAL, 0, config); + ASSERT_TRUE(clear("TIMED")); + failed("TIMED", Fault::SEVERITY_WARN, 1, config); + ASSERT_EQ(fault("TIMED").status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(confirmed_at("TIMED"), at(1)) << "confirmed by the report"; + + EXPECT_TRUE(storage_->check_time_based_confirmation(at(10)).empty()); + EXPECT_EQ(confirmed_at("TIMED"), at(1)) << "the sweep does not rewrite it"; + + reopen(); + storage_->set_debounce_config(config); + EXPECT_TRUE(storage_->check_time_based_confirmation(at(20)).empty()); + EXPECT_EQ(confirmed_at("TIMED"), at(1)); +} + +TEST_F(SqliteSeverityTest, PendingCriticalRowConfirmsOnTheNextFailedOfAnySeverity) { + const auto config = confirm_at(-3); + for (const uint8_t severity : + {Fault::SEVERITY_INFO, Fault::SEVERITY_WARN, Fault::SEVERITY_ERROR, Fault::SEVERITY_CRITICAL}) { + const std::string code = "LEGACY_FAILED_" + std::to_string(severity); + SCOPED_TRACE(code); + open_legacy_database(code, Fault::SEVERITY_CRITICAL, Fault::STATUS_PREFAILED, at(0)); + + failed(code, severity, 2, config); + const auto f = fault(code); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(f.occurrence_count, 2u) << "the row is not CLEARED, so this is the same occurrence"; + EXPECT_EQ(confirmed_at(code), at(2)); + } +} + +TEST_F(SqliteSeverityTest, PendingCriticalRowCanMoveToPrepassedOnPassed) { + const auto config = confirm_at(-3); + open_legacy_database("LEGACY_PASSED", Fault::SEVERITY_CRITICAL, Fault::STATUS_PREFAILED, at(0)); + + passed("LEGACY_PASSED", 2, config); + EXPECT_EQ(fault("LEGACY_PASSED").status, Fault::STATUS_PREFAILED) << "counter 0 keeps the status"; + passed("LEGACY_PASSED", 3, config); + auto f = fault("LEGACY_PASSED"); + EXPECT_EQ(f.status, Fault::STATUS_PREPASSED) << "a PASSED report does not apply the CRITICAL bypass"; + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + + failed("LEGACY_PASSED", Fault::SEVERITY_WARN, 4, config); + f = fault("LEGACY_PASSED"); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED) << "the next FAILED confirms it"; + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); +} + +// --------------------------------------------------------------------------- +// Exhaustive sweep: every event sequence of a given length over a small alphabet, applied to both +// backends and checked after every step, for both records of one code. +// --------------------------------------------------------------------------- + +enum class Kind { kFailed, kPassed, kClear }; + +struct Event { + Kind kind; + uint8_t severity; + const char * owner; +}; + +constexpr std::array kOwners = {"/a", "/b"}; + +constexpr std::array kAlphabet = {{ + {Kind::kFailed, Fault::SEVERITY_INFO, "/a"}, + {Kind::kFailed, Fault::SEVERITY_WARN, "/a"}, + {Kind::kFailed, Fault::SEVERITY_CRITICAL, "/a"}, + {Kind::kFailed, Fault::SEVERITY_WARN, "/b"}, + {Kind::kFailed, Fault::SEVERITY_CRITICAL, "/b"}, + {Kind::kPassed, 0, "/a"}, + {Kind::kClear, 0, "/a"}, + {Kind::kClear, 0, "/b"}, +}}; + +/// The fields this contract fixes for one record, independent of the debounce state machine. +struct Model { + bool exists{false}; + bool cleared{false}; + uint8_t severity{0}; + uint32_t occurrence_count{0}; + + void apply(const Event & e) { + switch (e.kind) { + case Kind::kFailed: + if (!exists || cleared) { + ++occurrence_count; + } + severity = exists ? std::max(severity, e.severity) : e.severity; + exists = true; + cleared = false; + break; + case Kind::kPassed: + break; + case Kind::kClear: + cleared = exists; + break; + } + } +}; + +std::string describe(const std::vector & seq, int upto) { + std::ostringstream out; + for (int i = 0; i <= upto; ++i) { + const auto & e = kAlphabet[seq[static_cast(i)]]; + if (i > 0) { + out << ", "; + } + switch (e.kind) { + case Kind::kFailed: + out << "FAILED(" << static_cast(e.severity) << "," << e.owner << ")"; + break; + case Kind::kPassed: + out << "PASSED(" << e.owner << ")"; + break; + case Kind::kClear: + out << "CLEAR(" << e.owner << ")"; + break; + } + } + return out.str(); +} + +/// Empty when the record holds the contract after @p e; otherwise what broke. +/// @param addressed Whether @p e was addressed to this record +std::string check(const fd::FaultRecord & f, const Model & m, const Event & e, bool addressed, + const DebounceConfig & config, const std::string & where) { + std::ostringstream out; + if (f.severity != m.severity) { + out << where << " severity " << static_cast(f.severity) << " != " << static_cast(m.severity) << "; "; + } + if (f.occurrence_count != m.occurrence_count) { + out << where << " occurrence_count " << f.occurrence_count << " != " << m.occurrence_count << "; "; + } + if (m.cleared != (f.status == Fault::STATUS_CLEARED)) { + out << where << " status " << f.status << "; "; + } + if (config.critical_immediate_confirm && f.severity == Fault::SEVERITY_CRITICAL) { + if (pending(f.status)) { + out << where << " CRITICAL in " << f.status << "; "; + } + if (addressed && e.kind == Kind::kFailed && f.status != Fault::STATUS_CONFIRMED) { + out << where << " FAILED on a CRITICAL record left " << f.status << "; "; + } + } + return out.str(); +} + +std::string check_parity(const fd::FaultRecord & mem, const fd::FaultRecord & sql, const std::string & owner) { + std::ostringstream out; + if (mem.severity != sql.severity || mem.status != sql.status || mem.description != sql.description || + mem.occurrence_count != sql.occurrence_count || mem.reporting_sources != sql.reporting_sources || + mem.first_occurred_ns != sql.first_occurred_ns || mem.last_occurred_ns != sql.last_occurred_ns || + mem.last_passed_ns != sql.last_passed_ns) { + out << owner << " backends disagree (memory: sev=" << static_cast(mem.severity) << " status=" << mem.status + << " occ=" << mem.occurrence_count << "; sqlite: sev=" << static_cast(sql.severity) + << " status=" << sql.status << " occ=" << sql.occurrence_count << "); "; + } + return out.str(); +} + +/// @param length Events per sequence; long enough to reach HEALED and report FAILED on it. +void run_sweep(const DebounceConfig & config, const std::string & tag, int length) { + InMemoryFaultStorage memory; + SqliteFaultStorage sqlite(":memory:", test_log_sink()); + + size_t total = 1; + for (int i = 0; i < length; ++i) { + total *= kAlphabet.size(); + } + + size_t failures = 0; + std::string first_failure; + std::vector seq(static_cast(length), 0); + for (size_t index = 0; index < total; ++index) { + size_t rest = index; + for (int i = 0; i < length; ++i) { + seq[static_cast(i)] = rest % kAlphabet.size(); + rest /= kAlphabet.size(); + } + const std::string code = tag + "_" + std::to_string(index); + std::array models{}; + for (int step = 0; step < length; ++step) { + const auto & e = kAlphabet[seq[static_cast(step)]]; + const auto t = at(step); + if (e.kind == Kind::kClear) { + memory.clear_fault(FaultId{code, e.owner}); + sqlite.clear_fault(FaultId{code, e.owner}); + } else { + const fd::ReportKind type = e.kind == Kind::kFailed ? fd::ReportKind::kFailed : fd::ReportKind::kPassed; + memory.report_fault_event(code, type, e.severity, "step " + std::to_string(step), e.owner, t, config); + sqlite.report_fault_event(code, type, e.severity, "step " + std::to_string(step), e.owner, t, config); + } + + std::string diff; + for (size_t o = 0; o < kOwners.size(); ++o) { + const std::string owner = kOwners[o]; + const bool addressed = owner == e.owner; + if (addressed) { + models[o].apply(e); + } + const auto mem = memory.get_fault(FaultId{code, owner}); + const auto sql = sqlite.get_fault(FaultId{code, owner}); + if (mem.has_value() != models[o].exists || sql.has_value() != models[o].exists) { + diff += owner + " existence differs; "; + } else if (models[o].exists) { + diff += check(*mem, models[o], e, addressed, config, "memory " + owner) + + check(*sql, models[o], e, addressed, config, "sqlite " + owner) + check_parity(*mem, *sql, owner); + } + } + if (!diff.empty()) { + if (failures == 0) { + first_failure = "[" + describe(seq, step) + "] " + diff; + } + ++failures; + break; + } + } + } + + EXPECT_EQ(failures, 0u) << failures << " of " << total << " sequences diverged; first: " << first_failure; +} + +TEST(FaultSeveritySweep, ImmediateConfirmation) { + run_sweep(DebounceConfig{}, "IC", 4); +} + +TEST(FaultSeveritySweep, ImmediateConfirmationHealOnSinglePassed) { + DebounceConfig config; + config.healing_enabled = true; + config.healing_threshold = 0; + run_sweep(config, "IH", 4); +} + +TEST(FaultSeveritySweep, DebouncedWithShortHealing) { + // HEALED within three events, so a FAILED on a HEALED record fits in four. + DebounceConfig config; + config.confirmation_threshold = -2; + config.healing_enabled = true; + config.healing_threshold = 1; + run_sweep(config, "DS", 4); +} + +TEST(FaultSeveritySweep, DeepDebounceWithHealing) { + // CRITICAL confirms at -1 and heals after three PASSED, so a FAILED on it needs five events. + DebounceConfig config; + config.confirmation_threshold = -3; + config.healing_enabled = true; + config.healing_threshold = 2; + run_sweep(config, "DH", 5); +} + +TEST(FaultSeveritySweep, CriticalBypassOff) { + // Without the bypass nothing forces a CRITICAL record to confirm; the other checks still hold. + DebounceConfig config; + config.confirmation_threshold = -3; + config.critical_immediate_confirm = false; + run_sweep(config, "NB", 4); +} + +} // namespace diff --git a/src/ros2_medkit_fault_manager/test/test_fault_thresholds.yaml b/src/ros2_medkit_fault_manager/test/test_fault_thresholds.yaml index 9753b5889..f27761052 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_thresholds.yaml +++ b/src/ros2_medkit_fault_manager/test/test_fault_thresholds.yaml @@ -13,7 +13,6 @@ SHARED.JAM: healing_threshold: 8 # Pins only the confirmation direction. The healing fields still come from -# whichever entity reported, so this code is only half settled - and the node -# still says so. +# whichever entity reported. SHARED.PARTIAL: confirmation_threshold: -2 diff --git a/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp b/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp index 6f86b69bf..73635ec97 100644 --- a/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp +++ b/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -52,6 +53,7 @@ #include "ros2_medkit_msgs/srv/report_fault.hpp" #include "test_log_sink.hpp" +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::FaultStorage; using ros2_medkit_fault_manager::InMemoryFaultStorage; using ros2_medkit_fault_manager::RosbagCapture; @@ -68,6 +70,15 @@ void spin_some_once(const rclcpp::Node::SharedPtr & node) { executor.remove_node(node); } +/// Every record this suite drives belongs to one owner. These tests are about the +/// recording state machine and the bag lifecycle, which the record identity does not +/// change. The cross-owner cases live with the storage and node tests. +constexpr const char * kOwner = "/reporter"; + +FaultId rec(const std::string & code) { + return FaultId{code, kOwner}; +} + /// Captures rcutils log output while alive and restores the console handler on /// every exit path - leaving the process-global handler installed would swallow /// the output of every later case in this binary. @@ -186,8 +197,8 @@ class RosbagCaptureTest : public ::testing::Test { node_ = std::make_shared("test_rosbag_capture_node"); storage_ = std::make_unique(); - // Create temp directory for test bags - temp_dir_ = std::filesystem::temp_directory_path() / "rosbag_capture_test"; + // Per process, so concurrent runs of this suite do not delete each other's bags. + temp_dir_ = std::filesystem::temp_directory_path() / ("rosbag_capture_test_" + std::to_string(::getpid())); std::filesystem::create_directories(temp_dir_); } @@ -849,7 +860,7 @@ TEST_F(RosbagCaptureTest, OnFaultPrefailedWhileDisabled) { auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); // Should not throw when disabled - EXPECT_NO_THROW(capture.on_fault_prefailed("TEST_FAULT")); + EXPECT_NO_THROW(capture.on_fault_prefailed(rec("TEST_FAULT"))); } TEST_F(RosbagCaptureTest, OnFaultConfirmedWhileDisabled) { @@ -857,7 +868,7 @@ TEST_F(RosbagCaptureTest, OnFaultConfirmedWhileDisabled) { auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); // Should not throw when disabled - EXPECT_NO_THROW(capture.on_fault_confirmed("TEST_FAULT")); + EXPECT_NO_THROW(capture.on_fault_confirmed(rec("TEST_FAULT"))); } TEST_F(RosbagCaptureTest, OnFaultClearedWhileDisabled) { @@ -865,7 +876,7 @@ TEST_F(RosbagCaptureTest, OnFaultClearedWhileDisabled) { auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); // Should not throw when disabled - EXPECT_NO_THROW(capture.on_fault_cleared("TEST_FAULT")); + EXPECT_NO_THROW(capture.on_fault_cleared(rec("TEST_FAULT"))); } TEST_F(RosbagCaptureTest, OnFaultPrefailedStartsLazyCapture) { @@ -874,7 +885,7 @@ TEST_F(RosbagCaptureTest, OnFaultPrefailedStartsLazyCapture) { auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); EXPECT_FALSE(capture.is_running()); - capture.on_fault_prefailed("TEST_FAULT"); + capture.on_fault_prefailed(rec("TEST_FAULT")); EXPECT_TRUE(capture.is_running()); } @@ -953,6 +964,7 @@ TEST_F(RosbagCaptureTest, SharedBagTotalTakesTheLargestRowPerPath) { auto store = [this](const char * code, const char * path, size_t bytes) { ros2_medkit_fault_manager::RosbagFileInfo info; info.fault_code = code; + info.owner = kOwner; info.file_path = path; info.format = "mcap"; info.duration_sec = 5.0; @@ -986,11 +998,13 @@ TEST_F(RosbagCaptureTest, EvictionFreesASharedBagOnceAndKeepsTheQuotaHonest) { shared.created_at_ns = 1000; for (const char * code : {"ROOT_CAUSE", "CORRELATED_A", "CORRELATED_B"}) { shared.fault_code = code; + shared.owner = kOwner; storage_->store_rosbag_file(shared); } ros2_medkit_fault_manager::RosbagFileInfo solo; solo.fault_code = "UNRELATED"; + solo.owner = kOwner; solo.file_path = solo_path; solo.format = "mcap"; solo.duration_sec = 5.0; @@ -1007,12 +1021,12 @@ TEST_F(RosbagCaptureTest, EvictionFreesASharedBagOnceAndKeepsTheQuotaHonest) { EXPECT_EQ(evicted, std::vector{shared_path}); EXPECT_EQ(storage_->get_total_rosbag_storage_bytes(), 600u); EXPECT_FALSE(std::filesystem::exists(shared_path)); - EXPECT_FALSE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); - EXPECT_FALSE(storage_->get_rosbag_file("CORRELATED_A").has_value()); - EXPECT_FALSE(storage_->get_rosbag_file("CORRELATED_B").has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("ROOT_CAUSE")).has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("CORRELATED_A")).has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("CORRELATED_B")).has_value()); // The newer bag fits under the quota and must survive. - EXPECT_TRUE(storage_->get_rosbag_file("UNRELATED").has_value()); + EXPECT_TRUE(storage_->get_rosbag_file(rec("UNRELATED")).has_value()); EXPECT_TRUE(std::filesystem::exists(solo_path)); } @@ -1034,14 +1048,17 @@ TEST_F(RosbagCaptureTest, EvictionKeepsGoingWhenTheSharedBagIsNotEnough) { shared.duration_sec = 5.0; shared.created_at_ns = 1000; shared.fault_code = "ROOT_CAUSE"; + shared.owner = kOwner; shared.size_bytes = 500; storage_->store_rosbag_file(shared); shared.fault_code = "CORRELATED_A"; + shared.owner = kOwner; shared.size_bytes = 400; storage_->store_rosbag_file(shared); ros2_medkit_fault_manager::RosbagFileInfo solo; solo.fault_code = "UNRELATED"; + solo.owner = kOwner; solo.file_path = solo_path; solo.format = "mcap"; solo.duration_sec = 5.0; @@ -1070,6 +1087,7 @@ TEST_F(RosbagCaptureTest, EvictionNeverDeletesTheRecordingJustStored) { ros2_medkit_fault_manager::RosbagFileInfo older; older.fault_code = "OLDER"; + older.owner = kOwner; older.file_path = older_path; older.format = "mcap"; older.size_bytes = 300; @@ -1078,6 +1096,7 @@ TEST_F(RosbagCaptureTest, EvictionNeverDeletesTheRecordingJustStored) { ros2_medkit_fault_manager::RosbagFileInfo newest; newest.fault_code = "NEWEST"; + newest.owner = kOwner; newest.file_path = newest_path; newest.format = "mcap"; newest.size_bytes = 2000; @@ -1089,7 +1108,7 @@ TEST_F(RosbagCaptureTest, EvictionNeverDeletesTheRecordingJustStored) { EXPECT_EQ(evicted, std::vector{older_path}); EXPECT_FALSE(std::filesystem::exists(older_path)); EXPECT_TRUE(std::filesystem::exists(newest_path)); - ASSERT_TRUE(storage_->get_rosbag_file("NEWEST").has_value()); + ASSERT_TRUE(storage_->get_rosbag_file(rec("NEWEST")).has_value()); // Still over the quota: that is the state the caller warns about. EXPECT_EQ(storage_->get_total_rosbag_storage_bytes(), 2000u); } @@ -1104,6 +1123,7 @@ TEST_F(RosbagCaptureTest, EvictionSkipsTheKeptRecordingWhereverItSorts) { ros2_medkit_fault_manager::RosbagFileInfo kept; kept.fault_code = "KEPT"; + kept.owner = kOwner; kept.file_path = kept_path; kept.format = "mcap"; kept.size_bytes = 900; @@ -1112,6 +1132,7 @@ TEST_F(RosbagCaptureTest, EvictionSkipsTheKeptRecordingWhereverItSorts) { ros2_medkit_fault_manager::RosbagFileInfo later; later.fault_code = "LATER"; + later.owner = kOwner; later.file_path = later_path; later.format = "mcap"; later.size_bytes = 900; @@ -1253,8 +1274,10 @@ TEST_F(RosbagCaptureTest, ABatchStoreUnlinksASharedReplacedBagExactlyOnce) { shared.size_bytes = 10; shared.created_at_ns = 1000; shared.fault_code = "SHARED_A"; + shared.owner = kOwner; storage_->store_rosbag_file(shared); shared.fault_code = "SHARED_B"; + shared.owner = kOwner; storage_->store_rosbag_file(shared); ASSERT_TRUE(std::filesystem::exists(old_bag)); @@ -1264,15 +1287,17 @@ TEST_F(RosbagCaptureTest, ABatchStoreUnlinksASharedReplacedBagExactlyOnce) { std::vector rows; rows.reserve(2); moved.fault_code = "SHARED_A"; + moved.owner = kOwner; rows.push_back(moved); moved.fault_code = "SHARED_B"; + moved.owner = kOwner; rows.push_back(moved); storage_->store_rosbag_files(rows); EXPECT_FALSE(std::filesystem::exists(old_bag)) << "the replaced bag leaked: nothing references it any more"; EXPECT_TRUE(std::filesystem::exists(new_bag)) << "the bag the batch points at was removed"; - auto row_a = storage_->get_rosbag_file("SHARED_A"); - auto row_b = storage_->get_rosbag_file("SHARED_B"); + auto row_a = storage_->get_rosbag_file(rec("SHARED_A")); + auto row_b = storage_->get_rosbag_file(rec("SHARED_B")); ASSERT_TRUE(row_a.has_value()); ASSERT_TRUE(row_b.has_value()); EXPECT_EQ(row_a->file_path, new_bag.string()); @@ -1361,9 +1386,14 @@ class RosbagNonStandardThrowStorage : public InMemoryFaultStorage { /// Storage counting fault lookups, which is what resolving an entity scope costs. class RosbagFaultLookupCountingStorage : public InMemoryFaultStorage { public: - std::optional get_fault(const std::string & fault_code) const override { + std::optional get_fault(const FaultId & id) const override { + ++lookups; + return InMemoryFaultStorage::get_fault(id); + } + + bool contains(const FaultId & id) const override { ++lookups; - return InMemoryFaultStorage::get_fault(fault_code); + return InMemoryFaultStorage::contains(id); } mutable size_t lookups = 0; @@ -1372,7 +1402,7 @@ class RosbagFaultLookupCountingStorage : public InMemoryFaultStorage { /// Storage that cannot read faults, as the PostgreSQL backend does while its server is down. class RosbagFaultLookupFailingStorage : public InMemoryFaultStorage { public: - std::optional get_fault(const std::string & /*fault_code*/) const override { + std::optional get_fault(const FaultId & /*id*/) const override { ++lookups; throw FaultStorage::IgnorableConnectionException("database unreachable"); } @@ -1383,7 +1413,7 @@ class RosbagFaultLookupFailingStorage : public InMemoryFaultStorage { /// Storage whose bag cleanup fails, as the PostgreSQL backend does while its server is down. class RosbagCleanupFailingStorage : public InMemoryFaultStorage { public: - bool delete_rosbag_file(const std::string & /*fault_code*/) override { + bool delete_rosbag_file(const FaultId & /*id*/) override { ++delete_attempts; throw FaultStorage::IgnorableConnectionException("database unreachable"); } @@ -1466,11 +1496,11 @@ class RosbagCaptureIntegrationTest : public RosbagCaptureTest { } } - /// Spin until the post-roll finalises and @p fault_code has its metadata row. - bool wait_for_row(const std::string & fault_code, std::chrono::milliseconds timeout) { + /// Spin until the post-roll finalises and @p id has its metadata row. + bool wait_for_row(const FaultId & id, std::chrono::milliseconds timeout) { auto deadline = std::chrono::steady_clock::now() + timeout; while (std::chrono::steady_clock::now() < deadline) { - if (storage_->get_rosbag_file(fault_code).has_value()) { + if (storage_->get_rosbag_file(id).has_value()) { return true; } spin_for(std::chrono::milliseconds(100)); @@ -1557,18 +1587,18 @@ TEST_F(RosbagCaptureIntegrationTest, FullFaultLifecycleWithNoMessages) { EXPECT_TRUE(capture.is_running()); // Simulate fault lifecycle - capture.on_fault_prefailed("TEST_FAULT_001"); + capture.on_fault_prefailed(rec("TEST_FAULT_001")); // Let some time pass spin_for(std::chrono::milliseconds(100)); - capture.on_fault_confirmed("TEST_FAULT_001"); + capture.on_fault_confirmed(rec("TEST_FAULT_001")); // Wait for post-fault timer spin_for(std::chrono::milliseconds(300)); // Clear the fault - capture.on_fault_cleared("TEST_FAULT_001"); + capture.on_fault_cleared(rec("TEST_FAULT_001")); // Stop capture capture.stop(); @@ -1593,15 +1623,15 @@ TEST_F(RosbagCaptureIntegrationTest, AcknowledgingKeepsAHistoryTheOperatorConfig RosbagCapture capture(node_.get(), &storage, rosbag_config, snapshot_config); capture.start(); fill_buffer("/keep_history_probe"); - capture.on_fault_confirmed("KEEPS_HISTORY"); + capture.on_fault_confirmed(rec("KEEPS_HISTORY")); spin_for(std::chrono::milliseconds(300)); - ASSERT_FALSE(storage.get_rosbag_files("KEEPS_HISTORY").empty()) << "nothing was recorded to keep"; - const auto before = storage.get_rosbag_files("KEEPS_HISTORY").size(); + ASSERT_FALSE(storage.get_rosbag_files(rec("KEEPS_HISTORY")).empty()) << "nothing was recorded to keep"; + const auto before = storage.get_rosbag_files(rec("KEEPS_HISTORY")).size(); - capture.on_fault_cleared("KEEPS_HISTORY"); + capture.on_fault_cleared(rec("KEEPS_HISTORY")); - EXPECT_EQ(storage.get_rosbag_files("KEEPS_HISTORY").size(), before) + EXPECT_EQ(storage.get_rosbag_files(rec("KEEPS_HISTORY")).size(), before) << "acknowledging the fault destroyed the recordings the cap was raised to keep"; capture.stop(); } @@ -1629,17 +1659,17 @@ TEST_F(RosbagCaptureIntegrationTest, AcknowledgingDuringThePostRollKeepsAConfigu RosbagCapture capture(node_.get(), &storage, rosbag_config, snapshot_config); capture.start(); fill_buffer("/ack_mid_roll_probe"); - capture.on_fault_confirmed("ACK_MID_ROLL"); + capture.on_fault_confirmed({"ACK_MID_ROLL", "/node"}); // The acknowledgement arrives while the post-fault window still runs, so the // store reports the fault as CLEARED by the time finalize asks. - ASSERT_TRUE(storage.clear_fault("ACK_MID_ROLL")); - capture.on_fault_cleared("ACK_MID_ROLL"); + ASSERT_TRUE(storage.clear_fault({"ACK_MID_ROLL", "/node"})); + capture.on_fault_cleared({"ACK_MID_ROLL", "/node"}); spin_for(std::chrono::milliseconds(900)); // past the post-roll and the finalize capture.wait_for_stored_recordings(); - EXPECT_FALSE(storage.get_rosbag_files("ACK_MID_ROLL").empty()) + EXPECT_FALSE(storage.get_rosbag_files({"ACK_MID_ROLL", "/node"}).empty()) << "the post-roll finalize deleted a history on_fault_cleared had decided to keep"; capture.stop(); } @@ -1658,14 +1688,14 @@ TEST_F(RosbagCaptureIntegrationTest, AcknowledgingStillCleansUpWhenNoHistoryIsCo RosbagCapture capture(node_.get(), &storage, rosbag_config, snapshot_config); capture.start(); fill_buffer("/cleanup_probe"); - capture.on_fault_confirmed("CLEANED_UP"); + capture.on_fault_confirmed(rec("CLEANED_UP")); spin_for(std::chrono::milliseconds(300)); - ASSERT_FALSE(storage.get_rosbag_files("CLEANED_UP").empty()); + ASSERT_FALSE(storage.get_rosbag_files(rec("CLEANED_UP")).empty()); - capture.on_fault_cleared("CLEANED_UP"); + capture.on_fault_cleared(rec("CLEANED_UP")); - EXPECT_TRUE(storage.get_rosbag_files("CLEANED_UP").empty()) << "auto_cleanup stopped working at the default cap"; + EXPECT_TRUE(storage.get_rosbag_files(rec("CLEANED_UP")).empty()) << "auto_cleanup stopped working at the default cap"; capture.stop(); } @@ -1679,16 +1709,16 @@ TEST_F(RosbagCaptureIntegrationTest, MultipleFaultsHandled) { capture.start(); // Multiple faults in sequence - capture.on_fault_prefailed("FAULT_A"); + capture.on_fault_prefailed(rec("FAULT_A")); spin_for(std::chrono::milliseconds(50)); - capture.on_fault_prefailed("FAULT_B"); + capture.on_fault_prefailed(rec("FAULT_B")); spin_for(std::chrono::milliseconds(50)); - capture.on_fault_confirmed("FAULT_A"); + capture.on_fault_confirmed(rec("FAULT_A")); spin_for(std::chrono::milliseconds(50)); - capture.on_fault_confirmed("FAULT_B"); + capture.on_fault_confirmed(rec("FAULT_B")); spin_for(std::chrono::milliseconds(200)); capture.stop(); @@ -1710,7 +1740,7 @@ TEST_F(RosbagCaptureIntegrationTest, StoringARecordingDoesNotHoldTheExecutor) { capture.start(); fill_buffer("/store_off_executor_probe"); - capture.on_fault_confirmed("STORE_OFF_EXECUTOR"); + capture.on_fault_confirmed(rec("STORE_OFF_EXECUTOR")); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); auto longest_spin = std::chrono::steady_clock::duration::zero(); @@ -1734,11 +1764,11 @@ TEST_F(RosbagCaptureIntegrationTest, StoringARecordingDoesNotHoldTheExecutor) { bag_path = gated.bag_path; } EXPECT_TRUE(read_manifest(bag_path).is_object()) << "the row was being stored before the manifest existed"; - EXPECT_FALSE(gated.get_rosbag_file("STORE_OFF_EXECUTOR").has_value()); + EXPECT_FALSE(gated.get_rosbag_file(rec("STORE_OFF_EXECUTOR")).has_value()); gated.release(); capture.wait_for_stored_recordings(); - auto row = gated.get_rosbag_file("STORE_OFF_EXECUTOR"); + auto row = gated.get_rosbag_file(rec("STORE_OFF_EXECUTOR")); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->file_path, bag_path); capture.stop(); @@ -1757,14 +1787,14 @@ TEST_F(RosbagCaptureIntegrationTest, AClearWhileTheRowIsStoredTakesTheRowWithIt) rosbag_config.max_bags_per_fault = 1; rclcpp::Clock clock; gated.report_fault_event("CLEARED_IN_STORE", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "store race", "/node", clock.now().nanoseconds(), + "store race", kOwner, clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), &gated, rosbag_config, snapshot_config); capture.start(); fill_buffer("/clear_during_store_probe"); - capture.on_fault_confirmed("CLEARED_IN_STORE"); + capture.on_fault_confirmed(rec("CLEARED_IN_STORE")); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); while (!gated.wait_entered(std::chrono::milliseconds(0)) && std::chrono::steady_clock::now() < deadline) { spin_for(std::chrono::milliseconds(10)); @@ -1778,9 +1808,9 @@ TEST_F(RosbagCaptureIntegrationTest, AClearWhileTheRowIsStoredTakesTheRowWithIt) // The worker found the fault not cleared and is inside the insert. The clear marks // the fault in the store, then runs the hook, as FaultManagerCore does. - ASSERT_TRUE(gated.clear_fault("CLEARED_IN_STORE")); + ASSERT_TRUE(gated.clear_fault(rec("CLEARED_IN_STORE"))); auto cleared = std::async(std::launch::async, [&capture]() { - capture.on_fault_cleared("CLEARED_IN_STORE"); + capture.on_fault_cleared(rec("CLEARED_IN_STORE")); }); EXPECT_EQ(cleared.wait_for(std::chrono::milliseconds(500)), std::future_status::timeout) << "the clear's delete ran while the row was still being inserted"; @@ -1788,7 +1818,7 @@ TEST_F(RosbagCaptureIntegrationTest, AClearWhileTheRowIsStoredTakesTheRowWithIt) gated.release(); cleared.get(); capture.wait_for_stored_recordings(); - EXPECT_TRUE(gated.get_rosbag_files("CLEARED_IN_STORE").empty()) << "a cleared fault kept its row"; + EXPECT_TRUE(gated.get_rosbag_files(rec("CLEARED_IN_STORE")).empty()) << "a cleared fault kept its row"; EXPECT_FALSE(std::filesystem::exists(bag_path)) << "a cleared fault kept its bag"; capture.stop(); } @@ -1806,16 +1836,16 @@ TEST_F(RosbagCaptureIntegrationTest, FaultConfirmedDuringPostRollGetsTheSameBag) capture.start(); fill_buffer("/rosbag_burst_probe"); - capture.on_fault_confirmed("ROOT_CAUSE"); - capture.on_fault_confirmed("CORRELATED"); + capture.on_fault_confirmed(rec("ROOT_CAUSE")); + capture.on_fault_confirmed(rec("CORRELATED")); // Let the post-fault timer fire and finalise the recording. spin_for(std::chrono::milliseconds(1600)); capture.wait_for_stored_recordings(); - auto root = storage_->get_rosbag_file("ROOT_CAUSE"); + auto root = storage_->get_rosbag_file(rec("ROOT_CAUSE")); ASSERT_TRUE(root.has_value()) << "the first fault must produce a bag"; - auto correlated = storage_->get_rosbag_file("CORRELATED"); + auto correlated = storage_->get_rosbag_file(rec("CORRELATED")); ASSERT_TRUE(correlated.has_value()) << "the fault confirmed during the post-roll lost its recording"; EXPECT_EQ(correlated->file_path, root->file_path); EXPECT_TRUE(std::filesystem::exists(root->file_path)); @@ -1833,28 +1863,73 @@ TEST_F(RosbagCaptureIntegrationTest, ClearingOneFaultOfABurstKeepsTheSharedBag) capture.start(); fill_buffer("/rosbag_burst_cleanup_probe"); - capture.on_fault_confirmed("ROOT_CAUSE"); - capture.on_fault_confirmed("CORRELATED"); + capture.on_fault_confirmed(rec("ROOT_CAUSE")); + capture.on_fault_confirmed(rec("CORRELATED")); spin_for(std::chrono::milliseconds(1600)); capture.wait_for_stored_recordings(); - auto root = storage_->get_rosbag_file("ROOT_CAUSE"); + auto root = storage_->get_rosbag_file(rec("ROOT_CAUSE")); ASSERT_TRUE(root.has_value()); - ASSERT_TRUE(storage_->get_rosbag_file("CORRELATED").has_value()); + ASSERT_TRUE(storage_->get_rosbag_file(rec("CORRELATED")).has_value()); const std::string bag_path = root->file_path; // Auto-cleanup of one fault must not pull the bag out from under its sibling. - capture.on_fault_cleared("CORRELATED"); + capture.on_fault_cleared(rec("CORRELATED")); EXPECT_TRUE(std::filesystem::exists(bag_path)); - EXPECT_TRUE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); + EXPECT_TRUE(storage_->get_rosbag_file(rec("ROOT_CAUSE")).has_value()); // The last reference going away does delete it. - capture.on_fault_cleared("ROOT_CAUSE"); + capture.on_fault_cleared(rec("ROOT_CAUSE")); EXPECT_FALSE(std::filesystem::exists(bag_path)); capture.stop(); } +// The bag directory name and the recording id stay keyed by the CODE, so two +// owners confirming one code inside a burst attach to the one in-flight recording - +// and each of them still gets its own rosbag_files row, because the download and the +// acknowledgement are addressed to a record, not to a code. +TEST_F(RosbagCaptureIntegrationTest, TwoOwnersOfOneCodeShareTheBagAndGetARowEach) { + auto rosbag_config = create_rosbag_config(); + rosbag_config.duration_sec = 10.0; + rosbag_config.duration_after_sec = 1.0; + auto snapshot_config = create_snapshot_config(); + RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); + + capture.start(); + fill_buffer("/rosbag_two_owners_probe"); + + const FaultId owner_a{"BURST_CODE", "/owner_a"}; + const FaultId owner_b{"BURST_CODE", "/owner_b"}; + capture.on_fault_confirmed(owner_a); + capture.on_fault_confirmed(owner_b); // lands inside A's post-roll, attaches + + ASSERT_TRUE(wait_for_row(owner_b, std::chrono::milliseconds(8000))); + + auto row_a = storage_->get_rosbag_file(owner_a); + auto row_b = storage_->get_rosbag_file(owner_b); + ASSERT_TRUE(row_a.has_value()); + ASSERT_TRUE(row_b.has_value()); + EXPECT_EQ(row_a->owner, "/owner_a"); + EXPECT_EQ(row_b->owner, "/owner_b"); + EXPECT_EQ(row_b->file_path, row_a->file_path) << "one recording, one bag"; + EXPECT_EQ(row_b->recording_id, row_a->recording_id); + EXPECT_EQ(storage_->get_rosbag_files_by_recording(row_a->recording_id).size(), 2u); + + // Acknowledging one owner's record unlinks only that row. The bag survives while + // the other owner still holds it. + capture.on_fault_cleared(owner_a); + EXPECT_FALSE(storage_->get_rosbag_file(owner_a).has_value()); + ASSERT_TRUE(storage_->get_rosbag_file(owner_b).has_value()); + EXPECT_TRUE(std::filesystem::exists(row_a->file_path)) << "the other owner still references the bag"; + + capture.on_fault_cleared(owner_b); + EXPECT_FALSE(storage_->get_rosbag_file(owner_b).has_value()); + EXPECT_FALSE(std::filesystem::exists(row_a->file_path)); + + capture.stop(); +} + TEST_F(RosbagCaptureIntegrationTest, FaultClearedDuringPostRollNeverGetsARow) { auto rosbag_config = create_rosbag_config(); rosbag_config.duration_sec = 10.0; @@ -1865,23 +1940,23 @@ TEST_F(RosbagCaptureIntegrationTest, FaultClearedDuringPostRollNeverGetsARow) { capture.start(); fill_buffer("/rosbag_cleared_mid_roll_probe"); - capture.on_fault_confirmed("ROOT_CAUSE"); - capture.on_fault_confirmed("CORRELATED"); + capture.on_fault_confirmed(rec("ROOT_CAUSE")); + capture.on_fault_confirmed(rec("CORRELATED")); // The attached fault clears before the post-roll timer fires. Its row has not // been written yet - it must be forgotten, not resurrected at finalize, where // a leftover row would pin the shared bag forever. - capture.on_fault_cleared("CORRELATED"); + capture.on_fault_cleared(rec("CORRELATED")); - ASSERT_TRUE(wait_for_row("ROOT_CAUSE", std::chrono::milliseconds(8000))); - EXPECT_FALSE(storage_->get_rosbag_file("CORRELATED").has_value()) + ASSERT_TRUE(wait_for_row(rec("ROOT_CAUSE"), std::chrono::milliseconds(8000))); + EXPECT_FALSE(storage_->get_rosbag_file(rec("CORRELATED")).has_value()) << "a fault cleared during the post-roll must not get a row at finalize"; // With no leftover row the bag belongs to the primary alone: clearing it // must unlink the recording. - auto root = storage_->get_rosbag_file("ROOT_CAUSE"); + auto root = storage_->get_rosbag_file(rec("ROOT_CAUSE")); ASSERT_TRUE(root.has_value()); - capture.on_fault_cleared("ROOT_CAUSE"); - EXPECT_FALSE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); + capture.on_fault_cleared(rec("ROOT_CAUSE")); + EXPECT_FALSE(storage_->get_rosbag_file(rec("ROOT_CAUSE")).has_value()); EXPECT_FALSE(std::filesystem::exists(root->file_path)); capture.stop(); @@ -1898,10 +1973,10 @@ TEST_F(RosbagCaptureIntegrationTest, PostRollKeepsTheRowWhenTheFaultCannotBeRead capture.start(); fill_buffer("/rosbag_lookup_failure_probe"); - capture.on_fault_confirmed("UNREADABLE"); + capture.on_fault_confirmed(rec("UNREADABLE")); // The post-roll timer runs inside spin_some, so a throw from the finalize ends the test here. - ASSERT_TRUE(wait_for_row("UNREADABLE", std::chrono::milliseconds(8000))); + ASSERT_TRUE(wait_for_row(rec("UNREADABLE"), std::chrono::milliseconds(8000))); EXPECT_GT(failing_storage->lookups, 0u) << "the finalize never asked the store"; capture.stop(); @@ -1913,7 +1988,7 @@ TEST_F(RosbagCaptureIntegrationTest, ClearSurvivesAFailingBagCleanup) { auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), &storage, rosbag_config, snapshot_config); - EXPECT_NO_THROW(capture.on_fault_cleared("UNREACHABLE")); + EXPECT_NO_THROW(capture.on_fault_cleared(rec("UNREACHABLE"))); EXPECT_EQ(storage.delete_attempts, 1u) << "the cleanup never reached the store"; } @@ -1927,21 +2002,21 @@ TEST_F(RosbagCaptureIntegrationTest, PrimaryClearedDuringPostRollLeavesTheBagToI capture.start(); fill_buffer("/rosbag_primary_cleared_probe"); - capture.on_fault_confirmed("ROOT_CAUSE"); - capture.on_fault_confirmed("CORRELATED"); + capture.on_fault_confirmed(rec("ROOT_CAUSE")); + capture.on_fault_confirmed(rec("CORRELATED")); // The fault that opened the recording clears mid post-roll; the attached // sibling still needs the bag, so the recording finalises for it alone. - capture.on_fault_cleared("ROOT_CAUSE"); + capture.on_fault_cleared(rec("ROOT_CAUSE")); - ASSERT_TRUE(wait_for_row("CORRELATED", std::chrono::milliseconds(8000))); - EXPECT_FALSE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()) + ASSERT_TRUE(wait_for_row(rec("CORRELATED"), std::chrono::milliseconds(8000))); + EXPECT_FALSE(storage_->get_rosbag_file(rec("ROOT_CAUSE")).has_value()) << "the cleared primary must not get a row at finalize"; - auto correlated = storage_->get_rosbag_file("CORRELATED"); + auto correlated = storage_->get_rosbag_file(rec("CORRELATED")); ASSERT_TRUE(correlated.has_value()); EXPECT_TRUE(std::filesystem::exists(correlated->file_path)); - capture.on_fault_cleared("CORRELATED"); + capture.on_fault_cleared(rec("CORRELATED")); EXPECT_FALSE(std::filesystem::exists(correlated->file_path)); capture.stop(); @@ -1957,17 +2032,17 @@ TEST_F(RosbagCaptureIntegrationTest, BurstFullyClearedDuringPostRollDiscardsTheB capture.start(); fill_buffer("/rosbag_all_cleared_probe"); - capture.on_fault_confirmed("ROOT_CAUSE"); - capture.on_fault_confirmed("CORRELATED"); - capture.on_fault_cleared("CORRELATED"); - capture.on_fault_cleared("ROOT_CAUSE"); + capture.on_fault_confirmed(rec("ROOT_CAUSE")); + capture.on_fault_confirmed(rec("CORRELATED")); + capture.on_fault_cleared(rec("CORRELATED")); + capture.on_fault_cleared(rec("ROOT_CAUSE")); // Let the post-roll timer fire with nothing left to register. spin_for(std::chrono::milliseconds(1500)); capture.wait_for_stored_recordings(); - EXPECT_FALSE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); - EXPECT_FALSE(storage_->get_rosbag_file("CORRELATED").has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("ROOT_CAUSE")).has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("CORRELATED")).has_value()); EXPECT_EQ(count_bag_dirs(), 0u) << "a bag nobody references must not be left on disk"; capture.stop(); @@ -1992,19 +2067,19 @@ TEST_F(RosbagCaptureIntegrationTest, AttachmentCapDropsFaultsPastIt) { codes.push_back("BURST_" + std::string(i < 10 ? "0" : "") + std::to_string(i)); } for (const auto & code : codes) { - capture.on_fault_confirmed(code); + capture.on_fault_confirmed(rec(code)); } - ASSERT_TRUE(wait_for_row(codes[0], std::chrono::milliseconds(10000))); + ASSERT_TRUE(wait_for_row(rec(codes[0]), std::chrono::milliseconds(10000))); - auto primary = storage_->get_rosbag_file(codes[0]); + auto primary = storage_->get_rosbag_file(rec(codes[0])); ASSERT_TRUE(primary.has_value()); for (size_t i = 1; i < 33; ++i) { - auto row = storage_->get_rosbag_file(codes[i]); + auto row = storage_->get_rosbag_file(rec(codes[i])); ASSERT_TRUE(row.has_value()) << codes[i] << " is within the cap and must resolve to the recording"; EXPECT_EQ(row->file_path, primary->file_path); } - EXPECT_FALSE(storage_->get_rosbag_file(codes[33]).has_value()) + EXPECT_FALSE(storage_->get_rosbag_file(rec(codes[33])).has_value()) << "fault 34 of the burst is past the cap and is dropped with a WARN"; EXPECT_EQ(count_bag_dirs(), 1u) << "the dropped fault must not open a second bag"; @@ -2048,8 +2123,8 @@ TEST_F(RosbagCaptureIntegrationTest, AttachedFaultTopicsJoinTheEntityScopedRecor spin_for(std::chrono::milliseconds(50)); } - capture.on_fault_confirmed("FAULT_A"); - capture.on_fault_confirmed("FAULT_B"); // lands inside A's post-roll, attaches + capture.on_fault_confirmed({"FAULT_A", "/scope_source_a"}); + capture.on_fault_confirmed({"FAULT_B", "/scope_source_b"}); // lands inside A's post-roll, attaches // Keep both topics flowing during the post-roll so the widened filter has // B's data to write. @@ -2059,10 +2134,10 @@ TEST_F(RosbagCaptureIntegrationTest, AttachedFaultTopicsJoinTheEntityScopedRecor spin_for(std::chrono::milliseconds(50)); } - ASSERT_TRUE(wait_for_row("FAULT_B", std::chrono::milliseconds(12000))); + ASSERT_TRUE(wait_for_row({"FAULT_B", "/scope_source_b"}, std::chrono::milliseconds(12000))); - auto row_a = storage_->get_rosbag_file("FAULT_A"); - auto row_b = storage_->get_rosbag_file("FAULT_B"); + auto row_a = storage_->get_rosbag_file({"FAULT_A", "/scope_source_a"}); + auto row_b = storage_->get_rosbag_file({"FAULT_B", "/scope_source_b"}); ASSERT_TRUE(row_a.has_value()); ASSERT_TRUE(row_b.has_value()); EXPECT_EQ(row_b->file_path, row_a->file_path); @@ -2095,20 +2170,20 @@ TEST_F(RosbagCaptureIntegrationTest, StopDuringPostRollFinalisesTheRecording) { capture.start(); fill_buffer("/rosbag_stop_probe"); - capture.on_fault_confirmed("BEFORE_STOP"); + capture.on_fault_confirmed(rec("BEFORE_STOP")); capture.stop(); - auto before = storage_->get_rosbag_file("BEFORE_STOP"); + auto before = storage_->get_rosbag_file(rec("BEFORE_STOP")); ASSERT_TRUE(before.has_value()) << "a post-roll cut short by stop() must still leave a usable bag"; // Restart and confirm again: the new fault must get its own recording. capture.start(); fill_buffer("/rosbag_stop_probe_2"); - capture.on_fault_confirmed("AFTER_RESTART"); + capture.on_fault_confirmed(rec("AFTER_RESTART")); capture.stop(); - auto after = storage_->get_rosbag_file("AFTER_RESTART"); + auto after = storage_->get_rosbag_file(rec("AFTER_RESTART")); ASSERT_TRUE(after.has_value()) << "confirmation after restart was swallowed by stale recording state"; EXPECT_NE(after->file_path, before->file_path); } @@ -2133,18 +2208,18 @@ TEST_F(RosbagCaptureIntegrationTest, ConfirmRightAfterFinalizeGetsAPostFaultOnly // buffer after the flush - the boundary state is reached deterministically. fill_buffer("/rosbag_boundary_probe"); - capture.on_fault_confirmed("BOUNDARY_A"); - ASSERT_TRUE(wait_for_row("BOUNDARY_A", std::chrono::milliseconds(8000))); + capture.on_fault_confirmed(rec("BOUNDARY_A")); + ASSERT_TRUE(wait_for_row(rec("BOUNDARY_A"), std::chrono::milliseconds(8000))); // A's flush drained the deque, its post-roll diverted direct writes, and no // publisher exists any more: the buffer is empty. Confirming now used to be // a warn-and-return no-op that left the fault with no recording at all. - capture.on_fault_confirmed("BOUNDARY_B"); + capture.on_fault_confirmed(rec("BOUNDARY_B")); - ASSERT_TRUE(wait_for_row("BOUNDARY_B", std::chrono::milliseconds(8000))) + ASSERT_TRUE(wait_for_row(rec("BOUNDARY_B"), std::chrono::milliseconds(8000))) << "a fault confirmed right after the previous post-roll finalised must get a post-fault-only bag"; - auto row_a = storage_->get_rosbag_file("BOUNDARY_A"); - auto row_b = storage_->get_rosbag_file("BOUNDARY_B"); + auto row_a = storage_->get_rosbag_file(rec("BOUNDARY_A")); + auto row_b = storage_->get_rosbag_file(rec("BOUNDARY_B")); ASSERT_TRUE(row_a.has_value()); ASSERT_TRUE(row_b.has_value()); EXPECT_NE(row_b->file_path, row_a->file_path) << "the boundary fault opens its own recording, not the closed one"; @@ -2171,15 +2246,15 @@ TEST_F(RosbagCaptureIntegrationTest, BurstFaultAttachesToThePostFaultOnlyRecordi capture.start(); fill_buffer("/rosbag_boundary_attach_probe"); - capture.on_fault_confirmed("BOUNDARY_A"); - ASSERT_TRUE(wait_for_row("BOUNDARY_A", std::chrono::milliseconds(8000))); + capture.on_fault_confirmed(rec("BOUNDARY_A")); + ASSERT_TRUE(wait_for_row(rec("BOUNDARY_A"), std::chrono::milliseconds(8000))); - capture.on_fault_confirmed("BOUNDARY_B"); // empty buffer -> post-fault-only recording - capture.on_fault_confirmed("BOUNDARY_C"); // lands inside B's post-roll -> attaches + capture.on_fault_confirmed(rec("BOUNDARY_B")); // empty buffer -> post-fault-only recording + capture.on_fault_confirmed(rec("BOUNDARY_C")); // lands inside B's post-roll -> attaches - ASSERT_TRUE(wait_for_row("BOUNDARY_B", std::chrono::milliseconds(8000))); - auto row_b = storage_->get_rosbag_file("BOUNDARY_B"); - auto row_c = storage_->get_rosbag_file("BOUNDARY_C"); + ASSERT_TRUE(wait_for_row(rec("BOUNDARY_B"), std::chrono::milliseconds(8000))); + auto row_b = storage_->get_rosbag_file(rec("BOUNDARY_B")); + auto row_c = storage_->get_rosbag_file(rec("BOUNDARY_C")); ASSERT_TRUE(row_b.has_value()); ASSERT_TRUE(row_c.has_value()) << "a burst fault confirming inside the post-fault-only window lost its attachment"; EXPECT_EQ(row_c->file_path, row_b->file_path); @@ -2199,10 +2274,10 @@ TEST_F(RosbagCaptureIntegrationTest, EmptyBufferWithNoPostFaultWindowGetsNoBag) capture.start(); // No publisher at all: the buffer is empty at confirmation. - capture.on_fault_confirmed("NO_WINDOW"); + capture.on_fault_confirmed(rec("NO_WINDOW")); spin_for(std::chrono::milliseconds(300)); - EXPECT_FALSE(storage_->get_rosbag_file("NO_WINDOW").has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("NO_WINDOW")).has_value()); EXPECT_EQ(count_bag_dirs(), 0u); capture.stop(); @@ -2221,10 +2296,10 @@ TEST_F(RosbagCaptureIntegrationTest, FullBagDurationReflectsActualContentNotConf capture.start(); fill_buffer("/rosbag_duration_probe"); - capture.on_fault_confirmed("SHORT_CONTENT"); - ASSERT_TRUE(wait_for_row("SHORT_CONTENT", std::chrono::milliseconds(8000))); + capture.on_fault_confirmed(rec("SHORT_CONTENT")); + ASSERT_TRUE(wait_for_row(rec("SHORT_CONTENT"), std::chrono::milliseconds(8000))); - auto row = storage_->get_rosbag_file("SHORT_CONTENT"); + auto row = storage_->get_rosbag_file(rec("SHORT_CONTENT")); ASSERT_TRUE(row.has_value()); EXPECT_GT(row->duration_sec, 0.0); EXPECT_LE(row->duration_sec, 5.0) << "duration_sec must reflect the ~2.5s actually recorded, not config pre+post"; @@ -2244,9 +2319,9 @@ TEST_F(RosbagCaptureIntegrationTest, ImmediateFinalizeDurationReflectsActualCont capture.start(); fill_buffer("/rosbag_immediate_duration_probe"); - capture.on_fault_confirmed("IMMEDIATE_CONTENT"); + capture.on_fault_confirmed(rec("IMMEDIATE_CONTENT")); - auto row = storage_->get_rosbag_file("IMMEDIATE_CONTENT"); + auto row = storage_->get_rosbag_file(rec("IMMEDIATE_CONTENT")); ASSERT_TRUE(row.has_value()); EXPECT_GT(row->duration_sec, 0.0); EXPECT_LE(row->duration_sec, 5.0) << "duration_sec must reflect the ~2s actually buffered, not config duration_sec"; @@ -2276,26 +2351,26 @@ TEST_F(RosbagCaptureIntegrationTest, WriterOpenFailureNeverOpensAPostRollAndReco fill_buffer("/rosbag_io_error_probe"); // Buffered flush hits the I/O error. - capture.on_fault_confirmed("IO_FAIL_FULL"); + capture.on_fault_confirmed(rec("IO_FAIL_FULL")); // Wait past duration_after_sec: a wrongly-opened post-roll would finalise // and store a row for the fault. spin_for(std::chrono::milliseconds(900)); - EXPECT_FALSE(storage_->get_rosbag_file("IO_FAIL_FULL").has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("IO_FAIL_FULL")).has_value()); // Empty buffer at the boundary + broken path: the post-fault-only writer // open fails. Never open a post-roll after an I/O failure. - capture.on_fault_confirmed("IO_FAIL_EMPTY"); + capture.on_fault_confirmed(rec("IO_FAIL_EMPTY")); spin_for(std::chrono::milliseconds(900)); - EXPECT_FALSE(storage_->get_rosbag_file("IO_FAIL_EMPTY").has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("IO_FAIL_EMPTY")).has_value()); // Repair the path (now a directory) - later faults must capture normally. std::filesystem::remove(blocked); std::filesystem::create_directories(blocked); fill_buffer("/rosbag_io_error_probe_2"); - capture.on_fault_confirmed("IO_RECOVERED"); - ASSERT_TRUE(wait_for_row("IO_RECOVERED", std::chrono::milliseconds(8000))) + capture.on_fault_confirmed(rec("IO_RECOVERED")); + ASSERT_TRUE(wait_for_row(rec("IO_RECOVERED"), std::chrono::milliseconds(8000))) << "an earlier I/O failure must not corrupt the state machine for a fault with a working path"; - EXPECT_TRUE(std::filesystem::exists(storage_->get_rosbag_file("IO_RECOVERED")->file_path)); + EXPECT_TRUE(std::filesystem::exists(storage_->get_rosbag_file(rec("IO_RECOVERED"))->file_path)); capture.stop(); } @@ -2316,11 +2391,11 @@ TEST_F(RosbagCaptureIntegrationTest, ZeroMessagePostFaultOnlyBagFinalizesCleanly capture.start(); // No publisher at all: the buffer is empty and the post-roll records nothing. const std::string fault_code = "ZERO_MSG_" + format; - capture.on_fault_confirmed(fault_code); + capture.on_fault_confirmed(rec(fault_code)); - ASSERT_TRUE(wait_for_row(fault_code, std::chrono::milliseconds(8000))) + ASSERT_TRUE(wait_for_row(rec(fault_code), std::chrono::milliseconds(8000))) << "zero-message post-fault-only bag did not finalise on " << format; - auto row = storage_->get_rosbag_file(fault_code); + auto row = storage_->get_rosbag_file(rec(fault_code)); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->format, format); ASSERT_TRUE(std::filesystem::is_directory(row->file_path)); @@ -2366,16 +2441,16 @@ TEST_F(RosbagCaptureIntegrationTest, PostFaultOnlyBagContainsThePostFaultWindow) // A's finalise and B's flush, or B is not the boundary case at all. publish_for(pub, std::chrono::milliseconds(2000)); - capture.on_fault_confirmed("CONTENT_A"); - ASSERT_TRUE(wait_for_row("CONTENT_A", std::chrono::milliseconds(10000))); + capture.on_fault_confirmed(rec("CONTENT_A")); + ASSERT_TRUE(wait_for_row(rec("CONTENT_A"), std::chrono::milliseconds(10000))); - capture.on_fault_confirmed("CONTENT_B"); + capture.on_fault_confirmed(rec("CONTENT_B")); // on_fault_confirmed armed B's window before returning, so everything published // from here lands inside it and has to reach B's bag. publish_for(pub, std::chrono::milliseconds(1000)); - ASSERT_TRUE(wait_for_row("CONTENT_B", std::chrono::milliseconds(10000))); - auto row_b = storage_->get_rosbag_file("CONTENT_B"); + ASSERT_TRUE(wait_for_row(rec("CONTENT_B"), std::chrono::milliseconds(10000))); + auto row_b = storage_->get_rosbag_file(rec("CONTENT_B")); ASSERT_TRUE(row_b.has_value()); EXPECT_TRUE(bag_has_topic(row_b->file_path, topic)) << "the post-fault-only bag never recorded the topic - an empty black box is the failure #574 is about"; @@ -2419,19 +2494,19 @@ TEST_F(RosbagCaptureIntegrationTest, EntityScopedPostFaultOnlyBagCarriesOnlyTheF spin_for(std::chrono::milliseconds(50)); } - capture.on_fault_confirmed("SCOPED_A"); - ASSERT_TRUE(wait_for_row("SCOPED_A", std::chrono::milliseconds(12000))); + capture.on_fault_confirmed({"SCOPED_A", "/boundary_scope_source_a"}); + ASSERT_TRUE(wait_for_row({"SCOPED_A", "/boundary_scope_source_a"}, std::chrono::milliseconds(12000))); // Nothing was published since A's flush drained the buffer: B is the boundary. - capture.on_fault_confirmed("SCOPED_B"); + capture.on_fault_confirmed({"SCOPED_B", "/boundary_scope_source_b"}); for (int i = 0; i < 20; ++i) { pub_a->publish(msg); pub_b->publish(msg); spin_for(std::chrono::milliseconds(50)); } - ASSERT_TRUE(wait_for_row("SCOPED_B", std::chrono::milliseconds(12000))); - auto row_b = storage_->get_rosbag_file("SCOPED_B"); + ASSERT_TRUE(wait_for_row({"SCOPED_B", "/boundary_scope_source_b"}, std::chrono::milliseconds(12000))); + auto row_b = storage_->get_rosbag_file({"SCOPED_B", "/boundary_scope_source_b"}); ASSERT_TRUE(row_b.has_value()); EXPECT_GT(bag_message_count(row_b->file_path), 0) << "the entity-scoped post-fault-only bag finalised empty"; EXPECT_TRUE(bag_has_topic(row_b->file_path, "/boundary_scope_b")) @@ -2485,7 +2560,7 @@ TEST_F(RosbagCaptureIntegrationTest, ABoundaryRecordingKeepsItsScopeWhileTheOldF spin_for(std::chrono::milliseconds(50)); } - capture.on_fault_confirmed("SCOPE_RACE_A"); + capture.on_fault_confirmed({"SCOPE_RACE_A", "/scope_race_source_a"}); // The racer is the capture pool and this thread is the executor, the production // shape. Only the racer ever confirms B, so no two threads call on_fault_confirmed @@ -2504,7 +2579,7 @@ TEST_F(RosbagCaptureIntegrationTest, ABoundaryRecordingKeepsItsScopeWhileTheOldF }; std::thread racer([&capture, &stop_racer, &boundary_bag_opened]() { while (!stop_racer.load()) { - capture.on_fault_confirmed("SCOPE_RACE_B"); + capture.on_fault_confirmed({"SCOPE_RACE_B", "/scope_race_source_b"}); if (boundary_bag_opened()) { return; } @@ -2527,8 +2602,8 @@ TEST_F(RosbagCaptureIntegrationTest, ABoundaryRecordingKeepsItsScopeWhileTheOldF spin_for(std::chrono::milliseconds(50)); } - ASSERT_TRUE(wait_for_row("SCOPE_RACE_B", std::chrono::milliseconds(12000))); - auto row_b = storage_->get_rosbag_file("SCOPE_RACE_B"); + ASSERT_TRUE(wait_for_row({"SCOPE_RACE_B", "/scope_race_source_b"}, std::chrono::milliseconds(12000))); + auto row_b = storage_->get_rosbag_file({"SCOPE_RACE_B", "/scope_race_source_b"}); ASSERT_TRUE(row_b.has_value()); size_t b_recordings = 0; for (const auto & entry : std::filesystem::directory_iterator(temp_dir_)) { @@ -2580,7 +2655,7 @@ TEST_F(RosbagCaptureIntegrationTest, BoundaryConfirmationRacingAFinaliseKeepsIts capture.start(); publish_for(pub, std::chrono::milliseconds(1500)); - capture.on_fault_confirmed("RACE_A"); + capture.on_fault_confirmed(rec("RACE_A")); // Hammer the boundary for the whole of A's window: every call while A records // attaches and returns, and the first one after the guard drops takes the @@ -2588,7 +2663,7 @@ TEST_F(RosbagCaptureIntegrationTest, BoundaryConfirmationRacingAFinaliseKeepsIts std::atomic stop_racer{false}; std::thread racer([&capture, &stop_racer]() { while (!stop_racer.load()) { - capture.on_fault_confirmed("RACE_B"); + capture.on_fault_confirmed(rec("RACE_B")); std::this_thread::sleep_for(std::chrono::microseconds(200)); } }); @@ -2598,8 +2673,8 @@ TEST_F(RosbagCaptureIntegrationTest, BoundaryConfirmationRacingAFinaliseKeepsIts racer.join(); publish_for(pub, std::chrono::milliseconds(600)); - ASSERT_TRUE(wait_for_row("RACE_B", std::chrono::milliseconds(12000))); - auto row_b = storage_->get_rosbag_file("RACE_B"); + ASSERT_TRUE(wait_for_row(rec("RACE_B"), std::chrono::milliseconds(12000))); + auto row_b = storage_->get_rosbag_file(rec("RACE_B")); ASSERT_TRUE(row_b.has_value()); EXPECT_GT(bag_message_count(row_b->file_path), 0) << "the boundary recording finalised empty - its writer was taken by the finalise it raced"; @@ -2625,18 +2700,18 @@ TEST_F(RosbagCaptureIntegrationTest, AttachmentCapAppliesToAPostFaultOnlyRecordi codes.push_back("POSTONLY_BURST_" + std::string(i < 10 ? "0" : "") + std::to_string(i)); } for (const auto & code : codes) { - capture.on_fault_confirmed(code); + capture.on_fault_confirmed(rec(code)); } - ASSERT_TRUE(wait_for_row(codes[0], std::chrono::milliseconds(12000))); - auto primary = storage_->get_rosbag_file(codes[0]); + ASSERT_TRUE(wait_for_row(rec(codes[0]), std::chrono::milliseconds(12000))); + auto primary = storage_->get_rosbag_file(rec(codes[0])); ASSERT_TRUE(primary.has_value()); for (size_t i = 1; i < 33; ++i) { - auto row = storage_->get_rosbag_file(codes[i]); + auto row = storage_->get_rosbag_file(rec(codes[i])); ASSERT_TRUE(row.has_value()) << codes[i] << " is within the cap and must resolve to the recording"; EXPECT_EQ(row->file_path, primary->file_path); } - EXPECT_FALSE(storage_->get_rosbag_file(codes[33]).has_value()) + EXPECT_FALSE(storage_->get_rosbag_file(rec(codes[33])).has_value()) << "fault 34 of the burst is past the cap and is dropped with a WARN"; EXPECT_EQ(count_bag_dirs(), 1u) << "the dropped fault must not open a second bag"; @@ -2654,21 +2729,21 @@ TEST_F(RosbagCaptureIntegrationTest, BoundaryFaultClearedDuringItsOwnWindowDisca RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); - capture.on_fault_confirmed("CLEARED_BOUNDARY"); // empty buffer -> post-fault-only + capture.on_fault_confirmed(rec("CLEARED_BOUNDARY")); // empty buffer -> post-fault-only spin_for(std::chrono::milliseconds(200)); - capture.on_fault_cleared("CLEARED_BOUNDARY"); + capture.on_fault_cleared(rec("CLEARED_BOUNDARY")); spin_for(std::chrono::milliseconds(2000)); // past the window, finalise ran capture.wait_for_stored_recordings(); - EXPECT_FALSE(storage_->get_rosbag_file("CLEARED_BOUNDARY").has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("CLEARED_BOUNDARY")).has_value()); EXPECT_EQ(count_bag_dirs(), 0u) << "the bag of a fault cleared inside its own window must be discarded"; capture.stop(); } TEST_F(RosbagCaptureIntegrationTest, MetadataStoreFailureDiscardsTheBagInsteadOfOrphaningIt) { - // If the row cannot be written, nothing can ever reach the bag: retrieval is - // keyed by fault code and quota accounting enumerates rows, so a kept directory + // If the row cannot be written, nothing can ever reach the bag: retrieval goes + // through the rows and quota accounting enumerates rows, so a kept directory // would occupy disk that nothing can find and nothing can evict. RosbagMetadataFailingStorage failing_storage; auto rosbag_config = create_rosbag_config(); @@ -2678,13 +2753,13 @@ TEST_F(RosbagCaptureIntegrationTest, MetadataStoreFailureDiscardsTheBagInsteadOf RosbagCapture capture(node_.get(), &failing_storage, rosbag_config, snapshot_config); capture.start(); - capture.on_fault_confirmed("STORE_FAILS"); // empty buffer -> post-fault-only - spin_for(std::chrono::milliseconds(1200)); // past the window, finalise ran + capture.on_fault_confirmed(rec("STORE_FAILS")); // empty buffer -> post-fault-only + spin_for(std::chrono::milliseconds(1200)); // past the window, finalise ran capture.wait_for_stored_recordings(); ASSERT_GT(failing_storage.store_attempts, 0u) << "the recording never reached the metadata store, so no row and no directory prove nothing"; - EXPECT_FALSE(failing_storage.get_rosbag_file("STORE_FAILS").has_value()); + EXPECT_FALSE(failing_storage.get_rosbag_file(rec("STORE_FAILS")).has_value()); EXPECT_EQ(count_bag_dirs(), 0u) << "a bag that no row can reference must not be left on disk"; capture.stop(); @@ -2694,7 +2769,7 @@ TEST_F(RosbagCaptureIntegrationTest, AFailingQuotaSweepKeepsTheBagItJustStored) // The mirror image of the test above, and the case its double cannot reach. The // sweep runs AFTER store_rosbag_files() has committed, so a failure there says // nothing about this recording. Discarding the bag then strands the rows just - // written: unreadable for good, because retrieval is keyed by fault code, and + // written: unreadable for good, because retrieval goes through the rows, and // still charged against max_total_storage_mb, which sums rows - the very pressure // that made the sweep run. RosbagQuotaSweepFailingStorage sweep_failing_storage; @@ -2710,6 +2785,7 @@ TEST_F(RosbagCaptureIntegrationTest, AFailingQuotaSweepKeepsTheBagItJustStored) // reach its delete. ros2_medkit_fault_manager::RosbagFileInfo older; older.fault_code = "OLDER"; + older.owner = kOwner; older.file_path = (temp_dir_ / "fault_OLDER_1000").string(); older.format = "mcap"; older.size_bytes = 1; @@ -2719,11 +2795,11 @@ TEST_F(RosbagCaptureIntegrationTest, AFailingQuotaSweepKeepsTheBagItJustStored) capture.start(); fill_buffer("/sweep_failure_probe"); - capture.on_fault_confirmed("SWEEP_FAILS"); + capture.on_fault_confirmed(rec("SWEEP_FAILS")); spin_for(std::chrono::milliseconds(1200)); // past the window, finalise ran capture.wait_for_stored_recordings(); - auto row = sweep_failing_storage.get_rosbag_file("SWEEP_FAILS"); + auto row = sweep_failing_storage.get_rosbag_file(rec("SWEEP_FAILS")); ASSERT_TRUE(row.has_value()) << "the metadata store succeeded, so its row must be there"; // Without this the test would pass on a build where eviction never engaged, and // would be proving nothing about a failing sweep at all. @@ -2739,7 +2815,7 @@ TEST_F(RosbagCaptureIntegrationTest, AFailingStoreLeavesNoBagOnTheImmediatePath) // rather than in the post-roll finalise, and only the finalise was ever covered // for a failing store. Nothing catches it here: in production the capture pool // logs the exception and moves on, leaving a directory no row names - unreachable, - // because retrieval is keyed by fault code, and uncounted, because the quota + // because retrieval goes through the rows, and uncounted, because the quota // enumerates rows, so it is never evicted either and accumulates per failure. RosbagMetadataFailingStorage failing_storage; auto rosbag_config = create_rosbag_config(); @@ -2752,13 +2828,13 @@ TEST_F(RosbagCaptureIntegrationTest, AFailingStoreLeavesNoBagOnTheImmediatePath) fill_buffer("/immediate_store_probe"); // The buffer has to hold something, or the confirmation returns before it ever // opens a bag and there is nothing for the store to fail on. - EXPECT_NO_THROW(capture.on_fault_confirmed("IMMEDIATE_STORE_FAILS")); + EXPECT_NO_THROW(capture.on_fault_confirmed(rec("IMMEDIATE_STORE_FAILS"))); spin_for(std::chrono::milliseconds(200)); ASSERT_GT(failing_storage.store_attempts, 0u) << "fill_buffer() buffered nothing, so the confirmation returned before opening a bag and the " "assertions below hold for the wrong reason"; - EXPECT_FALSE(failing_storage.get_rosbag_file("IMMEDIATE_STORE_FAILS").has_value()); + EXPECT_FALSE(failing_storage.get_rosbag_file(rec("IMMEDIATE_STORE_FAILS")).has_value()); EXPECT_EQ(count_bag_dirs(), 0u) << "a bag that no row can reference must not be left on disk"; capture.stop(); @@ -2779,9 +2855,9 @@ TEST_F(RosbagCaptureIntegrationTest, TheStorageQuotaEngagesOnABoundaryRecording) capture.start(); // Nothing has published yet, so the buffer is empty and this takes the boundary // path - the startup case, not just the one after a window closes. - capture.on_fault_confirmed("QUOTA_ROOMY"); - ASSERT_TRUE(wait_for_row("QUOTA_ROOMY", std::chrono::milliseconds(8000))); - auto row = storage_->get_rosbag_file("QUOTA_ROOMY"); + capture.on_fault_confirmed(rec("QUOTA_ROOMY")); + ASSERT_TRUE(wait_for_row(rec("QUOTA_ROOMY"), std::chrono::milliseconds(8000))); + auto row = storage_->get_rosbag_file(rec("QUOTA_ROOMY")); ASSERT_TRUE(row.has_value()); EXPECT_TRUE(std::filesystem::exists(row->file_path)); capture.stop(); @@ -2795,14 +2871,14 @@ TEST_F(RosbagCaptureIntegrationTest, TheStorageQuotaEngagesOnABoundaryRecording) strict.max_total_storage_mb = 0; RosbagCapture capture(node_.get(), storage_.get(), strict, snapshot_config); capture.start(); - capture.on_fault_confirmed("QUOTA_TIGHT"); - ASSERT_TRUE(wait_for_row("QUOTA_TIGHT", std::chrono::milliseconds(8000))) + capture.on_fault_confirmed(rec("QUOTA_TIGHT")); + ASSERT_TRUE(wait_for_row(rec("QUOTA_TIGHT"), std::chrono::milliseconds(8000))) << "a recording larger than the quota was deleted by its own sweep"; // The sweep runs on the store worker right after the row is stored. capture.wait_for_stored_recordings(); - EXPECT_TRUE(std::filesystem::exists(storage_->get_rosbag_file("QUOTA_TIGHT")->file_path)); + EXPECT_TRUE(std::filesystem::exists(storage_->get_rosbag_file(rec("QUOTA_TIGHT"))->file_path)); - EXPECT_FALSE(storage_->get_rosbag_file("QUOTA_ROOMY").has_value()) + EXPECT_FALSE(storage_->get_rosbag_file(rec("QUOTA_ROOMY")).has_value()) << "the quota did not engage on a post-fault-only recording"; capture.stop(); } @@ -2819,9 +2895,9 @@ TEST_F(RosbagCaptureIntegrationTest, ARecordingWithoutAPostRollCarriesAManifestT capture.start(); fill_buffer("/manifest_immediate_probe"); - capture.on_fault_confirmed("MANIFEST_IMMEDIATE"); + capture.on_fault_confirmed(rec("MANIFEST_IMMEDIATE")); - auto row = storage_->get_rosbag_file("MANIFEST_IMMEDIATE"); + auto row = storage_->get_rosbag_file(rec("MANIFEST_IMMEDIATE")); ASSERT_TRUE(row.has_value()); const auto manifest = read_manifest(row->file_path); ASSERT_TRUE(manifest.is_object()) << "a recording closed without a post-roll carries no manifest.json"; @@ -2866,9 +2942,9 @@ TEST_F(RosbagCaptureIntegrationTest, ARecordingNeverTakesOverAnExistingBagDirect std::ofstream(taken.back() / "earlier_recording") << "kept"; } - capture.on_fault_confirmed("PATH_COLLISION"); + capture.on_fault_confirmed(rec("PATH_COLLISION")); - auto row = storage_->get_rosbag_file("PATH_COLLISION"); + auto row = storage_->get_rosbag_file(rec("PATH_COLLISION")); ASSERT_TRUE(row.has_value()) << "a taken bag name cost the fault its recording"; EXPECT_TRUE(std::filesystem::exists(std::filesystem::path(row->file_path) / "metadata.yaml")); for (const auto & earlier : taken) { @@ -2892,7 +2968,7 @@ TEST_F(RosbagCaptureIntegrationTest, ABoundaryRecordingSplitsAndReportsTheWholeB RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); - capture.on_fault_confirmed("SPLIT_BOUNDARY"); // empty buffer -> post-fault-only + capture.on_fault_confirmed(rec("SPLIT_BOUNDARY")); // empty buffer -> post-fault-only auto pub = node_->create_publisher("/split_probe", 10); std_msgs::msg::String msg; @@ -2902,8 +2978,8 @@ TEST_F(RosbagCaptureIntegrationTest, ABoundaryRecordingSplitsAndReportsTheWholeB spin_for(std::chrono::milliseconds(50)); } - ASSERT_TRUE(wait_for_row("SPLIT_BOUNDARY", std::chrono::milliseconds(12000))); - auto row = storage_->get_rosbag_file("SPLIT_BOUNDARY"); + ASSERT_TRUE(wait_for_row(rec("SPLIT_BOUNDARY"), std::chrono::milliseconds(12000))); + auto row = storage_->get_rosbag_file(rec("SPLIT_BOUNDARY")); ASSERT_TRUE(row.has_value()); size_t data_files = 0; @@ -2997,14 +3073,14 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultAttachingToABoundaryRecordingWidensIt spin_for(std::chrono::milliseconds(50)); } - capture.on_fault_confirmed("WIDEN_C"); - ASSERT_TRUE(wait_for_row("WIDEN_C", std::chrono::milliseconds(12000))); + capture.on_fault_confirmed({"WIDEN_C", "/widen_source_c"}); + ASSERT_TRUE(wait_for_row({"WIDEN_C", "/widen_source_c"}, std::chrono::milliseconds(12000))); // Buffer drained by C's flush: this one is the boundary, scoped to C alone. - capture.on_fault_confirmed("WIDEN_C2"); + capture.on_fault_confirmed({"WIDEN_C2", "/widen_source_c"}); spin_for(std::chrono::milliseconds(300)); // D confirms inside that window and attaches, so its topics join from here on. - capture.on_fault_confirmed("WIDEN_D"); + capture.on_fault_confirmed({"WIDEN_D", "/widen_source_d"}); for (int i = 0; i < 30; ++i) { pub_c->publish(msg); pub_d->publish(msg); @@ -3012,8 +3088,8 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultAttachingToABoundaryRecordingWidensIt spin_for(std::chrono::milliseconds(50)); } - ASSERT_TRUE(wait_for_row("WIDEN_D", std::chrono::milliseconds(12000))); - auto row_d = storage_->get_rosbag_file("WIDEN_D"); + ASSERT_TRUE(wait_for_row({"WIDEN_D", "/widen_source_d"}, std::chrono::milliseconds(12000))); + auto row_d = storage_->get_rosbag_file({"WIDEN_D", "/widen_source_d"}); ASSERT_TRUE(row_d.has_value()); EXPECT_TRUE(bag_has_topic(row_d->file_path, "/widen_d")) << "the attached fault's row serves a boundary recording holding none of its topics"; @@ -3032,9 +3108,9 @@ TEST_F(RosbagCaptureIntegrationTest, FaultClearedBeforeConfirmed) { capture.start(); // Prefail then clear (fault didn't confirm) - capture.on_fault_prefailed("CLEARED_EARLY"); + capture.on_fault_prefailed(rec("CLEARED_EARLY")); spin_for(std::chrono::milliseconds(50)); - capture.on_fault_cleared("CLEARED_EARLY"); + capture.on_fault_cleared(rec("CLEARED_EARLY")); // Should not crash spin_for(std::chrono::milliseconds(100)); @@ -3050,7 +3126,7 @@ TEST_F(RosbagCaptureIntegrationTest, ConfirmedWithoutPrefailed) { capture.start(); // Direct confirm (edge case) - capture.on_fault_confirmed("DIRECT_CONFIRM"); + capture.on_fault_confirmed(rec("DIRECT_CONFIRM")); spin_for(std::chrono::milliseconds(200)); capture.stop(); @@ -3136,12 +3212,12 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultClearedWhileItsBagWasOpeningGetsNoRow RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); - capture.on_fault_confirmed("CLEARED_WHILE_OPENING"); - storage_->clear_fault("CLEARED_WHILE_OPENING"); + capture.on_fault_confirmed({"CLEARED_WHILE_OPENING", "/test_source"}); + storage_->clear_fault({"CLEARED_WHILE_OPENING", "/test_source"}); spin_for(std::chrono::milliseconds(1500)); capture.wait_for_stored_recordings(); - EXPECT_FALSE(storage_->get_rosbag_file("CLEARED_WHILE_OPENING").has_value()) + EXPECT_FALSE(storage_->get_rosbag_file({"CLEARED_WHILE_OPENING", "/test_source"}).has_value()) << "a fault cleared before its recording finalised kept a row nothing will ever remove"; EXPECT_EQ(count_bag_dirs(), 0u) << "the bag nobody references any more was left on disk"; @@ -3182,7 +3258,7 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultPastTheAttachmentCapStillGetsItsTopic spin_for(std::chrono::milliseconds(50)); } - capture.on_fault_confirmed("CAP_PRIMARY"); + capture.on_fault_confirmed({"CAP_PRIMARY", "/cap_primary_source"}); // Fill the attachment set past its cap, then confirm the one whose topics matter. // The fillers report the PRIMARY node as their source on purpose: a fault whose // scope cannot be resolved widens the recording to every topic, which would let the @@ -3191,19 +3267,20 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultPastTheAttachmentCapStillGetsItsTopic const std::string filler = "CAP_FILLER_" + std::string(i < 10 ? "0" : "") + std::to_string(i); storage_->report_fault_event(filler, fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "cap filler", "/cap_primary_source", clock.now().nanoseconds(), debounce); - capture.on_fault_confirmed(filler); + capture.on_fault_confirmed({filler, "/cap_primary_source"}); } - capture.on_fault_confirmed("CAP_LATE"); + capture.on_fault_confirmed({"CAP_LATE", "/cap_late_source"}); for (int i = 0; i < 30; ++i) { pub_primary->publish(msg); pub_late->publish(msg); spin_for(std::chrono::milliseconds(50)); } - ASSERT_TRUE(wait_for_row("CAP_PRIMARY", std::chrono::milliseconds(12000))); - auto row = storage_->get_rosbag_file("CAP_PRIMARY"); + ASSERT_TRUE(wait_for_row({"CAP_PRIMARY", "/cap_primary_source"}, std::chrono::milliseconds(12000))); + auto row = storage_->get_rosbag_file({"CAP_PRIMARY", "/cap_primary_source"}); ASSERT_TRUE(row.has_value()); - EXPECT_FALSE(storage_->get_rosbag_file("CAP_LATE").has_value()) << "the cap is supposed to withhold the lookup key"; + EXPECT_FALSE(storage_->get_rosbag_file({"CAP_LATE", "/cap_late_source"}).has_value()) + << "the cap is supposed to withhold the lookup key"; EXPECT_TRUE(bag_has_topic(row->file_path, "/cap_primary")) << "the recording lost the topics it was opened for"; EXPECT_TRUE(bag_has_topic(row->file_path, "/cap_late")) << "an over-cap fault's topics never reached the bag, so the burst's black box is missing its data"; @@ -3224,10 +3301,10 @@ TEST_F(RosbagCaptureIntegrationTest, AnUnusualExceptionFromTheStoreDoesNotEscape capture.start(); fill_buffer("/odd_throw_probe"); - EXPECT_NO_THROW(capture.on_fault_confirmed("ODD_THROW")); + EXPECT_NO_THROW(capture.on_fault_confirmed(rec("ODD_THROW"))); ASSERT_GT(odd_storage.store_attempts, 0u) << "the store was never reached, so nothing was caught"; - EXPECT_FALSE(odd_storage.get_rosbag_file("ODD_THROW").has_value()); + EXPECT_FALSE(odd_storage.get_rosbag_file(rec("ODD_THROW")).has_value()); EXPECT_EQ(count_bag_dirs(), 0u) << "the bag survived a failed store"; EXPECT_NO_THROW(capture.stop()); @@ -3264,12 +3341,12 @@ TEST_F(RosbagCaptureIntegrationTest, ReconfirmingTheRecordingsOwnFaultResolvesNo spin_for(std::chrono::milliseconds(50)); } - capture.on_fault_confirmed("RECONFIRMED"); + capture.on_fault_confirmed({"RECONFIRMED", "/reconfirm_source"}); const size_t after_first = storage_ptr->lookups; constexpr int kRepeats = 20; for (int i = 0; i < kRepeats; ++i) { - capture.on_fault_confirmed("RECONFIRMED"); + capture.on_fault_confirmed({"RECONFIRMED", "/reconfirm_source"}); } EXPECT_LT(storage_ptr->lookups - after_first, static_cast(kRepeats)) @@ -3352,7 +3429,7 @@ TEST_F(RosbagCaptureIntegrationTest, ConcurrentCapturesInOneProcessSurviveEachOt std::thread confirmer([&]() { int n = 0; while (!stop_racers.load()) { - capture.on_fault_confirmed("PLUGIN_RACE_" + std::to_string(n++)); + capture.on_fault_confirmed(rec("PLUGIN_RACE_" + std::to_string(n++))); confirms_done.fetch_add(1); std::this_thread::sleep_for(std::chrono::milliseconds(5)); } @@ -3381,7 +3458,7 @@ TEST_F(RosbagCaptureIntegrationTest, ConcurrentCapturesInOneProcessSurviveEachOt << "the confirming thread barely ran, so few bags were opened off-executor"; size_t rows = 0; for (int i = 0; i < confirms_done.load(); ++i) { - if (storage_->get_rosbag_file("PLUGIN_RACE_" + std::to_string(i)).has_value()) { + if (storage_->get_rosbag_file(rec("PLUGIN_RACE_" + std::to_string(i))).has_value()) { ++rows; } } diff --git a/src/ros2_medkit_fault_manager/test/test_rosbag_storage_parity.cpp b/src/ros2_medkit_fault_manager/test/test_rosbag_storage_parity.cpp index 7cb06c262..ecc6028a0 100644 --- a/src/ros2_medkit_fault_manager/test/test_rosbag_storage_parity.cpp +++ b/src/ros2_medkit_fault_manager/test/test_rosbag_storage_parity.cpp @@ -44,6 +44,7 @@ namespace { +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::FaultStorage; using ros2_medkit_fault_manager::InMemoryFaultStorage; using ros2_medkit_fault_manager::rosbag_recording_id; @@ -96,9 +97,19 @@ class RosbagRetentionParityTest : public ::testing::Test { return path.string(); } + /// Every row in this suite belongs to one owner: these tests are about retention + /// and ordering, which the record identity does not change. Cross-owner separation + /// is covered where it belongs, in the storage identity tests. + static constexpr const char * kOwner = "/node"; + + static FaultId id(const std::string & code) { + return FaultId{code, kOwner}; + } + RosbagFileInfo row(const std::string & code, const std::string & bag_name, int64_t created_ns) { RosbagFileInfo info; info.fault_code = code; + info.owner = kOwner; info.file_path = make_bag(bag_name); info.recording_id = rosbag_recording_id(info.file_path); info.format = "mcap"; @@ -122,7 +133,7 @@ TYPED_TEST(RosbagRetentionParityTest, RecordingIdIsTheBasenameOfEveryStoredRow) this->storage_->store_rosbag_file(this->row("BASENAME", "fault_BASENAME_1", 1000)); this->storage_->store_rosbag_file(this->row("BASENAME", "fault_BASENAME_2", 2000)); - const auto rows = this->storage_->get_rosbag_files("BASENAME"); + const auto rows = this->storage_->get_rosbag_files(this->id("BASENAME")); ASSERT_EQ(rows.size(), 2u); for (const auto & r : rows) { EXPECT_EQ(r.recording_id, std::filesystem::path(r.file_path).filename().string()); @@ -137,7 +148,7 @@ TYPED_TEST(RosbagRetentionParityTest, RecordingIdIsDerivedWhenTheCallerLeftItEmp info.recording_id.clear(); this->storage_->store_rosbag_file(info); - const auto rows = this->storage_->get_rosbag_files("DERIVED"); + const auto rows = this->storage_->get_rosbag_files(this->id("DERIVED")); ASSERT_EQ(rows.size(), 1u); EXPECT_EQ(rows[0].recording_id, "fault_DERIVED_1"); } @@ -149,7 +160,7 @@ TYPED_TEST(RosbagRetentionParityTest, SeveralRecordingsForOneFaultSurviveUnderAC this->storage_->store_rosbag_file(this->row("FLAP", "fault_FLAP_2", 2000)); this->storage_->store_rosbag_file(this->row("FLAP", "fault_FLAP_3", 3000)); - const auto rows = this->storage_->get_rosbag_files("FLAP"); + const auto rows = this->storage_->get_rosbag_files(this->id("FLAP")); ASSERT_EQ(rows.size(), 3u); EXPECT_EQ(rows[0].recording_id, "fault_FLAP_3") << "newest first"; EXPECT_EQ(rows[2].recording_id, "fault_FLAP_1"); @@ -165,7 +176,7 @@ TYPED_TEST(RosbagRetentionParityTest, CapKeepsTheNewestAndUnlinksTheEvicted) { this->storage_->store_rosbag_file(this->row("CAP", "fault_CAP_2", 2000)); this->storage_->store_rosbag_file(this->row("CAP", "fault_CAP_3", 3000)); - const auto rows = this->storage_->get_rosbag_files("CAP"); + const auto rows = this->storage_->get_rosbag_files(this->id("CAP")); ASSERT_EQ(rows.size(), 2u); EXPECT_EQ(rows[0].recording_id, "fault_CAP_3"); EXPECT_EQ(rows[1].recording_id, "fault_CAP_2"); @@ -180,12 +191,12 @@ TYPED_TEST(RosbagRetentionParityTest, ACapOfOneReproducesThePreviousBehaviourExa this->storage_->store_rosbag_file(first); this->storage_->store_rosbag_file(this->row("ONE", "fault_ONE_2", 2000)); - const auto rows = this->storage_->get_rosbag_files("ONE"); + const auto rows = this->storage_->get_rosbag_files(this->id("ONE")); ASSERT_EQ(rows.size(), 1u); EXPECT_EQ(rows[0].recording_id, "fault_ONE_2"); EXPECT_FALSE(std::filesystem::exists(first.file_path)); - const auto newest = this->storage_->get_rosbag_file("ONE"); + const auto newest = this->storage_->get_rosbag_file(this->id("ONE")); ASSERT_TRUE(newest.has_value()); EXPECT_EQ(newest->recording_id, "fault_ONE_2"); } @@ -197,7 +208,7 @@ TYPED_TEST(RosbagRetentionParityTest, ACapOfOneIsTheDefaultWithoutConfiguringAny this->storage_->store_rosbag_file(first); this->storage_->store_rosbag_file(this->row("DEFAULT", "fault_DEFAULT_2", 2000)); - EXPECT_EQ(this->storage_->get_rosbag_files("DEFAULT").size(), 1u); + EXPECT_EQ(this->storage_->get_rosbag_files(this->id("DEFAULT")).size(), 1u); EXPECT_FALSE(std::filesystem::exists(first.file_path)); } @@ -206,7 +217,7 @@ TYPED_TEST(RosbagRetentionParityTest, ZeroMeansUnlimited) { for (int i = 1; i <= 6; ++i) { this->storage_->store_rosbag_file(this->row("UNBOUNDED", "fault_UNBOUNDED_" + std::to_string(i), i * 1000)); } - EXPECT_EQ(this->storage_->get_rosbag_files("UNBOUNDED").size(), 6u); + EXPECT_EQ(this->storage_->get_rosbag_files(this->id("UNBOUNDED")).size(), 6u); } TYPED_TEST(RosbagRetentionParityTest, ReStoringTheSamePathIsAnUpsertNotASecondRecording) { @@ -219,7 +230,7 @@ TYPED_TEST(RosbagRetentionParityTest, ReStoringTheSamePathIsAnUpsertNotASecondRe info.duration_sec = 9.0; this->storage_->store_rosbag_file(info); - const auto rows = this->storage_->get_rosbag_files("UPSERT"); + const auto rows = this->storage_->get_rosbag_files(this->id("UPSERT")); ASSERT_EQ(rows.size(), 1u); EXPECT_EQ(rows[0].size_bytes, 4096u); EXPECT_DOUBLE_EQ(rows[0].duration_sec, 9.0); @@ -239,7 +250,7 @@ TYPED_TEST(RosbagRetentionParityTest, RefreshingALinkDoesNotMoveItWithinATieGrou first.size_bytes = 8192; this->storage_->store_rosbag_file(first); - const auto rows = this->storage_->get_rosbag_files("REFRESH"); + const auto rows = this->storage_->get_rosbag_files(this->id("REFRESH")); ASSERT_EQ(rows.size(), 2u); EXPECT_EQ(rows[0].recording_id, "fault_REFRESH_2") << "the refresh must not reorder the tie group"; EXPECT_EQ(rows[1].recording_id, "fault_REFRESH_1"); @@ -256,7 +267,7 @@ TYPED_TEST(RosbagRetentionParityTest, NewestFirstIsStableWhenAWholeBurstSharesAT this->storage_->store_rosbag_file(this->row("TIE", "fault_TIE_" + std::to_string(i), 7000)); } - const auto rows = this->storage_->get_rosbag_files("TIE"); + const auto rows = this->storage_->get_rosbag_files(this->id("TIE")); ASSERT_EQ(rows.size(), 4u); EXPECT_EQ(rows[0].recording_id, "fault_TIE_4") << "insertion order breaks the tie, newest first"; EXPECT_EQ(rows[1].recording_id, "fault_TIE_3"); @@ -273,7 +284,7 @@ TYPED_TEST(RosbagRetentionParityTest, ACapTieIsBrokenByInsertionOrderNotArbitrar const auto second = this->row("TIECAP", "fault_TIECAP_2", 7000); this->storage_->store_rosbag_file(second); - const auto rows = this->storage_->get_rosbag_files("TIECAP"); + const auto rows = this->storage_->get_rosbag_files(this->id("TIECAP")); ASSERT_EQ(rows.size(), 1u); EXPECT_EQ(rows[0].recording_id, "fault_TIECAP_2"); EXPECT_TRUE(std::filesystem::exists(second.file_path)); @@ -292,7 +303,7 @@ TYPED_TEST(RosbagRetentionParityTest, ABurstRecordingSurvivesWhileASiblingRowRef this->storage_->store_rosbag_file(this->row("BURST_A", "fault_BURST_2", 2000)); EXPECT_TRUE(std::filesystem::exists(shared_path)) << "the sibling fault still owns the shared bag"; - const auto sibling = this->storage_->get_rosbag_files("BURST_B"); + const auto sibling = this->storage_->get_rosbag_files(this->id("BURST_B")); ASSERT_EQ(sibling.size(), 1u); EXPECT_EQ(sibling[0].file_path, shared_path); } @@ -323,8 +334,8 @@ TYPED_TEST(RosbagRetentionParityTest, AnUnknownRecordingLooksUpEmptyRatherThanTh // The gateway's compatibility path depends on this: empty means "not a recording, // try it as a fault code", so it must be a normal answer. EXPECT_TRUE(this->storage_->get_rosbag_files_by_recording("fault_NOPE_1").empty()); - EXPECT_TRUE(this->storage_->get_rosbag_files("NOPE").empty()); - EXPECT_FALSE(this->storage_->get_rosbag_file("NOPE").has_value()); + EXPECT_TRUE(this->storage_->get_rosbag_files(this->id("NOPE")).empty()); + EXPECT_FALSE(this->storage_->get_rosbag_file(this->id("NOPE")).has_value()); } TYPED_TEST(RosbagRetentionParityTest, DeletingARecordingRemovesEveryLinkAndTheBag) { @@ -340,9 +351,9 @@ TYPED_TEST(RosbagRetentionParityTest, DeletingARecordingRemovesEveryLinkAndTheBa EXPECT_EQ(this->storage_->delete_rosbag_recording("fault_DELREC_1"), 2u); EXPECT_FALSE(std::filesystem::exists(shared.file_path)); - EXPECT_TRUE(this->storage_->get_rosbag_files("DELREC_B").empty()); + EXPECT_TRUE(this->storage_->get_rosbag_files(this->id("DELREC_B")).empty()); - const auto survivors = this->storage_->get_rosbag_files("DELREC_A"); + const auto survivors = this->storage_->get_rosbag_files(this->id("DELREC_A")); ASSERT_EQ(survivors.size(), 1u) << "the other fault's other recording is untouched"; EXPECT_EQ(survivors[0].recording_id, "fault_DELREC_2"); EXPECT_TRUE(std::filesystem::exists(kept.file_path)); @@ -357,8 +368,8 @@ TYPED_TEST(RosbagRetentionParityTest, DeletingAFaultDropsAllItsRecordings) { this->storage_->store_rosbag_file(a); this->storage_->store_rosbag_file(b); - EXPECT_TRUE(this->storage_->delete_rosbag_file("DELALL")); - EXPECT_TRUE(this->storage_->get_rosbag_files("DELALL").empty()); + EXPECT_TRUE(this->storage_->delete_rosbag_file(this->id("DELALL"))); + EXPECT_TRUE(this->storage_->get_rosbag_files(this->id("DELALL")).empty()); EXPECT_FALSE(std::filesystem::exists(a.file_path)); EXPECT_FALSE(std::filesystem::exists(b.file_path)); } @@ -417,6 +428,7 @@ class SnapshotRetentionParityTest : public ::testing::Test { for (size_t i = 0; i < topics; ++i) { SnapshotData row; row.fault_code = code; + row.owner = kOwner; row.topic = "/topic_" + std::to_string(i); row.message_type = "std_msgs/msg/Float64"; row.data = R"({"data": )" + std::to_string(capture_id) + "}"; @@ -427,10 +439,18 @@ class SnapshotRetentionParityTest : public ::testing::Test { return rows; } + /// Every record in this suite belongs to one owner: these tests are about retention + /// and ordering, which the record identity does not change. + static constexpr const char * kOwner = "/node"; + + static FaultId id(const std::string & code) { + return FaultId{code, kOwner}; + } + void confirm(const std::string & code) { rclcpp::Clock clock; storage_->report_fault_event(code, fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "parity", - "/node", clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + kOwner, clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); } std::filesystem::path dir_; @@ -448,7 +468,7 @@ TYPED_TEST(SnapshotRetentionParityTest, ACaptureLargerThanTheCapIsKeptWholeRathe this->storage_->store_snapshots(this->capture("BIG", 1, 5)); - const auto rows = this->storage_->get_snapshots("BIG"); + const auto rows = this->storage_->get_snapshots(this->id("BIG")); EXPECT_EQ(rows.size(), 5u) << "a capture over the cap on its own is kept entire"; for (const auto & row : rows) { EXPECT_EQ(row.capture_id, 1); @@ -463,7 +483,7 @@ TYPED_TEST(SnapshotRetentionParityTest, TheOldestWholeCaptureGoesFirstPastTheCap this->storage_->store_snapshots(this->capture("EVICT", 2, 2)); this->storage_->store_snapshots(this->capture("EVICT", 3, 2)); - const auto rows = this->storage_->get_snapshots("EVICT"); + const auto rows = this->storage_->get_snapshots(this->id("EVICT")); ASSERT_EQ(rows.size(), 4u); for (const auto & row : rows) { EXPECT_NE(row.capture_id, 1) << "the oldest set goes whole, not row by row"; @@ -480,7 +500,7 @@ TYPED_TEST(SnapshotRetentionParityTest, GetSnapshotsReturnsTheNewestCaptureFirst this->storage_->store_snapshots(this->capture("ORDER", 1, 2)); this->storage_->store_snapshots(this->capture("ORDER", 2, 2)); - const auto rows = this->storage_->get_snapshots("ORDER"); + const auto rows = this->storage_->get_snapshots(this->id("ORDER")); ASSERT_EQ(rows.size(), 4u); EXPECT_EQ(rows.front().capture_id, 2) << "newest capture leads"; EXPECT_EQ(rows.back().capture_id, 1); @@ -507,9 +527,9 @@ TYPED_TEST(SnapshotRetentionParityTest, ClearFaultKeepsSnapshotsWhenEvidenceIsRe this->storage_->set_retain_snapshots_on_clear(true); this->storage_->store_snapshots(this->capture("KEEP", 1, 2)); - ASSERT_TRUE(this->storage_->clear_fault("KEEP")); + ASSERT_TRUE(this->storage_->clear_fault(this->id("KEEP"))); - EXPECT_EQ(this->storage_->get_snapshots("KEEP").size(), 2u); + EXPECT_EQ(this->storage_->get_snapshots(this->id("KEEP")).size(), 2u); EXPECT_TRUE(this->storage_->retains_snapshots_on_clear()); } @@ -518,9 +538,9 @@ TYPED_TEST(SnapshotRetentionParityTest, ClearFaultDropsSnapshotsByDefault) { this->confirm("DROP"); this->storage_->store_snapshots(this->capture("DROP", 1, 2)); - ASSERT_TRUE(this->storage_->clear_fault("DROP")); + ASSERT_TRUE(this->storage_->clear_fault(this->id("DROP"))); - EXPECT_TRUE(this->storage_->get_snapshots("DROP").empty()); + EXPECT_TRUE(this->storage_->get_snapshots(this->id("DROP")).empty()); EXPECT_FALSE(this->storage_->retains_snapshots_on_clear()); } diff --git a/src/ros2_medkit_fault_manager/test/test_snapshot_capture.cpp b/src/ros2_medkit_fault_manager/test/test_snapshot_capture.cpp index fa8f656d1..70b32d099 100644 --- a/src/ros2_medkit_fault_manager/test/test_snapshot_capture.cpp +++ b/src/ros2_medkit_fault_manager/test/test_snapshot_capture.cpp @@ -62,6 +62,15 @@ void await_topic_type(const rclcpp::Node::SharedPtr & node, const std::string & FAIL() << "the graph never reported a type for " << topic; } +/// The owner every record in this suite belongs to. Snapshot capture resolves topics +/// by fault CODE and writes per RECORD, so one owner is enough except in the +/// entity-default suite, which resolves the scope from the owner itself. +constexpr const char * kOwner = "/node"; + +ros2_medkit_fault_manager::FaultId rec(const std::string & code) { + return ros2_medkit_fault_manager::FaultId{code, kOwner}; +} + } // namespace class SnapshotCaptureTest : public ::testing::Test { @@ -108,10 +117,10 @@ TEST_F(SnapshotCaptureTest, DisabledCaptureSkipsProcessing) { config.default_topics = {"/test_topic"}; SnapshotCapture capture(node_.get(), storage_.get(), config); - capture.capture("TEST_FAULT"); + capture.capture(rec("TEST_FAULT")); // No snapshots should be stored when disabled - auto snapshots = storage_->get_snapshots("TEST_FAULT"); + auto snapshots = storage_->get_snapshots(rec("TEST_FAULT")); EXPECT_TRUE(snapshots.empty()); } @@ -227,10 +236,10 @@ TEST_F(SnapshotCaptureTest, EmptyConfigurationHandledGracefully) { // No topics configured at all SnapshotCapture capture(node_.get(), storage_.get(), config); - capture.capture("ANY_FAULT"); + capture.capture(rec("ANY_FAULT")); // Should not crash, just log that no topics configured - auto snapshots = storage_->get_snapshots("ANY_FAULT"); + auto snapshots = storage_->get_snapshots(rec("ANY_FAULT")); EXPECT_TRUE(snapshots.empty()); } @@ -256,14 +265,14 @@ TEST_F(SnapshotCaptureTest, OnDemandCaptureHandlesNonExistentTopic) { config.default_topics = {"/nonexistent_topic"}; SnapshotCapture capture(node_.get(), storage_.get(), config); - capture.capture("TEST_FAULT"); + capture.capture(rec("TEST_FAULT")); // Should timeout gracefully, no snapshot stored - auto snapshots = storage_->get_snapshots("TEST_FAULT"); + auto snapshots = storage_->get_snapshots(rec("TEST_FAULT")); EXPECT_TRUE(snapshots.empty()); // A configured capture that sampled nothing still records an empty {} frame. - auto frame = storage_->get_freeze_frame("TEST_FAULT"); + auto frame = storage_->get_freeze_frame(rec("TEST_FAULT")); ASSERT_TRUE(frame.has_value()); EXPECT_EQ(frame->data, "{}"); } @@ -323,9 +332,9 @@ TEST_F(SnapshotCaptureTest, CaptureWritesFreezeFrameFromConfiguredTopic) { } ASSERT_GT(node_->count_publishers("/plc/pressure"), 0u); - capture.capture("PLC_PRESSURE_HIGH"); + capture.capture(rec("PLC_PRESSURE_HIGH")); - auto frame = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto frame = storage_->get_freeze_frame(rec("PLC_PRESSURE_HIGH")); ASSERT_TRUE(frame.has_value()); auto parsed = nlohmann::json::parse(frame->data); ASSERT_TRUE(parsed.contains("/plc/pressure")); @@ -362,10 +371,10 @@ TEST_F(SnapshotCaptureTest, EmptyRecaptureKeepsRetainedFreezeFrame) { } ASSERT_GT(node_->count_publishers("/plc/flow"), 0u); - capture.capture("PLC_FLOW_LOW"); + capture.capture(rec("PLC_FLOW_LOW")); } - auto frame = storage_->get_freeze_frame("PLC_FLOW_LOW"); + auto frame = storage_->get_freeze_frame(rec("PLC_FLOW_LOW")); ASSERT_TRUE(frame.has_value()); ASSERT_NE(frame->data, "{}"); @@ -378,9 +387,9 @@ TEST_F(SnapshotCaptureTest, EmptyRecaptureKeepsRetainedFreezeFrame) { } ASSERT_EQ(node_->count_publishers("/plc/flow"), 0u); - capture.capture("PLC_FLOW_LOW"); + capture.capture(rec("PLC_FLOW_LOW")); - auto retained = storage_->get_freeze_frame("PLC_FLOW_LOW"); + auto retained = storage_->get_freeze_frame(rec("PLC_FLOW_LOW")); ASSERT_TRUE(retained.has_value()); auto parsed = nlohmann::json::parse(retained->data); ASSERT_TRUE(parsed.contains("/plc/flow")); @@ -418,8 +427,8 @@ TEST_F(SnapshotCaptureTest, BackgroundCaptureCachesFreezeFrame) { auto start = std::chrono::steady_clock::now(); while (!got_frame && std::chrono::steady_clock::now() - start < std::chrono::seconds(10)) { spin_some_once(node_); - capture.capture("PLC_TEMP_HIGH"); - auto frame = storage_->get_freeze_frame("PLC_TEMP_HIGH"); + capture.capture(rec("PLC_TEMP_HIGH")); + auto frame = storage_->get_freeze_frame(rec("PLC_TEMP_HIGH")); got_frame = frame.has_value() && frame->data != "{}"; if (!got_frame) { std::this_thread::sleep_for(std::chrono::milliseconds(20)); @@ -427,14 +436,14 @@ TEST_F(SnapshotCaptureTest, BackgroundCaptureCachesFreezeFrame) { } ASSERT_TRUE(got_frame); - auto frame = storage_->get_freeze_frame("PLC_TEMP_HIGH"); + auto frame = storage_->get_freeze_frame(rec("PLC_TEMP_HIGH")); ASSERT_TRUE(frame.has_value()); auto parsed = nlohmann::json::parse(frame->data); ASSERT_TRUE(parsed.contains("/plc/temperature")); EXPECT_DOUBLE_EQ(parsed["/plc/temperature"]["data"].get(), 91.5); // The cache path also stores a per-topic snapshot. - auto snapshots = storage_->get_snapshots("PLC_TEMP_HIGH"); + auto snapshots = storage_->get_snapshots(rec("PLC_TEMP_HIGH")); ASSERT_FALSE(snapshots.empty()); EXPECT_EQ(snapshots.back().topic, "/plc/temperature"); EXPECT_EQ(snapshots.back().message_type, "std_msgs/msg/Float64"); @@ -466,8 +475,8 @@ TEST_F(SnapshotCaptureTest, BackgroundCaptureSubscribesToATopicThatAppearsAfterS const auto start = std::chrono::steady_clock::now(); while (!got_frame && std::chrono::steady_clock::now() - start < std::chrono::seconds(10)) { spin_some_once(node_); - capture.capture("PLC_LATE_TEMP_HIGH"); - auto frame = storage_->get_freeze_frame("PLC_LATE_TEMP_HIGH"); + capture.capture(rec("PLC_LATE_TEMP_HIGH")); + auto frame = storage_->get_freeze_frame(rec("PLC_LATE_TEMP_HIGH")); got_frame = frame.has_value() && frame->data != "{}"; if (!got_frame) { std::this_thread::sleep_for(std::chrono::milliseconds(20)); @@ -475,7 +484,7 @@ TEST_F(SnapshotCaptureTest, BackgroundCaptureSubscribesToATopicThatAppearsAfterS } ASSERT_TRUE(got_frame) << "the background cache never carried " << topic << ", which appeared after startup"; - auto parsed = nlohmann::json::parse(storage_->get_freeze_frame("PLC_LATE_TEMP_HIGH")->data); + auto parsed = nlohmann::json::parse(storage_->get_freeze_frame(rec("PLC_LATE_TEMP_HIGH"))->data); ASSERT_TRUE(parsed.contains(topic)); EXPECT_DOUBLE_EQ(parsed[topic]["data"].get(), 77.25); @@ -493,9 +502,9 @@ TEST_F(SnapshotCaptureTest, UnconfiguredFaultWritesNoFreezeFrame) { // No default_topics: an unrelated fault code resolves to an empty capture set. SnapshotCapture capture(node_.get(), storage_.get(), config); - capture.capture("UNMAPPED_FAULT"); + capture.capture(rec("UNMAPPED_FAULT")); - EXPECT_FALSE(storage_->get_freeze_frame("UNMAPPED_FAULT").has_value()); + EXPECT_FALSE(storage_->get_freeze_frame(rec("UNMAPPED_FAULT")).has_value()); } // @verifies REQ_INTEROP_088 @@ -505,9 +514,9 @@ TEST_F(SnapshotCaptureTest, DisabledCaptureWritesNoFreezeFrame) { config.default_topics = {"/plc/pressure"}; SnapshotCapture capture(node_.get(), storage_.get(), config); - capture.capture("ANY_FAULT"); + capture.capture(rec("ANY_FAULT")); - EXPECT_FALSE(storage_->get_freeze_frame("ANY_FAULT").has_value()); + EXPECT_FALSE(storage_->get_freeze_frame(rec("ANY_FAULT")).has_value()); } // Entity-default (zero-config) capture tests. The reporting source is this @@ -523,6 +532,11 @@ class EntityDefaultCaptureTest : public SnapshotCaptureTest { debounce); } + /// The record this node owns: the entity-default scope is resolved from the owner. + ros2_medkit_fault_manager::FaultId id(const std::string & fault_code) const { + return ros2_medkit_fault_manager::FaultId{fault_code, node_->get_fully_qualified_name()}; + } + /// Wait until this node's publisher on @p topic is visible on the graph. void wait_for_publisher(const std::string & topic) { auto start = std::chrono::steady_clock::now(); @@ -557,9 +571,9 @@ TEST_F(EntityDefaultCaptureTest, EntityDefaultCapturesSourceNodeTopics) { }); wait_for_publisher("/entity/own_metric"); - capture.capture("ENTITY_FAULT"); + capture.capture(id("ENTITY_FAULT")); - auto frame = storage_->get_freeze_frame("ENTITY_FAULT"); + auto frame = storage_->get_freeze_frame(id("ENTITY_FAULT")); ASSERT_TRUE(frame.has_value()); auto parsed = nlohmann::json::parse(frame->data); ASSERT_TRUE(parsed.contains("/entity/own_metric")); @@ -581,9 +595,9 @@ TEST_F(EntityDefaultCaptureTest, EntityDefaultDisabledWritesNoFreezeFrame) { store_fault_from_this_node("OPTED_OUT_FAULT"); wait_for_publisher("/entity/opted_out"); - capture.capture("OPTED_OUT_FAULT"); + capture.capture(id("OPTED_OUT_FAULT")); - EXPECT_FALSE(storage_->get_freeze_frame("OPTED_OUT_FAULT").has_value()); + EXPECT_FALSE(storage_->get_freeze_frame(id("OPTED_OUT_FAULT")).has_value()); } // @verifies REQ_INTEROP_088 @@ -611,9 +625,9 @@ TEST_F(EntityDefaultCaptureTest, ExplicitConfigWinsOverEntityDefault) { }); wait_for_publisher("/entity/configured"); - capture.capture("CONFIGURED_FAULT"); + capture.capture(id("CONFIGURED_FAULT")); - auto frame = storage_->get_freeze_frame("CONFIGURED_FAULT"); + auto frame = storage_->get_freeze_frame(id("CONFIGURED_FAULT")); ASSERT_TRUE(frame.has_value()); auto parsed = nlohmann::json::parse(frame->data); // Only the explicitly configured topic is captured, never the node's other topics. @@ -649,11 +663,11 @@ TEST_F(EntityDefaultCaptureTest, ExplicitEmptyConfigEntryOptsOutOfEntityDefault) }); wait_for_publisher("/entity/own_optout"); - capture.capture("OPTED_OUT_SPECIFIC"); - capture.capture("OPTED_OUT_PATTERN_X"); + capture.capture(id("OPTED_OUT_SPECIFIC")); + capture.capture(id("OPTED_OUT_PATTERN_X")); - EXPECT_FALSE(storage_->get_freeze_frame("OPTED_OUT_SPECIFIC").has_value()); - EXPECT_FALSE(storage_->get_freeze_frame("OPTED_OUT_PATTERN_X").has_value()); + EXPECT_FALSE(storage_->get_freeze_frame(id("OPTED_OUT_SPECIFIC")).has_value()); + EXPECT_FALSE(storage_->get_freeze_frame(id("OPTED_OUT_PATTERN_X")).has_value()); } // @verifies REQ_INTEROP_088 @@ -665,11 +679,11 @@ TEST_F(EntityDefaultCaptureTest, UnresolvableSourceWritesNoRow) { // Plugin-style bare entity id: not a live node, resolves to no topics. ros2_medkit_fault_manager::DebounceConfig debounce; storage_->report_fault_event("PLC_FAULT", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "plc fault", "beckhoff_plc_app", rclcpp::Clock().now().nanoseconds(), debounce); + "plc fault", "plc_app", rclcpp::Clock().now().nanoseconds(), debounce); - capture.capture("PLC_FAULT"); + capture.capture({"PLC_FAULT", "plc_app"}); - EXPECT_FALSE(storage_->get_freeze_frame("PLC_FAULT").has_value()); + EXPECT_FALSE(storage_->get_freeze_frame({"PLC_FAULT", "plc_app"}).has_value()); } // @verifies REQ_INTEROP_088 @@ -699,9 +713,9 @@ TEST_F(EntityDefaultCaptureTest, BarePluginIdNeverMatchesSameNamedNode) { }); wait_for_publisher("/entity/bare_id_metric"); - capture.capture("BARE_ID_FAULT"); + capture.capture({"BARE_ID_FAULT", node_->get_name()}); - EXPECT_FALSE(storage_->get_freeze_frame("BARE_ID_FAULT").has_value()); + EXPECT_FALSE(storage_->get_freeze_frame({"BARE_ID_FAULT", node_->get_name()}).has_value()); } namespace { @@ -763,9 +777,9 @@ TEST_F(SnapshotCaptureTest, CaptureIdContinuesFromWhatStorageAlreadyHolds) { await_publisher(node_, "/plc/seeded"); confirm_fault(storage_.get(), "SEEDED_FAULT"); - capture.capture("SEEDED_FAULT"); + capture.capture(rec("SEEDED_FAULT")); - const auto rows = storage_->get_snapshots("SEEDED_FAULT"); + const auto rows = storage_->get_snapshots(rec("SEEDED_FAULT")); ASSERT_FALSE(rows.empty()); EXPECT_GT(rows.front().capture_id, 41) << "the new capture outranks everything already stored"; } @@ -824,12 +838,12 @@ TEST_F(SnapshotCaptureTest, CaptureIdIsSeededOnTheFirstCaptureWhenTheStoreWasUnr confirm_fault(&storage, "LATE_SEED_FAULT"); // Still unreachable: no capture runs with an unseeded counter. - capture->capture("LATE_SEED_FAULT"); - EXPECT_TRUE(storage.get_snapshots("LATE_SEED_FAULT").empty()); + capture->capture(rec("LATE_SEED_FAULT")); + EXPECT_TRUE(storage.get_snapshots(rec("LATE_SEED_FAULT")).empty()); storage.reachable = true; - capture->capture("LATE_SEED_FAULT"); - const auto rows = storage.get_snapshots("LATE_SEED_FAULT"); + capture->capture(rec("LATE_SEED_FAULT")); + const auto rows = storage.get_snapshots(rec("LATE_SEED_FAULT")); ASSERT_FALSE(rows.empty()); EXPECT_GT(rows.front().capture_id, 41) << "the new capture outranks everything already stored"; } @@ -858,11 +872,11 @@ TEST_F(SnapshotCaptureTest, ACaptureFinishingAfterAcknowledgementIsNotStored) { await_publisher(node_, "/plc/acked"); confirm_fault(storage_.get(), "ACKED_FAULT"); - ASSERT_TRUE(storage_->clear_fault("ACKED_FAULT")); + ASSERT_TRUE(storage_->clear_fault(rec("ACKED_FAULT"))); - capture.capture("ACKED_FAULT"); + capture.capture(rec("ACKED_FAULT")); - EXPECT_TRUE(storage_->get_snapshots("ACKED_FAULT").empty()) + EXPECT_TRUE(storage_->get_snapshots(rec("ACKED_FAULT")).empty()) << "acknowledgement promised these were gone; the batch must not resurrect them"; } @@ -891,11 +905,11 @@ TEST_F(SnapshotCaptureTest, ACaptureFinishingAfterAcknowledgementIsStoredWhenEvi await_publisher(node_, "/plc/retained"); confirm_fault(storage_.get(), "RETAINED_FAULT"); - ASSERT_TRUE(storage_->clear_fault("RETAINED_FAULT")); + ASSERT_TRUE(storage_->clear_fault(rec("RETAINED_FAULT"))); - capture.capture("RETAINED_FAULT"); + capture.capture(rec("RETAINED_FAULT")); - EXPECT_FALSE(storage_->get_snapshots("RETAINED_FAULT").empty()); + EXPECT_FALSE(storage_->get_snapshots(rec("RETAINED_FAULT")).empty()); } int main(int argc, char ** argv) { diff --git a/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp b/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp index 509f1a3ea..0bfb24958 100644 --- a/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp +++ b/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp @@ -16,16 +16,23 @@ #include +#include +#include +#include #include #include #include #include #include +#include #include #include +#include +#include #include #include "rclcpp/rclcpp.hpp" +#include "rcutils/logging.h" #include "ros2_medkit_fault_manager/sqlite_fault_storage.hpp" #include "ros2_medkit_msgs/msg/fault.hpp" #include "ros2_medkit_msgs/srv/report_fault.hpp" @@ -76,7 +83,7 @@ TEST_F(SqliteFaultStorageTest, ReportNewFaultEvent) { EXPECT_TRUE(is_new); EXPECT_EQ(storage_->size(), 1u); - EXPECT_TRUE(storage_->contains("MOTOR_OVERHEAT")); + EXPECT_TRUE(storage_->contains({"MOTOR_OVERHEAT", "/powertrain/motor"})); } TEST_F(SqliteFaultStorageTest, PassedEventForNonExistentFaultIgnored) { @@ -100,18 +107,70 @@ TEST_F(SqliteFaultStorageTest, ReportExistingFaultEventUpdates) { bool is_new = storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Second report", - "/powertrain/motor2", timestamp2.nanoseconds(), default_config()); + "/powertrain/motor1", timestamp2.nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_->size(), 1u); - auto fault = storage_->get_fault("MOTOR_OVERHEAT"); + auto fault = storage_->get_fault({"MOTOR_OVERHEAT", "/powertrain/motor1"}); ASSERT_TRUE(fault.has_value()); // Still the same continuous occurrence (not CLEARED in between): occurrence_count - // does not bump on every report, only severity/sources/description update. + // does not bump on every report, only severity/description update. EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->severity, Fault::SEVERITY_ERROR); // Updated to higher severity - EXPECT_EQ(fault->reporting_sources.size(), 2u); + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources.front(), "/powertrain/motor1"); +} + +// SQLite twin of the in-memory identity test: the composite index has to keep the two +// records apart in the table the way the map does in memory. +TEST_F(SqliteFaultStorageTest, TwoSourcesReportingOneCodeAreTwoRecords) { + DebounceConfig config; + config.confirmation_threshold = -2; + + const rclcpp::Time a_time(1000, 0, RCL_SYSTEM_TIME); + const rclcpp::Time b_time(2000, 0, RCL_SYSTEM_TIME); + + EXPECT_TRUE(storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "from a", + "/powertrain/motor1", a_time.nanoseconds(), config)); + storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "from a", + "/powertrain/motor1", a_time.nanoseconds(), config); + EXPECT_TRUE(storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "from b", + "/powertrain/motor2", b_time.nanoseconds(), config)); + + EXPECT_EQ(storage_->size(), 2u); + + auto a = storage_->get_fault({"MOTOR_OVERHEAT", "/powertrain/motor1"}); + auto b = storage_->get_fault({"MOTOR_OVERHEAT", "/powertrain/motor2"}); + ASSERT_TRUE(a.has_value()); + ASSERT_TRUE(b.has_value()); + EXPECT_EQ(a->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(b->status, Fault::STATUS_PREFAILED); + EXPECT_EQ(a->severity, Fault::SEVERITY_WARN) << "B's ERROR must not escalate A's record"; + EXPECT_EQ(b->severity, Fault::SEVERITY_ERROR); + EXPECT_EQ(a->description, "from a"); + EXPECT_EQ(b->description, "from b"); + EXPECT_EQ(a->first_occurred_ns, a_time.nanoseconds()); + EXPECT_EQ(b->first_occurred_ns, b_time.nanoseconds()); +} + +TEST_F(SqliteFaultStorageTest, GetFaultsByCodeReturnsEveryOwnerAndGetFaultExactlyOne) { + rclcpp::Clock clock; + storage_->report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_b", + clock.now().nanoseconds(), default_config()); + storage_->report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_a", + clock.now().nanoseconds(), default_config()); + + const auto shared = storage_->get_faults_by_code("SHARED"); + ASSERT_EQ(shared.size(), 2u); + ASSERT_EQ(shared[0].reporting_sources.size(), 1u); + ASSERT_EQ(shared[1].reporting_sources.size(), 1u); + EXPECT_EQ(shared[0].reporting_sources.front(), "/owner_a") << "ordered by owner, like the in-memory backend"; + EXPECT_EQ(shared[1].reporting_sources.front(), "/owner_b"); + + EXPECT_TRUE(storage_->get_faults_by_code("NEVER_REPORTED").empty()); + ASSERT_TRUE(storage_->get_fault({"SHARED", "/owner_a"}).has_value()); + EXPECT_FALSE(storage_->get_fault({"SHARED", "/nobody"}).has_value()); } TEST_F(SqliteFaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCount) { @@ -124,7 +183,7 @@ TEST_F(SqliteFaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCo "/tank", clock.now().nanoseconds(), default_config()); } - auto fault = storage_->get_fault("TANK_OVERFILL"); + auto fault = storage_->get_fault({"TANK_OVERFILL", "/tank"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -186,16 +245,16 @@ TEST_F(SqliteFaultStorageTest, ClearFault) { storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", timestamp.nanoseconds(), default_config()); - bool cleared = storage_->clear_fault("MOTOR_OVERHEAT"); + bool cleared = storage_->clear_fault({"MOTOR_OVERHEAT", "/node1"}); EXPECT_TRUE(cleared); - auto fault = storage_->get_fault("MOTOR_OVERHEAT"); + auto fault = storage_->get_fault({"MOTOR_OVERHEAT", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); } TEST_F(SqliteFaultStorageTest, ClearNonExistentFault) { - bool cleared = storage_->clear_fault("NON_EXISTENT"); + bool cleared = storage_->clear_fault({"NON_EXISTENT", "/node1"}); EXPECT_FALSE(cleared); } @@ -211,7 +270,7 @@ TEST_F(SqliteFaultStorageTest, PassedEventDoesNotAdvanceLastOccurred) { storage_->report_fault_event("FAULT_LO", fd::ReportKind::kPassed, 0, "", "/node1", passed_at.nanoseconds(), default_config()); - auto fault = storage_->get_fault("FAULT_LO"); + auto fault = storage_->get_fault({"FAULT_LO", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // healing disabled: latched, by design EXPECT_EQ(fault->last_occurred_ns, failed_at.nanoseconds()); @@ -239,7 +298,7 @@ TEST_F(SqliteFaultStorageTest, ReopenRepairsLastOccurredInflatedByOldPassedBug) storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - auto fault = storage_->get_fault("FAULT_MIG"); + auto fault = storage_->get_fault({"FAULT_MIG", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->last_occurred_ns, failed_at.nanoseconds()); } @@ -250,7 +309,7 @@ TEST_F(SqliteFaultStorageTest, GetClearedFaults) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", timestamp.nanoseconds(), default_config()); - storage_->clear_fault("FAULT_1"); + storage_->clear_fault({"FAULT_1", "/node1"}); // Query cleared faults auto faults = storage_->list_faults(false, 0, {Fault::STATUS_CLEARED}); @@ -283,7 +342,7 @@ TEST_F(SqliteFaultStorageTest, PersistenceAcrossRestarts) { "/node1", timestamp.nanoseconds(), default_config()); storage_->report_fault_event("FAULT_2", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Persistent fault 2", "/node2", timestamp.nanoseconds(), default_config()); - storage_->clear_fault("FAULT_2"); + storage_->clear_fault({"FAULT_2", "/node2"}); // Verify initial state EXPECT_EQ(storage_->size(), 2u); @@ -296,16 +355,16 @@ TEST_F(SqliteFaultStorageTest, PersistenceAcrossRestarts) { // Verify faults persisted EXPECT_EQ(storage_->size(), 2u); - EXPECT_TRUE(storage_->contains("FAULT_1")); - EXPECT_TRUE(storage_->contains("FAULT_2")); + EXPECT_TRUE(storage_->contains({"FAULT_1", "/node1"})); + EXPECT_TRUE(storage_->contains({"FAULT_2", "/node2"})); - auto fault1 = storage_->get_fault("FAULT_1"); + auto fault1 = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault1.has_value()); EXPECT_EQ(fault1->severity, Fault::SEVERITY_ERROR); EXPECT_EQ(fault1->status, Fault::STATUS_CONFIRMED); // Immediately confirmed with threshold=-1 EXPECT_EQ(fault1->description, "Persistent fault 1"); - auto fault2 = storage_->get_fault("FAULT_2"); + auto fault2 = storage_->get_fault({"FAULT_2", "/node2"}); ASSERT_TRUE(fault2.has_value()); EXPECT_EQ(fault2->status, Fault::STATUS_CLEARED); } @@ -319,7 +378,7 @@ TEST_F(SqliteFaultStorageTest, TimestampPrecision) { storage_->report_fault_event("FAULT_TS", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Timestamp test", "/node1", timestamp.nanoseconds(), default_config()); - auto fault = storage_->get_fault("FAULT_TS"); + auto fault = storage_->get_fault({"FAULT_TS", "/node1"}); ASSERT_TRUE(fault.has_value()); // Wrap the stored nanoseconds in rclcpp::Time for comparison @@ -341,7 +400,7 @@ TEST(SqliteInMemoryTest, InMemoryDatabase) { timestamp.nanoseconds(), default_config()); EXPECT_EQ(storage.size(), 1u); - EXPECT_TRUE(storage.contains("MEM_FAULT")); + EXPECT_TRUE(storage.contains({"MEM_FAULT", "/test"})); } // Test reporting sources JSON handling @@ -349,23 +408,26 @@ TEST_F(SqliteFaultStorageTest, ReportingSourcesJsonHandling) { rclcpp::Clock clock; auto timestamp = clock.now(); - // Add multiple sources for the same fault - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", - "/node/path/with/slashes", timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", - "/another/node", timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", - "/special\"chars", timestamp.nanoseconds(), default_config()); + // Three awkward source ids, each owning its own record of the same code. Owners are + // an identity column now, so an embedded quote has to survive the round trip through + // the key rather than through a JSON array. + const std::vector owners = {"/node/path/with/slashes", "/another/node", "/special\"chars"}; + for (const auto & owner : owners) { + storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", owner, + timestamp.nanoseconds(), default_config()); + } - auto fault = storage_->get_fault("MULTI_SRC"); - ASSERT_TRUE(fault.has_value()); - EXPECT_EQ(fault->reporting_sources.size(), 3u); + EXPECT_EQ(storage_->size(), owners.size()); - // Verify all sources are present (order may vary due to set) - std::set sources(fault->reporting_sources.begin(), fault->reporting_sources.end()); - EXPECT_TRUE(sources.count("/node/path/with/slashes") > 0); - EXPECT_TRUE(sources.count("/another/node") > 0); - EXPECT_TRUE(sources.count("/special\"chars") > 0); + std::set seen; + for (const auto & owner : owners) { + auto fault = storage_->get_fault({"MULTI_SRC", owner}); + ASSERT_TRUE(fault.has_value()) << owner; + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources.front(), owner); + seen.insert(fault->reporting_sources.front()); + } + EXPECT_EQ(seen.size(), owners.size()); } // Test database path accessor @@ -406,12 +468,13 @@ TEST_F(SqliteFaultStorageTest, FaultStaysPrefailedAboveThreshold) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); + // Both reports from one source: the counter belongs to that source's record. + for (int i = 0; i < 2; ++i) { + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", + clock.now().nanoseconds(), config); + } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Still debouncing towards confirmation, same occurrence EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); @@ -425,14 +488,12 @@ TEST_F(SqliteFaultStorageTest, FaultConfirmsAtThreshold) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), config); + for (int i = 0; i < 3; ++i) { + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", + clock.now().nanoseconds(), config); + } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Debounce build-up is still one occurrence EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -448,7 +509,7 @@ TEST_F(SqliteFaultStorageTest, ImmediateConfirmationWithThresholdZero) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -461,7 +522,7 @@ TEST_F(SqliteFaultStorageTest, CriticalSeverityBypassesDebounce) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "Critical test", "/node1", clock.now().nanoseconds(), default_config()); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -476,30 +537,31 @@ TEST_F(SqliteFaultStorageTest, ClearedFaultCanBeReactivated) { "/node1", first_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->first_occurred_ns, first_ts.nanoseconds()); - // Clear the fault - storage_->clear_fault("FAULT_1"); - fault = storage_->get_fault("FAULT_1"); + // Clear the record + storage_->clear_fault({"FAULT_1", "/node1"}); + fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); - // Report again after a gap - should reactivate as a new cycle + // The same owner reports again after a gap - reactivates its own record rclcpp::Time second_ts(first_ts.nanoseconds() + 1'000'000'000LL); // +1s is_new = storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Reactivated", - "/node2", second_ts.nanoseconds(), default_config()); + "/node1", second_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); // Should return true like a new fault - fault = storage_->get_fault("FAULT_1"); + fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Should be reconfirmed EXPECT_EQ(fault->occurrence_count, 2u); // Should increment - EXPECT_EQ(fault->reporting_sources.size(), 2u); // Both sources - EXPECT_EQ(fault->description, "Reactivated"); // Updated description + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources.front(), "/node1"); + EXPECT_EQ(fault->description, "Reactivated"); // Updated description // #25: first_occurred must reflect the new cycle, not the outage that already cleared. EXPECT_EQ(fault->first_occurred_ns, second_ts.nanoseconds()); } @@ -512,14 +574,14 @@ TEST_F(SqliteFaultStorageTest, PassedEventForClearedFaultIgnored) { clock.now().nanoseconds(), default_config()); // Clear the fault - storage_->clear_fault("FAULT_1"); + storage_->clear_fault({"FAULT_1", "/node1"}); // PASSED event should be ignored for CLEARED fault bool result = storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), default_config()); EXPECT_FALSE(result); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); // Should stay cleared } @@ -532,36 +594,34 @@ TEST_F(SqliteFaultStorageTest, ClearedFaultReactivationRestartsDebounce) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - // Report 3 times to confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), config); + // One source reports 3 times to confirm its own record + for (int i = 0; i < 3; ++i) { + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", + clock.now().nanoseconds(), config); + } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); - // Clear the fault - storage_->clear_fault("FAULT_1"); + // Clear the record + storage_->clear_fault({"FAULT_1", "/node1"}); // Reactivate - should start in PREFAILED with counter=-1 - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node4", + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - fault = storage_->get_fault("FAULT_1"); + fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Not yet confirmed, needs 2 more FAILED // Report 2 more times to re-confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node5", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node6", - clock.now().nanoseconds(), config); + for (int i = 0; i < 2; ++i) { + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", + clock.now().nanoseconds(), config); + } - fault = storage_->get_fault("FAULT_1"); + fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Now confirmed } @@ -569,20 +629,18 @@ TEST_F(SqliteFaultStorageTest, ClearedFaultReactivationRestartsDebounce) { TEST_F(SqliteFaultStorageTest, ConfirmationPersistsAfterReopen) { rclcpp::Clock clock; - // Report 3 times to confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), default_config()); + // One source reports 3 times to confirm its own record + for (int i = 0; i < 3; ++i) { + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", + clock.now().nanoseconds(), default_config()); + } // Close and reopen storage storage_.reset(); storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); // Verify status persisted - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -594,11 +652,11 @@ TEST_F(SqliteFaultStorageTest, PassedEventIncrementsCounter) { DebounceConfig config; config.confirmation_threshold = -3; - // Report 2 FAILED events (counter -2, PREFAILED) - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); + // Report 2 FAILED events from one source (its counter -2, PREFAILED) + for (int i = 0; i < 2; ++i) { + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", + clock.now().nanoseconds(), config); + } // Report 3 PASSED events (counter -2 -> +1) for (int i = 0; i < 3; ++i) { @@ -606,7 +664,7 @@ TEST_F(SqliteFaultStorageTest, PassedEventIncrementsCounter) { config); } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREPASSED); // Counter > 0, never confirmed so not latched } @@ -624,14 +682,14 @@ TEST_F(SqliteFaultStorageTest, HeartbeatHealClampedAndStatusLatched) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); // Long heal heartbeat: counter clamps at healing_threshold instead of running off. for (int i = 0; i < 100; ++i) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); } - auto healed = storage_->get_fault("FAULT_1"); + auto healed = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(healed.has_value()); EXPECT_EQ(healed->status, Fault::STATUS_HEALED); @@ -640,11 +698,12 @@ TEST_F(SqliteFaultStorageTest, HeartbeatHealClampedAndStatusLatched) { for (int i = 0; i < 3; ++i) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); + EXPECT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_HEALED) + << "latch broke after " << (i + 1); } storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - auto reconfirmed = storage_->get_fault("FAULT_1"); + auto reconfirmed = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(reconfirmed.has_value()); EXPECT_EQ(reconfirmed->status, Fault::STATUS_CONFIRMED); } @@ -660,19 +719,20 @@ TEST_F(SqliteFaultStorageTest, ConfirmedFaultSurvivesHealHeartbeat) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 20; ++i) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED) << "un-confirmed after " << (i + 1); + EXPECT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED) + << "un-confirmed after " << (i + 1); } // The counter is now positive (clamped at +3). One FAILED must keep it CONFIRMED, not flip it to // PREPASSED via the `counter > 0` branch - that was the asymmetric-latch bug. storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); } // Latch must hold in BOTH directions and identically on both backends (regression for the @@ -687,16 +747,16 @@ TEST_F(SqliteFaultStorageTest, ConfirmedLatchSurvivesFailedAtPositiveCounter) { // CRITICAL confirms immediately at counter -1. storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); // PASSED events raise the counter into positive territory; latch keeps it CONFIRMED. for (int i = 0; i < 3; ++i) { storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); // One normal FAILED: counter still positive, must NOT become PREPASSED. storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); } // A HEALED fault with a negative counter stays HEALED on PASSED (opposite direction of the bug above). @@ -712,16 +772,16 @@ TEST_F(SqliteFaultStorageTest, HealedLatchSurvivesPassedAtNegativeCounter) { for (int i = 0; i < 4; ++i) { // counter -1 -> +3 -> HEALED storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); // FAILED events drive the counter negative while HEALED is latched (not yet at confirmation -3). for (int i = 0; i < 5; ++i) { // +3 -> -2 storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); + ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); } // One PASSED: counter goes -2 -> -1, still negative; must stay HEALED, not flip to PREFAILED. storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } TEST_F(SqliteFaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePassed) { @@ -737,19 +797,19 @@ TEST_F(SqliteFaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePasse for (int i = 0; i < 4; ++i) { // -1 -> +3 -> HEALED storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); // CRITICAL confirms immediately from HEALED; counter must not stay at +2 where one PASSED heals it. storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { // -1 -> +2, still CONFIRMED storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); } storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } // A database written by an older build can hold a runaway counter. It must be clamped back on first @@ -777,7 +837,7 @@ TEST_F(SqliteFaultStorageTest, RunawayCounterFromOldDbRecovers) { storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); } - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); } TEST_F(SqliteFaultStorageTest, ReclassifyHealedAsCleared) { @@ -791,12 +851,12 @@ TEST_F(SqliteFaultStorageTest, ReclassifyHealedAsCleared) { for (int i = 0; i < 4; ++i) { storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); const auto reclassified = storage_->reclassify_healed_as_cleared(); ASSERT_EQ(reclassified.size(), 1u); - EXPECT_EQ(reclassified[0], "F"); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(reclassified[0], (ros2_medkit_fault_manager::FaultId{"F", "/n"})); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CLEARED); EXPECT_TRUE(storage_->reclassify_healed_as_cleared().empty()); } @@ -812,16 +872,16 @@ TEST_F(SqliteFaultStorageTest, ReclassifyKeepsCodesTheCallerKeepsHealed) { for (int i = 0; i < 4; ++i) { storage_->report_fault_event(code, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault(code)->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault({code, "/n"})->status, Fault::STATUS_HEALED); } const auto reclassified = storage_->reclassify_healed_as_cleared([](const std::string & code) { return code == "KEEP"; }); ASSERT_EQ(reclassified.size(), 1u); - EXPECT_EQ(reclassified[0], "FLIP"); - EXPECT_EQ(storage_->get_fault("KEEP")->status, Fault::STATUS_HEALED); - EXPECT_EQ(storage_->get_fault("FLIP")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(reclassified[0], (ros2_medkit_fault_manager::FaultId{"FLIP", "/n"})); + EXPECT_EQ(storage_->get_fault({"KEEP", "/n"})->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_->get_fault({"FLIP", "/n"})->status, Fault::STATUS_CLEARED); } TEST_F(SqliteFaultStorageTest, HealingWhenEnabled) { @@ -841,7 +901,7 @@ TEST_F(SqliteFaultStorageTest, HealingWhenEnabled) { config); } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); } @@ -866,9 +926,9 @@ TEST_F(SqliteFaultStorageTest, TimeBasedConfirmationWhenEnabled) { auto after_timeout = rclcpp::Time(now.nanoseconds() + static_cast(15e9)); auto confirmed = storage_->check_time_based_confirmation(after_timeout.nanoseconds()); ASSERT_EQ(confirmed.size(), 1u); - EXPECT_EQ(confirmed[0], "FAULT_1"); + EXPECT_EQ(confirmed[0], (ros2_medkit_fault_manager::FaultId{"FAULT_1", "/node1"})); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -899,7 +959,7 @@ TEST_F(SqliteFaultStorageTest, ConfirmedAtRecordedOnImmediateConfirmation) { // Default config confirms on the first FAILED event. storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t.nanoseconds(), default_config()); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(read_confirmed_at(temp_db_path_, "F"), t.nanoseconds()); // A later FAILED on an already-confirmed fault must NOT move the timestamp. @@ -917,13 +977,13 @@ TEST_F(SqliteFaultStorageTest, ConfirmedAtRecordedOnDebouncedConfirmation) { auto t1 = clock.now(); storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t1.nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_PREFAILED); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_PREFAILED); EXPECT_EQ(read_confirmed_at(temp_db_path_, "F"), 0) << "not confirmed yet: no confirmation timestamp"; auto t2 = rclcpp::Time(t1.nanoseconds() + static_cast(1e9)); storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t2.nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(read_confirmed_at(temp_db_path_, "F"), t2.nanoseconds()); } @@ -965,7 +1025,7 @@ TEST_F(SqliteFaultStorageTest, ConfirmedAtColumnMigratedIntoOldDatabase) { sqlite3_close(raw); } storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - EXPECT_TRUE(storage_->contains("OLD")); + EXPECT_TRUE(storage_->contains({"OLD", "/n"})) << "the owner is backfilled from the legacy first source"; EXPECT_EQ(read_confirmed_at(temp_db_path_, "OLD"), 0) << "pre-migration rows carry no confirmation time"; rclcpp::Clock clock; @@ -975,6 +1035,159 @@ TEST_F(SqliteFaultStorageTest, ConfirmedAtColumnMigratedIntoOldDatabase) { EXPECT_EQ(read_confirmed_at(temp_db_path_, "NEW"), t.nanoseconds()); } +namespace { + +/// Whether @p table carries an index over exactly @p columns, whatever its name. +bool has_index_over(const std::filesystem::path & db_path, const std::string & table, + const std::vector & columns, bool * unique_out = nullptr) { + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path.string().c_str(), &raw), SQLITE_OK); + + std::vector> indexes; + { + sqlite3_stmt * stmt = nullptr; + const std::string sql = "PRAGMA index_list(" + table + ")"; + EXPECT_EQ(sqlite3_prepare_v2(raw, sql.c_str(), -1, &stmt, nullptr), SQLITE_OK); + while (sqlite3_step(stmt) == SQLITE_ROW) { + const auto * name = reinterpret_cast(sqlite3_column_text(stmt, 1)); + indexes.emplace_back(name != nullptr ? name : "", sqlite3_column_int(stmt, 2) != 0); + } + sqlite3_finalize(stmt); + } + + bool found = false; + for (const auto & [name, unique] : indexes) { + std::vector cols; + sqlite3_stmt * stmt = nullptr; + const std::string sql = "PRAGMA index_info(" + name + ")"; + EXPECT_EQ(sqlite3_prepare_v2(raw, sql.c_str(), -1, &stmt, nullptr), SQLITE_OK); + while (sqlite3_step(stmt) == SQLITE_ROW) { + const auto * col = reinterpret_cast(sqlite3_column_text(stmt, 2)); + cols.emplace_back(col != nullptr ? col : ""); + } + sqlite3_finalize(stmt); + if (cols == columns) { + found = true; + if (unique_out != nullptr) { + *unique_out = unique; + } + break; + } + } + sqlite3_close(raw); + return found; +} + +/// One scalar out of a legacy-schema database, read outside the storage class. +std::string read_text(const std::filesystem::path & db_path, const std::string & sql) { + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path.string().c_str(), &raw), SQLITE_OK); + sqlite3_stmt * stmt = nullptr; + EXPECT_EQ(sqlite3_prepare_v2(raw, sql.c_str(), -1, &stmt, nullptr), SQLITE_OK); + std::string value; + if (sqlite3_step(stmt) == SQLITE_ROW) { + const auto * text = reinterpret_cast(sqlite3_column_text(stmt, 0)); + value = text != nullptr ? text : ""; + } + sqlite3_finalize(stmt); + sqlite3_close(raw); + return value; +} + +} // namespace + +// The identity of a faults row changed, and SQLite cannot change a PRIMARY KEY in +// place while CREATE TABLE IF NOT EXISTS silently skips an existing table. A database +// written by an earlier release therefore has to be rebuilt on open, and its child +// rows have to learn which record they belong to. +TEST_F(SqliteFaultStorageTest, LegacyDatabaseGainsTheOwnerColumnOnOpen) { + storage_.reset(); + std::filesystem::remove(temp_db_path_); + { + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec( + raw, + "CREATE TABLE faults (fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, " + "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " + "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " + "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " + "debounce_counter INTEGER NOT NULL DEFAULT 0, " + "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " + "confirmed_at_ns INTEGER NOT NULL DEFAULT 0);" + // Two sources on one legacy row: the old schema held no per-source counter, + // status or timestamps, so there is nothing to split it by and it folds onto + // its FIRST source. + "INSERT INTO faults VALUES ('SHARED', 2, 'shared', 1, 1, 1, 'CONFIRMED', " + "'[\"/owner_a\",\"/owner_b\"]', -1, 1, 0, 0);" + "INSERT INTO faults VALUES ('SOLO', 1, 'solo', 2, 2, 1, 'CONFIRMED', '[\"/owner_c\"]', -1, 2, 0, 0);" + "CREATE TABLE freeze_frames (fault_code TEXT PRIMARY KEY, data TEXT NOT NULL, " + "captured_at_ns INTEGER NOT NULL);" + "INSERT INTO freeze_frames VALUES ('SHARED', '{\"/t\":1}', 7);" + "CREATE TABLE snapshots (id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, " + "topic TEXT NOT NULL, message_type TEXT NOT NULL, data TEXT NOT NULL, " + "captured_at_ns INTEGER NOT NULL);" + "INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) " + "VALUES ('SHARED', '/t', 'std_msgs/msg/Float64', '{}', 9);" + "CREATE TABLE rosbag_files (id INTEGER PRIMARY KEY AUTOINCREMENT, " + "fault_code TEXT NOT NULL UNIQUE, file_path TEXT NOT NULL, format TEXT NOT NULL, " + "duration_sec REAL NOT NULL, size_bytes INTEGER NOT NULL, created_at_ns INTEGER NOT NULL);" + "INSERT INTO rosbag_files (fault_code, file_path, format, duration_sec, size_bytes, created_at_ns) " + "VALUES ('SHARED', '/bags/fault_SHARED_1', 'mcap', 5.0, 1024, 11);", + nullptr, nullptr, nullptr), + SQLITE_OK); + sqlite3_close(raw); + } + + storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); + + // The record identity is an index over (fault_code, owner), and it is unique. + bool unique = false; + ASSERT_TRUE(has_index_over(temp_db_path_, "faults", {"fault_code", "owner"}, &unique)); + EXPECT_TRUE(unique); + ASSERT_TRUE(has_index_over(temp_db_path_, "freeze_frames", {"fault_code", "owner"}, &unique)); + EXPECT_TRUE(unique); + // The rosbag unique index widened with the identity. Without the DROP INDEX in the + // migration, CREATE INDEX IF NOT EXISTS would leave the old two-column shape here. + ASSERT_TRUE(has_index_over(temp_db_path_, "rosbag_files", {"fault_code", "owner", "file_path"}, &unique)); + EXPECT_TRUE(unique); + + // owner is backfilled with the FIRST legacy reporting source, and reporting_sources + // is rewritten to match so the column and the identity cannot disagree. + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = 'SHARED'"), "/owner_a"); + EXPECT_EQ(read_text(temp_db_path_, "SELECT reporting_sources FROM faults WHERE fault_code = 'SHARED'"), + "[\"/owner_a\"]"); + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = 'SOLO'"), "/owner_c"); + + const ros2_medkit_fault_manager::FaultId shared{"SHARED", "/owner_a"}; + auto migrated = storage_->get_fault(shared); + ASSERT_TRUE(migrated.has_value()); + EXPECT_EQ(migrated->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(migrated->reporting_sources, std::vector{"/owner_a"}); + EXPECT_FALSE(storage_->get_fault({"SHARED", "/owner_b"}).has_value()) << "the second source had no state to migrate"; + EXPECT_TRUE(storage_->get_fault({"SOLO", "/owner_c"}).has_value()); + + // The child rows learned which record they belong to. + ASSERT_TRUE(storage_->get_freeze_frame(shared).has_value()); + EXPECT_EQ(storage_->get_freeze_frame(shared)->captured_at_ns, 7); + ASSERT_EQ(storage_->get_snapshots(shared).size(), 1u); + EXPECT_EQ(storage_->get_snapshots(shared).front().owner, "/owner_a"); + ASSERT_TRUE(storage_->get_rosbag_file(shared).has_value()); + EXPECT_EQ(storage_->get_rosbag_file(shared)->owner, "/owner_a"); + + // Re-opening a migrated database changes nothing: every probe is per table, so the + // second open neither re-ALTERs a column that is there nor rebuilds a table again. + storage_.reset(); + storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); + + EXPECT_EQ(storage_->size(), 2u); + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = 'SHARED'"), "/owner_a"); + ASSERT_TRUE(storage_->get_freeze_frame(shared).has_value()); + EXPECT_EQ(storage_->get_snapshots(shared).size(), 1u); + ASSERT_TRUE(storage_->get_rosbag_file(shared).has_value()); + EXPECT_TRUE(has_index_over(temp_db_path_, "faults", {"fault_code", "owner"})); +} + // --- #620: many recordings per fault ----------------------------------------- namespace { @@ -999,10 +1212,15 @@ bool has_unique_constraint_index(const std::filesystem::path & db_path, const st return found; } +/// Rows in this file belong to one owner unless a test says otherwise: they exercise +/// retention and ordering, which the record identity does not change. +constexpr const char * kRowOwner = "/test_node"; + RosbagFileInfo make_rosbag(const std::string & code, const std::string & path, int64_t created_ns, size_t bytes = 1024) { RosbagFileInfo info; info.fault_code = code; + info.owner = kRowOwner; info.file_path = path; info.format = "mcap"; info.duration_sec = 6.0; @@ -1055,6 +1273,447 @@ TEST_F(SqliteFaultStorageTest, LegacyUniqueConstraintIsRebuiltAwayAndRecordingId EXPECT_EQ(all[1].recording_id, "fault_B_200"); } +namespace { + +/// One control byte, kept out of the string literals that use it: written inline, +/// "\x01" followed by a hex digit would be read as one wider escape. +constexpr char kCtrl = '\x01'; + +/// Run one statement on a raw handle, binding @p text as the single parameter. Legacy +/// fixtures carry bytes that cannot survive being pasted into a SQL literal, so the +/// values that matter go in through a binding. +void exec_bound(sqlite3 * raw, const char * sql, const std::string & text) { + sqlite3_stmt * stmt = nullptr; + ASSERT_EQ(sqlite3_prepare_v2(raw, sql, -1, &stmt, nullptr), SQLITE_OK) << sqlite3_errmsg(raw); + ASSERT_EQ(sqlite3_bind_text(stmt, 1, text.c_str(), static_cast(text.size()), SQLITE_TRANSIENT), SQLITE_OK); + ASSERT_EQ(sqlite3_step(stmt), SQLITE_DONE) << sqlite3_errmsg(raw); + sqlite3_finalize(stmt); +} + +/// Captures rcutils log output while alive and restores the console handler on every +/// exit path, so a live capture never swallows the output of later cases in this +/// binary. Same shape as the one the rosbag suite uses: the handler is a plain C +/// function pointer with no user-data slot, so the live capture is reached through a +/// file-static, atomic because the handler is process-global. +class LogCapture { + public: + LogCapture() { + active().store(this); + rcutils_logging_set_output_handler(&LogCapture::handler); + } + ~LogCapture() { + rcutils_logging_set_output_handler(rcutils_logging_console_output_handler); + active().store(nullptr); + } + LogCapture(const LogCapture &) = delete; + LogCapture & operator=(const LogCapture &) = delete; + LogCapture(LogCapture &&) = delete; + LogCapture & operator=(LogCapture &&) = delete; + + int count(const std::string & needle) const { + std::lock_guard lk(mutex_); + return static_cast(std::count_if(lines_.begin(), lines_.end(), [&needle](const std::string & line) { + return line.find(needle) != std::string::npos; + })); + } + + private: + static std::atomic & active() { + static std::atomic current{nullptr}; + return current; + } + + static void handler(const rcutils_log_location_t * /*location*/, int /*severity*/, const char * /*name*/, + rcutils_time_point_value_t /*timestamp*/, const char * format, va_list * args) { + char buf[1024]; + va_list copy; + va_copy(copy, *args); + // The format string arrives through the handler signature, so there is no literal + // to write here. Scoped to the single call, as in the rosbag suite. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" + vsnprintf(buf, sizeof(buf), format, copy); +#pragma GCC diagnostic pop + va_end(copy); + LogCapture * capture = active().load(); + if (capture == nullptr) { + return; + } + std::lock_guard lk(capture->mutex_); + capture->lines_.emplace_back(buf); + } + + mutable std::mutex mutex_; + std::vector lines_; +}; + +/// The legacy faults table exactly as the previous releases created it. +constexpr const char * kLegacyFaultsDdl = + "CREATE TABLE faults (fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, " + "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " + "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " + "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " + "debounce_counter INTEGER NOT NULL DEFAULT 0, " + "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " + "confirmed_at_ns INTEGER NOT NULL DEFAULT 0);"; + +} // namespace + +// A migration that refuses a row refuses the database. The rebuild runs inside one +// transaction, so a row it cannot read rolls the whole thing back, and because the +// migration is re-attempted on every open the node then fails to start for as long as +// that row exists. The column it reads was written by a serializer that escaped only +// the quote, the backslash and \b \f \n \r \t, so any other control byte in a +// source_id produced text no JSON parser accepts. Recovery therefore reads the text +// itself instead of requiring it to be valid JSON, and a row from which nothing can be +// read is migrated with an empty owner rather than taking the database down. +TEST_F(SqliteFaultStorageTest, LegacyReportingSourcesThatAreNotValidJsonStillMigrate) { + storage_.reset(); + std::filesystem::remove(temp_db_path_); + + // The five shapes, with the owner each must produce. + const std::string ctrl_pair = std::string("a") + kCtrl + "b"; + const std::string device = std::string("/dev/ttyUSB0") + kCtrl + "reader"; + const std::string kLegacy = ros2_medkit_fault_manager::kLegacyOwner; + const std::vector> cases = { + {"", kLegacy}, // nothing recorded + {"sensor_a", "sensor_a"}, // bare word, never an array + {"[]", kLegacy}, // array with no source + {"[\"" + ctrl_pair + "\"]", ctrl_pair}, // control byte inside the array + {"[\"" + device + "\"]", device}, // the reported real-world case + {"\xEF\xBB\xBF[\"/bom_source\"]", kLegacy}, // array behind a byte-order mark + {"{\"source\": \"/wrapped\"}", kLegacy}, // an object, not the array this reads + }; + + { + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, kLegacyFaultsDdl, nullptr, nullptr, nullptr), SQLITE_OK); + for (size_t i = 0; i < cases.size(); ++i) { + const std::string insert = "INSERT INTO faults VALUES ('CODE_" + std::to_string(i) + + "', 2, 'legacy', 1, 1, 1, 'CONFIRMED', ?, -1, 1, 0, 0);"; + exec_bound(raw, insert.c_str(), cases[i].first); + } + sqlite3_close(raw); + } + + // Opening must succeed. On the json_extract path this throws and the process exits. + ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())); + + for (size_t i = 0; i < cases.size(); ++i) { + const std::string code = "CODE_" + std::to_string(i); + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = '" + code + "'"), cases[i].second) + << "owner recovered from " << cases[i].first; + EXPECT_TRUE(storage_->contains({code, cases[i].second})) << code << " must be reachable by its migrated owner"; + } + // No migrated record is left unaddressable. The empty owner belongs to child rows + // alone, and a fault carrying it could not be named by any source_id. + EXPECT_EQ(read_text(temp_db_path_, "SELECT COUNT(*) FROM faults WHERE owner = ''"), "0"); + EXPECT_EQ(storage_->size(), cases.size()); + + // Whatever came in, what goes back out is valid JSON, so the next release's reader + // is not handed the same problem. + EXPECT_EQ(read_text(temp_db_path_, "SELECT MIN(json_valid(reporting_sources)) FROM faults"), "1"); + + // Idempotent: a second open neither rebuilds nor re-derives. + storage_.reset(); + ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())); + EXPECT_EQ(storage_->size(), cases.size()); + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = 'CODE_4'"), device); +} + +// End to end: a record whose source cannot be read is still a record. It gets the +// synthetic owner, its evidence is filed under that owner like any other record's, and +// the store then reaches a resting state. The resting state is the point: while a fault +// carried an empty owner its child rows could never be assigned to anyone, yet they +// still looked like pending work, so every single open re-entered the migration write +// transaction and logged a warning that nothing would ever resolve. +TEST_F(SqliteFaultStorageTest, AnUnreadableSourceYieldsALegacyRecordAndThenSettles) { + storage_.reset(); + std::filesystem::remove(temp_db_path_); + { + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, kLegacyFaultsDdl, nullptr, nullptr, nullptr), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, + "INSERT INTO faults VALUES ('ORPHANED', 2, 'legacy', 1, 1, 1, 'CONFIRMED', " + "'', -1, 1, 0, 0);" + "CREATE TABLE snapshots (id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, " + "topic TEXT NOT NULL, message_type TEXT NOT NULL, data TEXT NOT NULL, " + "captured_at_ns INTEGER NOT NULL);" + "INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) " + "VALUES ('ORPHANED', '/t', 'std_msgs/msg/Float64', '{}', 9);", + nullptr, nullptr, nullptr), + SQLITE_OK); + sqlite3_close(raw); + } + + ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())); + + const std::string legacy = ros2_medkit_fault_manager::kLegacyOwner; + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = 'ORPHANED'"), legacy); + EXPECT_EQ(read_text(temp_db_path_, "SELECT reporting_sources FROM faults WHERE fault_code = 'ORPHANED'"), + "[\"" + legacy + "\"]") + << "the column and the owner must agree, so a raw reader sees the same name"; + EXPECT_TRUE(storage_->contains({"ORPHANED", legacy})) << "the record must be addressable by source_id"; + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM snapshots WHERE fault_code = 'ORPHANED'"), legacy); + ASSERT_EQ(storage_->get_snapshots({"ORPHANED", legacy}).size(), 1u); + + // A tripwire that fires on any write to the child table, armed only now, so it sees + // the SECOND open and nothing the first one did. + { + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, + "CREATE TABLE tripwire (note TEXT);" + "CREATE TRIGGER tripwire_snapshots AFTER UPDATE ON snapshots " + "BEGIN INSERT INTO tripwire VALUES ('backfill ran'); END;", + nullptr, nullptr, nullptr), + SQLITE_OK) + << sqlite3_errmsg(raw); + sqlite3_close(raw); + } + + storage_.reset(); + ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())); + + EXPECT_EQ(read_text(temp_db_path_, "SELECT COUNT(*) FROM tripwire"), "0") + << "a settled store must not rewrite child rows on every open"; + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM snapshots WHERE fault_code = 'ORPHANED'"), legacy); +} + +// A rebuild that was interrupted, or a manual recovery with an external tool, can leave +// the scratch table behind. CREATE TABLE (no IF NOT EXISTS, deliberately, so a real +// collision is never silently reused) then fails on every open and the schema stays +// legacy forever. Dropping the scratch table first inside the same transaction is what +// makes the migration able to finish the job it started. +TEST_F(SqliteFaultStorageTest, LeftoverScratchTablesDoNotBlockTheMigration) { + for (const char * leftover : {"faults_new", "freeze_frames_new"}) { + storage_.reset(); + std::filesystem::remove(temp_db_path_); + { + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, kLegacyFaultsDdl, nullptr, nullptr, nullptr), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, + "INSERT INTO faults VALUES ('LEFTOVER', 2, 'legacy', 1, 1, 1, 'CONFIRMED', " + "'[\"/owner_a\"]', -1, 1, 0, 0);" + "CREATE TABLE freeze_frames (fault_code TEXT PRIMARY KEY, data TEXT NOT NULL, " + "captured_at_ns INTEGER NOT NULL);" + "INSERT INTO freeze_frames VALUES ('LEFTOVER', '{\"/t\":1}', 7);", + nullptr, nullptr, nullptr), + SQLITE_OK); + // The debris: a scratch table with a shape that does not even match the real one, + // so reusing it instead of dropping it could not go unnoticed either. + const std::string debris = std::string("CREATE TABLE ") + leftover + " (junk TEXT);"; + ASSERT_EQ(sqlite3_exec(raw, debris.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + sqlite3_close(raw); + } + + // The drop is announced, because throwing away a table an operator may have been + // in the middle of recovering by hand has to be visible in the log. + std::unique_ptr log = std::make_unique(); + ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())) + << "leftover " << leftover << " must not block the rebuild"; + const int named = log->count(std::string("scratch table '") + leftover + "'"); + log.reset(); + EXPECT_EQ(named, 1) << "the dropped table must be named in the log: " << leftover; + + const ros2_medkit_fault_manager::FaultId id{"LEFTOVER", "/owner_a"}; + EXPECT_TRUE(storage_->contains(id)) << "with leftover " << leftover; + EXPECT_TRUE(has_index_over(temp_db_path_, "faults", {"fault_code", "owner"})); + ASSERT_TRUE(storage_->get_freeze_frame(id).has_value()) << "with leftover " << leftover; + EXPECT_EQ(storage_->get_freeze_frame(id)->captured_at_ns, 7); + } +} + +// Evidence is never assigned to an owner the database cannot prove. A code carrying two +// owners has no single answer, so the child rows keep the empty owner they came with, +// and neither owner is handed the other's snapshots. +TEST_F(SqliteFaultStorageTest, ChildRowsOfASharedCodeAreNotGivenAnArbitraryOwner) { + storage_.reset(); + std::filesystem::remove(temp_db_path_); + { + // faults already on the new schema with TWO owners of one code, snapshots still + // legacy. Reachable by an open that was interrupted, and by a database an operator + // repaired by hand. + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, + "CREATE TABLE faults (fault_code TEXT NOT NULL, severity INTEGER NOT NULL, " + "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " + "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " + "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " + "debounce_counter INTEGER NOT NULL DEFAULT 0, " + "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " + "confirmed_at_ns INTEGER NOT NULL DEFAULT 0, owner TEXT NOT NULL);" + "INSERT INTO faults VALUES ('SHARED', 2, 'a', 1, 1, 1, 'CONFIRMED', " + "'[\"/owner_a\"]', -1, 1, 0, 0, '/owner_a');" + "INSERT INTO faults VALUES ('SHARED', 2, 'b', 1, 1, 1, 'CONFIRMED', " + "'[\"/owner_b\"]', -1, 1, 0, 0, '/owner_b');" + "CREATE TABLE snapshots (id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, " + "topic TEXT NOT NULL, message_type TEXT NOT NULL, data TEXT NOT NULL, " + "captured_at_ns INTEGER NOT NULL);" + "INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) " + "VALUES ('SHARED', '/t', 'std_msgs/msg/Float64', '{}', 9);", + nullptr, nullptr, nullptr), + SQLITE_OK); + sqlite3_close(raw); + } + + ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())); + + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM snapshots WHERE fault_code = 'SHARED'"), "") + << "an ambiguous code must leave the row unassigned, not pick a winner"; + EXPECT_TRUE(storage_->get_snapshots({"SHARED", "/owner_a"}).empty()) << "A must not be handed unproven evidence"; + EXPECT_TRUE(storage_->get_snapshots({"SHARED", "/owner_b"}).empty()) << "and neither must B"; + // The row is kept, not deleted: it is still an operator's evidence. + EXPECT_EQ(read_text(temp_db_path_, "SELECT COUNT(*) FROM snapshots"), "1"); +} + +// The mirror case: a child table that already carries the column but whose rows were +// never filled, with exactly one owner to attribute them to. The backfill is not tied +// to the ALTER that adds the column, so the next open heals them. +TEST_F(SqliteFaultStorageTest, OwnerlessChildRowsAreHealedOnTheNextOpen) { + storage_.reset(); + std::filesystem::remove(temp_db_path_); + { + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, + "CREATE TABLE faults (fault_code TEXT NOT NULL, severity INTEGER NOT NULL, " + "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " + "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " + "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " + "debounce_counter INTEGER NOT NULL DEFAULT 0, " + "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " + "confirmed_at_ns INTEGER NOT NULL DEFAULT 0, owner TEXT NOT NULL);" + "INSERT INTO faults VALUES ('SOLO', 2, 'a', 1, 1, 1, 'CONFIRMED', " + "'[\"/owner_a\"]', -1, 1, 0, 0, '/owner_a');" + // snapshots is AHEAD of the old migration: the column is there, + // the value never arrived. + "CREATE TABLE snapshots (id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, " + "owner TEXT NOT NULL DEFAULT '', topic TEXT NOT NULL, message_type TEXT NOT NULL, " + "data TEXT NOT NULL, captured_at_ns INTEGER NOT NULL);" + "INSERT INTO snapshots (fault_code, owner, topic, message_type, data, captured_at_ns) " + "VALUES ('SOLO', '', '/t', 'std_msgs/msg/Float64', '{}', 9);", + nullptr, nullptr, nullptr), + SQLITE_OK); + sqlite3_close(raw); + } + + ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())); + + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM snapshots WHERE fault_code = 'SOLO'"), "/owner_a"); + ASSERT_EQ(storage_->get_snapshots({"SOLO", "/owner_a"}).size(), 1u) << "the owner must now see its own evidence"; + EXPECT_EQ(storage_->get_snapshots({"SOLO", "/owner_a"}).front().owner, "/owner_a"); +} + +// New rows must never reproduce the problem the migration had to recover from: a +// source_id carrying a control byte round-trips, and what lands in the column is JSON +// every reader can parse. +TEST_F(SqliteFaultStorageTest, AControlByteInASourceIdStoresAsValidJson) { + const std::string source = std::string("/dev/ttyUSB0") + kCtrl + "reader"; + rclcpp::Clock clock; + storage_->report_fault_event("CTRL_BYTE", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "d", source, + clock.now().nanoseconds(), default_config()); + + EXPECT_EQ(read_text(temp_db_path_, "SELECT json_valid(reporting_sources) FROM faults WHERE fault_code='CTRL_BYTE'"), + "1") + << "a raw control byte inside a JSON string is what made the legacy column unreadable"; + EXPECT_EQ(read_text(temp_db_path_, + "SELECT json_extract(reporting_sources, '$[0]') FROM faults " + "WHERE fault_code='CTRL_BYTE'"), + source) + << "the escape must be reversible, not lossy"; + + auto stored = storage_->get_fault({"CTRL_BYTE", source}); + ASSERT_TRUE(stored.has_value()) << "the record is keyed by the byte-exact source"; + ASSERT_EQ(stored->reporting_sources.size(), 1u); + EXPECT_EQ(stored->reporting_sources.front(), source); +} + +// The rosbag indexes that carry fault_code have to widen with the record identity, and +// widening them is the one step no rebuild does for free. LegacyDatabaseGainsTheOwner- +// ColumnOnOpen cannot show it: its fixture still carries a column UNIQUE on fault_code, +// so the earlier rosbag migration rebuilds the table, takes every index with it, and +// the widened shape appears whether or not the owner migration drops anything. The +// shape that does exercise the drops is the PREVIOUS release, where rosbag_files is +// already rebuilt and already carries the named indexes and only owner is missing. +// CREATE INDEX IF NOT EXISTS is a no-op on a name that exists, so without the drops +// the two-column indexes survive and the upsert has no unique constraint to target. +TEST_F(SqliteFaultStorageTest, PreviousReleaseRosbagIndexesWidenWithTheRecordIdentity) { + storage_.reset(); + std::filesystem::remove(temp_db_path_); + { + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, + "CREATE TABLE faults (fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, " + "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " + "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " + "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " + "debounce_counter INTEGER NOT NULL DEFAULT 0, " + "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " + "confirmed_at_ns INTEGER NOT NULL DEFAULT 0);" + "INSERT INTO faults VALUES ('BURST', 3, 'burst', 1, 1, 1, 'CONFIRMED', " + "'[\"/owner_a\"]', -1, 1, 0, 0);" + // rosbag_files as the previous release left it: no column UNIQUE, a + // recording_id column, and the named indexes over fault_code alone. + "CREATE TABLE rosbag_files (id INTEGER PRIMARY KEY AUTOINCREMENT, " + "fault_code TEXT NOT NULL, recording_id TEXT NOT NULL DEFAULT '', " + "file_path TEXT NOT NULL, format TEXT NOT NULL, duration_sec REAL NOT NULL, " + "size_bytes INTEGER NOT NULL, created_at_ns INTEGER NOT NULL);" + "CREATE INDEX idx_rosbag_files_fault_code ON rosbag_files(fault_code);" + "CREATE INDEX idx_rosbag_files_created_at ON rosbag_files(created_at_ns);" + "CREATE INDEX idx_rosbag_files_fault_created ON rosbag_files(fault_code, created_at_ns, id);" + "CREATE INDEX idx_rosbag_files_recording ON rosbag_files(recording_id);" + "CREATE INDEX idx_rosbag_files_path ON rosbag_files(file_path);" + "CREATE UNIQUE INDEX idx_rosbag_files_fault_path ON rosbag_files(fault_code, file_path);" + "INSERT INTO rosbag_files (fault_code, recording_id, file_path, format, duration_sec, " + "size_bytes, created_at_ns) " + "VALUES ('BURST', 'fault_BURST_1', '/bags/fault_BURST_1', 'mcap', 6.0, 512, 11);", + nullptr, nullptr, nullptr), + SQLITE_OK); + sqlite3_close(raw); + } + ASSERT_FALSE(has_unique_constraint_index(temp_db_path_, "rosbag_files")) + << "fixture must start already rebuilt, or the rebuild would widen the indexes for free"; + ASSERT_TRUE(has_index_over(temp_db_path_, "rosbag_files", {"fault_code", "file_path"})) + << "fixture must start on the previous release's two-column unique index"; + + storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); + + bool unique = false; + ASSERT_TRUE(has_index_over(temp_db_path_, "rosbag_files", {"fault_code", "owner", "file_path"}, &unique)); + EXPECT_TRUE(unique); + EXPECT_FALSE(has_index_over(temp_db_path_, "rosbag_files", {"fault_code", "file_path"})) + << "the narrow unique index must be gone, not left beside the wide one"; + EXPECT_TRUE(has_index_over(temp_db_path_, "rosbag_files", {"fault_code", "owner", "created_at_ns", "id"})) + << "the per-record read order index widened too"; + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM rosbag_files WHERE fault_code = 'BURST'"), "/owner_a"); + + // What the widened index buys: two owners of one code link the same bag. Under the + // previous release's index the second store would collide, and the upsert cannot + // even be prepared without a unique constraint over (fault_code, owner, file_path). + RosbagFileInfo link; + link.fault_code = "BURST"; + link.file_path = "/bags/fault_BURST_1"; + link.format = "mcap"; + link.duration_sec = 6.0; + link.size_bytes = 512; + link.created_at_ns = 11; + link.owner = "/owner_a"; + storage_->store_rosbag_file(link); + link.owner = "/owner_b"; + storage_->store_rosbag_file(link); + + const auto rows = storage_->get_rosbag_files_by_recording("fault_BURST_1"); + ASSERT_EQ(rows.size(), 2u) << "one link per record, both pointing at the one bag"; + EXPECT_EQ(rows[0].owner, "/owner_a"); + EXPECT_EQ(rows[1].owner, "/owner_b"); +} + TEST_F(SqliteFaultStorageTest, ReopeningAMigratedDatabaseIsANoOp) { // The migration runs on every open, so it has to be idempotent - a second and // third open must not rebuild again, lose rows or re-backfill. @@ -1076,13 +1735,13 @@ TEST_F(SqliteFaultStorageTest, OneFaultKeepsSeveralRecordingsWhenTheCapAllows) { storage_->store_rosbag_file(make_rosbag("FLAP", "/bags/fault_FLAP_100", 100)); storage_->store_rosbag_file(make_rosbag("FLAP", "/bags/fault_FLAP_200", 200)); - const auto rows = storage_->get_rosbag_files("FLAP"); + const auto rows = storage_->get_rosbag_files({"FLAP", kRowOwner}); ASSERT_EQ(rows.size(), 2u) << "the second recording must not replace the first"; EXPECT_EQ(rows[0].recording_id, "fault_FLAP_200") << "newest first"; EXPECT_EQ(rows[1].recording_id, "fault_FLAP_100"); // get_rosbag_file is "the newest", deterministically. - const auto newest = storage_->get_rosbag_file("FLAP"); + const auto newest = storage_->get_rosbag_file({"FLAP", kRowOwner}); ASSERT_TRUE(newest.has_value()); EXPECT_EQ(newest->recording_id, "fault_FLAP_200"); } @@ -1098,7 +1757,7 @@ TEST_F(SqliteFaultStorageTest, CapKeepsTheNewestAndUnlinksTheEvictedBag) { storage_->store_rosbag_file(make_rosbag("CAP", dir_a.string(), 100)); storage_->store_rosbag_file(make_rosbag("CAP", dir_b.string(), 200)); - const auto rows = storage_->get_rosbag_files("CAP"); + const auto rows = storage_->get_rosbag_files({"CAP", kRowOwner}); ASSERT_EQ(rows.size(), 1u) << "cap 1 is the historical behaviour: one recording per fault"; EXPECT_EQ(rows[0].file_path, dir_b.string()); EXPECT_FALSE(std::filesystem::exists(dir_a)) << "the evicted bag must be unlinked"; @@ -1120,7 +1779,7 @@ TEST_F(SqliteFaultStorageTest, EvictingOneFaultsLinkKeepsABagASiblingStillRefere storage_->store_rosbag_file(make_rosbag("A", later.string(), 200)); EXPECT_TRUE(std::filesystem::exists(shared)) << "B still references it"; - const auto b_rows = storage_->get_rosbag_files("B"); + const auto b_rows = storage_->get_rosbag_files({"B", kRowOwner}); ASSERT_EQ(b_rows.size(), 1u); EXPECT_EQ(b_rows[0].file_path, shared.string()); } @@ -1143,8 +1802,8 @@ TEST_F(SqliteFaultStorageTest, DeleteRecordingRemovesEveryLinkAndTheBag) { storage_->store_rosbag_files({make_rosbag("A", dir.string(), 100), make_rosbag("B", dir.string(), 100)}); EXPECT_EQ(storage_->delete_rosbag_recording("fault_A_100"), 2u); - EXPECT_TRUE(storage_->get_rosbag_files("A").empty()); - EXPECT_TRUE(storage_->get_rosbag_files("B").empty()); + EXPECT_TRUE(storage_->get_rosbag_files({"A", kRowOwner}).empty()); + EXPECT_TRUE(storage_->get_rosbag_files({"B", kRowOwner}).empty()); EXPECT_FALSE(std::filesystem::exists(dir)); } @@ -1153,8 +1812,8 @@ TEST_F(SqliteFaultStorageTest, DeletingAFaultDropsAllItsRecordings) { storage_->store_rosbag_file(make_rosbag("A", "/bags/fault_A_100", 100)); storage_->store_rosbag_file(make_rosbag("A", "/bags/fault_A_200", 200)); - EXPECT_TRUE(storage_->delete_rosbag_file("A")); - EXPECT_TRUE(storage_->get_rosbag_files("A").empty()) << "auto_cleanup drops the fault's whole history"; + EXPECT_TRUE(storage_->delete_rosbag_file({"A", kRowOwner})); + EXPECT_TRUE(storage_->get_rosbag_files({"A", kRowOwner}).empty()) << "auto_cleanup drops the fault's whole history"; } TEST_F(SqliteFaultStorageTest, SharedRecordingStillCountsOnceTowardsStorageWithSeveralRecordings) { @@ -1179,6 +1838,7 @@ TEST_F(SqliteFaultStorageTest, StoreAndRetrieveSnapshot) { // Store a snapshot SnapshotData snapshot; snapshot.fault_code = "MOTOR_OVERHEAT"; + snapshot.owner = "/motor_node"; snapshot.topic = "/motor/temperature"; snapshot.message_type = "sensor_msgs/msg/Temperature"; snapshot.data = R"({"temperature": 85.5, "variance": 0.1})"; @@ -1187,7 +1847,7 @@ TEST_F(SqliteFaultStorageTest, StoreAndRetrieveSnapshot) { storage_->store_snapshot(snapshot); // Retrieve snapshots - auto snapshots = storage_->get_snapshots("MOTOR_OVERHEAT"); + auto snapshots = storage_->get_snapshots({"MOTOR_OVERHEAT", "/motor_node"}); ASSERT_EQ(snapshots.size(), 1u); EXPECT_EQ(snapshots[0].fault_code, "MOTOR_OVERHEAT"); @@ -1208,6 +1868,7 @@ TEST_F(SqliteFaultStorageTest, MultipleSnapshotsForSameFault) { // Store multiple snapshots for the same fault SnapshotData snapshot1; snapshot1.fault_code = "MOTOR_OVERHEAT"; + snapshot1.owner = "/motor_node"; snapshot1.topic = "/motor/temperature"; snapshot1.message_type = "sensor_msgs/msg/Temperature"; snapshot1.data = R"({"temperature": 85.5})"; @@ -1215,6 +1876,7 @@ TEST_F(SqliteFaultStorageTest, MultipleSnapshotsForSameFault) { SnapshotData snapshot2; snapshot2.fault_code = "MOTOR_OVERHEAT"; + snapshot2.owner = "/motor_node"; snapshot2.topic = "/motor/rpm"; snapshot2.message_type = "std_msgs/msg/Float64"; snapshot2.data = R"({"data": 5500.0})"; @@ -1223,7 +1885,7 @@ TEST_F(SqliteFaultStorageTest, MultipleSnapshotsForSameFault) { storage_->store_snapshot(snapshot1); storage_->store_snapshot(snapshot2); - auto snapshots = storage_->get_snapshots("MOTOR_OVERHEAT"); + auto snapshots = storage_->get_snapshots({"MOTOR_OVERHEAT", "/motor_node"}); EXPECT_EQ(snapshots.size(), 2u); } @@ -1237,6 +1899,7 @@ TEST_F(SqliteFaultStorageTest, FilterSnapshotsByTopic) { SnapshotData snapshot1; snapshot1.fault_code = "MOTOR_OVERHEAT"; + snapshot1.owner = "/motor_node"; snapshot1.topic = "/motor/temperature"; snapshot1.message_type = "sensor_msgs/msg/Temperature"; snapshot1.data = R"({"temperature": 85.5})"; @@ -1244,6 +1907,7 @@ TEST_F(SqliteFaultStorageTest, FilterSnapshotsByTopic) { SnapshotData snapshot2; snapshot2.fault_code = "MOTOR_OVERHEAT"; + snapshot2.owner = "/motor_node"; snapshot2.topic = "/motor/rpm"; snapshot2.message_type = "std_msgs/msg/Float64"; snapshot2.data = R"({"data": 5500.0})"; @@ -1253,14 +1917,14 @@ TEST_F(SqliteFaultStorageTest, FilterSnapshotsByTopic) { storage_->store_snapshot(snapshot2); // Filter by topic - auto filtered = storage_->get_snapshots("MOTOR_OVERHEAT", "/motor/temperature"); + auto filtered = storage_->get_snapshots({"MOTOR_OVERHEAT", "/motor_node"}, "/motor/temperature"); ASSERT_EQ(filtered.size(), 1u); EXPECT_EQ(filtered[0].topic, "/motor/temperature"); } // @verifies REQ_INTEROP_088 TEST_F(SqliteFaultStorageTest, NoSnapshotsForUnknownFault) { - auto snapshots = storage_->get_snapshots("UNKNOWN_FAULT"); + auto snapshots = storage_->get_snapshots({"UNKNOWN_FAULT", "/motor_node"}); EXPECT_TRUE(snapshots.empty()); } @@ -1277,6 +1941,7 @@ TEST_F(SqliteFaultStorageTest, ClearFaultDeletesAssociatedSnapshots) { // Store snapshots for this fault SnapshotData snapshot1; snapshot1.fault_code = "SNAPSHOT_CLEAR_TEST"; + snapshot1.owner = "/test_node"; snapshot1.topic = "/test/topic1"; snapshot1.message_type = "std_msgs/msg/String"; snapshot1.data = R"({"data": "test1"})"; @@ -1285,6 +1950,7 @@ TEST_F(SqliteFaultStorageTest, ClearFaultDeletesAssociatedSnapshots) { SnapshotData snapshot2; snapshot2.fault_code = "SNAPSHOT_CLEAR_TEST"; + snapshot2.owner = "/test_node"; snapshot2.topic = "/test/topic2"; snapshot2.message_type = "std_msgs/msg/String"; snapshot2.data = R"({"data": "test2"})"; @@ -1292,15 +1958,15 @@ TEST_F(SqliteFaultStorageTest, ClearFaultDeletesAssociatedSnapshots) { storage_->store_snapshot(snapshot2); // Verify snapshots exist - auto snapshots_before = storage_->get_snapshots("SNAPSHOT_CLEAR_TEST"); + auto snapshots_before = storage_->get_snapshots({"SNAPSHOT_CLEAR_TEST", "/test_node"}); ASSERT_EQ(snapshots_before.size(), 2u); // Clear the fault - bool cleared = storage_->clear_fault("SNAPSHOT_CLEAR_TEST"); + bool cleared = storage_->clear_fault({"SNAPSHOT_CLEAR_TEST", "/test_node"}); EXPECT_TRUE(cleared); // Verify snapshots are deleted - auto snapshots_after = storage_->get_snapshots("SNAPSHOT_CLEAR_TEST"); + auto snapshots_after = storage_->get_snapshots({"SNAPSHOT_CLEAR_TEST", "/test_node"}); EXPECT_TRUE(snapshots_after.empty()); } @@ -1316,6 +1982,7 @@ TEST_F(SqliteFaultStorageTest, ClearFaultKeepsSnapshotsWhenEvidenceIsRetained) { SnapshotData row; row.fault_code = "KEEP"; + row.owner = "/n"; row.topic = "/t"; row.message_type = "std_msgs/msg/Float64"; row.data = R"({"data": 1.0})"; @@ -1323,12 +1990,12 @@ TEST_F(SqliteFaultStorageTest, ClearFaultKeepsSnapshotsWhenEvidenceIsRetained) { row.capture_id = 1; storage_->store_snapshots({row}); - ASSERT_TRUE(storage_->clear_fault("KEEP")); + ASSERT_TRUE(storage_->clear_fault({"KEEP", "/n"})); // Recordings survive an acknowledgement once a history is configured, so the // readings captured beside them have to as well - otherwise the fault is left // holding bags whose values are gone, which is worse than losing both. - EXPECT_EQ(storage_->get_snapshots("KEEP").size(), 1u); + EXPECT_EQ(storage_->get_snapshots({"KEEP", "/n"}).size(), 1u); } TEST_F(SqliteFaultStorageTest, StoreAndRetrieveFreezeFrame) { @@ -1340,11 +2007,12 @@ TEST_F(SqliteFaultStorageTest, StoreAndRetrieveFreezeFrame) { FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc_node"; frame.data = R"({"/plc/pressure":{"data":8.4},"/plc/valve":{"data":true}})"; frame.captured_at_ns = clock.now().nanoseconds(); storage_->store_freeze_frame(frame); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage_->get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc_node"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->fault_code, "PLC_PRESSURE_HIGH"); EXPECT_EQ(retrieved->data, frame.data); @@ -1353,7 +2021,7 @@ TEST_F(SqliteFaultStorageTest, StoreAndRetrieveFreezeFrame) { // @verifies REQ_INTEROP_088 TEST_F(SqliteFaultStorageTest, NoFreezeFrameForUnknownFault) { - auto retrieved = storage_->get_freeze_frame("NEVER_CAPTURED"); + auto retrieved = storage_->get_freeze_frame({"NEVER_CAPTURED", "/plc_node"}); EXPECT_FALSE(retrieved.has_value()); } @@ -1364,17 +2032,19 @@ TEST_F(SqliteFaultStorageTest, FreezeFrameReplacedOnRecapture) { FreezeFrameData first; first.fault_code = "PLC_PRESSURE_HIGH"; + first.owner = "/plc_node"; first.data = R"({"/plc/pressure":{"data":8.4}})"; first.captured_at_ns = 1000; storage_->store_freeze_frame(first); FreezeFrameData second; second.fault_code = "PLC_PRESSURE_HIGH"; + second.owner = "/plc_node"; second.data = R"({"/plc/pressure":{"data":9.9}})"; second.captured_at_ns = 2000; storage_->store_freeze_frame(second); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage_->get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc_node"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, second.data); EXPECT_EQ(retrieved->captured_at_ns, 2000); @@ -1392,6 +2062,7 @@ TEST_F(SqliteFaultStorageTest, FreezeFrameSurvivesClearFault) { // A per-topic snapshot (removed on clear) plus a freeze-frame (retained on clear). SnapshotData snapshot; snapshot.fault_code = "PLC_PRESSURE_HIGH"; + snapshot.owner = "/plc_node"; snapshot.topic = "/plc/pressure"; snapshot.message_type = "std_msgs/msg/Float64"; snapshot.data = R"({"data":8.4})"; @@ -1400,15 +2071,16 @@ TEST_F(SqliteFaultStorageTest, FreezeFrameSurvivesClearFault) { FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc_node"; frame.data = R"({"/plc/pressure":{"data":8.4}})"; frame.captured_at_ns = clock.now().nanoseconds(); storage_->store_freeze_frame(frame); - ASSERT_TRUE(storage_->clear_fault("PLC_PRESSURE_HIGH")); + ASSERT_TRUE(storage_->clear_fault({"PLC_PRESSURE_HIGH", "/plc_node"})); // Snapshots are wiped on clear, the freeze-frame is retained and still retrievable. - EXPECT_TRUE(storage_->get_snapshots("PLC_PRESSURE_HIGH").empty()); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + EXPECT_TRUE(storage_->get_snapshots({"PLC_PRESSURE_HIGH", "/plc_node"}).empty()); + auto retrieved = storage_->get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc_node"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, frame.data); } @@ -1419,6 +2091,7 @@ TEST_F(SqliteFaultStorageTest, FreezeFramePersistsAcrossReopen) { FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc_node"; frame.data = R"({"/plc/pressure":{"data":8.4}})"; frame.captured_at_ns = 4242; storage_->store_freeze_frame(frame); @@ -1427,7 +2100,7 @@ TEST_F(SqliteFaultStorageTest, FreezeFramePersistsAcrossReopen) { storage_.reset(); storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage_->get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc_node"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, frame.data); EXPECT_EQ(retrieved->captured_at_ns, 4242); @@ -1452,6 +2125,7 @@ TEST_F(SqliteFaultStorageTest, ListRosbagsForEntityFiltersCorrectly) { // Store rosbags for both faults RosbagFileInfo info1; info1.fault_code = "ENTITY_FAULT_1"; + info1.owner = "/powertrain/motor"; info1.file_path = "/tmp/entity1.mcap"; info1.format = "mcap"; info1.duration_sec = 5.0; @@ -1461,6 +2135,7 @@ TEST_F(SqliteFaultStorageTest, ListRosbagsForEntityFiltersCorrectly) { RosbagFileInfo info2; info2.fault_code = "ENTITY_FAULT_2"; + info2.owner = "/chassis/brake"; info2.file_path = "/tmp/entity2.mcap"; info2.format = "mcap"; info2.duration_sec = 3.0; @@ -1497,6 +2172,7 @@ TEST_F(SqliteFaultStorageTest, AFailingRowDeleteKeepsTheBagOnDisk) { RosbagFileInfo info; info.fault_code = "DELETE_FAILS"; + info.owner = kRowOwner; info.file_path = bag_dir.string(); info.format = "mcap"; info.duration_sec = 1.0; @@ -1515,9 +2191,10 @@ TEST_F(SqliteFaultStorageTest, AFailingRowDeleteKeepsTheBagOnDisk) { SQLITE_OK); sqlite3_close(raw); - EXPECT_THROW(storage_->delete_rosbag_file("DELETE_FAILS"), std::runtime_error); + EXPECT_THROW(storage_->delete_rosbag_file({"DELETE_FAILS", kRowOwner}), std::runtime_error); - EXPECT_TRUE(storage_->get_rosbag_file("DELETE_FAILS").has_value()) << "the DELETE failed, so its row must remain"; + EXPECT_TRUE(storage_->get_rosbag_file({"DELETE_FAILS", kRowOwner}).has_value()) + << "the DELETE failed, so its row must remain"; EXPECT_TRUE(std::filesystem::exists(bag_dir)) << "the bag was unlinked before its row was deleted, so the surviving row now points at nothing"; @@ -1531,6 +2208,7 @@ TEST_F(SqliteFaultStorageTest, GetAllRosbagFilesReturnsSortedByCreatedAt) { RosbagFileInfo info1; info1.fault_code = "FAULT_A"; + info1.owner = kRowOwner; info1.file_path = "/tmp/a.mcap"; info1.format = "mcap"; info1.duration_sec = 1.0; @@ -1540,6 +2218,7 @@ TEST_F(SqliteFaultStorageTest, GetAllRosbagFilesReturnsSortedByCreatedAt) { RosbagFileInfo info2; info2.fault_code = "FAULT_B"; + info2.owner = kRowOwner; info2.file_path = "/tmp/b.mcap"; info2.format = "mcap"; info2.duration_sec = 2.0; @@ -1567,6 +2246,7 @@ TEST_F(SqliteFaultStorageTest, SharedRosbagSurvivesUntilTheLastFaultIsDeleted) { RosbagFileInfo info; info.fault_code = "ROOT_CAUSE"; + info.owner = kRowOwner; info.file_path = bag_path; info.format = "mcap"; info.duration_sec = 5.0; @@ -1575,13 +2255,14 @@ TEST_F(SqliteFaultStorageTest, SharedRosbagSurvivesUntilTheLastFaultIsDeleted) { storage_->store_rosbag_file(info); info.fault_code = "CORRELATED"; + info.owner = kRowOwner; storage_->store_rosbag_file(info); - EXPECT_TRUE(storage_->delete_rosbag_file("CORRELATED")); + EXPECT_TRUE(storage_->delete_rosbag_file({"CORRELATED", kRowOwner})); EXPECT_TRUE(std::filesystem::exists(bag_path)); - EXPECT_TRUE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); + EXPECT_TRUE(storage_->get_rosbag_file({"ROOT_CAUSE", kRowOwner}).has_value()); - EXPECT_TRUE(storage_->delete_rosbag_file("ROOT_CAUSE")); + EXPECT_TRUE(storage_->delete_rosbag_file({"ROOT_CAUSE", kRowOwner})); EXPECT_FALSE(std::filesystem::exists(bag_path)); } @@ -1595,6 +2276,7 @@ TEST_F(SqliteFaultStorageTest, SharedRosbagCountsOnceTowardsStorageTotal) { auto store = [this](const char * code, const char * path, size_t bytes) { RosbagFileInfo info; info.fault_code = code; + info.owner = kRowOwner; info.file_path = path; info.format = "mcap"; info.duration_sec = 5.0; @@ -1623,6 +2305,7 @@ TEST_F(SqliteFaultStorageTest, RestoreWithNewPathUnlinksTheOldExclusiveBag) { RosbagFileInfo info; info.fault_code = "X"; + info.owner = kRowOwner; info.file_path = old_path; info.format = "mcap"; info.duration_sec = 5.0; @@ -1634,7 +2317,7 @@ TEST_F(SqliteFaultStorageTest, RestoreWithNewPathUnlinksTheOldExclusiveBag) { storage_->store_rosbag_file(info); EXPECT_FALSE(std::filesystem::exists(old_path)) << "nobody references the old bag, it must be unlinked"; - auto row = storage_->get_rosbag_file("X"); + auto row = storage_->get_rosbag_file({"X", kRowOwner}); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->file_path, old_path + "_new"); } @@ -1652,16 +2335,19 @@ TEST_F(SqliteFaultStorageTest, RestoreWithNewPathKeepsTheBagASiblingStillReferen info.size_bytes = 100; info.created_at_ns = 1000; info.fault_code = "X"; + info.owner = kRowOwner; storage_->store_rosbag_file(info); info.fault_code = "Y"; + info.owner = kRowOwner; storage_->store_rosbag_file(info); info.fault_code = "X"; + info.owner = kRowOwner; info.file_path = shared_path + "_new"; storage_->store_rosbag_file(info); EXPECT_TRUE(std::filesystem::exists(shared_path)) << "the sibling fault still owns the shared bag"; - auto sibling = storage_->get_rosbag_file("Y"); + auto sibling = storage_->get_rosbag_file({"Y", kRowOwner}); ASSERT_TRUE(sibling.has_value()); EXPECT_EQ(sibling->file_path, shared_path); @@ -1684,12 +2370,13 @@ TEST_F(SqliteFaultStorageTest, BulkStoreRegistersEveryFaultOfTheBurst) { std::vector rows; for (const char * code : {"ROOT_CAUSE", "CORRELATED_A", "CORRELATED_B"}) { info.fault_code = code; + info.owner = kRowOwner; rows.push_back(info); } storage_->store_rosbag_files(rows); for (const char * code : {"ROOT_CAUSE", "CORRELATED_A", "CORRELATED_B"}) { - auto row = storage_->get_rosbag_file(code); + auto row = storage_->get_rosbag_file({code, kRowOwner}); ASSERT_TRUE(row.has_value()) << code; EXPECT_EQ(row->file_path, "/tmp/burst_bag"); } @@ -1711,18 +2398,21 @@ TEST_F(SqliteFaultStorageTest, BulkDeleteRemovesTheBurstAndUnlinksTheBagOnce) { std::vector rows; for (const char * code : {"ROOT_CAUSE", "CORRELATED_A", "CORRELATED_B"}) { info.fault_code = code; + info.owner = kRowOwner; rows.push_back(info); } storage_->store_rosbag_files(rows); // A partial delete leaves the bag on disk for the remaining fault. - EXPECT_EQ(storage_->delete_rosbag_files({"CORRELATED_A", "CORRELATED_B", "NEVER_STORED"}), 2u); + EXPECT_EQ(storage_->delete_rosbag_files( + {{"CORRELATED_A", kRowOwner}, {"CORRELATED_B", kRowOwner}, {"NEVER_STORED", kRowOwner}}), + 2u); EXPECT_TRUE(std::filesystem::exists(bag_path)); - EXPECT_TRUE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); - EXPECT_FALSE(storage_->get_rosbag_file("CORRELATED_A").has_value()); + EXPECT_TRUE(storage_->get_rosbag_file({"ROOT_CAUSE", kRowOwner}).has_value()); + EXPECT_FALSE(storage_->get_rosbag_file({"CORRELATED_A", kRowOwner}).has_value()); // The last reference going away unlinks the directory. - EXPECT_EQ(storage_->delete_rosbag_files({"ROOT_CAUSE"}), 1u); + EXPECT_EQ(storage_->delete_rosbag_files({{"ROOT_CAUSE", kRowOwner}}), 1u); EXPECT_FALSE(std::filesystem::exists(bag_path)); EXPECT_EQ(storage_->get_total_rosbag_storage_bytes(), 0u); } @@ -1746,6 +2436,7 @@ TEST_F(SqliteFaultStorageTest, SnapshotCapKeepsTheNewestCaptureWholeAndDropsTheO for (const char * topic : {"/motor/temp", "/motor/rpm"}) { SnapshotData row; row.fault_code = "MOTOR_OVERHEAT"; + row.owner = "/motor_node"; row.topic = topic; row.message_type = "std_msgs/msg/Float64"; row.data = R"({"data": 1.0})"; @@ -1760,7 +2451,7 @@ TEST_F(SqliteFaultStorageTest, SnapshotCapKeepsTheNewestCaptureWholeAndDropsTheO storage_->store_snapshots(capture(2, 2000)); storage_->store_snapshots(capture(3, 3000)); - auto snapshots = storage_->get_snapshots("MOTOR_OVERHEAT"); + auto snapshots = storage_->get_snapshots({"MOTOR_OVERHEAT", "/motor_node"}); // The third capture is stored WHOLE and the first goes whole. The old rule // counted rows and rejected the new one once full, so capture 3 landed with one @@ -1794,6 +2485,7 @@ TEST_F(SqliteFaultStorageTest, ACaptureLargerThanTheCapIsKeptWholeRatherThanTorn for (int i = 0; i < 4; ++i) { SnapshotData row; row.fault_code = "WIDE"; + row.owner = "/n"; row.topic = "/t" + std::to_string(i); row.message_type = "std_msgs/msg/Float64"; row.data = "{}"; @@ -1806,7 +2498,7 @@ TEST_F(SqliteFaultStorageTest, ACaptureLargerThanTheCapIsKeptWholeRatherThanTorn // The cap is smaller than this fault's topic count. Trimming to it would mean // storing the reading with holes, which is the failure being fixed; the capture // stays whole and the operator can see the cap is too small. - EXPECT_EQ(storage_->get_snapshots("WIDE").size(), 4u); + EXPECT_EQ(storage_->get_snapshots({"WIDE", "/n"}).size(), 4u); } TEST_F(SqliteFaultStorageTest, SnapshotLimitZeroMeansUnlimited) { @@ -1822,6 +2514,7 @@ TEST_F(SqliteFaultStorageTest, SnapshotLimitZeroMeansUnlimited) { for (int i = 0; i < 20; ++i) { SnapshotData snap; snap.fault_code = "FAULT_A"; + snap.owner = "/node"; snap.topic = "/topic"; snap.message_type = "std_msgs/msg/String"; snap.data = "{}"; @@ -1829,7 +2522,7 @@ TEST_F(SqliteFaultStorageTest, SnapshotLimitZeroMeansUnlimited) { storage_->store_snapshot(snap); } - auto snapshots = storage_->get_snapshots("FAULT_A"); + auto snapshots = storage_->get_snapshots({"FAULT_A", "/node"}); EXPECT_EQ(snapshots.size(), 20u) << "Unlimited mode should store all snapshots"; } @@ -1846,6 +2539,7 @@ TEST_F(SqliteFaultStorageTest, SnapshotLimitPerFaultNotGlobal) { SnapshotData snap_a; snap_a.fault_code = "FAULT_A"; + snap_a.owner = "/node"; snap_a.topic = "/topic"; snap_a.message_type = "std_msgs/msg/String"; snap_a.data = "{}"; @@ -1853,13 +2547,14 @@ TEST_F(SqliteFaultStorageTest, SnapshotLimitPerFaultNotGlobal) { SnapshotData snap_b = snap_a; snap_b.fault_code = "FAULT_B"; + snap_b.owner = "/node"; storage_->store_snapshot(snap_a); storage_->store_snapshot(snap_b); // Both faults should have 1 snapshot each (limit is per-fault) - EXPECT_EQ(storage_->get_snapshots("FAULT_A").size(), 1u); - EXPECT_EQ(storage_->get_snapshots("FAULT_B").size(), 1u); + EXPECT_EQ(storage_->get_snapshots({"FAULT_A", "/node"}).size(), 1u); + EXPECT_EQ(storage_->get_snapshots({"FAULT_B", "/node"}).size(), 1u); } // --- Near-miss series --- @@ -1890,11 +2585,11 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesIsAppendedNotOverwritten) { "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto fault = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_NE(fault->status, Fault::STATUS_CONFIRMED) << "test setup: these reports must not confirm"; - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "each near miss must append an entry, not overwrite the last"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[1].debounce_counter, -2); @@ -1915,12 +2610,12 @@ TEST_F(SqliteFaultStorageTest, ConfirmingReportIsNotANearMiss) { "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto fault = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); // The fourth report is the fault happening, not nearly happening. - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); } TEST_F(SqliteFaultStorageTest, NearMissSeriesSurvivesClearFault) { @@ -1930,11 +2625,11 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesSurvivesClearFault) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); - ASSERT_TRUE(storage_->clear_fault("PUMP_PRESSURE_LOW")); + ASSERT_TRUE(storage_->clear_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "acknowledging the fault destroyed the near-miss record"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[2].debounce_counter, -3); @@ -1947,13 +2642,13 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesContinuesAcrossReactivation) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - ASSERT_TRUE(storage_->clear_fault("PUMP_PRESSURE_LOW")); + ASSERT_TRUE(storage_->clear_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); // A new outage cycle starts: the reactivating report resets the counter to -1 without confirming. storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping again", "/hydraulics/pump", nth_report_time(10).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 4u) << "the series must span fault cycles, one entry per occurrence"; EXPECT_EQ(series[3].debounce_counter, -1); EXPECT_EQ(series[3].occurred_at_ns, nth_report_time(10).nanoseconds()); @@ -1966,12 +2661,12 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesSurvivesReopen) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - ASSERT_TRUE(storage_->clear_fault("PUMP_PRESSURE_LOW")); + ASSERT_TRUE(storage_->clear_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); storage_.reset(); storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "the series must outlive the process, not just the fault cycle"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[2].debounce_counter, -3); @@ -1984,13 +2679,13 @@ TEST_F(SqliteFaultStorageTest, PassedReportIsNotANearMiss) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); + ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); // A PASSED report moves the counter in the healing direction: the fault receding, not nearing. storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); } TEST_F(SqliteFaultStorageTest, CriticalImmediateConfirmIsNotANearMiss) { @@ -2000,10 +2695,10 @@ TEST_F(SqliteFaultStorageTest, CriticalImmediateConfirmIsNotANearMiss) { storage_->report_fault_event("BATTERY_THERMAL_RUNAWAY", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "cell over temperature", "/power/bms", nth_report_time(0).nanoseconds(), config); - auto fault = storage_->get_fault("BATTERY_THERMAL_RUNAWAY"); + auto fault = storage_->get_fault({"BATTERY_THERMAL_RUNAWAY", "/power/bms"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); - EXPECT_TRUE(storage_->get_near_misses("BATTERY_THERMAL_RUNAWAY").empty()); + EXPECT_TRUE(storage_->get_near_misses({"BATTERY_THERMAL_RUNAWAY", "/power/bms"}).empty()); } TEST_F(SqliteFaultStorageTest, ImmediateConfirmThresholdRecordsNoNearMiss) { @@ -2014,10 +2709,10 @@ TEST_F(SqliteFaultStorageTest, ImmediateConfirmThresholdRecordsNoNearMiss) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure lost", "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); - auto fault = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); - EXPECT_TRUE(storage_->get_near_misses("PUMP_PRESSURE_LOW").empty()); + EXPECT_TRUE(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).empty()); } TEST_F(SqliteFaultStorageTest, FailedReportUnderHealedLatchIsNearMiss) { @@ -2027,13 +2722,13 @@ TEST_F(SqliteFaultStorageTest, FailedReportUnderHealedLatchIsNearMiss) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); for (int i = 1; i <= 2; ++i) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto healed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; @@ -2041,7 +2736,7 @@ TEST_F(SqliteFaultStorageTest, FailedReportUnderHealedLatchIsNearMiss) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(3).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[1].debounce_counter, 0); } @@ -2056,7 +2751,7 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesBoundedKeepingNewest) { "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u); // Oldest-first eviction: a series frozen at boot would say nothing about a trend. EXPECT_EQ(series[0].debounce_counter, -3); @@ -2074,7 +2769,7 @@ TEST_F(SqliteFaultStorageTest, NearMissBoundOfOneKeepsLatestOnly) { "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 1u); EXPECT_EQ(series[0].debounce_counter, -4); } @@ -2091,8 +2786,8 @@ TEST_F(SqliteFaultStorageTest, NearMissBoundIsPerFaultCode) { "/powertrain/motor", nth_report_time(i).nanoseconds(), config); } - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); - EXPECT_EQ(storage_->get_near_misses("MOTOR_OVERHEAT").size(), 2u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); + EXPECT_EQ(storage_->get_near_misses({"MOTOR_OVERHEAT", "/powertrain/motor"}).size(), 2u); } TEST_F(SqliteFaultStorageTest, NearMissBoundZeroIsUnlimited) { @@ -2105,7 +2800,7 @@ TEST_F(SqliteFaultStorageTest, NearMissBoundZeroIsUnlimited) { "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 150u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 150u); } TEST_F(SqliteFaultStorageTest, NearMissTableCreatedOnDatabaseFromOlderBuild) { @@ -2124,11 +2819,11 @@ TEST_F(SqliteFaultStorageTest, NearMissTableCreatedOnDatabaseFromOlderBuild) { } storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - EXPECT_TRUE(storage_->get_near_misses("PUMP_PRESSURE_LOW").empty()); + EXPECT_TRUE(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).empty()); storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(1).nanoseconds(), config); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); } TEST_F(SqliteFaultStorageTest, NearMissSeriesSurvivesHealedReclassification) { @@ -2144,14 +2839,14 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesSurvivesHealedReclassification) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto healed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); ASSERT_EQ(storage_->reclassify_healed_as_cleared().size(), 1u); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); } TEST_F(SqliteFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { @@ -2169,7 +2864,7 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[0].occurred_at_ns, nth_report_time(11).nanoseconds()); EXPECT_EQ(series[1].occurred_at_ns, nth_report_time(0).nanoseconds()) @@ -2178,8 +2873,8 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { } TEST_F(SqliteFaultStorageTest, NearMissEntriesDescribeTheirOwnReport) { - // Each entry must describe the report that produced it, not the fault's current state, or a - // series spanning a threshold change or a new reporting source reads as if nothing changed. + // Each entry must describe the report that produced it, not the record's current state, or a + // series spanning a threshold change reads as if nothing changed. DebounceConfig first = four_strike_config(); storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(0).nanoseconds(), first); @@ -2187,16 +2882,35 @@ TEST_F(SqliteFaultStorageTest, NearMissEntriesDescribeTheirOwnReport) { DebounceConfig second = four_strike_config(); second.confirmation_threshold = -8; storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure dipping", - "/hydraulics/backup_pump", nth_report_time(1).nanoseconds(), second); + "/hydraulics/pump", nth_report_time(1).nanoseconds(), second); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[0].confirmation_threshold, -4); EXPECT_EQ(series[0].severity, Fault::SEVERITY_WARN); EXPECT_EQ(series[0].source_id, "/hydraulics/pump"); EXPECT_EQ(series[1].confirmation_threshold, -8); EXPECT_EQ(series[1].severity, Fault::SEVERITY_ERROR); - EXPECT_EQ(series[1].source_id, "/hydraulics/backup_pump"); + EXPECT_EQ(series[1].source_id, "/hydraulics/pump"); +} + +// The series is per RECORD: another source reporting the same code keeps its own, +// so a plot of one reporter's approaches to confirmation is not diluted by another's. +TEST_F(SqliteFaultStorageTest, NearMissSeriesIsPerRecord) { + const auto config = four_strike_config(); + storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "dip", + "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "dip", + "/hydraulics/backup_pump", nth_report_time(1).nanoseconds(), config); + + const auto pump = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); + const auto backup = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}); + ASSERT_EQ(pump.size(), 1u); + ASSERT_EQ(backup.size(), 1u); + EXPECT_EQ(pump.front().source_id, "/hydraulics/pump"); + EXPECT_EQ(pump.front().severity, Fault::SEVERITY_WARN); + EXPECT_EQ(backup.front().source_id, "/hydraulics/backup_pump"); + EXPECT_EQ(backup.front().severity, Fault::SEVERITY_ERROR); } TEST_F(SqliteFaultStorageTest, NearMissSeriesContinuesAfterReopen) { @@ -2213,7 +2927,7 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesContinuesAfterReopen) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "a restart must extend the series, not restart or reorder it"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[1].debounce_counter, -2); @@ -2235,15 +2949,16 @@ TEST_F(SqliteFaultStorageTest, ApplyingSmallerBoundTrimsExistingSeries) { storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", nth_report_time(i).nanoseconds(), config); } - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 5u); + ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 5u); storage_->set_max_near_misses_per_fault(2); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u) << "applying the bound left the stored series over it"; EXPECT_EQ(series[0].debounce_counter, -4); EXPECT_EQ(series[1].debounce_counter, -5); - EXPECT_EQ(storage_->get_near_misses("MOTOR_OVERHEAT").size(), 2u) << "the bound is applied per fault code"; + EXPECT_EQ(storage_->get_near_misses({"MOTOR_OVERHEAT", "/powertrain/motor"}).size(), 2u) + << "the bound is applied per record"; } TEST_F(SqliteFaultStorageTest, ApplyingUnlimitedBoundKeepsExistingSeries) { @@ -2258,7 +2973,7 @@ TEST_F(SqliteFaultStorageTest, ApplyingUnlimitedBoundKeepsExistingSeries) { storage_->set_max_near_misses_per_fault(0); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 4u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 4u); } TEST_F(SqliteFaultStorageTest, UnlimitedBoundSpeltAsSizeMaxKeepsTheSeries) { @@ -2273,11 +2988,11 @@ TEST_F(SqliteFaultStorageTest, UnlimitedBoundSpeltAsSizeMaxKeepsTheSeries) { } EXPECT_EQ(storage_->set_max_near_misses_per_fault(std::numeric_limits::max()), 0u); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(3).nanoseconds(), config); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 4u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 4u); } TEST_F(SqliteFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { @@ -2292,7 +3007,7 @@ TEST_F(SqliteFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { "/powertrain/motor", nth_report_time(i).nanoseconds(), config); } - // Two codes, five entries each, bound of 2: three dropped per code. + // Two records, five entries each, bound of 2: three dropped per record. EXPECT_EQ(storage_->set_max_near_misses_per_fault(2), 6u); // Applying the same bound again has nothing left to drop. EXPECT_EQ(storage_->set_max_near_misses_per_fault(2), 0u); @@ -2304,7 +3019,7 @@ TEST_F(SqliteFaultStorageTest, PassedReportOnUnknownFaultWritesNothing) { EXPECT_FALSE(storage_->report_fault_event("NEVER_REPORTED", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/test_node", nth_report_time(0).nanoseconds(), default_config())); EXPECT_EQ(storage_->size(), 0u); - EXPECT_TRUE(storage_->get_near_misses("NEVER_REPORTED").empty()); + EXPECT_TRUE(storage_->get_near_misses({"NEVER_REPORTED", "/test_node"}).empty()); } TEST_F(SqliteFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) { @@ -2322,7 +3037,7 @@ TEST_F(SqliteFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto healed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; @@ -2333,11 +3048,11 @@ TEST_F(SqliteFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto confirmed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto confirmed = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(confirmed.has_value()); ASSERT_EQ(confirmed->status, Fault::STATUS_CONFIRMED) << "test setup: the ramp must end in a real fault"; - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_FALSE(series.empty()); EXPECT_EQ(series[0].resulting_status, Fault::STATUS_PREFAILED) << "the first report was a genuine approach"; @@ -2359,7 +3074,7 @@ TEST_F(SqliteFaultStorageTest, NearMissResultingStatusSurvivesReopen) { storage_.reset(); storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 1u); EXPECT_EQ(series[0].resulting_status, Fault::STATUS_PREFAILED); } @@ -2381,19 +3096,19 @@ TEST_F(SqliteFaultStorageTest, NearMissResultingStatusColumnAddedToOlderTable) { storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 1u) << "the migration must keep the rows it already had"; EXPECT_TRUE(series[0].resulting_status.empty()) << "an unrecorded latch state reads as empty, not as a status"; storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(1).nanoseconds(), config); - auto extended = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto extended = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(extended.size(), 2u); EXPECT_EQ(extended[1].resulting_status, Fault::STATUS_PREFAILED); } TEST_F(SqliteFaultStorageTest, NearMissSeriesEmptyForUnknownFault) { - EXPECT_TRUE(storage_->get_near_misses("NEVER_REPORTED").empty()); + EXPECT_TRUE(storage_->get_near_misses({"NEVER_REPORTED", "/hydraulics/pump"}).empty()); } // --- Snapshot retention through the startup reclassification --- @@ -2407,6 +3122,7 @@ static void store_snapshots_for(ros2_medkit_fault_manager::FaultStorage & storag for (int i = 0; i < count; ++i) { ros2_medkit_fault_manager::SnapshotData snapshot; snapshot.fault_code = fault_code; + snapshot.owner = "/test_node"; snapshot.topic = "/test/topic" + std::to_string(i); snapshot.message_type = "std_msgs/msg/String"; snapshot.data = R"({"data": "value"})"; @@ -2435,14 +3151,14 @@ TEST_F(SqliteFaultStorageTest, SnapshotsRetainedThroughHealedReclassification) { storage_->set_retain_snapshots_on_clear(true); drive_to_healed(*storage_, "SNAPSHOT_RETAIN_TEST"); - auto healed = storage_->get_fault("SNAPSHOT_RETAIN_TEST"); + auto healed = storage_->get_fault({"SNAPSHOT_RETAIN_TEST", "/test_node"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; - ASSERT_EQ(storage_->get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u); + ASSERT_EQ(storage_->get_snapshots({"SNAPSHOT_RETAIN_TEST", "/test_node"}).size(), 2u); ASSERT_EQ(storage_->reclassify_healed_as_cleared().size(), 1u); - EXPECT_EQ(storage_->get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u) + EXPECT_EQ(storage_->get_snapshots({"SNAPSHOT_RETAIN_TEST", "/test_node"}).size(), 2u) << "the restart reclassification deleted snapshots the configuration asked to keep"; } @@ -2450,7 +3166,7 @@ TEST_F(SqliteFaultStorageTest, SnapshotsDroppedByHealedReclassificationByDefault drive_to_healed(*storage_, "SNAPSHOT_DROP_TEST"); ASSERT_EQ(storage_->reclassify_healed_as_cleared().size(), 1u); - EXPECT_TRUE(storage_->get_snapshots("SNAPSHOT_DROP_TEST").empty()); + EXPECT_TRUE(storage_->get_snapshots({"SNAPSHOT_DROP_TEST", "/test_node"}).empty()); } int main(int argc, char ** argv) { diff --git a/src/ros2_medkit_fault_reporter/test/test_integration.test.py b/src/ros2_medkit_fault_reporter/test/test_integration.test.py index fa6233ab8..c0816b178 100644 --- a/src/ros2_medkit_fault_reporter/test/test_integration.test.py +++ b/src/ros2_medkit_fault_reporter/test/test_integration.test.py @@ -162,25 +162,26 @@ def test_02_report_fault_is_stored(self): fault_codes = [f.fault_code for f in faults_response.faults] self.assertIn('REPORTER_TEST_001', fault_codes) - def test_03_multiple_reports_aggregate(self): - """Test that multiple reports of same fault aggregate.""" - # First report - use CRITICAL to bypass debounce + def test_03_two_sources_of_one_code_keep_one_record_each(self): + """Two sources reporting one code get a record each, counted and escalated apart.""" + # The default confirmation threshold confirms on the first FAILED + # report, so every record below is CONFIRMED after its first report. self._report_fault( fault_code='REPORTER_TEST_002', - severity=Fault.SEVERITY_CRITICAL, + severity=Fault.SEVERITY_ERROR, description='First occurrence', source_id='/node_a' ) - # Second report from different source + # Second report, from a different source self._report_fault( fault_code='REPORTER_TEST_002', - severity=Fault.SEVERITY_CRITICAL, + severity=Fault.SEVERITY_WARN, description='Second occurrence', source_id='/node_b' ) - # Third report + # Third report, /node_a again and more severe self._report_fault( fault_code='REPORTER_TEST_002', severity=Fault.SEVERITY_CRITICAL, @@ -188,20 +189,24 @@ def test_03_multiple_reports_aggregate(self): source_id='/node_a' ) - # Verify aggregation faults_response = self._list_faults(statuses=['CONFIRMED']) - fault = next( - (f for f in faults_response.faults if f.fault_code == 'REPORTER_TEST_002'), - None - ) + records = [f for f in faults_response.faults if f.fault_code == 'REPORTER_TEST_002'] - self.assertIsNotNone(fault) - # Edge-counting: re-reports of a still-active fault do not bump the - # count; severity escalation and source aggregation still apply. - self.assertEqual(fault.occurrence_count, 1) - self.assertEqual(fault.severity, Fault.SEVERITY_CRITICAL) - self.assertIn('/node_a', fault.reporting_sources) - self.assertIn('/node_b', fault.reporting_sources) + # One record per reporting source, each naming only its own owner. + self.assertEqual( + sorted(list(f.reporting_sources) for f in records), + [['/node_a'], ['/node_b']], + ) + by_owner = {f.reporting_sources[0]: f for f in records} + + # Edge counting per record: the third report re-reports /node_a's + # still-active record, so neither count moves past the first edge. + self.assertEqual(by_owner['/node_a'].occurrence_count, 1) + self.assertEqual(by_owner['/node_b'].occurrence_count, 1) + # Severity escalates inside /node_a's record only. The CRITICAL report + # from /node_a leaves /node_b's record at the severity /node_b sent. + self.assertEqual(by_owner['/node_a'].severity, Fault.SEVERITY_CRITICAL) + self.assertEqual(by_owner['/node_b'].severity, Fault.SEVERITY_WARN) def test_04_source_id_tracked(self): """Test that source_id is correctly tracked.""" diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index 26a034d71..2f1e129bf 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -1047,7 +1047,7 @@ Real-time fault event stream using Server-Sent Events (SSE). Clients receive ins **Event Types:** - `fault_confirmed` - Fault transitioned to CONFIRMED status -- `fault_updated` - Fault data changed without a status transition (`last_occurred`, severity, reporting sources). Not `occurrence_count`, which only changes on an occurrence edge +- `fault_updated` - Fault data changed without a status transition (`last_occurred`, severity). Not the reporting sources, which name the record's one owner and never change, and not `occurrence_count`, which only changes on an occurrence edge - `fault_cleared` - Fault ended: cleared via ClearFault service, or auto-healed by PASSED events (check `fault.status` for `CLEARED` vs `HEALED`) **Example:** @@ -1125,7 +1125,7 @@ Snapshots are configured via FaultManager parameters: | `snapshots.max_message_size` | int | `65536` | Maximum message size in bytes (larger messages skipped) | | `snapshots.default_topics` | string[] | `[]` | Topics to capture for all faults | | `snapshots.config_file` | string | `""` | Path to YAML config file for `fault_specific` and `patterns` | -| `snapshots.recapture_cooldown_sec` | double | `60.0` | Min seconds between captures for the same fault code. | +| `snapshots.recapture_cooldown_sec` | double | `60.0` | Min seconds between captures for the same fault record. | | `snapshots.max_per_fault` | int | `10` | Max snapshots retained per fault. | | `snapshots.capture_pool_size` | int | `2` | Max concurrent capture threads under a fault storm (>= 1). Parallelizes snapshot capture only; rosbag stays single-writer (one fault at a time). | | `snapshots.capture_queue_depth` | int | `16` | Max pending captures before the full-queue policy applies (>= 1). | diff --git a/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py b/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py new file mode 100644 index 000000000..f7ede9451 --- /dev/null +++ b/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A fault record shows one severity on every gateway surface. + +Real fault manager (SQLite, confirmation threshold -3), real gateway, reports +through the ReportFault service. Each code has one owner, so the gateway's +code-addressed routes reach its one record. A CRITICAL fault is acknowledged +and then reported again as WARN. The record keeps CRITICAL, so the WARN report +confirms it at once, and GET /faults, the fault detail, the /faults/stream +frame and the payload of an OnChange trigger all carry the same severity and +status. +""" + +import json +import os +import shutil +import tempfile +import threading +import time +import unittest + +import launch_testing +import launch_testing.actions +import rclpy +from rclpy.node import Node +import requests +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ReportFault + +from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES, API_BASE_PATH, FAULT_TIMEOUT +from ros2_medkit_test_utils.gateway_test_case import GatewayTestCase +from ros2_medkit_test_utils.launch_helpers import create_test_launch + + +APP_ID = 'temp_sensor' +OWNER = '/powertrain/engine/temp_sensor' +CODE = 'SEV_CONSISTENCY' +PRIME_CODE = 'SEV_CONSISTENCY_PRIME' +DB_DIR = tempfile.mkdtemp(prefix='medkit_severity_e2e_') + + +def generate_test_description(): + return create_test_launch( + demo_nodes=['temp_sensor'], + fault_manager=True, + fault_manager_params={ + 'storage_type': 'sqlite', + 'database_path': os.path.join(DB_DIR, 'faults.db'), + 'confirmation_threshold': -3, + 'snapshots.rosbag.enabled': False, + }, + gateway_params={ + # Two SSE streams at once: /faults/stream and one trigger stream. + 'sse.max_clients': 4, + 'server.http_thread_pool_size': 10, + # A stream reader only sees its stop flag when a line arrives. + 'sse.keepalive_interval_sec': 1, + }, + ) + + +class SseCollector: + """Collect SSE frames from one stream on a background thread.""" + + def __init__(self, url): + self.frames = [] + self._stop = threading.Event() + self._response = requests.get(url, stream=True, timeout=(5, 120)) + self.status_code = self._response.status_code + self._thread = threading.Thread(target=self._pump, daemon=True) + self._thread.start() + + def _pump(self): + current = {} + try: + for line in self._response.iter_lines(decode_unicode=True): + if self._stop.is_set(): + break + if line is None: + continue + if line == '': + if current: + self.frames.append(current) + current = {} + continue + if line.startswith(':'): + continue + key, _, value = line.partition(':') + current[key.strip()] = value.strip() + except Exception: # noqa: BLE001 - closed socket on teardown + pass + + def payloads(self): + return [json.loads(f['data']) for f in list(self.frames) if 'data' in f] + + def close(self): + self._stop.set() + self._response.close() + self._thread.join(timeout=5) + + +class TestFaultSeverityConsistency(GatewayTestCase): + """Subscribers are told the severity that GET shows.""" + + MIN_EXPECTED_APPS = 1 + REQUIRED_APPS = {APP_ID} + + @classmethod + def setUpClass(cls): + super().setUpClass() + rclpy.init() + cls._node = Node('fault_severity_consistency_reporter') + cls._report_client = cls._node.create_client(ReportFault, '/fault_manager/report_fault') + assert cls._report_client.wait_for_service(timeout_sec=15.0), \ + 'report_fault service not available' + + @classmethod + def tearDownClass(cls): + cls._node.destroy_node() + rclpy.shutdown() + + def _failed(self, code, severity, description): + req = ReportFault.Request() + req.fault_code = code + req.event_type = ReportFault.Request.EVENT_FAILED + req.severity = severity + req.description = description + req.source_id = OWNER + future = self._report_client.call_async(req) + deadline = time.monotonic() + 5.0 + while not future.done() and time.monotonic() < deadline: + rclpy.spin_once(self._node, timeout_sec=0.1) + self.assertTrue(future.done(), 'report_fault timed out') + self.assertTrue(future.result().accepted) + + def _wait_for(self, predicate, what): + deadline = time.monotonic() + FAULT_TIMEOUT + while time.monotonic() < deadline: + found = predicate() + if found is not None: + return found + time.sleep(0.2) + self.fail(f'timed out waiting for {what}') + + def _prime_fault_stream(self, stream): + """Report a sacrificial CRITICAL fault until its frame arrives on ``stream``.""" + deadline = time.monotonic() + FAULT_TIMEOUT + while time.monotonic() < deadline: + self._failed(PRIME_CODE, Fault.SEVERITY_CRITICAL, 'prime') + settle = min(time.monotonic() + 1.0, deadline) + while time.monotonic() < settle: + if any(p.get('fault', {}).get('fault_code') == PRIME_CODE + for p in stream.payloads()): + return + time.sleep(0.1) + self.fail(f'no frame for {PRIME_CODE} on /faults/stream within {FAULT_TIMEOUT}s') + + def _list_item(self): + r = requests.get(f'{self.BASE_URL}/faults', params={'status': 'all'}, timeout=10) + self.assertEqual(r.status_code, 200, r.text) + return next((i for i in r.json()['items'] if i['fault_code'] == CODE), None) + + def _detail(self): + r = requests.get(f'{self.BASE_URL}/apps/{APP_ID}/faults/{CODE}', timeout=10) + self.assertEqual(r.status_code, 200, r.text) + return r.json() + + def _create_trigger(self): + body = { + 'resource': f'{API_BASE_PATH}/apps/{APP_ID}/faults/{CODE}', + 'trigger_condition': {'condition_type': 'OnChange'}, + 'multishot': True, + 'lifetime': 300, + } + r = requests.post(f'{self.BASE_URL}/apps/{APP_ID}/triggers', json=body, timeout=5) + self.assertEqual(r.status_code, 201, r.text) + return r.json() + + def test_one_severity_on_every_surface(self): + trig = self._create_trigger() + trigger_stream = SseCollector( + f'{self.BASE_URL}{trig["event_source"].removeprefix(API_BASE_PATH)}') + fault_stream = SseCollector(f'{self.BASE_URL}/faults/stream') + try: + self._check_surfaces(trigger_stream, fault_stream) + finally: + # Deleting the trigger ends its stream on the server side. + requests.delete(f'{self.BASE_URL}/apps/{APP_ID}/triggers/{trig["id"]}', timeout=5) + trigger_stream.close() + fault_stream.close() + + def _check_surfaces(self, trigger_stream, fault_stream): + self.assertEqual(trigger_stream.status_code, 200) + self.assertEqual(fault_stream.status_code, 200) + self._prime_fault_stream(fault_stream) + + def frame(event, occurrence): + def find(): + for f in list(fault_stream.frames): + if f.get('event') != event or 'data' not in f: + continue + fault = json.loads(f['data'])['fault'] + if fault['fault_code'] == CODE and fault['occurrence_count'] == occurrence: + return fault + return None + return find + + def trigger_payload(event_type, occurrence): + def find(): + for frame_data in trigger_stream.payloads(): + p = frame_data.get('payload', {}) + key = (p.get('event_type'), p.get('occurrence_count')) + if key == (event_type, occurrence): + return p + return None + return find + + # Occurrence 1: CRITICAL confirms at once. The owner acknowledges it. + self._failed(CODE, Fault.SEVERITY_CRITICAL, 'occurrence 1') + self._wait_for(frame('fault_confirmed', 1), 'fault_confirmed of occurrence 1') + r = requests.delete(f'{self.BASE_URL}/apps/{APP_ID}/faults/{CODE}', timeout=10) + self.assertEqual(r.status_code, 204, r.text) + self._wait_for(frame('fault_cleared', 1), 'fault_cleared of occurrence 1') + + # Occurrence 2 is reported as WARN; one report confirms it. + self._failed(CODE, Fault.SEVERITY_WARN, 'occurrence 2') + surfaces = {} + with self.subTest(surface='/faults/stream'): + surfaces['stream'] = self._wait_for( + frame('fault_confirmed', 2), 'fault_confirmed of occurrence 2') + with self.subTest(surface='OnChange trigger'): + surfaces['trigger'] = self._wait_for( + trigger_payload('fault_confirmed', 2), 'trigger event of occurrence 2') + with self.subTest(surface='GET /faults'): + surfaces['list'] = self._list_item() + self.assertIsNotNone(surfaces['list']) + self.assertEqual(surfaces['list']['occurrence_count'], 2) + with self.subTest(surface='fault detail'): + detail = self._detail() + surfaces['detail'] = { + 'severity': detail['item']['severity'], + 'severity_label': detail['x-medkit']['severity_label'], + 'status': detail['x-medkit']['status_raw'], + } + + for name, shown in surfaces.items(): + with self.subTest(agree=name): + self.assertIsNotNone(shown) + self.assertEqual(shown['severity'], Fault.SEVERITY_CRITICAL, shown) + self.assertEqual(shown['severity_label'], 'CRITICAL', shown) + self.assertEqual(shown['status'], 'CONFIRMED', shown) + + # A lower report in the occurrence updates the fault; every surface still shows CRITICAL. + self._failed(CODE, Fault.SEVERITY_ERROR, 'occurrence 2, lower') + updated = self._wait_for(frame('fault_updated', 2), 'fault_updated of occurrence 2') + fired = self._wait_for( + trigger_payload('fault_updated', 2), 'trigger event for the update') + item = self._list_item() + for name, shown in {'stream': updated, 'trigger': fired, 'list': item}.items(): + with self.subTest(after_update=name): + self.assertEqual(shown['severity'], Fault.SEVERITY_CRITICAL, shown) + self.assertEqual(shown['description'], 'occurrence 2, lower', shown) + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + + def test_exit_codes(self, proc_info): + shutil.rmtree(DB_DIR, ignore_errors=True) + for info in proc_info: + self.assertIn( + info.returncode, ALLOWED_EXIT_CODES, + f'{info.process_name} exited with code {info.returncode}' + ) diff --git a/src/ros2_medkit_msgs/README.md b/src/ros2_medkit_msgs/README.md index fe8cf92a7..9ad70535e 100644 --- a/src/ros2_medkit_msgs/README.md +++ b/src/ros2_medkit_msgs/README.md @@ -6,7 +6,7 @@ ROS 2 message and service definitions for the ros2_medkit fault management syste This package provides the interface definitions used by the fault management components: -- **FaultManager** (`ros2_medkit_fault_manager`) - Central fault aggregation and lifecycle management +- **FaultManager** (`ros2_medkit_fault_manager`) - Central fault record store and lifecycle management - **FaultReporter** (`ros2_medkit_fault_reporter`) - Client library for fault reporting - **Gateway** (`ros2_medkit_gateway`) - REST API endpoints for fault access @@ -14,7 +14,15 @@ This package provides the interface definitions used by the fault management com ### Fault.msg -Core fault data model representing an aggregated fault condition with AUTOSAR DEM-style debounce filtering. +Core fault data model representing one fault record with AUTOSAR DEM-style debounce filtering. + +A record is identified by the pair (`fault_code`, owning reporting source). The owner is the +`source_id` a `ReportFault` call carried. Two sources reporting one `fault_code` are two records, +each with its own status, debounce counter, `occurrence_count`, severity and timestamps, and each +cleared on its own. The services that act on a single record (`ClearFault`, `GetFault`, +`GetSnapshots`, `GetRosbag`) carry a `source_id` request field naming the owner. An empty +`source_id` is unscoped: the call applies only when exactly one record carries the `fault_code`, +and fails as ambiguous when several do. | Field | Type | Description | |-------|------|-------------| @@ -26,7 +34,7 @@ Core fault data model representing an aggregated fault condition with AUTOSAR DE | `last_passed` | builtin_interfaces/Time | When fault last reported PASSED (zero = never) | | `occurrence_count` | uint32 | Times this fault has occurred, counted on edges (first FAILED, then each FAILED that arrives while CLEARED). Repeats within one occurrence do not increment it | | `status` | string | Current status (see STATUS_* constants) | -| `reporting_sources` | string[] | List of source identifiers that reported this fault | +| `reporting_sources` | string[] | The reporting source that owns this record, as a one-element list | **Severity Levels:** | Constant | Value | Description | @@ -121,12 +129,14 @@ Query faults with optional filtering. ### ClearFault.srv -Clear/acknowledge a fault. Cleared faults are retained and queryable with `statuses=["CLEARED"]`. +Clear/acknowledge one fault record. Cleared records are retained and queryable with +`statuses=["CLEARED"]`. **Request:** | Field | Type | Description | |-------|------|-------------| -| `fault_code` | string | The fault to clear | +| `fault_code` | string | The fault code to clear | +| `source_id` | string | Reporting source that owns the record. Empty means unscoped: the call applies only when exactly one record carries the `fault_code`, otherwise it fails with a message beginning `ambiguous:` and clears nothing | | `skip_correlation_auto_clear` | bool | When `true`, only the requested fault_code is cleared; symptom faults that the correlation engine would normally auto-clear via `auto_clear_with_root` rules are left untouched. Default `false` (cascade clear). The gateway sets this to `true` on per-entity `DELETE /{entity-path}/faults/{fault_code}` so that an operator with access to one entity cannot cascade-clear correlated symptoms reported by apps in other entities. | **Response:** @@ -136,7 +146,11 @@ Clear/acknowledge a fault. Cleared faults are retained and queryable with `statu | `message` | string | Status or error message | | `auto_cleared_codes` | string[] | Symptom fault codes auto-cleared with the root cause (empty when `skip_correlation_auto_clear=true`) | -> **Note:** `skip_correlation_auto_clear` was added in `ros2_medkit_msgs` post-0.4.0. Adding a request field changes the service type hash, so out-of-tree callers that invoke `/fault_manager/clear_fault` directly (via `ros2 service call` or a generated client) must rebuild against the new `ros2_medkit_msgs` release to keep talking to `fault_manager`. +> **Note:** `skip_correlation_auto_clear` was added in `ros2_medkit_msgs` post-0.4.0, and `source_id` +> after it. `source_id` was added the same way to `GetFault`, `GetSnapshots` and `GetRosbag`. Adding a +> request field changes the service type hash, so out-of-tree callers that invoke those services +> directly (via `ros2 service call` or a generated client) must rebuild against the new +> `ros2_medkit_msgs` release to keep talking to `fault_manager`. ## Usage diff --git a/src/ros2_medkit_msgs/msg/Fault.msg b/src/ros2_medkit_msgs/msg/Fault.msg index aa24ec629..99227411b 100644 --- a/src/ros2_medkit_msgs/msg/Fault.msg +++ b/src/ros2_medkit_msgs/msg/Fault.msg @@ -14,16 +14,19 @@ # # Fault.msg - Core fault data model for ros2_medkit fault management system. # -# A Fault represents an aggregated fault condition identified by a global fault_code. -# Multiple sources can report the same fault_code, and they are aggregated into a -# single Fault with tracked occurrence_count and reporting_sources. +# A Fault is one fault record, identified by the pair (fault_code, reporting source). +# The reporting source is the source_id a ReportFault call carried, and it is the +# owner of the record. Two sources reporting one fault_code are two records, each +# with its own status, debounce counter, occurrence_count, severity and timestamps, +# and each cleared on its own. reporting_sources holds the one owner. # # Debounce model (AUTOSAR DEM-style): # - FAILED events decrement internal counter (towards confirmation) # - PASSED events increment internal counter (towards healing) # - Status reflects the current debounce state -# Global fault identifier (e.g., "MOTOR_OVERHEAT", "SENSOR_FAILURE_001") +# Global fault identifier (e.g., "MOTOR_OVERHEAT", "SENSOR_FAILURE_001"). +# Half of the record identity: the other half is the owning reporting source. string fault_code # Fault severity level (use SEVERITY_* constants) @@ -53,7 +56,9 @@ uint32 occurrence_count # Current fault status (use STATUS_* constants) string status -# List of source identifiers that have reported this fault +# The reporting source that owns this record, as a one-element list. Together with +# fault_code it identifies the record. Services that act on a single record take +# this value as their source_id request field. string[] reporting_sources # Severity level constants diff --git a/src/ros2_medkit_msgs/msg/FaultEvent.msg b/src/ros2_medkit_msgs/msg/FaultEvent.msg index 40cebe6bb..7fa119289 100644 --- a/src/ros2_medkit_msgs/msg/FaultEvent.msg +++ b/src/ros2_medkit_msgs/msg/FaultEvent.msg @@ -35,7 +35,8 @@ string EVENT_CONFIRMED = "fault_confirmed" # the two apart. string EVENT_CLEARED = "fault_cleared" # Emitted when fault data changes without status transition (e.g., last_occurred -# advanced, severity escalated, new source added to reporting_sources). +# advanced, severity escalated). A report from a source that owns no record yet +# is not one of them: it creates a record of its own. # occurrence_count is not one of them: it only changes on an occurrence edge # (the first FAILED, or a FAILED that arrives while the fault is CLEARED). string EVENT_UPDATED = "fault_updated" diff --git a/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg b/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg index f5fa8e044..c87d67d61 100644 --- a/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg +++ b/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg @@ -12,11 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# MutedFaultInfo.msg - Information about a muted (correlated) fault. +# MutedFaultInfo.msg - Information about a muted (correlated) fault record. # -# When fault correlation is enabled, symptom faults are muted (not published +# When fault correlation is enabled, symptom records are muted (not published # as events) but still stored. This message provides information about -# muted faults when queried with include_muted=true. +# muted records when queried with include_muted=true. +# +# One entry per muted RECORD. A root cause mutes the symptoms of its own +# reporting source only, so two sources muted on one fault_code produce two +# entries carrying that code, told apart by source_id. # The fault code of the muted symptom string fault_code @@ -29,3 +33,8 @@ string rule_id # Time delay from root cause to this symptom [ms] uint32 delay_ms + +# Reporting source that owns the muted record (the source_id used in ReportFault). +# Together with fault_code it identifies the one record this entry describes, and +# it is the value the single-record services take to address it. +string source_id diff --git a/src/ros2_medkit_msgs/srv/ClearFault.srv b/src/ros2_medkit_msgs/srv/ClearFault.srv index cfa0b778f..340c58ace 100644 --- a/src/ros2_medkit_msgs/srv/ClearFault.srv +++ b/src/ros2_medkit_msgs/srv/ClearFault.srv @@ -14,8 +14,10 @@ # # ClearFault.srv - Clear/acknowledge a fault in the FaultManager. # -# Marks the specified fault as CLEARED. The fault record is retained for historical -# purposes and can be retrieved by calling ListFaults with statuses=["CLEARED"]. +# Marks one fault record as CLEARED. A record is identified by (fault_code, source_id), +# so clearing one owner's record never touches another owner's record of the same code. +# The record is retained for historical purposes and can be retrieved by calling +# ListFaults with statuses=["CLEARED"]. # Cleared faults are retained indefinitely unless storage cleanup is configured. # A FaultEvent with EVENT_CLEARED is published when a fault is successfully cleared. @@ -29,6 +31,11 @@ string fault_code # The fault_code to clear # Set this from scoped per-entity DELETE routes so a viewer of one entity # cannot cascade-clear symptom faults reported by apps in other entities. bool skip_correlation_auto_clear + +# Reporting source that owns the fault record (the source_id used in ReportFault). +# Together with fault_code it identifies one record. Empty: the call applies only +# when exactly one record carries fault_code, otherwise it fails as ambiguous. +string source_id --- # Response fields bool success # True if the fault was found and cleared diff --git a/src/ros2_medkit_msgs/srv/GetFault.srv b/src/ros2_medkit_msgs/srv/GetFault.srv index 337c2b5f6..427af3f7f 100644 --- a/src/ros2_medkit_msgs/srv/GetFault.srv +++ b/src/ros2_medkit_msgs/srv/GetFault.srv @@ -15,13 +15,19 @@ # GetFault.srv - Get single fault by code with environment data. # # Unlike ListFaults.srv (plural) which returns a list of faults for filtering, -# this service returns a single fault with full environment data including +# this service returns a single fault record with full environment data including # snapshots and extended data records for SOVD-compliant fault responses. +# A record is identified by (fault_code, source_id). # Request fields # Fault code to retrieve (e.g., "MOTOR_OVERHEAT") string fault_code + +# Reporting source that owns the fault record (the source_id used in ReportFault). +# Together with fault_code it identifies one record. Empty: the call applies only +# when exactly one record carries fault_code, otherwise it fails as ambiguous. +string source_id --- # Response fields diff --git a/src/ros2_medkit_msgs/srv/GetRosbag.srv b/src/ros2_medkit_msgs/srv/GetRosbag.srv index 76ef27cc0..5079f1b9f 100644 --- a/src/ros2_medkit_msgs/srv/GetRosbag.srv +++ b/src/ros2_medkit_msgs/srv/GetRosbag.srv @@ -14,7 +14,8 @@ # # GetRosbag.srv - Retrieve rosbag file information for a fault. # -# Returns the path to the rosbag file that was recorded when a fault was confirmed. +# Returns the path to the rosbag file that was recorded when a fault record was +# confirmed. A recording is linked per record, identified by (fault_code, source_id). # The rosbag contains time-window recording of configured topics before and after # the fault confirmation event. @@ -29,6 +30,12 @@ string fault_code # fault_code alone keeps its historical meaning, "the newest recording of this # fault". string recording_id + +# Reporting source that owns the fault record (the source_id used in ReportFault). +# Together with fault_code it identifies one record. Empty: the call applies only +# when exactly one record carries fault_code, otherwise it fails as ambiguous. +# It scopes the fault_code lookup only. The recording_id path ignores it. +string source_id --- # Response fields diff --git a/src/ros2_medkit_msgs/srv/GetSnapshots.srv b/src/ros2_medkit_msgs/srv/GetSnapshots.srv index de1bf5f71..fcd8dfe91 100644 --- a/src/ros2_medkit_msgs/srv/GetSnapshots.srv +++ b/src/ros2_medkit_msgs/srv/GetSnapshots.srv @@ -14,7 +14,8 @@ # # GetSnapshots.srv - Retrieve topic snapshots captured when a fault was confirmed. # -# Returns topic data that was captured at the moment a fault transitioned to CONFIRMED. +# Returns topic data that was captured at the moment a fault record transitioned to +# CONFIRMED. Snapshots belong to one record, identified by (fault_code, source_id). # Snapshots provide debugging context by preserving system state at fault time. # Request fields @@ -25,6 +26,11 @@ string fault_code # Optional topic filter. If empty, returns snapshots for all captured topics. # If specified, returns only the snapshot for that specific topic. string topic + +# Reporting source that owns the fault record (the source_id used in ReportFault). +# Together with fault_code it identifies one record. Empty: the call applies only +# when exactly one record carries fault_code, otherwise it fails as ambiguous. +string source_id --- # Response fields diff --git a/src/ros2_medkit_msgs/srv/ListFaultsForEntity.srv b/src/ros2_medkit_msgs/srv/ListFaultsForEntity.srv index c58bdcc14..5ae09449a 100644 --- a/src/ros2_medkit_msgs/srv/ListFaultsForEntity.srv +++ b/src/ros2_medkit_msgs/srv/ListFaultsForEntity.srv @@ -14,7 +14,8 @@ # # ListFaultsForEntity.srv - Request list of faults for a specific entity. # -# Filters faults by checking if entity FQN appears in fault.reporting_sources[]. +# Filters fault records by their owning reporting source: the entity FQN must match +# the single entry of fault.reporting_sources[], exactly or as an FQN suffix. # Used by the gateway to serve entity-specific fault lists. # Request fields diff --git a/src/ros2_medkit_msgs/srv/ListRosbags.srv b/src/ros2_medkit_msgs/srv/ListRosbags.srv index 5595816d0..c520bb82e 100644 --- a/src/ros2_medkit_msgs/srv/ListRosbags.srv +++ b/src/ros2_medkit_msgs/srv/ListRosbags.srv @@ -20,7 +20,7 @@ # Request fields # Entity FQN (fully qualified name) to filter rosbags by. -# Matches against fault reporting_sources using prefix matching. +# Matches the owning reporting source of each recording's fault record, exactly. string entity_fqn --- # Response fields diff --git a/src/ros2_medkit_msgs/srv/ReportFault.srv b/src/ros2_medkit_msgs/srv/ReportFault.srv index c6403ca9f..e26fda5d9 100644 --- a/src/ros2_medkit_msgs/srv/ReportFault.srv +++ b/src/ros2_medkit_msgs/srv/ReportFault.srv @@ -15,14 +15,14 @@ # ReportFault.srv - Report a fault event to the FaultManager. # # Called by FaultReporter clients or directly by nodes to report fault conditions. -# The FaultManager aggregates reports by fault_code across all sources and applies -# debounce filtering based on FAILED/PASSED event counts. +# The FaultManager keeps one record per (fault_code, source_id) pair and applies +# debounce filtering per record, based on FAILED/PASSED event counts. # Request fields # Global fault identifier. Convention: UPPER_SNAKE_CASE (e.g., "MOTOR_OVERHEAT"). # Recommended max length: 64 characters. Allowed: A-Z, 0-9, underscore. -# Same fault_code from different sources = single aggregated fault. +# Same fault_code from different sources = one record each, filtered independently. string fault_code # Event type indicating whether the fault condition is detected or cleared. @@ -47,7 +47,9 @@ string description # Identifier of the reporting node/entity. Recommended: fully qualified ROS 2 node name # including namespace (e.g., "/powertrain/engine/temp_sensor"). -# This value is added to the fault's reporting_sources array for multi-source tracking. +# This value owns the record: (fault_code, source_id) is the record identity, and the +# services that read or clear one record take it back as their source_id field. +# Required: a report with an empty source_id is rejected. string source_id --- # Response fields From 8442bdd6d3199870491f68f7d90736716b589b2e Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 15:10:31 +0100 Subject: [PATCH 02/52] fault_manager: key PostgreSQL fault records by fault code and owner The PostgreSQL backend identifies a record by (fault_code, owner) like the SQLite one. owner is added to faults, freeze_frames, snapshots and rosbag_files, the primary keys on the fault code become unique indexes that include it, and every read, write, trim and delete is scoped to one record. get_faults_by_code and the FaultId-keyed reclassify and time-based confirmation come with it. Tables written by the earlier layout are migrated in place on the first connection, in one transaction: the owner is the first entry of a row's reporting_sources (legacy when none is readable), pending rows lose their confirmation time, and evidence rows are assigned only where their code has exactly one owner. A stored CRITICAL severity confirms the record and a reactivation clears its confirmation time, as in the SQLite backend. --- docs/config/fault-manager.rst | 14 +- src/ros2_medkit_fault_manager/README.md | 6 + .../fault_storage.hpp | 10 + .../postgres_fault_storage.hpp | 46 +- .../sqlite_fault_storage.hpp | 10 - .../src/postgres_fault_storage.cpp | 519 ++++++++---- .../test/test_postgres_storage.cpp | 801 +++++++++++++----- 7 files changed, 989 insertions(+), 417 deletions(-) diff --git a/docs/config/fault-manager.rst b/docs/config/fault-manager.rst index 2075dfab4..d0af73c87 100644 --- a/docs/config/fault-manager.rst +++ b/docs/config/fault-manager.rst @@ -58,10 +58,16 @@ it statically, so it needs network access at configure time. rosdep installs not A build without the option stops at startup when ``storage_type`` is ``postgres``. - Supported servers: PostgreSQL 14 and newer. -- One database belongs to exactly one fault manager. Fault codes are the primary key, so two fault - managers on one database merge their faults and delete each other's rosbag rows. Give each fault - manager its own database, or its own schema with ``options=-csearch_path=`` in the - connection string. +- One database belongs to exactly one fault manager. A record is keyed by ``(fault_code, owner)``, + so two fault managers on one database merge their records and delete each other's rosbag rows. + Give each fault manager its own database, or its own schema with + ``options=-csearch_path=`` in the connection string. +- Tables written before records were keyed by owner (one row per fault code) are migrated in place + on the first connection: ``owner`` is added to ``faults``, ``freeze_frames``, ``snapshots`` and + ``rosbag_files``, the primary keys on the fault code are replaced by unique indexes that include + it, and each row takes the first entry of its ``reporting_sources``. The same rules as for + SQLite apply, see the package README. The migration runs in one transaction. Keep a copy of the + database first: an earlier release cannot use the migrated tables. - The audit log stays a local SQLite file, see ``audit_log.database_path``. - The node never logs ``database_url``. It logs host, port, database and user, or the service name. Error text from the server is logged and returned with the password from ``database_url`` diff --git a/src/ros2_medkit_fault_manager/README.md b/src/ros2_medkit_fault_manager/README.md index 38dbf66e1..19773f672 100644 --- a/src/ros2_medkit_fault_manager/README.md +++ b/src/ros2_medkit_fault_manager/README.md @@ -60,6 +60,12 @@ on any open that finds a row it can actually assign, not only on the open that a a store left half-migrated by an interrupted run is healed the next time it is opened, while a row nothing can resolve is left alone instead of being retried forever. +A PostgreSQL database written by an earlier release is migrated in place on the first connection, +by the same rules: the owner is the first legacy `reporting_sources` entry (`legacy` when none is +readable), the primary keys on the fault code become unique indexes over `(fault_code, owner)`, and +evidence rows are assigned only where their code has exactly one owner. It runs in one transaction, +so a failure leaves the tables as they were. + The table names `faults_new` and `freeze_frames_new` are reserved for the rebuild. Anything found under those names is debris from a rebuild that did not finish and is dropped on open, with a warning naming the table. diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp index 078abb4ae..dff17263a 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp @@ -116,6 +116,16 @@ struct FaultId { } }; +/// Owner given to a record migrated out of a database that recorded no readable +/// reporting source. +/// +/// A migrated record is never left with an empty owner. It would be unreachable +/// through the source_id every single-record service takes, and it would leave the +/// child backfill looking for work on every open. The empty owner means one thing +/// only, a child row (freeze frame, snapshot, rosbag link) not yet assigned to a +/// record, and keeping that meaning unambiguous is what this constant is for. +inline constexpr const char * kLegacyOwner = "legacy"; + /// Internal fault state of one record, stored in memory. Times are wall-clock ns >= 0; 0 means never. struct FaultState { std::string fault_code; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/postgres_fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/postgres_fault_storage.hpp index 360082c63..000997c8e 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/postgres_fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/postgres_fault_storage.hpp @@ -69,15 +69,17 @@ class PgFaultStorage : public FaultStorage { std::vector list_faults(bool filter_by_severity, uint8_t severity, const std::vector & statuses) const override; - std::optional get_fault(const std::string & fault_code) const override; + std::optional get_fault(const FaultId & id) const override; - bool clear_fault(const std::string & fault_code) override; + std::vector get_faults_by_code(const std::string & fault_code) const override; + + bool clear_fault(const FaultId & id) override; size_t size() const override; - bool contains(const std::string & fault_code) const override; + bool contains(const FaultId & id) const override; - std::vector check_time_based_confirmation(int64_t now_ns) override; + std::vector check_time_based_confirmation(int64_t now_ns) override; void set_max_snapshots_per_fault(size_t max_count) override; void set_retain_snapshots_on_clear(bool retain) override; @@ -87,28 +89,27 @@ class PgFaultStorage : public FaultStorage { void store_snapshot(const SnapshotData & snapshot) override; void store_snapshots(const std::vector & snapshots) override; - std::vector get_snapshots(const std::string & fault_code, - const std::string & topic_filter = "") const override; + std::vector get_snapshots(const FaultId & id, const std::string & topic_filter = "") const override; int64_t get_max_capture_id() const override; void store_freeze_frame(const FreezeFrameData & frame) override; - std::optional get_freeze_frame(const std::string & fault_code) const override; + std::optional get_freeze_frame(const FaultId & id) const override; size_t set_max_near_misses_per_fault(size_t max_count) override; - std::vector get_near_misses(const std::string & fault_code) const override; + std::vector get_near_misses(const FaultId & id) const override; void store_rosbag_file(const RosbagFileInfo & info) override; void store_rosbag_files(const std::vector & infos) override; - std::optional get_rosbag_file(const std::string & fault_code) const override; - std::vector get_rosbag_files(const std::string & fault_code) const override; + std::optional get_rosbag_file(const FaultId & id) const override; + std::vector get_rosbag_files(const FaultId & id) const override; std::vector get_rosbag_files_by_recording(const std::string & recording_id) const override; - bool delete_rosbag_file(const std::string & fault_code) override; + bool delete_rosbag_file(const FaultId & id) override; size_t delete_rosbag_recording(const std::string & recording_id) override; - size_t delete_rosbag_files(const std::vector & fault_codes) override; + size_t delete_rosbag_files(const std::vector & ids) override; size_t get_total_rosbag_storage_bytes() const override; std::vector get_all_rosbag_files() const override; std::vector list_rosbags_for_entity(const std::string & entity_fqn) const override; std::vector get_all_faults() const override; - std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) override; + std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) override; /// Get the connection info string used to initialize the database const std::string & conn_info() const { @@ -137,6 +138,25 @@ class PgFaultStorage : public FaultStorage { /// Create the tables and indexes that do not exist yet, in @p tx, and check the columns of existing ones. void create_schema(pqxx::work & tx) const; + /// Bring tables written before records were keyed by (fault_code, owner) to the current layout, in @p tx. + /// Does nothing on a fresh database or one already migrated. + void migrate_to_owner_identity(pqxx::work & tx) const; + + /// Whether @p table exists in the current schema. + bool table_exists(pqxx::work & tx, const char * table) const; + + /// Whether @p table has @p column in the current schema. + bool table_has_column(pqxx::work & tx, const char * table, const char * column) const; + + /// Drop the PRIMARY KEY of @p table, whatever it is named. + void drop_primary_key(pqxx::work & tx, const char * table) const; + + /// Whether a child row with an empty owner belongs to a code that has exactly one owner. + bool has_assignable_child_rows(pqxx::work & tx) const; + + /// Give child rows with an empty owner the owner of their code, where the code has exactly one. + void backfill_child_owners(pqxx::work & tx) const; + /// @p text with every copy of the known password replaced by "***". std::string redact(std::string text) const; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp index af15f551d..daf2a90d6 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp @@ -26,16 +26,6 @@ namespace ros2_medkit_fault_manager { -/// Owner given to a record migrated out of a database that recorded no readable -/// reporting source. -/// -/// A migrated record is never left with an empty owner. It would be unreachable -/// through the source_id every single-record service takes, and it would leave the -/// child backfill looking for work on every open. The empty owner means one thing -/// only, a child row (freeze frame, snapshot, rosbag link) not yet assigned to a -/// record, and keeping that meaning unambiguous is what this constant is for. -inline constexpr const char * kLegacyOwner = "legacy"; - /// SQLite-based fault storage implementation with persistence /// Thread-safe implementation using mutex protection class SqliteFaultStorage : public FaultStorage { diff --git a/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp b/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp index e130fbc7b..645d87a08 100644 --- a/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp @@ -305,10 +305,144 @@ DebounceConfig PgFaultStorage::get_debounce_config() const { return config_; } +bool PgFaultStorage::table_exists(pqxx::work & tx, const char * table) const { + return !execute(tx, + "SELECT 1 FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = $1", + std::string(table)) + .empty(); +} + +bool PgFaultStorage::table_has_column(pqxx::work & tx, const char * table, const char * column) const { + return !execute(tx, + "SELECT 1 FROM information_schema.columns WHERE table_schema = current_schema() " + "AND table_name = $1 AND column_name = $2", + std::string(table), std::string(column)) + .empty(); +} + +void PgFaultStorage::drop_primary_key(pqxx::work & tx, const char * table) const { + const auto res = execute(tx, "SELECT conname FROM pg_constraint WHERE conrelid = to_regclass($1) AND contype = 'p'", + std::string(table)); + for (const auto & r : res) { + execute(tx, + std::string("ALTER TABLE ") + table + " DROP CONSTRAINT " + tx.quote_name(r["conname"].as())); + } +} + +bool PgFaultStorage::has_assignable_child_rows(pqxx::work & tx) const { + if (!table_exists(tx, "faults") || !table_has_column(tx, "faults", "owner")) { + return false; + } + for (const char * table : {"freeze_frames", "snapshots", "rosbag_files"}) { + if (!table_exists(tx, table) || !table_has_column(tx, table, "owner")) { + continue; // the column arrives with the migration, which then fills it + } + // The same condition the backfill's UPDATE applies, so "is there work" and "what the work + // does" cannot disagree. + const std::string sql = std::string("SELECT 1 FROM ") + table + + " t WHERE t.owner = '' AND (SELECT COUNT(DISTINCT f.owner) FROM faults f " + "WHERE f.fault_code = t.fault_code) = 1 LIMIT 1"; + if (!execute(tx, sql).empty()) { + return true; + } + } + return false; +} + +void PgFaultStorage::backfill_child_owners(pqxx::work & tx) const { + for (const char * table : {"freeze_frames", "snapshots", "rosbag_files"}) { + if (!table_exists(tx, table) || !table_has_column(tx, table, "owner")) { + continue; + } + + // Correlated on the child row's own fault_code and applied only where that code has exactly + // ONE owner, so a code with two owners never hands one owner's evidence to the other. + execute(tx, std::string("UPDATE ") + table + + " SET owner = (SELECT MIN(f.owner) FROM faults f WHERE f.fault_code = " + table + + ".fault_code) WHERE owner = '' AND (SELECT COUNT(DISTINCT f.owner) FROM faults f WHERE " + "f.fault_code = " + + table + ".fault_code) = 1"); + + // What is left is evidence for a code no fault row carries any more, or for a code several + // owners share. Neither can be assigned without guessing, so it is named and kept. + const auto leftover = execute(tx, std::string("SELECT fault_code, COUNT(*) AS n FROM ") + table + + " WHERE owner = '' GROUP BY fault_code"); + for (const auto & r : leftover) { + core::logf(log_sink_, core::LogLevel::kWarn, "postgres_fault_storage", + "%s: %lld row(s) for fault '%s' keep an empty owner, the code has no single owner to assign them to", + table, static_cast(r["n"].as()), r["fault_code"].as().c_str()); + } + } +} + +void PgFaultStorage::migrate_to_owner_identity(pqxx::work & tx) const { + // Probed per table: a table that does not exist yet is built in the new layout by create_schema. + const auto needs_owner = [&](const char * table) { + return table_exists(tx, table) && !table_has_column(tx, table, "owner"); + }; + const bool faults_needs_owner = needs_owner("faults"); + const bool freeze_frames_needs_owner = needs_owner("freeze_frames"); + const bool snapshots_needs_owner = needs_owner("snapshots"); + const bool rosbag_files_needs_owner = needs_owner("rosbag_files"); + + if (!faults_needs_owner && !freeze_frames_needs_owner && !snapshots_needs_owner && !rosbag_files_needs_owner && + !has_assignable_child_rows(tx)) { + return; // fresh database, or fully migrated - safe to re-run on every connection + } + + if (faults_needs_owner) { + drop_primary_key(tx, "faults"); + execute(tx, "ALTER TABLE faults ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + + // The owner is the first reporting source the row recorded. reporting_sources is rewritten to + // that one owner, as every write keeps it afterwards. + const auto rows = execute(tx, "SELECT fault_code, reporting_sources, status, confirmed_at_ns FROM faults"); + for (const auto & r : rows) { + const std::string fault_code = r["fault_code"].as(); + const auto sources = parse_json_array(r["reporting_sources"].as()); + const std::string owner = + (sources.empty() || sources.front().empty()) ? std::string(kLegacyOwner) : sources.front(); + if (owner == kLegacyOwner) { + core::logf(log_sink_, core::LogLevel::kWarn, "postgres_fault_storage", + "Fault '%s' carried no readable reporting source, migrated under owner '%s'", fault_code.c_str(), + kLegacyOwner); + } + // A pending row carries a confirmation time from the occurrence before a reactivation. + const std::string status = r["status"].as(); + const bool pending = status == fd::fault_status::kPrefailed || status == fd::fault_status::kPrepassed; + execute(tx, "UPDATE faults SET owner = $1, reporting_sources = $2, confirmed_at_ns = $3 WHERE fault_code = $4", + owner, serialize_json_array({owner}), pending ? int64_t{0} : r["confirmed_at_ns"].as(), + fault_code); + } + } + + if (freeze_frames_needs_owner) { + drop_primary_key(tx, "freeze_frames"); + execute(tx, "ALTER TABLE freeze_frames ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + } + if (snapshots_needs_owner) { + execute(tx, "ALTER TABLE snapshots ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + } + if (rosbag_files_needs_owner) { + execute(tx, "ALTER TABLE rosbag_files ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + // CREATE INDEX IF NOT EXISTS in create_schema keeps an index that exists, so the two indexes + // that carry fault_code are dropped to be rebuilt with owner. + execute(tx, "DROP INDEX IF EXISTS idx_rosbag_files_fault_path"); + execute(tx, "DROP INDEX IF EXISTS idx_rosbag_files_fault_created"); + } + + backfill_child_owners(tx); +} + void PgFaultStorage::create_schema(pqxx::work & tx) const { + migrate_to_owner_identity(tx); + + // A row is one fault RECORD, identified by (fault_code, owner). owner is the reporting + // source; reporting_sources keeps it serialised so a raw-table reader still sees the + // field it always saw. Uniqueness through the index below, the same shape as SQLite. execute(tx, R"( CREATE TABLE IF NOT EXISTS faults ( - fault_code TEXT PRIMARY KEY, + fault_code TEXT NOT NULL, severity INTEGER NOT NULL, description TEXT NOT NULL, first_occurred_ns BIGINT NOT NULL, @@ -319,15 +453,17 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { debounce_counter INTEGER NOT NULL DEFAULT 0, last_failed_ns BIGINT NOT NULL DEFAULT 0, last_passed_ns BIGINT NOT NULL DEFAULT 0, - confirmed_at_ns BIGINT NOT NULL DEFAULT 0 + confirmed_at_ns BIGINT NOT NULL DEFAULT 0, + owner TEXT NOT NULL DEFAULT '' ); - )"); + CREATE UNIQUE INDEX IF NOT EXISTS idx_faults_code_owner ON faults(fault_code, owner);)"); // Create snapshots table for storing topic data captured when faults are confirmed execute(tx, R"( CREATE TABLE IF NOT EXISTS snapshots ( id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', topic TEXT NOT NULL, message_type TEXT NOT NULL, data TEXT NOT NULL, @@ -338,15 +474,16 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { CREATE INDEX IF NOT EXISTS idx_snapshots_fault_topic ON snapshots(fault_code, topic);)"); // Create freeze_frames table: one compact JSON dict of captured topic values per fault - // code. Unlike snapshots, freeze frames are keyed by fault_code and are NOT removed on - // clear_fault, so the confirmed-state record is retained after acknowledgement. + // RECORD. Unlike snapshots, freeze frames are keyed by the record identity and are NOT + // removed on clear_fault, so the confirmed-state record is retained after acknowledgement. execute(tx, R"( CREATE TABLE IF NOT EXISTS freeze_frames ( - fault_code TEXT PRIMARY KEY, + fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', data TEXT NOT NULL, captured_at_ns BIGINT NOT NULL ); - )"); + CREATE UNIQUE INDEX IF NOT EXISTS idx_freeze_frames_code_owner ON freeze_frames(fault_code, owner);)"); // Create near_misses table: append-only series of FAILED reports that moved the debounce // counter without confirming the fault. One row per qualifying report, never updated in @@ -374,6 +511,7 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { CREATE TABLE IF NOT EXISTS rosbag_files ( id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', recording_id TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL, format TEXT NOT NULL, @@ -383,23 +521,23 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { ); CREATE INDEX IF NOT EXISTS idx_rosbag_files_fault_code ON rosbag_files(fault_code); CREATE INDEX IF NOT EXISTS idx_rosbag_files_created_at ON rosbag_files(created_at_ns); - CREATE INDEX IF NOT EXISTS idx_rosbag_files_fault_created ON rosbag_files(fault_code, created_at_ns, id); + CREATE INDEX IF NOT EXISTS idx_rosbag_files_fault_created ON rosbag_files(fault_code, owner, created_at_ns, id); CREATE INDEX IF NOT EXISTS idx_rosbag_files_recording ON rosbag_files(recording_id); CREATE INDEX IF NOT EXISTS idx_rosbag_files_path ON rosbag_files(file_path); - CREATE UNIQUE INDEX IF NOT EXISTS idx_rosbag_files_fault_path ON rosbag_files(fault_code, file_path); + CREATE UNIQUE INDEX IF NOT EXISTS idx_rosbag_files_fault_path ON rosbag_files(fault_code, owner, file_path); )"); // CREATE TABLE IF NOT EXISTS keeps a table of another layout. Check the columns so that startup fails. execute(tx, R"( SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, occurrence_count, status, - reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, confirmed_at_ns + reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, confirmed_at_ns, owner FROM faults LIMIT 0; - SELECT id, fault_code, topic, message_type, data, captured_at_ns, capture_id FROM snapshots LIMIT 0; - SELECT fault_code, data, captured_at_ns FROM freeze_frames LIMIT 0; + SELECT id, fault_code, owner, topic, message_type, data, captured_at_ns, capture_id FROM snapshots LIMIT 0; + SELECT fault_code, owner, data, captured_at_ns FROM freeze_frames LIMIT 0; SELECT id, fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, severity, source_id, resulting_status FROM near_misses LIMIT 0; - SELECT id, fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns + SELECT id, fault_code, owner, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns FROM rosbag_files LIMIT 0; )"); } @@ -560,17 +698,17 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f // No row lock needed: mutex_ serializes every call on this connection, and one database belongs to exactly one // fault manager. - // Check if fault exists + // Check if the record exists. The report is addressed to (fault_code, source_id): + // another source's record of the same code is a different row and is not touched. auto res = execute(tx, - "SELECT severity, occurrence_count, reporting_sources, status, debounce_counter, confirmed_at_ns, " - "first_occurred_ns FROM faults WHERE fault_code = $1", - fault_code); + "SELECT severity, occurrence_count, status, debounce_counter, confirmed_at_ns, " + "first_occurred_ns FROM faults WHERE fault_code = $1 AND owner = $2", + fault_code, source_id); if (!res.empty()) { - // Fault exists - update it + // Record exists - update it int existing_severity = res[0]["severity"].as(); int64_t existing_count = res[0]["occurrence_count"].as(); - std::string sources_json = res[0]["reporting_sources"].as(); std::string current_status = res[0]["status"].as(); int32_t debounce_counter = res[0]["debounce_counter"].as(); int64_t confirmed_at_ns = res[0]["confirmed_at_ns"].as(); @@ -591,20 +729,17 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f // decrements it to -1, then reuse the existing FAILED logic below. Also // reset first_occurred: this is a new outage cycle, not a continuation // of the one that just cleared. + // The new occurrence has no confirmation time until it confirms. debounce_counter = 0; first_occurred_ns = timestamp_ns; + confirmed_at_ns = 0; is_reactivation = true; } if (is_failed) { // FAILED event - // Parse existing sources and add new one - std::vector sources = parse_json_array(sources_json); - std::set sources_set(sources.begin(), sources.end()); - sources_set.insert(source_id); - sources.assign(sources_set.begin(), sources_set.end()); - - // Escalate severity if new severity is higher + // Escalate severity if new severity is higher. Per record: a CRITICAL from one + // owner does not escalate another owner's record of the same code. int new_severity = std::max(existing_severity, static_cast(severity)); // Increment count with saturation - only on a genuine new occurrence (reactivation @@ -618,9 +753,10 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f // Decrement towards confirmation, clamped to the thresholds. debounce_counter = clamp_debounce_counter(debounce_counter - 1, config); - // CRITICAL bypasses debounce; otherwise the shared state machine decides (with hysteresis). + // A CRITICAL record bypasses debounce and the HEALED latch. The stored severity decides, not the + // report's. Otherwise the shared state machine decides (with hysteresis). std::string new_status; - if (config.critical_immediate_confirm && severity == fd::severity::kCritical) { + if (config.critical_immediate_confirm && new_severity == fd::severity::kCritical) { debounce_counter = std::min(debounce_counter, -1); new_status = fd::fault_status::kConfirmed; } else { @@ -634,20 +770,22 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f confirmed_at_ns = timestamp_ns; } + // reporting_sources is not touched: it is the one owner, fixed at insert, and this + // row is already that owner's. if (description.empty()) { execute(tx, "UPDATE faults SET severity = $1, last_occurred_ns = $2, last_failed_ns = $3, occurrence_count = $4, " - "reporting_sources = $5, status = $6, debounce_counter = $7, confirmed_at_ns = $8, " - "first_occurred_ns = $9 WHERE fault_code = $10", - new_severity, timestamp_ns, timestamp_ns, new_count, serialize_json_array(sources), new_status, - debounce_counter, confirmed_at_ns, first_occurred_ns, fault_code); + "status = $5, debounce_counter = $6, confirmed_at_ns = $7, " + "first_occurred_ns = $8 WHERE fault_code = $9 AND owner = $10", + new_severity, timestamp_ns, timestamp_ns, new_count, new_status, debounce_counter, confirmed_at_ns, + first_occurred_ns, fault_code, source_id); } else { execute(tx, "UPDATE faults SET severity = $1, description = $2, last_occurred_ns = $3, last_failed_ns = $4, " - "occurrence_count = $5, reporting_sources = $6, status = $7, debounce_counter = $8, " - "confirmed_at_ns = $9, first_occurred_ns = $10 WHERE fault_code = $11", - new_severity, description, timestamp_ns, timestamp_ns, new_count, serialize_json_array(sources), - new_status, debounce_counter, confirmed_at_ns, first_occurred_ns, fault_code); + "occurrence_count = $5, status = $6, debounce_counter = $7, " + "confirmed_at_ns = $8, first_occurred_ns = $9 WHERE fault_code = $10 AND owner = $11", + new_severity, description, timestamp_ns, timestamp_ns, new_count, new_status, debounce_counter, + confirmed_at_ns, first_occurred_ns, fault_code, source_id); } if (is_near_miss(true, new_status)) { @@ -663,8 +801,10 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f // last_occurred_ns is deliberately NOT touched: a PASSED event is the fault // ENDING, not occurring. Bumping it made a long-stale CONFIRMED fault look // freshly active. The PASSED instant is recorded in last_passed_ns. - execute(tx, "UPDATE faults SET last_passed_ns = $1, status = $2, debounce_counter = $3 WHERE fault_code = $4", - timestamp_ns, new_status, debounce_counter, fault_code); + execute(tx, + "UPDATE faults SET last_passed_ns = $1, status = $2, debounce_counter = $3 " + "WHERE fault_code = $4 AND owner = $5", + timestamp_ns, new_status, debounce_counter, fault_code, source_id); } return is_reactivation; // Reactivation treated as new occurrence for event publishing @@ -688,11 +828,11 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f execute( tx, "INSERT INTO faults (fault_code, severity, description, first_occurred_ns, last_occurred_ns, occurrence_count, " - "status, reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, confirmed_at_ns) " - "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", + "status, reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, confirmed_at_ns, owner) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", fault_code, static_cast(severity), description, timestamp_ns, timestamp_ns, 1 /* occurrence_count = 1 */, initial_status, serialize_json_array({source_id}), initial_counter, timestamp_ns, 0, - confirmed_now ? timestamp_ns : 0); + confirmed_now ? timestamp_ns : 0, source_id); if (is_near_miss(true, initial_status)) { record_near_miss_locked(fault_code, timestamp_ns, initial_counter, config, severity, source_id, initial_status, tx); @@ -752,14 +892,15 @@ std::vector PgFaultStorage::list_faults(bool filter_by_severity }); } -std::optional PgFaultStorage::get_fault(const std::string & fault_code) const { +std::optional PgFaultStorage::get_fault(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("get_fault", [&](pqxx::work & tx) -> std::optional { auto res = execute(tx, "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, last_passed_ns FROM faults WHERE fault_code = $1", - fault_code); + "occurrence_count, status, reporting_sources, last_passed_ns FROM faults " + "WHERE fault_code = $1 AND owner = $2", + id.fault_code, id.owner); if (res.empty()) { return std::nullopt; } @@ -778,52 +919,77 @@ std::optional PgFaultStorage::get_fault(const std::string & fau }); } -bool PgFaultStorage::clear_fault(const std::string & fault_code) { +std::vector PgFaultStorage::get_faults_by_code(const std::string & fault_code) const { + std::lock_guard lock(mutex_); + + return run_in_transaction("get_faults_by_code", [&](pqxx::work & tx) { + auto res = execute(tx, + "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " + "occurrence_count, status, reporting_sources, last_passed_ns FROM faults " + "WHERE fault_code = $1 ORDER BY owner ASC", + fault_code); + std::vector result; + for (const auto & r : res) { + fd::FaultRecord fault; + fault.fault_code = r["fault_code"].as(); + fault.severity = static_cast(r["severity"].as()); + fault.description = r["description"].as(); + fault.first_occurred_ns = r["first_occurred_ns"].as(); + fault.last_occurred_ns = r["last_occurred_ns"].as(); + fault.occurrence_count = static_cast(r["occurrence_count"].as()); + fault.status = r["status"].as(); + fault.reporting_sources = parse_json_array(r["reporting_sources"].as()); + fault.last_passed_ns = r["last_passed_ns"].as(); + result.push_back(std::move(fault)); + } + return result; + }); +} + +bool PgFaultStorage::clear_fault(const FaultId & id) { std::lock_guard lock(mutex_); return run_in_transaction("clear_fault", [&](pqxx::work & tx) { - // The near_misses rows for this code are deliberately left alone. Clearing acknowledges one - // fault cycle; the record of how often the code approached confirmation spans cycles and - // cannot be reconstructed once deleted. + // The near_misses rows for this record are deliberately left alone. Clearing acknowledges + // one fault cycle; the record of how often the code approached confirmation spans cycles. - // Acknowledging a fault drops its value snapshots, unless a history was asked - // for: with recordings retained past a clear, deleting the readings that go with - // them leaves a fault holding bags whose matching values are gone. + // Acknowledging a record drops its value snapshots, scoped to the record so another + // owner's readings for the same code stay. Unless a history was asked for: with + // recordings retained past a clear, deleting the readings leaves bags whose values are gone. if (!retain_snapshots_on_clear_) { - execute(tx, "DELETE FROM snapshots WHERE fault_code = $1", fault_code); + execute(tx, "DELETE FROM snapshots WHERE fault_code = $1 AND owner = $2", id.fault_code, id.owner); } - auto res = - execute(tx, "UPDATE faults SET status = $1 WHERE fault_code = $2", fd::fault_status::kCleared, fault_code); + auto res = execute(tx, "UPDATE faults SET status = $1 WHERE fault_code = $2 AND owner = $3", + fd::fault_status::kCleared, id.fault_code, id.owner); return res.affected_rows() > 0; }); } -std::vector PgFaultStorage::reclassify_healed_as_cleared(const KeepHealedFn & keep_healed) { +std::vector PgFaultStorage::reclassify_healed_as_cleared(const KeepHealedFn & keep_healed) { std::lock_guard lock(mutex_); return run_in_transaction("reclassify_healed_as_cleared", [&](pqxx::work & tx) { - // Collect the codes that will flip first so the caller can audit each one. Codes the + // Collect the records that will flip first so the caller can audit each one. Codes the // caller keeps HEALED are left out. Everything runs in the same transaction, so the // returned list matches the rows actually reclassified below. - auto res = execute(tx, "SELECT fault_code FROM faults WHERE status = $1", fd::fault_status::kHealed); - std::vector reclassified; + auto res = execute(tx, "SELECT fault_code, owner FROM faults WHERE status = $1", fd::fault_status::kHealed); + std::vector reclassified; for (const auto & r : res) { - auto code = r["fault_code"].as(); - if (!(keep_healed && keep_healed(code))) { - reclassified.push_back(std::move(code)); + FaultId id{r["fault_code"].as(), r["owner"].as()}; + if (!(keep_healed && keep_healed(id.fault_code))) { + reclassified.push_back(std::move(id)); } } - for (const auto & code : reclassified) { - // Drop snapshots for the affected faults so a reclassified row matches CLEARED semantics. - // clear_fault is not the only place that takes a fault's readings, so retain_snapshots_on_clear_ - // has to reach here too: otherwise the setting holds until the next restart and then the - // reclassification deletes exactly what it was set to keep. + for (const auto & id : reclassified) { + // Drop snapshots for the affected records so a reclassified row matches CLEARED + // semantics. clear_fault is not the only place that takes a record's readings, so + // retain_snapshots_on_clear_ has to reach here too. if (!retain_snapshots_on_clear_) { - execute(tx, "DELETE FROM snapshots WHERE fault_code = $1", code); + execute(tx, "DELETE FROM snapshots WHERE fault_code = $1 AND owner = $2", id.fault_code, id.owner); } - execute(tx, "UPDATE faults SET status = $1 WHERE fault_code = $2 AND status = $3", fd::fault_status::kCleared, - code, fd::fault_status::kHealed); + execute(tx, "UPDATE faults SET status = $1 WHERE fault_code = $2 AND owner = $3 AND status = $4", + fd::fault_status::kCleared, id.fault_code, id.owner, fd::fault_status::kHealed); } return reclassified; }); @@ -838,16 +1004,17 @@ size_t PgFaultStorage::size() const { }); } -bool PgFaultStorage::contains(const std::string & fault_code) const { +bool PgFaultStorage::contains(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("contains", [&](pqxx::work & tx) { - auto res = execute(tx, "SELECT 1 FROM faults WHERE fault_code = $1 LIMIT 1", fault_code); + auto res = + execute(tx, "SELECT 1 FROM faults WHERE fault_code = $1 AND owner = $2 LIMIT 1", id.fault_code, id.owner); return !res.empty(); }); } -std::vector PgFaultStorage::check_time_based_confirmation(int64_t now_ns) { +std::vector PgFaultStorage::check_time_based_confirmation(int64_t now_ns) { std::lock_guard lock(mutex_); if (config_.auto_confirm_after_sec <= 0.0) { @@ -856,16 +1023,16 @@ std::vector PgFaultStorage::check_time_based_confirmation(int64_t n const int64_t cutoff_ns = now_ns - std::llround(config_.auto_confirm_after_sec * 1e9); return run_in_transaction("check_time_based_confirmation", [&](pqxx::work & tx) { - // Collect the codes that will flip first so the caller can audit each one. The + // Collect the records that will flip first so the caller can audit each one. The // SELECT predicate mirrors the UPDATE exactly, and both run under the same lock, // so the returned list matches the rows actually confirmed below. - auto res = - execute(tx, "SELECT fault_code FROM faults WHERE status = $1 AND last_failed_ns <= $2 AND last_failed_ns > 0 ", - fd::fault_status::kPrefailed, cutoff_ns); + auto res = execute( + tx, "SELECT fault_code, owner FROM faults WHERE status = $1 AND last_failed_ns <= $2 AND last_failed_ns > 0 ", + fd::fault_status::kPrefailed, cutoff_ns); - std::vector confirmed; + std::vector confirmed; for (const auto & r : res) { - confirmed.push_back(r["fault_code"].as()); + confirmed.push_back(FaultId{r["fault_code"].as(), r["owner"].as()}); } if (!confirmed.empty()) { @@ -904,44 +1071,46 @@ void PgFaultStorage::store_snapshots(const std::vector & snapshots std::lock_guard lock(mutex_); const std::string & fault_code = snapshots.front().fault_code; + const std::string & owner = snapshots.front().owner; // One transaction for the whole capture: a capture is all-or-nothing, and the // old row-at-a-time path could leave a confirmation's values half stored. run_in_transaction("store_snapshots", [&](pqxx::work & tx) { for (const auto & snapshot : snapshots) { - execute( - tx, - "INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns, capture_id) VALUES ($1, $2, " - "$3, $4, $5, $6)", - snapshot.fault_code, snapshot.topic, snapshot.message_type, snapshot.data, snapshot.captured_at_ns, - snapshot.capture_id); + execute(tx, + "INSERT INTO snapshots (fault_code, owner, topic, message_type, data, captured_at_ns, capture_id) " + "VALUES ($1, $2, $3, $4, $5, $6, $7)", + snapshot.fault_code, snapshot.owner, snapshot.topic, snapshot.message_type, snapshot.data, + snapshot.captured_at_ns, snapshot.capture_id); } if (max_snapshots_per_fault_ > 0) { - // Trim whole capture sets, oldest first, until the fault fits. The old rule - // counted rows and rejected the NEW row once full, so a capture straddling + // Trim whole capture sets of THIS record, oldest first, until it fits. The old + // rule counted rows and rejected the NEW row once full, so a capture straddling // the cap was stored in part - some topics present, the rest silently gone, // indistinguishable from "that topic was not publishing". Keep-newest also // stops this cap from opposing the rosbag one. // // The newest capture is never trimmed: if it alone exceeds the cap, the cap - // is smaller than this fault's topic count and tearing it would be the very + // is smaller than this record's topic count and tearing it would be the very // thing being fixed. auto res = - execute(tx, "SELECT MAX(capture_id) AS max_capture_id FROM snapshots WHERE fault_code = $1", fault_code); + execute(tx, "SELECT MAX(capture_id) AS max_capture_id FROM snapshots WHERE fault_code = $1 AND owner = $2", + fault_code, owner); int64_t newest_capture = 0; if (!res.empty() && !res[0]["max_capture_id"].is_null()) { newest_capture = res[0]["max_capture_id"].as(); } while (true) { - auto count_res = execute(tx, "SELECT COUNT(*) AS sz FROM snapshots WHERE fault_code = $1", fault_code); + auto count_res = + execute(tx, "SELECT COUNT(*) AS sz FROM snapshots WHERE fault_code = $1 AND owner = $2", fault_code, owner); if (count_res.empty() || count_res[0]["sz"].as() <= max_snapshots_per_fault_) { break; } auto trim_res = execute(tx, - "DELETE FROM snapshots WHERE fault_code = $1 AND capture_id = " - "(SELECT MIN(capture_id) FROM snapshots WHERE fault_code = $1) " - "AND capture_id <> $2", - fault_code, newest_capture); + "DELETE FROM snapshots WHERE fault_code = $1 AND owner = $2 AND capture_id = " + "(SELECT MIN(capture_id) FROM snapshots WHERE fault_code = $1 AND owner = $2) " + "AND capture_id <> $3", + fault_code, owner, newest_capture); if (trim_res.affected_rows() == 0) { break; } @@ -950,14 +1119,14 @@ void PgFaultStorage::store_snapshots(const std::vector & snapshots }); } -std::vector PgFaultStorage::get_snapshots(const std::string & fault_code, - const std::string & topic_filter) const { +std::vector PgFaultStorage::get_snapshots(const FaultId & id, const std::string & topic_filter) const { std::lock_guard lock(mutex_); std::string sql = - "SELECT fault_code, topic, message_type, data, captured_at_ns, capture_id FROM snapshots WHERE fault_code = $1"; + "SELECT fault_code, owner, topic, message_type, data, captured_at_ns, capture_id FROM snapshots " + "WHERE fault_code = $1 AND owner = $2"; if (!topic_filter.empty()) { - sql += " AND topic = $2"; + sql += " AND topic = $3"; } // capture_id before the timestamp: the rows of one capture are written seconds // apart under load and their timestamps interleave with a neighbouring capture's, @@ -965,12 +1134,14 @@ std::vector PgFaultStorage::get_snapshots(const std::string & faul sql += " ORDER BY capture_id DESC, captured_at_ns DESC"; return run_in_transaction("get_snapshots", [&](pqxx::work & tx) { - auto res = topic_filter.empty() ? execute(tx, sql, fault_code) : execute(tx, sql, fault_code, topic_filter); + auto res = topic_filter.empty() ? execute(tx, sql, id.fault_code, id.owner) + : execute(tx, sql, id.fault_code, id.owner, topic_filter); std::vector result; for (const auto & r : res) { SnapshotData snapshot; snapshot.fault_code = r["fault_code"].as(); + snapshot.owner = r["owner"].as(); snapshot.topic = r["topic"].as(); snapshot.message_type = r["message_type"].as(); snapshot.data = r["data"].as(); @@ -998,26 +1169,28 @@ void PgFaultStorage::store_freeze_frame(const FreezeFrameData & frame) { std::lock_guard lock(mutex_); run_in_transaction("store_freeze_frame", [&](pqxx::work & tx) { - // Keyed by fault_code (PRIMARY KEY): a re-confirm replaces the previous frame. + // Keyed by the record (idx_freeze_frames_code_owner): a re-confirm of the same record + // replaces its frame, and another owner's frame of the same code is its own row. execute(tx, - "INSERT INTO freeze_frames (fault_code, data, captured_at_ns) VALUES ($1, $2, $3) ON CONFLICT(fault_code) " - "DO " - "UPDATE SET data = $2, captured_at_ns = $3 ", - frame.fault_code, frame.data, frame.captured_at_ns); + "INSERT INTO freeze_frames (fault_code, owner, data, captured_at_ns) VALUES ($1, $2, $3, $4) " + "ON CONFLICT(fault_code, owner) DO UPDATE SET data = $3, captured_at_ns = $4 ", + frame.fault_code, frame.owner, frame.data, frame.captured_at_ns); }); } -std::optional PgFaultStorage::get_freeze_frame(const std::string & fault_code) const { +std::optional PgFaultStorage::get_freeze_frame(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("get_freeze_frame", [&](pqxx::work & tx) -> std::optional { - auto res = - execute(tx, "SELECT fault_code, data, captured_at_ns FROM freeze_frames WHERE fault_code = $1", fault_code); + auto res = execute( + tx, "SELECT fault_code, owner, data, captured_at_ns FROM freeze_frames WHERE fault_code = $1 AND owner = $2", + id.fault_code, id.owner); if (res.empty()) { return std::nullopt; } FreezeFrameData frame; frame.fault_code = res[0]["fault_code"].as(); + frame.owner = res[0]["owner"].as(); frame.data = res[0]["data"].as(); frame.captured_at_ns = res[0]["captured_at_ns"].as(); return frame; @@ -1042,7 +1215,7 @@ size_t PgFaultStorage::set_max_near_misses_per_fault(size_t max_count) { auto res = execute(tx, "DELETE FROM near_misses WHERE id IN (" "SELECT id FROM (SELECT id, ROW_NUMBER() OVER " - "(PARTITION BY fault_code ORDER BY id DESC) AS rn FROM near_misses) AS ranked " + "(PARTITION BY fault_code, source_id ORDER BY id DESC) AS rn FROM near_misses) AS ranked " "WHERE rn > $1)", static_cast(max_count)); // Returned rather than logged: the storage layer has no logger, and a bound applied by mistake @@ -1075,20 +1248,22 @@ void PgFaultStorage::record_near_miss_locked(const std::string & fault_code, int // by timestamp would then drop the row that was just appended and make the two backends, which // append in arrival order, disagree on the same input. execute(tx, - "DELETE FROM near_misses WHERE fault_code = $1 AND id NOT IN " - "(SELECT id FROM near_misses WHERE fault_code = $1 ORDER BY id DESC LIMIT $2)", - fault_code, static_cast(max_near_misses_per_fault_)); + "DELETE FROM near_misses WHERE fault_code = $1 AND source_id = $2 AND id NOT IN " + "(SELECT id FROM near_misses WHERE fault_code = $1 AND source_id = $2 ORDER BY id DESC LIMIT $3)", + fault_code, source_id, static_cast(max_near_misses_per_fault_)); } -std::vector PgFaultStorage::get_near_misses(const std::string & fault_code) const { +std::vector PgFaultStorage::get_near_misses(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("get_near_misses", [&](pqxx::work & tx) { + // source_id already IS the owner on this table, so the series only needs the extra + // predicate, not a new column. auto res = execute(tx, "SELECT fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, " - "severity, source_id, resulting_status FROM near_misses WHERE fault_code = $1 " - "ORDER BY id ASC", - fault_code); + "severity, source_id, resulting_status FROM near_misses " + "WHERE fault_code = $1 AND source_id = $2 ORDER BY id ASC", + id.fault_code, id.owner); std::vector result; for (const auto & r : res) { @@ -1175,25 +1350,25 @@ std::vector PgFaultStorage::store_rosbag_file_locked(const RosbagFi // re-stored row inside a tie group. execute(tx, "INSERT INTO rosbag_files " - "(fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns) " - "VALUES ($1, $2, $3, $4, $5, $6, $7) " - "ON CONFLICT (fault_code, file_path) DO UPDATE SET " + "(fault_code, owner, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8) " + "ON CONFLICT (fault_code, owner, file_path) DO UPDATE SET " "recording_id = EXCLUDED.recording_id, format = EXCLUDED.format, " "duration_sec = EXCLUDED.duration_sec, size_bytes = EXCLUDED.size_bytes, " "created_at_ns = EXCLUDED.created_at_ns", - row.fault_code, row.recording_id, row.file_path, row.format, row.duration_sec, + row.fault_code, row.owner, row.recording_id, row.file_path, row.format, row.duration_sec, static_cast(row.size_bytes), row.created_at_ns); if (max_rosbags_per_fault_ == 0) { - return {}; // unlimited per fault; only the global byte quota bounds this + return {}; // unlimited per record; only the global byte quota bounds this } - auto res = - execute(tx, - "DELETE FROM rosbag_files WHERE fault_code = $1 AND id NOT IN " - "(SELECT id FROM rosbag_files WHERE fault_code = $1 ORDER BY created_at_ns DESC, id DESC LIMIT $2) " - "RETURNING file_path", - row.fault_code, static_cast(max_rosbags_per_fault_)); + auto res = execute(tx, + "DELETE FROM rosbag_files WHERE fault_code = $1 AND owner = $2 AND id NOT IN " + "(SELECT id FROM rosbag_files WHERE fault_code = $1 AND owner = $2 ORDER BY created_at_ns DESC, " + "id DESC LIMIT $3) " + "RETURNING file_path", + row.fault_code, row.owner, static_cast(max_rosbags_per_fault_)); std::set evicted; for (const auto & r : res) { @@ -1207,6 +1382,7 @@ namespace { RosbagFileInfo read_rosbag_row(const pqxx::row & r) { RosbagFileInfo info; info.fault_code = r["fault_code"].as(); + info.owner = r["owner"].as(); info.recording_id = r["recording_id"].as(); info.file_path = r["file_path"].as(); info.format = r["format"].as(); @@ -1217,18 +1393,19 @@ RosbagFileInfo read_rosbag_row(const pqxx::row & r) { } constexpr const char * kRosbagColumns = - "fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns"; + "fault_code, owner, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns"; } // namespace -std::vector PgFaultStorage::get_rosbag_files(const std::string & fault_code) const { +std::vector PgFaultStorage::get_rosbag_files(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("get_rosbag_files", [&](pqxx::work & tx) { - auto res = execute(tx, - std::string("SELECT ") + kRosbagColumns + - " FROM rosbag_files WHERE fault_code = $1 ORDER BY created_at_ns DESC, id DESC", - fault_code); + auto res = + execute(tx, + std::string("SELECT ") + kRosbagColumns + + " FROM rosbag_files WHERE fault_code = $1 AND owner = $2 ORDER BY created_at_ns DESC, id DESC", + id.fault_code, id.owner); std::vector result; for (const auto & r : res) { @@ -1244,7 +1421,7 @@ std::vector PgFaultStorage::get_rosbag_files_by_recording(const return run_in_transaction("get_rosbag_files_by_recording", [&](pqxx::work & tx) { auto res = execute(tx, std::string("SELECT ") + kRosbagColumns + - " FROM rosbag_files WHERE recording_id = $1 ORDER BY fault_code ASC", + " FROM rosbag_files WHERE recording_id = $1 ORDER BY fault_code ASC, owner ASC", recording_id); std::vector result; @@ -1291,19 +1468,20 @@ size_t PgFaultStorage::delete_rosbag_recording(const std::string & recording_id) return outcome.second; } -std::optional PgFaultStorage::get_rosbag_file(const std::string & fault_code) const { +std::optional PgFaultStorage::get_rosbag_file(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("get_rosbag_file", [&](pqxx::work & tx) -> std::optional { - // ORDER BY is load-bearing now that a fault can hold several recordings. Without + // ORDER BY is load-bearing now that a record can hold several recordings. Without // it PostgreSQL may return any matching row, so the fault detail and the download // would serve an arbitrary recording - non-deterministically, which no test // catches reliably. id breaks the tie because a burst stamps one created_at_ns // across all its rows. auto res = execute(tx, - "SELECT fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns " - "FROM rosbag_files WHERE fault_code = $1 ORDER BY created_at_ns DESC, id DESC LIMIT 1", - fault_code); + std::string("SELECT ") + kRosbagColumns + + " FROM rosbag_files WHERE fault_code = $1 AND owner = $2 " + "ORDER BY created_at_ns DESC, id DESC LIMIT 1", + id.fault_code, id.owner); if (res.empty()) { return std::nullopt; } @@ -1311,7 +1489,7 @@ std::optional PgFaultStorage::get_rosbag_file(const std::string }); } -bool PgFaultStorage::delete_rosbag_file(const std::string & fault_code) { +bool PgFaultStorage::delete_rosbag_file(const FaultId & id) { std::lock_guard lock(mutex_); auto outcome = @@ -1320,8 +1498,9 @@ bool PgFaultStorage::delete_rosbag_file(const std::string & fault_code) { // DELETE, leave a surviving row pointing at a directory that is gone - // unreadable for good, and still charged against the storage quota, which sums // rows. This way the worst case is an orphaned directory instead. - auto res = execute(tx, "DELETE FROM rosbag_files WHERE fault_code = $1 RETURNING file_path", fault_code); - // Every path, not the first one: a fault holds as many recordings as its cap + auto res = execute(tx, "DELETE FROM rosbag_files WHERE fault_code = $1 AND owner = $2 RETURNING file_path", + id.fault_code, id.owner); + // Every path, not the first one: a record holds as many recordings as its cap // allows, and reading one row would unlink one bag and leak the rest - rows gone, // directories left behind, uncounted by a quota that sums rows. std::set paths; @@ -1347,8 +1526,8 @@ bool PgFaultStorage::delete_rosbag_file(const std::string & fault_code) { return outcome.second; } -size_t PgFaultStorage::delete_rosbag_files(const std::vector & fault_codes) { - if (fault_codes.empty()) { +size_t PgFaultStorage::delete_rosbag_files(const std::vector & ids) { + if (ids.empty()) { return 0; } std::lock_guard lock(mutex_); @@ -1360,11 +1539,12 @@ size_t PgFaultStorage::delete_rosbag_files(const std::vector & faul run_in_transaction("delete_rosbag_files", [&](pqxx::work & tx) -> std::pair, size_t> { std::set paths; size_t deleted = 0; - for (const auto & code : fault_codes) { - // RETURNING gives every path this code held, not just the first: one fault - // code can name several recordings now, and the sweep must be able to - // reclaim every one of their bags. - auto res = execute(tx, "DELETE FROM rosbag_files WHERE fault_code = $1 RETURNING file_path", code); + for (const auto & id : ids) { + // RETURNING gives every path this record held, not just the first: one record + // can name several recordings now, and the sweep must be able to reclaim every + // one of their bags. + auto res = execute(tx, "DELETE FROM rosbag_files WHERE fault_code = $1 AND owner = $2 RETURNING file_path", + id.fault_code, id.owner); for (const auto & r : res) { paths.insert(r["file_path"].as()); } @@ -1415,21 +1595,12 @@ std::vector PgFaultStorage::get_all_rosbag_files() const { std::lock_guard lock(mutex_); return run_in_transaction("get_all_rosbag_files", [&](pqxx::work & tx) { - auto res = execute(tx, - "SELECT fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns " - "FROM rosbag_files ORDER BY created_at_ns ASC, id ASC"); + auto res = + execute(tx, std::string("SELECT ") + kRosbagColumns + " FROM rosbag_files ORDER BY created_at_ns ASC, id ASC"); std::vector result; for (const auto & r : res) { - RosbagFileInfo info; - info.fault_code = r["fault_code"].as(); - info.recording_id = r["recording_id"].as(); - info.file_path = r["file_path"].as(); - info.format = r["format"].as(); - info.duration_sec = r["duration_sec"].as(); - info.size_bytes = static_cast(r["size_bytes"].as()); - info.created_at_ns = r["created_at_ns"].as(); - result.push_back(info); + result.push_back(read_rosbag_row(r)); } return result; }); @@ -1439,26 +1610,22 @@ std::vector PgFaultStorage::list_rosbags_for_entity(const std::s std::lock_guard lock(mutex_); return run_in_transaction("list_rosbags_for_entity", [&](pqxx::work & tx) { - auto res = execute( - tx, - "SELECT r.fault_code, r.recording_id, r.file_path, r.format, r.duration_sec, r.size_bytes, r.created_at_ns " - "FROM rosbag_files r " - "JOIN faults f ON r.fault_code = f.fault_code " - "JOIN jsonb_array_elements_text(f.reporting_sources::jsonb) AS j(value) ON j.value = $1 " - "ORDER BY r.created_at_ns DESC, r.id DESC", - entity_fqn); + // The row carries its record's owner, so the entity match is an equality on the row + // and the json scan over reporting_sources is gone. The join to faults stays as the + // existence check it always was: a row whose record is gone is not served. + auto res = + execute(tx, + "SELECT r.fault_code, r.owner, r.recording_id, r.file_path, r.format, r.duration_sec, r.size_bytes, " + "r.created_at_ns " + "FROM rosbag_files r " + "JOIN faults f ON r.fault_code = f.fault_code AND r.owner = f.owner " + "WHERE r.owner = $1 " + "ORDER BY r.created_at_ns DESC, r.id DESC", + entity_fqn); std::vector result; for (const auto & r : res) { - RosbagFileInfo info; - info.fault_code = r["fault_code"].as(); - info.recording_id = r["recording_id"].as(); - info.file_path = r["file_path"].as(); - info.format = r["format"].as(); - info.duration_sec = r["duration_sec"].as(); - info.size_bytes = static_cast(r["size_bytes"].as()); - info.created_at_ns = r["created_at_ns"].as(); - result.push_back(info); + result.push_back(read_rosbag_row(r)); } return result; }); diff --git a/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp b/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp index d3bdb419e..cebfa86c4 100644 --- a/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp +++ b/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp @@ -53,6 +53,7 @@ #include "test_log_sink.hpp" using ros2_medkit_fault_manager::DebounceConfig; +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::PgFaultStorage; using ros2_medkit_fault_manager::RosbagFileInfo; using ros2_medkit_msgs::msg::Fault; @@ -160,7 +161,7 @@ TEST_F(PgFaultStorageTest, ReportNewFaultEvent) { EXPECT_TRUE(is_new); EXPECT_EQ(storage_->size(), 1u); - EXPECT_TRUE(storage_->contains("MOTOR_OVERHEAT")); + EXPECT_TRUE(storage_->contains(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"})); } TEST_F(PgFaultStorageTest, PassedEventForNonExistentFaultIgnored) { @@ -174,28 +175,38 @@ TEST_F(PgFaultStorageTest, PassedEventForNonExistentFaultIgnored) { EXPECT_EQ(storage_->size(), 0u); } -TEST_F(PgFaultStorageTest, ReportExistingFaultEventUpdates) { +TEST_F(PgFaultStorageTest, SameCodeDifferentSourcesAreSeparateRecords) { rclcpp::Clock clock; auto timestamp1 = clock.now(); auto timestamp2 = clock.now(); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Initial report", - "/powertrain/motor1", timestamp1.nanoseconds(), default_config()); + bool first_new = + storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Initial report", + "/powertrain/motor1", timestamp1.nanoseconds(), default_config()); - bool is_new = + // A second source reporting the same code owns a NEW record, not an update of the first. + bool second_new = storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Second report", "/powertrain/motor2", timestamp2.nanoseconds(), default_config()); - EXPECT_FALSE(is_new); - EXPECT_EQ(storage_->size(), 1u); + EXPECT_TRUE(first_new); + EXPECT_TRUE(second_new); + EXPECT_EQ(storage_->size(), 2u); - auto fault = storage_->get_fault("MOTOR_OVERHEAT"); - ASSERT_TRUE(fault.has_value()); - // Still the same continuous occurrence (not CLEARED in between): occurrence_count - // does not bump on every report, only severity/sources/description update. - EXPECT_EQ(fault->occurrence_count, 1u); - EXPECT_EQ(fault->severity, Fault::SEVERITY_ERROR); // Updated to higher severity - EXPECT_EQ(fault->reporting_sources.size(), 2u); + auto by_code = storage_->get_faults_by_code("MOTOR_OVERHEAT"); + ASSERT_EQ(by_code.size(), 2u); + + auto first = storage_->get_fault(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}); + auto second = storage_->get_fault(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor2"}); + ASSERT_TRUE(first.has_value()); + ASSERT_TRUE(second.has_value()); + // No merge: each record carries only its own owner and its own severity. + EXPECT_EQ(first->occurrence_count, 1u); + EXPECT_EQ(second->occurrence_count, 1u); + ASSERT_EQ(first->reporting_sources.size(), 1u); + EXPECT_EQ(first->reporting_sources[0], "/powertrain/motor1"); + EXPECT_EQ(first->severity, Fault::SEVERITY_WARN); + EXPECT_EQ(second->severity, Fault::SEVERITY_ERROR); } TEST_F(PgFaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCount) { @@ -208,12 +219,126 @@ TEST_F(PgFaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCount) "/tank", clock.now().nanoseconds(), default_config()); } - auto fault = storage_->get_fault("TANK_OVERFILL"); + auto fault = storage_->get_fault(FaultId{"TANK_OVERFILL", "/tank"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } +// --- Record identity is (fault_code, owner) --- +// Two sources reporting one code own independent records. A single-record service driven by +// one owner never reaches another owner's record of the same code. + +TEST_F(PgFaultStorageTest, ClearingOneOwnerLeavesTheOtherOwnersRecord) { + rclcpp::Clock clock; + const FaultId a{"SHARED_CODE", "/node_a"}; + const FaultId b{"SHARED_CODE", "/node_b"}; + storage_->report_fault_event("SHARED_CODE", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + clock.now().nanoseconds(), default_config()); + storage_->report_fault_event("SHARED_CODE", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, + clock.now().nanoseconds(), default_config()); + ASSERT_EQ(storage_->size(), 2u); + + EXPECT_TRUE(storage_->clear_fault(a)); + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_CLEARED); + EXPECT_TRUE(storage_->contains(b)); + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->size(), 2u); +} + +TEST_F(PgFaultStorageTest, RecordsAreDistinctPerOwnerIncludingUnscoped) { + rclcpp::Clock clock; + const FaultId a{"DUP", "/owner_a"}; + const FaultId b{"DUP", "/owner_b"}; + const FaultId unscoped{"DUP", ""}; + storage_->report_fault_event("DUP", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "a", a.owner, + clock.now().nanoseconds(), default_config()); + storage_->report_fault_event("DUP", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, + clock.now().nanoseconds(), default_config()); + storage_->report_fault_event("DUP", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "u", unscoped.owner, + clock.now().nanoseconds(), default_config()); + + EXPECT_EQ(storage_->size(), 3u); + EXPECT_EQ(storage_->get_faults_by_code("DUP").size(), 3u); + EXPECT_EQ(storage_->get_fault(a)->severity, Fault::SEVERITY_WARN); + EXPECT_EQ(storage_->get_fault(b)->severity, Fault::SEVERITY_ERROR); + EXPECT_EQ(storage_->get_fault(unscoped)->severity, Fault::SEVERITY_CRITICAL); +} + +TEST_F(PgFaultStorageTest, DebounceCountersAreIndependentPerOwner) { + rclcpp::Clock clock; + DebounceConfig config; + config.confirmation_threshold = -3; // three of one owner's own events confirm + storage_->set_debounce_config(config); + const FaultId a{"GRIND", "/node_a"}; + const FaultId b{"GRIND", "/node_b"}; + + storage_->report_fault_event("GRIND", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + clock.now().nanoseconds(), config); + storage_->report_fault_event("GRIND", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_PREFAILED); + + storage_->report_fault_event("GRIND", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, + clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_PREFAILED); + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_PREFAILED); + + storage_->report_fault_event("GRIND", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_PREFAILED); +} + +TEST_F(PgFaultStorageTest, EvidenceAttachesToOneOwnerAndIsInvisibleUnderAnother) { + using ros2_medkit_fault_manager::FreezeFrameData; + using ros2_medkit_fault_manager::SnapshotData; + rclcpp::Clock clock; + const FaultId a{"EVID", "/node_a"}; + const FaultId b{"EVID", "/node_b"}; + storage_->report_fault_event("EVID", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + clock.now().nanoseconds(), default_config()); + storage_->report_fault_event("EVID", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, + clock.now().nanoseconds(), default_config()); + + SnapshotData snap; + snap.fault_code = a.fault_code; + snap.owner = a.owner; + snap.topic = "/t"; + snap.message_type = "std_msgs/msg/Float64"; + snap.data = R"({"data": 1.0})"; + snap.captured_at_ns = 1000; + snap.capture_id = 1; + storage_->store_snapshots({snap}); + + FreezeFrameData frame; + frame.fault_code = a.fault_code; + frame.owner = a.owner; + frame.data = R"({"/t": 1.0})"; + frame.captured_at_ns = 1000; + storage_->store_freeze_frame(frame); + + RosbagFileInfo bag; + bag.fault_code = a.fault_code; + bag.owner = a.owner; + bag.recording_id = "fault_EVID_1"; + bag.file_path = "/tmp/evid.mcap"; + bag.format = "mcap"; + bag.created_at_ns = 1000; + storage_->store_rosbag_files({bag}); + + EXPECT_EQ(storage_->get_snapshots(a).size(), 1u); + EXPECT_TRUE(storage_->get_snapshots(b).empty()); + EXPECT_TRUE(storage_->get_freeze_frame(a).has_value()); + EXPECT_FALSE(storage_->get_freeze_frame(b).has_value()); + EXPECT_TRUE(storage_->get_rosbag_file(a).has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(b).has_value()); + + EXPECT_TRUE(storage_->clear_fault(a)); + EXPECT_TRUE(storage_->get_snapshots(a).empty()); + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_CONFIRMED); +} + TEST_F(PgFaultStorageTest, ListFaultsDefaultReturnsConfirmedOnly) { rclcpp::Clock clock; auto timestamp = clock.now(); @@ -270,16 +395,16 @@ TEST_F(PgFaultStorageTest, ClearFault) { storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", timestamp.nanoseconds(), default_config()); - bool cleared = storage_->clear_fault("MOTOR_OVERHEAT"); + bool cleared = storage_->clear_fault(FaultId{"MOTOR_OVERHEAT", "/node1"}); EXPECT_TRUE(cleared); - auto fault = storage_->get_fault("MOTOR_OVERHEAT"); + auto fault = storage_->get_fault(FaultId{"MOTOR_OVERHEAT", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); } TEST_F(PgFaultStorageTest, ClearNonExistentFault) { - bool cleared = storage_->clear_fault("NON_EXISTENT"); + bool cleared = storage_->clear_fault(FaultId{"NON_EXISTENT", ""}); EXPECT_FALSE(cleared); } @@ -295,7 +420,7 @@ TEST_F(PgFaultStorageTest, PassedEventDoesNotAdvanceLastOccurred) { storage_->report_fault_event("FAULT_LO", fd::ReportKind::kPassed, 0, "", "/node1", passed_at.nanoseconds(), default_config()); - auto fault = storage_->get_fault("FAULT_LO"); + auto fault = storage_->get_fault(FaultId{"FAULT_LO", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // healing disabled: latched, by design EXPECT_EQ(fault->last_occurred_ns, failed_at.nanoseconds()); @@ -309,7 +434,7 @@ TEST_F(PgFaultStorageTest, GetClearedFaults) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", timestamp.nanoseconds(), default_config()); - storage_->clear_fault("FAULT_1"); + storage_->clear_fault(FaultId{"FAULT_1", "/node1"}); // Query cleared faults auto faults = storage_->list_faults(false, 0, {Fault::STATUS_CLEARED}); @@ -342,7 +467,7 @@ TEST_F(PgFaultStorageTest, PersistenceAcrossRestarts) { "/node1", timestamp.nanoseconds(), default_config()); storage_->report_fault_event("FAULT_2", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Persistent fault 2", "/node2", timestamp.nanoseconds(), default_config()); - storage_->clear_fault("FAULT_2"); + storage_->clear_fault(FaultId{"FAULT_2", "/node2"}); // Verify initial state EXPECT_EQ(storage_->size(), 2u); @@ -355,16 +480,16 @@ TEST_F(PgFaultStorageTest, PersistenceAcrossRestarts) { // Verify faults persisted EXPECT_EQ(storage_->size(), 2u); - EXPECT_TRUE(storage_->contains("FAULT_1")); - EXPECT_TRUE(storage_->contains("FAULT_2")); + EXPECT_TRUE(storage_->contains(FaultId{"FAULT_1", "/node1"})); + EXPECT_TRUE(storage_->contains(FaultId{"FAULT_2", "/node2"})); - auto fault1 = storage_->get_fault("FAULT_1"); + auto fault1 = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault1.has_value()); EXPECT_EQ(fault1->severity, Fault::SEVERITY_ERROR); EXPECT_EQ(fault1->status, Fault::STATUS_CONFIRMED); // Immediately confirmed with threshold=-1 EXPECT_EQ(fault1->description, "Persistent fault 1"); - auto fault2 = storage_->get_fault("FAULT_2"); + auto fault2 = storage_->get_fault(FaultId{"FAULT_2", "/node2"}); ASSERT_TRUE(fault2.has_value()); EXPECT_EQ(fault2->status, Fault::STATUS_CLEARED); } @@ -378,7 +503,7 @@ TEST_F(PgFaultStorageTest, TimestampPrecision) { storage_->report_fault_event("FAULT_TS", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Timestamp test", "/node1", timestamp.nanoseconds(), default_config()); - auto fault = storage_->get_fault("FAULT_TS"); + auto fault = storage_->get_fault(FaultId{"FAULT_TS", "/node1"}); ASSERT_TRUE(fault.has_value()); // Wrap the stored nanoseconds in rclcpp::Time for comparison @@ -1067,23 +1192,22 @@ TEST_F(PgFaultStorageTest, ReportingSourcesJsonHandling) { rclcpp::Clock clock; auto timestamp = clock.now(); - // Add multiple sources for the same fault - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", - "/node/path/with/slashes", timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", - "/another/node", timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", - "/special\"chars", timestamp.nanoseconds(), default_config()); - - auto fault = storage_->get_fault("MULTI_SRC"); - ASSERT_TRUE(fault.has_value()); - EXPECT_EQ(fault->reporting_sources.size(), 3u); + // Each source owns its own record of the same code. The owner round-trips through the JSON + // reporting_sources column, including slashes and an embedded quote. + const std::vector owners = {"/node/path/with/slashes", "/another/node", "/special\"chars"}; + for (const auto & owner : owners) { + storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", owner, + timestamp.nanoseconds(), default_config()); + } - // Verify all sources are present (order may vary due to set) - std::set sources(fault->reporting_sources.begin(), fault->reporting_sources.end()); - EXPECT_TRUE(sources.count("/node/path/with/slashes") > 0); - EXPECT_TRUE(sources.count("/another/node") > 0); - EXPECT_TRUE(sources.count("/special\"chars") > 0); + EXPECT_EQ(storage_->size(), owners.size()); + EXPECT_EQ(storage_->get_faults_by_code("MULTI_SRC").size(), owners.size()); + for (const auto & owner : owners) { + auto fault = storage_->get_fault(FaultId{"MULTI_SRC", owner}); + ASSERT_TRUE(fault.has_value()) << owner; + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources[0], owner); + } } // Test connection string accessor (equivalent of sqlite's DbPathAccessor). @@ -1126,10 +1250,10 @@ TEST_F(PgFaultStorageTest, FaultStaysPrefailedAboveThreshold) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Still debouncing towards confirmation, same occurrence EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); @@ -1145,12 +1269,12 @@ TEST_F(PgFaultStorageTest, FaultConfirmsAtThreshold) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Debounce build-up is still one occurrence EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -1166,7 +1290,7 @@ TEST_F(PgFaultStorageTest, ImmediateConfirmationWithThresholdZero) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -1179,7 +1303,7 @@ TEST_F(PgFaultStorageTest, CriticalSeverityBypassesDebounce) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "Critical test", "/node1", clock.now().nanoseconds(), default_config()); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -1189,35 +1313,37 @@ TEST_F(PgFaultStorageTest, ClearedFaultCanBeReactivated) { rclcpp::Clock clock; // Report to confirm (with default threshold=-1, single report confirms) + const FaultId id{"FAULT_1", "/node1"}; auto first_ts = clock.now(); bool is_new = storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Initial", "/node1", first_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(id); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->first_occurred_ns, first_ts.nanoseconds()); // Clear the fault - storage_->clear_fault("FAULT_1"); - fault = storage_->get_fault("FAULT_1"); + storage_->clear_fault(id); + fault = storage_->get_fault(id); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); - // Report again after a gap - should reactivate as a new cycle + // The SAME owner re-raising after a clear reopens its record as a new cycle. rclcpp::Time second_ts(first_ts.nanoseconds() + 1'000'000'000LL); // +1s is_new = storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Reactivated", - "/node2", second_ts.nanoseconds(), default_config()); - EXPECT_TRUE(is_new); // Should return true like a new fault + "/node1", second_ts.nanoseconds(), default_config()); + EXPECT_TRUE(is_new); // Reactivation returns true like a new fault - fault = storage_->get_fault("FAULT_1"); + fault = storage_->get_fault(id); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Should be reconfirmed EXPECT_EQ(fault->occurrence_count, 2u); // Should increment - EXPECT_EQ(fault->reporting_sources.size(), 2u); // Both sources - EXPECT_EQ(fault->description, "Reactivated"); // Updated description + ASSERT_EQ(fault->reporting_sources.size(), 1u); // One record, one owner + EXPECT_EQ(fault->reporting_sources[0], "/node1"); + EXPECT_EQ(fault->description, "Reactivated"); // Updated description // #25: first_occurred must reflect the new cycle, not the outage that already cleared. EXPECT_EQ(fault->first_occurred_ns, second_ts.nanoseconds()); } @@ -1230,14 +1356,14 @@ TEST_F(PgFaultStorageTest, PassedEventForClearedFaultIgnored) { clock.now().nanoseconds(), default_config()); // Clear the fault - storage_->clear_fault("FAULT_1"); + storage_->clear_fault(FaultId{"FAULT_1", "/node1"}); // PASSED event should be ignored for CLEARED fault bool result = storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), default_config()); EXPECT_FALSE(result); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); // Should stay cleared } @@ -1253,33 +1379,33 @@ TEST_F(PgFaultStorageTest, ClearedFaultReactivationRestartsDebounce) { // Report 3 times to confirm storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Clear the fault - storage_->clear_fault("FAULT_1"); + storage_->clear_fault(FaultId{"FAULT_1", "/node1"}); // Reactivate - should start in PREFAILED with counter=-1 - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node4", + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - fault = storage_->get_fault("FAULT_1"); + fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Not yet confirmed, needs 2 more FAILED // Report 2 more times to re-confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node5", + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node6", + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - fault = storage_->get_fault("FAULT_1"); + fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Now confirmed } @@ -1290,9 +1416,9 @@ TEST_F(PgFaultStorageTest, ConfirmationPersistsAfterReopen) { // Report 3 times to confirm storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), default_config()); // Close and reopen storage @@ -1300,7 +1426,7 @@ TEST_F(PgFaultStorageTest, ConfirmationPersistsAfterReopen) { storage_ = std::make_unique(conn_info_, test_log_sink()); // Verify status persisted - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -1315,7 +1441,7 @@ TEST_F(PgFaultStorageTest, PassedEventIncrementsCounter) { // Report 2 FAILED events (counter -2, PREFAILED) storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", + storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); // Report 3 PASSED events (counter -2 -> +1) @@ -1324,7 +1450,7 @@ TEST_F(PgFaultStorageTest, PassedEventIncrementsCounter) { config); } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREPASSED); // Counter > 0, never confirmed so not latched } @@ -1342,14 +1468,14 @@ TEST_F(PgFaultStorageTest, HeartbeatHealClampedAndStatusLatched) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); // Long heal heartbeat: counter clamps at healing_threshold instead of running off. for (int i = 0; i < 100; ++i) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); } - auto healed = storage_->get_fault("FAULT_1"); + auto healed = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(healed.has_value()); EXPECT_EQ(healed->status, Fault::STATUS_HEALED); @@ -1358,11 +1484,12 @@ TEST_F(PgFaultStorageTest, HeartbeatHealClampedAndStatusLatched) { for (int i = 0; i < 3; ++i) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); + EXPECT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_HEALED) + << "latch broke after " << (i + 1); } storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - auto reconfirmed = storage_->get_fault("FAULT_1"); + auto reconfirmed = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(reconfirmed.has_value()); EXPECT_EQ(reconfirmed->status, Fault::STATUS_CONFIRMED); } @@ -1378,19 +1505,20 @@ TEST_F(PgFaultStorageTest, ConfirmedFaultSurvivesHealHeartbeat) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 20; ++i) { storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED) << "un-confirmed after " << (i + 1); + EXPECT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED) + << "un-confirmed after " << (i + 1); } // The counter is now positive (clamped at +3). One FAILED must keep it CONFIRMED, not flip it to // PREPASSED via the `counter > 0` branch - that was the asymmetric-latch bug. storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); } // Latch must hold in BOTH directions and identically on all backends (regression for the @@ -1405,16 +1533,16 @@ TEST_F(PgFaultStorageTest, ConfirmedLatchSurvivesFailedAtPositiveCounter) { // CRITICAL confirms immediately at counter -1. storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); // PASSED events raise the counter into positive territory; latch keeps it CONFIRMED. for (int i = 0; i < 3; ++i) { storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); // One normal FAILED: counter still positive, must NOT become PREPASSED. storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); } // A HEALED fault with a negative counter stays HEALED on PASSED (opposite direction of the bug above). @@ -1430,16 +1558,16 @@ TEST_F(PgFaultStorageTest, HealedLatchSurvivesPassedAtNegativeCounter) { for (int i = 0; i < 4; ++i) { // counter -1 -> +3 -> HEALED storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); // FAILED events drive the counter negative while HEALED is latched (not yet at confirmation -3). for (int i = 0; i < 5; ++i) { // +3 -> -2 storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); + ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); } // One PASSED: counter goes -2 -> -1, still negative; must stay HEALED, not flip to PREFAILED. storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); } TEST_F(PgFaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePassed) { @@ -1455,19 +1583,19 @@ TEST_F(PgFaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePassed) { for (int i = 0; i < 4; ++i) { // -1 -> +3 -> HEALED storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); // CRITICAL confirms immediately from HEALED; counter must not stay at +2 where one PASSED heals it. storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { // -1 -> +2, still CONFIRMED storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); } storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); } // A database written by an older build can hold a runaway counter. It must be clamped back on first @@ -1489,7 +1617,7 @@ TEST_F(PgFaultStorageTest, RunawayCounterFromOldRowRecovers) { storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); } - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); } TEST_F(PgFaultStorageTest, ReclassifyHealedAsCleared) { @@ -1503,12 +1631,12 @@ TEST_F(PgFaultStorageTest, ReclassifyHealedAsCleared) { for (int i = 0; i < 4; ++i) { storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); const auto reclassified = storage_->reclassify_healed_as_cleared(); ASSERT_EQ(reclassified.size(), 1u); - EXPECT_EQ(reclassified[0], "F"); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(reclassified[0].fault_code, "F"); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CLEARED); EXPECT_TRUE(storage_->reclassify_healed_as_cleared().empty()); } @@ -1524,16 +1652,16 @@ TEST_F(PgFaultStorageTest, ReclassifyKeepsCodesTheCallerKeepsHealed) { for (int i = 0; i < 4; ++i) { storage_->report_fault_event(code, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault(code)->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault(FaultId{code, "/n"})->status, Fault::STATUS_HEALED); } const auto reclassified = storage_->reclassify_healed_as_cleared([](const std::string & code) { return code == "KEEP"; }); ASSERT_EQ(reclassified.size(), 1u); - EXPECT_EQ(reclassified[0], "FLIP"); - EXPECT_EQ(storage_->get_fault("KEEP")->status, Fault::STATUS_HEALED); - EXPECT_EQ(storage_->get_fault("FLIP")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(reclassified[0].fault_code, "FLIP"); + EXPECT_EQ(storage_->get_fault(FaultId{"KEEP", "/n"})->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_->get_fault(FaultId{"FLIP", "/n"})->status, Fault::STATUS_CLEARED); } TEST_F(PgFaultStorageTest, HealingWhenEnabled) { @@ -1553,7 +1681,7 @@ TEST_F(PgFaultStorageTest, HealingWhenEnabled) { config); } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); } @@ -1578,9 +1706,9 @@ TEST_F(PgFaultStorageTest, TimeBasedConfirmationWhenEnabled) { auto after_timeout = rclcpp::Time(now.nanoseconds() + static_cast(15e9)); auto confirmed = storage_->check_time_based_confirmation(after_timeout.nanoseconds()); ASSERT_EQ(confirmed.size(), 1u); - EXPECT_EQ(confirmed[0], "FAULT_1"); + EXPECT_EQ(confirmed[0].fault_code, "FAULT_1"); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -1604,9 +1732,9 @@ TEST_F(PgFaultStorageTest, TimeBasedConfirmationConfirmsOnlyReturnedFaults) { auto after_timeout = rclcpp::Time(now.nanoseconds() + static_cast(15e9)); auto confirmed = storage_->check_time_based_confirmation(after_timeout.nanoseconds()); ASSERT_EQ(confirmed.size(), 1u); - EXPECT_EQ(confirmed[0], "TIMED"); + EXPECT_EQ(confirmed[0].fault_code, "TIMED"); - auto skipped = storage_->get_fault("NO_FAILURE_TIME"); + auto skipped = storage_->get_fault(FaultId{"NO_FAILURE_TIME", "/node1"}); ASSERT_TRUE(skipped.has_value()); EXPECT_EQ(skipped->status, Fault::STATUS_PREFAILED); } @@ -1617,7 +1745,7 @@ TEST_F(PgFaultStorageTest, ConfirmedAtRecordedOnImmediateConfirmation) { // Default config confirms on the first FAILED event. storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t.nanoseconds(), default_config()); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(read_confirmed_at("F"), t.nanoseconds()); // A later FAILED on an already-confirmed fault must NOT move the timestamp. @@ -1635,13 +1763,13 @@ TEST_F(PgFaultStorageTest, ConfirmedAtRecordedOnDebouncedConfirmation) { auto t1 = clock.now(); storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t1.nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_PREFAILED); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_PREFAILED); EXPECT_EQ(read_confirmed_at("F"), 0) << "not confirmed yet: no confirmation timestamp"; auto t2 = rclcpp::Time(t1.nanoseconds() + static_cast(1e9)); storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t2.nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(read_confirmed_at("F"), t2.nanoseconds()); } @@ -1669,7 +1797,7 @@ TEST_F(PgFaultStorageTest, ConfirmedAtDefaultsToZeroForExternallyInsertedRow) { "occurrence_count, status, reporting_sources) " "VALUES ('OLD', 2, 'd', 1, 1, 1, 'CONFIRMED', '[\"/n\"]')"); - EXPECT_TRUE(storage_->contains("OLD")); + EXPECT_TRUE(storage_->contains(FaultId{"OLD", ""})); EXPECT_EQ(read_confirmed_at("OLD"), 0) << "rows carrying no confirmation time default to 0"; rclcpp::Clock clock; @@ -1701,13 +1829,13 @@ TEST_F(PgFaultStorageTest, OneFaultKeepsSeveralRecordingsWhenTheCapAllows) { storage_->store_rosbag_file(make_rosbag("FLAP", "/bags/fault_FLAP_100", 100)); storage_->store_rosbag_file(make_rosbag("FLAP", "/bags/fault_FLAP_200", 200)); - const auto rows = storage_->get_rosbag_files("FLAP"); + const auto rows = storage_->get_rosbag_files(FaultId{"FLAP", ""}); ASSERT_EQ(rows.size(), 2u) << "the second recording must not replace the first"; EXPECT_EQ(rows[0].recording_id, "fault_FLAP_200") << "newest first"; EXPECT_EQ(rows[1].recording_id, "fault_FLAP_100"); // get_rosbag_file is "the newest", deterministically. - const auto newest = storage_->get_rosbag_file("FLAP"); + const auto newest = storage_->get_rosbag_file(FaultId{"FLAP", ""}); ASSERT_TRUE(newest.has_value()); EXPECT_EQ(newest->recording_id, "fault_FLAP_200"); } @@ -1722,7 +1850,7 @@ TEST_F(PgFaultStorageTest, CapKeepsTheNewestAndUnlinksTheEvictedBag) { storage_->store_rosbag_file(make_rosbag("CAP", dir_a.string(), 100)); storage_->store_rosbag_file(make_rosbag("CAP", dir_b.string(), 200)); - const auto rows = storage_->get_rosbag_files("CAP"); + const auto rows = storage_->get_rosbag_files(FaultId{"CAP", ""}); ASSERT_EQ(rows.size(), 1u) << "cap 1 is the historical behaviour: one recording per fault"; EXPECT_EQ(rows[0].file_path, dir_b.string()); EXPECT_FALSE(std::filesystem::exists(dir_a)) << "the evicted bag must be unlinked"; @@ -1743,7 +1871,7 @@ TEST_F(PgFaultStorageTest, EvictingOneFaultsLinkKeepsABagASiblingStillReferences storage_->store_rosbag_file(make_rosbag("A", later.string(), 200)); EXPECT_TRUE(std::filesystem::exists(shared)) << "B still references it"; - const auto b_rows = storage_->get_rosbag_files("B"); + const auto b_rows = storage_->get_rosbag_files(FaultId{"B", ""}); ASSERT_EQ(b_rows.size(), 1u); EXPECT_EQ(b_rows[0].file_path, shared.string()); } @@ -1765,8 +1893,8 @@ TEST_F(PgFaultStorageTest, DeleteRecordingRemovesEveryLinkAndTheBag) { storage_->store_rosbag_files({make_rosbag("A", dir.string(), 100), make_rosbag("B", dir.string(), 100)}); EXPECT_EQ(storage_->delete_rosbag_recording("fault_A_100"), 2u); - EXPECT_TRUE(storage_->get_rosbag_files("A").empty()); - EXPECT_TRUE(storage_->get_rosbag_files("B").empty()); + EXPECT_TRUE(storage_->get_rosbag_files(FaultId{"A", ""}).empty()); + EXPECT_TRUE(storage_->get_rosbag_files(FaultId{"B", ""}).empty()); EXPECT_FALSE(std::filesystem::exists(dir)); } @@ -1775,8 +1903,8 @@ TEST_F(PgFaultStorageTest, DeletingAFaultDropsAllItsRecordings) { storage_->store_rosbag_file(make_rosbag("A", "/bags/fault_A_100", 100)); storage_->store_rosbag_file(make_rosbag("A", "/bags/fault_A_200", 200)); - EXPECT_TRUE(storage_->delete_rosbag_file("A")); - EXPECT_TRUE(storage_->get_rosbag_files("A").empty()) << "auto_cleanup drops the fault's whole history"; + EXPECT_TRUE(storage_->delete_rosbag_file(FaultId{"A", ""})); + EXPECT_TRUE(storage_->get_rosbag_files(FaultId{"A", ""}).empty()) << "auto_cleanup drops the fault's whole history"; } TEST_F(PgFaultStorageTest, SharedRecordingStillCountsOnceTowardsStorageWithSeveralRecordings) { @@ -1809,7 +1937,7 @@ TEST_F(PgFaultStorageTest, StoreAndRetrieveSnapshot) { storage_->store_snapshot(snapshot); // Retrieve snapshots - auto snapshots = storage_->get_snapshots("MOTOR_OVERHEAT"); + auto snapshots = storage_->get_snapshots(FaultId{"MOTOR_OVERHEAT", ""}); ASSERT_EQ(snapshots.size(), 1u); EXPECT_EQ(snapshots[0].fault_code, "MOTOR_OVERHEAT"); @@ -1845,7 +1973,7 @@ TEST_F(PgFaultStorageTest, MultipleSnapshotsForSameFault) { storage_->store_snapshot(snapshot1); storage_->store_snapshot(snapshot2); - auto snapshots = storage_->get_snapshots("MOTOR_OVERHEAT"); + auto snapshots = storage_->get_snapshots(FaultId{"MOTOR_OVERHEAT", ""}); EXPECT_EQ(snapshots.size(), 2u); } @@ -1875,14 +2003,14 @@ TEST_F(PgFaultStorageTest, FilterSnapshotsByTopic) { storage_->store_snapshot(snapshot2); // Filter by topic - auto filtered = storage_->get_snapshots("MOTOR_OVERHEAT", "/motor/temperature"); + auto filtered = storage_->get_snapshots(FaultId{"MOTOR_OVERHEAT", ""}, "/motor/temperature"); ASSERT_EQ(filtered.size(), 1u); EXPECT_EQ(filtered[0].topic, "/motor/temperature"); } // @verifies REQ_INTEROP_088 TEST_F(PgFaultStorageTest, NoSnapshotsForUnknownFault) { - auto snapshots = storage_->get_snapshots("UNKNOWN_FAULT"); + auto snapshots = storage_->get_snapshots(FaultId{"UNKNOWN_FAULT", ""}); EXPECT_TRUE(snapshots.empty()); } @@ -1893,8 +2021,7 @@ TEST_F(PgFaultStorageTest, ClearFaultDeletesAssociatedSnapshots) { // Create a fault using report_fault_event storage_->report_fault_event("SNAPSHOT_CLEAR_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "Test fault for snapshot cleanup", "/test_node", clock.now().nanoseconds(), - default_config()); + "Test fault for snapshot cleanup", "", clock.now().nanoseconds(), default_config()); // Store snapshots for this fault SnapshotData snapshot1; @@ -1914,15 +2041,15 @@ TEST_F(PgFaultStorageTest, ClearFaultDeletesAssociatedSnapshots) { storage_->store_snapshot(snapshot2); // Verify snapshots exist - auto snapshots_before = storage_->get_snapshots("SNAPSHOT_CLEAR_TEST"); + auto snapshots_before = storage_->get_snapshots(FaultId{"SNAPSHOT_CLEAR_TEST", ""}); ASSERT_EQ(snapshots_before.size(), 2u); // Clear the fault - bool cleared = storage_->clear_fault("SNAPSHOT_CLEAR_TEST"); + bool cleared = storage_->clear_fault(FaultId{"SNAPSHOT_CLEAR_TEST", ""}); EXPECT_TRUE(cleared); // Verify snapshots are deleted - auto snapshots_after = storage_->get_snapshots("SNAPSHOT_CLEAR_TEST"); + auto snapshots_after = storage_->get_snapshots(FaultId{"SNAPSHOT_CLEAR_TEST", ""}); EXPECT_TRUE(snapshots_after.empty()); } @@ -1932,7 +2059,7 @@ TEST_F(PgFaultStorageTest, ClearFaultKeepsSnapshotsWhenEvidenceIsRetained) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("KEEP", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "keep", "/n", + storage_->report_fault_event("KEEP", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "keep", "", clock.now().nanoseconds(), default_config()); storage_->set_retain_snapshots_on_clear(true); EXPECT_TRUE(storage_->retains_snapshots_on_clear()); @@ -1946,20 +2073,20 @@ TEST_F(PgFaultStorageTest, ClearFaultKeepsSnapshotsWhenEvidenceIsRetained) { row.capture_id = 1; storage_->store_snapshots({row}); - ASSERT_TRUE(storage_->clear_fault("KEEP")); + ASSERT_TRUE(storage_->clear_fault(FaultId{"KEEP", ""})); // Recordings survive an acknowledgement once a history is configured, so the // readings captured beside them have to as well - otherwise the fault is left // holding bags whose values are gone, which is worse than losing both. - EXPECT_EQ(storage_->get_snapshots("KEEP").size(), 1u); + EXPECT_EQ(storage_->get_snapshots(FaultId{"KEEP", ""}).size(), 1u); } TEST_F(PgFaultStorageTest, StoreAndRetrieveFreezeFrame) { using ros2_medkit_fault_manager::FreezeFrameData; rclcpp::Clock clock; - storage_->report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", - "/plc_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", "", + clock.now().nanoseconds(), default_config()); FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; @@ -1967,7 +2094,7 @@ TEST_F(PgFaultStorageTest, StoreAndRetrieveFreezeFrame) { frame.captured_at_ns = clock.now().nanoseconds(); storage_->store_freeze_frame(frame); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage_->get_freeze_frame(FaultId{"PLC_PRESSURE_HIGH", ""}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->fault_code, "PLC_PRESSURE_HIGH"); EXPECT_EQ(retrieved->data, frame.data); @@ -1976,7 +2103,7 @@ TEST_F(PgFaultStorageTest, StoreAndRetrieveFreezeFrame) { // @verifies REQ_INTEROP_088 TEST_F(PgFaultStorageTest, NoFreezeFrameForUnknownFault) { - auto retrieved = storage_->get_freeze_frame("NEVER_CAPTURED"); + auto retrieved = storage_->get_freeze_frame(FaultId{"NEVER_CAPTURED", ""}); EXPECT_FALSE(retrieved.has_value()); } @@ -1996,7 +2123,7 @@ TEST_F(PgFaultStorageTest, FreezeFrameReplacedOnRecapture) { second.captured_at_ns = 2000; storage_->store_freeze_frame(second); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage_->get_freeze_frame(FaultId{"PLC_PRESSURE_HIGH", ""}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, second.data); EXPECT_EQ(retrieved->captured_at_ns, 2000); @@ -2008,8 +2135,8 @@ TEST_F(PgFaultStorageTest, FreezeFrameSurvivesClearFault) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", - "/plc_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", "", + clock.now().nanoseconds(), default_config()); // A per-topic snapshot (removed on clear) plus a freeze-frame (retained on clear). SnapshotData snapshot; @@ -2026,11 +2153,11 @@ TEST_F(PgFaultStorageTest, FreezeFrameSurvivesClearFault) { frame.captured_at_ns = clock.now().nanoseconds(); storage_->store_freeze_frame(frame); - ASSERT_TRUE(storage_->clear_fault("PLC_PRESSURE_HIGH")); + ASSERT_TRUE(storage_->clear_fault(FaultId{"PLC_PRESSURE_HIGH", ""})); // Snapshots are wiped on clear, the freeze-frame is retained and still retrievable. - EXPECT_TRUE(storage_->get_snapshots("PLC_PRESSURE_HIGH").empty()); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + EXPECT_TRUE(storage_->get_snapshots(FaultId{"PLC_PRESSURE_HIGH", ""}).empty()); + auto retrieved = storage_->get_freeze_frame(FaultId{"PLC_PRESSURE_HIGH", ""}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, frame.data); } @@ -2049,7 +2176,7 @@ TEST_F(PgFaultStorageTest, FreezeFramePersistsAcrossReopen) { storage_.reset(); storage_ = std::make_unique(conn_info_, test_log_sink()); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage_->get_freeze_frame(FaultId{"PLC_PRESSURE_HIGH", ""}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, frame.data); EXPECT_EQ(retrieved->captured_at_ns, 4242); @@ -2074,6 +2201,7 @@ TEST_F(PgFaultStorageTest, ListRosbagsForEntityFiltersCorrectly) { // Store rosbags for both faults RosbagFileInfo info1; info1.fault_code = "ENTITY_FAULT_1"; + info1.owner = "/powertrain/motor"; info1.file_path = "/tmp/entity1.mcap"; info1.format = "mcap"; info1.duration_sec = 5.0; @@ -2083,6 +2211,7 @@ TEST_F(PgFaultStorageTest, ListRosbagsForEntityFiltersCorrectly) { RosbagFileInfo info2; info2.fault_code = "ENTITY_FAULT_2"; + info2.owner = "/chassis/brake"; info2.file_path = "/tmp/entity2.mcap"; info2.format = "mcap"; info2.duration_sec = 3.0; @@ -2128,11 +2257,12 @@ TEST_F(PgFaultStorageTest, AFailingRowDeleteKeepsTheBagOnDisk) { "CREATE TRIGGER block_rosbag_delete BEFORE DELETE ON rosbag_files " "FOR EACH ROW EXECUTE PROCEDURE block_rosbag_delete()"); - EXPECT_THROW(storage_->delete_rosbag_file("DELETE_FAILS"), std::runtime_error); + EXPECT_THROW(storage_->delete_rosbag_file(FaultId{"DELETE_FAILS", ""}), std::runtime_error); exec_raw("DROP TRIGGER block_rosbag_delete ON rosbag_files"); - EXPECT_TRUE(storage_->get_rosbag_file("DELETE_FAILS").has_value()) << "the DELETE failed, so its row must remain"; + EXPECT_TRUE(storage_->get_rosbag_file(FaultId{"DELETE_FAILS", ""}).has_value()) + << "the DELETE failed, so its row must remain"; EXPECT_TRUE(std::filesystem::exists(bag_dir)) << "the bag was unlinked before its row was deleted, so the surviving row now points at nothing"; } @@ -2189,11 +2319,11 @@ TEST_F(PgFaultStorageTest, SharedRosbagSurvivesUntilTheLastFaultIsDeleted) { info.fault_code = "CORRELATED"; storage_->store_rosbag_file(info); - EXPECT_TRUE(storage_->delete_rosbag_file("CORRELATED")); + EXPECT_TRUE(storage_->delete_rosbag_file(FaultId{"CORRELATED", ""})); EXPECT_TRUE(std::filesystem::exists(bag_path)); - EXPECT_TRUE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); + EXPECT_TRUE(storage_->get_rosbag_file(FaultId{"ROOT_CAUSE", ""}).has_value()); - EXPECT_TRUE(storage_->delete_rosbag_file("ROOT_CAUSE")); + EXPECT_TRUE(storage_->delete_rosbag_file(FaultId{"ROOT_CAUSE", ""})); EXPECT_FALSE(std::filesystem::exists(bag_path)); } @@ -2246,7 +2376,7 @@ TEST_F(PgFaultStorageTest, RestoreWithNewPathUnlinksTheOldExclusiveBag) { storage_->store_rosbag_file(info); EXPECT_FALSE(std::filesystem::exists(old_path)) << "nobody references the old bag, it must be unlinked"; - auto row = storage_->get_rosbag_file("X"); + auto row = storage_->get_rosbag_file(FaultId{"X", ""}); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->file_path, old_path + "_new"); } @@ -2273,7 +2403,7 @@ TEST_F(PgFaultStorageTest, RestoreWithNewPathKeepsTheBagASiblingStillReferences) storage_->store_rosbag_file(info); EXPECT_TRUE(std::filesystem::exists(shared_path)) << "the sibling fault still owns the shared bag"; - auto sibling = storage_->get_rosbag_file("Y"); + auto sibling = storage_->get_rosbag_file(FaultId{"Y", ""}); ASSERT_TRUE(sibling.has_value()); EXPECT_EQ(sibling->file_path, shared_path); } @@ -2298,7 +2428,7 @@ TEST_F(PgFaultStorageTest, BulkStoreRegistersEveryFaultOfTheBurst) { storage_->store_rosbag_files(rows); for (const char * code : {"ROOT_CAUSE", "CORRELATED_A", "CORRELATED_B"}) { - auto row = storage_->get_rosbag_file(code); + auto row = storage_->get_rosbag_file(FaultId{code, ""}); ASSERT_TRUE(row.has_value()) << code; EXPECT_EQ(row->file_path, "/tmp/burst_bag"); } @@ -2325,13 +2455,15 @@ TEST_F(PgFaultStorageTest, BulkDeleteRemovesTheBurstAndUnlinksTheBagOnce) { storage_->store_rosbag_files(rows); // A partial delete leaves the bag on disk for the remaining fault. - EXPECT_EQ(storage_->delete_rosbag_files({"CORRELATED_A", "CORRELATED_B", "NEVER_STORED"}), 2u); + EXPECT_EQ(storage_->delete_rosbag_files( + {FaultId{"CORRELATED_A", ""}, FaultId{"CORRELATED_B", ""}, FaultId{"NEVER_STORED", ""}}), + 2u); EXPECT_TRUE(std::filesystem::exists(bag_path)); - EXPECT_TRUE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); - EXPECT_FALSE(storage_->get_rosbag_file("CORRELATED_A").has_value()); + EXPECT_TRUE(storage_->get_rosbag_file(FaultId{"ROOT_CAUSE", ""}).has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(FaultId{"CORRELATED_A", ""}).has_value()); // The last reference going away unlinks the directory. - EXPECT_EQ(storage_->delete_rosbag_files({"ROOT_CAUSE"}), 1u); + EXPECT_EQ(storage_->delete_rosbag_files({FaultId{"ROOT_CAUSE", ""}}), 1u); EXPECT_FALSE(std::filesystem::exists(bag_path)); EXPECT_EQ(storage_->get_total_rosbag_storage_bytes(), 0u); } @@ -2369,7 +2501,7 @@ TEST_F(PgFaultStorageTest, SnapshotCapKeepsTheNewestCaptureWholeAndDropsTheOldes storage_->store_snapshots(capture(2, 2000)); storage_->store_snapshots(capture(3, 3000)); - auto snapshots = storage_->get_snapshots("MOTOR_OVERHEAT"); + auto snapshots = storage_->get_snapshots(FaultId{"MOTOR_OVERHEAT", ""}); // The third capture is stored WHOLE and the first goes whole. The old rule // counted rows and rejected the new one once full, so capture 3 landed with one @@ -2415,7 +2547,7 @@ TEST_F(PgFaultStorageTest, ACaptureLargerThanTheCapIsKeptWholeRatherThanTorn) { // The cap is smaller than this fault's topic count. Trimming to it would mean // storing the reading with holes, which is the failure being fixed; the capture // stays whole and the operator can see the cap is too small. - EXPECT_EQ(storage_->get_snapshots("WIDE").size(), 4u); + EXPECT_EQ(storage_->get_snapshots(FaultId{"WIDE", ""}).size(), 4u); } TEST_F(PgFaultStorageTest, SnapshotLimitZeroMeansUnlimited) { @@ -2438,7 +2570,7 @@ TEST_F(PgFaultStorageTest, SnapshotLimitZeroMeansUnlimited) { storage_->store_snapshot(snap); } - auto snapshots = storage_->get_snapshots("FAULT_A"); + auto snapshots = storage_->get_snapshots(FaultId{"FAULT_A", ""}); EXPECT_EQ(snapshots.size(), 20u) << "Unlimited mode should store all snapshots"; } @@ -2467,8 +2599,8 @@ TEST_F(PgFaultStorageTest, SnapshotLimitPerFaultNotGlobal) { storage_->store_snapshot(snap_b); // Both faults should have 1 snapshot each (limit is per-fault) - EXPECT_EQ(storage_->get_snapshots("FAULT_A").size(), 1u); - EXPECT_EQ(storage_->get_snapshots("FAULT_B").size(), 1u); + EXPECT_EQ(storage_->get_snapshots(FaultId{"FAULT_A", ""}).size(), 1u); + EXPECT_EQ(storage_->get_snapshots(FaultId{"FAULT_B", ""}).size(), 1u); } // --- Near-miss series --- @@ -2499,11 +2631,11 @@ TEST_F(PgFaultStorageTest, NearMissSeriesIsAppendedNotOverwritten) { "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto fault = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_NE(fault->status, Fault::STATUS_CONFIRMED) << "test setup: these reports must not confirm"; - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "each near miss must append an entry, not overwrite the last"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[1].debounce_counter, -2); @@ -2524,12 +2656,12 @@ TEST_F(PgFaultStorageTest, ConfirmingReportIsNotANearMiss) { "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto fault = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); // The fourth report is the fault happening, not nearly happening. - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); } TEST_F(PgFaultStorageTest, NearMissSeriesSurvivesClearFault) { @@ -2539,11 +2671,11 @@ TEST_F(PgFaultStorageTest, NearMissSeriesSurvivesClearFault) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); - ASSERT_TRUE(storage_->clear_fault("PUMP_PRESSURE_LOW")); + ASSERT_TRUE(storage_->clear_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "acknowledging the fault destroyed the near-miss record"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[2].debounce_counter, -3); @@ -2556,13 +2688,13 @@ TEST_F(PgFaultStorageTest, NearMissSeriesContinuesAcrossReactivation) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - ASSERT_TRUE(storage_->clear_fault("PUMP_PRESSURE_LOW")); + ASSERT_TRUE(storage_->clear_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); // A new outage cycle starts: the reactivating report resets the counter to -1 without confirming. storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping again", "/hydraulics/pump", nth_report_time(10).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 4u) << "the series must span fault cycles, one entry per occurrence"; EXPECT_EQ(series[3].debounce_counter, -1); EXPECT_EQ(series[3].occurred_at_ns, nth_report_time(10).nanoseconds()); @@ -2575,12 +2707,12 @@ TEST_F(PgFaultStorageTest, NearMissSeriesSurvivesReopen) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - ASSERT_TRUE(storage_->clear_fault("PUMP_PRESSURE_LOW")); + ASSERT_TRUE(storage_->clear_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); storage_.reset(); storage_ = std::make_unique(conn_info_, test_log_sink()); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "the series must outlive the process, not just the fault cycle"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[2].debounce_counter, -3); @@ -2593,13 +2725,13 @@ TEST_F(PgFaultStorageTest, PassedReportIsNotANearMiss) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); + ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); // A PASSED report moves the counter in the healing direction: the fault receding, not nearing. storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); } TEST_F(PgFaultStorageTest, CriticalImmediateConfirmIsNotANearMiss) { @@ -2609,10 +2741,10 @@ TEST_F(PgFaultStorageTest, CriticalImmediateConfirmIsNotANearMiss) { storage_->report_fault_event("BATTERY_THERMAL_RUNAWAY", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "cell over temperature", "/power/bms", nth_report_time(0).nanoseconds(), config); - auto fault = storage_->get_fault("BATTERY_THERMAL_RUNAWAY"); + auto fault = storage_->get_fault(FaultId{"BATTERY_THERMAL_RUNAWAY", "/power/bms"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); - EXPECT_TRUE(storage_->get_near_misses("BATTERY_THERMAL_RUNAWAY").empty()); + EXPECT_TRUE(storage_->get_near_misses(FaultId{"BATTERY_THERMAL_RUNAWAY", "/power/bms"}).empty()); } TEST_F(PgFaultStorageTest, ImmediateConfirmThresholdRecordsNoNearMiss) { @@ -2623,10 +2755,10 @@ TEST_F(PgFaultStorageTest, ImmediateConfirmThresholdRecordsNoNearMiss) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure lost", "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); - auto fault = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); - EXPECT_TRUE(storage_->get_near_misses("PUMP_PRESSURE_LOW").empty()); + EXPECT_TRUE(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).empty()); } TEST_F(PgFaultStorageTest, FailedReportUnderHealedLatchIsNearMiss) { @@ -2636,13 +2768,13 @@ TEST_F(PgFaultStorageTest, FailedReportUnderHealedLatchIsNearMiss) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); for (int i = 1; i <= 2; ++i) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto healed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; @@ -2650,7 +2782,7 @@ TEST_F(PgFaultStorageTest, FailedReportUnderHealedLatchIsNearMiss) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(3).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[1].debounce_counter, 0); } @@ -2665,7 +2797,7 @@ TEST_F(PgFaultStorageTest, NearMissSeriesBoundedKeepingNewest) { "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u); // Oldest-first eviction: a series frozen at boot would say nothing about a trend. EXPECT_EQ(series[0].debounce_counter, -3); @@ -2683,7 +2815,7 @@ TEST_F(PgFaultStorageTest, NearMissBoundOfOneKeepsLatestOnly) { "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 1u); EXPECT_EQ(series[0].debounce_counter, -4); } @@ -2700,8 +2832,8 @@ TEST_F(PgFaultStorageTest, NearMissBoundIsPerFaultCode) { "/powertrain/motor", nth_report_time(i).nanoseconds(), config); } - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); - EXPECT_EQ(storage_->get_near_misses("MOTOR_OVERHEAT").size(), 2u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}).size(), 2u); } TEST_F(PgFaultStorageTest, NearMissBoundZeroIsUnlimited) { @@ -2714,7 +2846,7 @@ TEST_F(PgFaultStorageTest, NearMissBoundZeroIsUnlimited) { "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 150u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 150u); } TEST_F(PgFaultStorageTest, NearMissSeriesSurvivesHealedReclassification) { @@ -2730,14 +2862,14 @@ TEST_F(PgFaultStorageTest, NearMissSeriesSurvivesHealedReclassification) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto healed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); ASSERT_EQ(storage_->reclassify_healed_as_cleared().size(), 1u); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); } TEST_F(PgFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { @@ -2755,7 +2887,7 @@ TEST_F(PgFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[0].occurred_at_ns, nth_report_time(11).nanoseconds()); EXPECT_EQ(series[1].occurred_at_ns, nth_report_time(0).nanoseconds()) @@ -2766,23 +2898,31 @@ TEST_F(PgFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { TEST_F(PgFaultStorageTest, NearMissEntriesDescribeTheirOwnReport) { // Each entry must describe the report that produced it, not the fault's current state, or a // series spanning a threshold change or a new reporting source reads as if nothing changed. + const FaultId pump{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}; + const FaultId backup{"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}; + DebounceConfig first = four_strike_config(); storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), first); + pump.owner, nth_report_time(0).nanoseconds(), first); DebounceConfig second = four_strike_config(); second.confirmation_threshold = -8; storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure dipping", - "/hydraulics/backup_pump", nth_report_time(1).nanoseconds(), second); + backup.owner, nth_report_time(1).nanoseconds(), second); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); - ASSERT_EQ(series.size(), 2u); - EXPECT_EQ(series[0].confirmation_threshold, -4); - EXPECT_EQ(series[0].severity, Fault::SEVERITY_WARN); - EXPECT_EQ(series[0].source_id, "/hydraulics/pump"); - EXPECT_EQ(series[1].confirmation_threshold, -8); - EXPECT_EQ(series[1].severity, Fault::SEVERITY_ERROR); - EXPECT_EQ(series[1].source_id, "/hydraulics/backup_pump"); + // Two sources of one code own separate records, so each near-miss series holds only its + // owner's reports and is not visible under the other owner. + auto pump_series = storage_->get_near_misses(pump); + ASSERT_EQ(pump_series.size(), 1u); + EXPECT_EQ(pump_series[0].confirmation_threshold, -4); + EXPECT_EQ(pump_series[0].severity, Fault::SEVERITY_WARN); + EXPECT_EQ(pump_series[0].source_id, "/hydraulics/pump"); + + auto backup_series = storage_->get_near_misses(backup); + ASSERT_EQ(backup_series.size(), 1u); + EXPECT_EQ(backup_series[0].confirmation_threshold, -8); + EXPECT_EQ(backup_series[0].severity, Fault::SEVERITY_ERROR); + EXPECT_EQ(backup_series[0].source_id, "/hydraulics/backup_pump"); } TEST_F(PgFaultStorageTest, NearMissSeriesContinuesAfterReopen) { @@ -2799,7 +2939,7 @@ TEST_F(PgFaultStorageTest, NearMissSeriesContinuesAfterReopen) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "a restart must extend the series, not restart or reorder it"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[1].debounce_counter, -2); @@ -2821,15 +2961,16 @@ TEST_F(PgFaultStorageTest, ApplyingSmallerBoundTrimsExistingSeries) { storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", nth_report_time(i).nanoseconds(), config); } - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 5u); + ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 5u); storage_->set_max_near_misses_per_fault(2); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u) << "applying the bound left the stored series over it"; EXPECT_EQ(series[0].debounce_counter, -4); EXPECT_EQ(series[1].debounce_counter, -5); - EXPECT_EQ(storage_->get_near_misses("MOTOR_OVERHEAT").size(), 2u) << "the bound is applied per fault code"; + EXPECT_EQ(storage_->get_near_misses(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}).size(), 2u) + << "the bound is applied per fault code"; } TEST_F(PgFaultStorageTest, ApplyingUnlimitedBoundKeepsExistingSeries) { @@ -2844,7 +2985,7 @@ TEST_F(PgFaultStorageTest, ApplyingUnlimitedBoundKeepsExistingSeries) { storage_->set_max_near_misses_per_fault(0); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 4u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 4u); } TEST_F(PgFaultStorageTest, UnlimitedBoundSpeltAsSizeMaxKeepsTheSeries) { @@ -2859,11 +3000,11 @@ TEST_F(PgFaultStorageTest, UnlimitedBoundSpeltAsSizeMaxKeepsTheSeries) { } EXPECT_EQ(storage_->set_max_near_misses_per_fault(std::numeric_limits::max()), 0u); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(3).nanoseconds(), config); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 4u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 4u); } TEST_F(PgFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { @@ -2889,7 +3030,7 @@ TEST_F(PgFaultStorageTest, PassedReportOnUnknownFaultWritesNothing) { EXPECT_FALSE(storage_->report_fault_event("NEVER_REPORTED", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/test_node", nth_report_time(0).nanoseconds(), default_config())); EXPECT_EQ(storage_->size(), 0u); - EXPECT_TRUE(storage_->get_near_misses("NEVER_REPORTED").empty()); + EXPECT_TRUE(storage_->get_near_misses(FaultId{"NEVER_REPORTED", "/test_node"}).empty()); } TEST_F(PgFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) { @@ -2907,7 +3048,7 @@ TEST_F(PgFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto healed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; @@ -2918,11 +3059,11 @@ TEST_F(PgFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) { storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); } - auto confirmed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto confirmed = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(confirmed.has_value()); ASSERT_EQ(confirmed->status, Fault::STATUS_CONFIRMED) << "test setup: the ramp must end in a real fault"; - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_FALSE(series.empty()); EXPECT_EQ(series[0].resulting_status, Fault::STATUS_PREFAILED) << "the first report was a genuine approach"; @@ -2944,13 +3085,13 @@ TEST_F(PgFaultStorageTest, NearMissResultingStatusSurvivesReopen) { storage_.reset(); storage_ = std::make_unique(conn_info_, test_log_sink()); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 1u); EXPECT_EQ(series[0].resulting_status, Fault::STATUS_PREFAILED); } TEST_F(PgFaultStorageTest, NearMissSeriesEmptyForUnknownFault) { - EXPECT_TRUE(storage_->get_near_misses("NEVER_REPORTED").empty()); + EXPECT_TRUE(storage_->get_near_misses(FaultId{"NEVER_REPORTED", ""}).empty()); } // --- Snapshot retention through the startup reclassification --- @@ -2978,12 +3119,12 @@ static void drive_to_healed(ros2_medkit_fault_manager::FaultStorage & storage, c config.healing_enabled = true; config.healing_threshold = 1; - storage.report_fault_event(fault_code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "/test_node", + storage.report_fault_event(fault_code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "", nth_report_time(0).nanoseconds(), config); store_snapshots_for(storage, fault_code, 2); // Two PASSED reports: the first only lifts the counter to 0, which the CONFIRMED latch holds. for (int i = 1; i <= 2; ++i) { - storage.report_fault_event(fault_code, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "/test_node", + storage.report_fault_event(fault_code, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "", nth_report_time(i).nanoseconds(), config); } } @@ -2992,14 +3133,14 @@ TEST_F(PgFaultStorageTest, SnapshotsRetainedThroughHealedReclassification) { storage_->set_retain_snapshots_on_clear(true); drive_to_healed(*storage_, "SNAPSHOT_RETAIN_TEST"); - auto healed = storage_->get_fault("SNAPSHOT_RETAIN_TEST"); + auto healed = storage_->get_fault(FaultId{"SNAPSHOT_RETAIN_TEST", ""}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; - ASSERT_EQ(storage_->get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u); + ASSERT_EQ(storage_->get_snapshots(FaultId{"SNAPSHOT_RETAIN_TEST", ""}).size(), 2u); ASSERT_EQ(storage_->reclassify_healed_as_cleared().size(), 1u); - EXPECT_EQ(storage_->get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u) + EXPECT_EQ(storage_->get_snapshots(FaultId{"SNAPSHOT_RETAIN_TEST", ""}).size(), 2u) << "the restart reclassification deleted snapshots the configuration asked to keep"; } @@ -3007,7 +3148,239 @@ TEST_F(PgFaultStorageTest, SnapshotsDroppedByHealedReclassificationByDefault) { drive_to_healed(*storage_, "SNAPSHOT_DROP_TEST"); ASSERT_EQ(storage_->reclassify_healed_as_cleared().size(), 1u); - EXPECT_TRUE(storage_->get_snapshots("SNAPSHOT_DROP_TEST").empty()); + EXPECT_TRUE(storage_->get_snapshots(FaultId{"SNAPSHOT_DROP_TEST", ""}).empty()); +} + +// --- Record identity: the same per-record contract as the SQLite backend --- + +TEST_F(PgFaultStorageTest, GetFaultsByCodeReturnsEveryOwnerAndGetFaultExactlyOne) { + rclcpp::Clock clock; + storage_->report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_b", + clock.now().nanoseconds(), default_config()); + storage_->report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_a", + clock.now().nanoseconds(), default_config()); + + const auto shared = storage_->get_faults_by_code("SHARED"); + ASSERT_EQ(shared.size(), 2u); + ASSERT_EQ(shared[0].reporting_sources.size(), 1u); + ASSERT_EQ(shared[1].reporting_sources.size(), 1u); + EXPECT_EQ(shared[0].reporting_sources.front(), "/owner_a") << "ordered by owner, like the other backends"; + EXPECT_EQ(shared[1].reporting_sources.front(), "/owner_b"); + + EXPECT_TRUE(storage_->get_faults_by_code("NEVER_REPORTED").empty()); + ASSERT_TRUE(storage_->get_fault(FaultId{"SHARED", "/owner_a"}).has_value()); + EXPECT_FALSE(storage_->get_fault(FaultId{"SHARED", "/nobody"}).has_value()); +} + +TEST_F(PgFaultStorageTest, NearMissSeriesIsPerRecord) { + const auto config = four_strike_config(); + storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "dip", + "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "dip", + "/hydraulics/backup_pump", nth_report_time(1).nanoseconds(), config); + + const auto pump = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); + const auto backup = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}); + ASSERT_EQ(pump.size(), 1u); + ASSERT_EQ(backup.size(), 1u); + EXPECT_EQ(pump.front().source_id, "/hydraulics/pump"); + EXPECT_EQ(pump.front().severity, Fault::SEVERITY_WARN); + EXPECT_EQ(backup.front().source_id, "/hydraulics/backup_pump"); + EXPECT_EQ(backup.front().severity, Fault::SEVERITY_ERROR); +} + +TEST_F(PgFaultStorageTest, ACriticalRecordConfirmsOnAnyFailedReportAndAnotherOwnersIsUntouched) { + rclcpp::Clock clock; + DebounceConfig config; + config.confirmation_threshold = -3; + ASSERT_TRUE(config.critical_immediate_confirm); + const FaultId a{"CRIT", "/a"}; + const FaultId b{"CRIT", "/b"}; + + storage_->report_fault_event("CRIT", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "", a.owner, + clock.now().nanoseconds(), config); + ASSERT_TRUE(storage_->clear_fault(a)); + + EXPECT_TRUE(storage_->report_fault_event("CRIT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "", a.owner, + clock.now().nanoseconds(), config)) + << "a report on a CLEARED record is an occurrence"; + EXPECT_EQ(storage_->get_fault(a)->severity, Fault::SEVERITY_CRITICAL) << "the record keeps its highest severity"; + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_CONFIRMED) << "the stored severity decides, not the report's"; + + storage_->report_fault_event("CRIT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "", b.owner, + clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(b)->severity, Fault::SEVERITY_WARN) << "severity is per record"; + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_PREFAILED) << "another owner's CRITICAL confirms nothing"; +} + +TEST_F(PgFaultStorageTest, HealingAndTimeBasedConfirmationActOnOneRecord) { + rclcpp::Clock clock; + DebounceConfig config; + config.confirmation_threshold = -3; + config.healing_enabled = true; + config.healing_threshold = 1; + config.auto_confirm_after_sec = 5.0; + storage_->set_debounce_config(config); + const FaultId a{"TWO", "/a"}; + const FaultId b{"TWO", "/b"}; + + const int64_t t0 = 1'000'000'000'000LL; + storage_->report_fault_event("TWO", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", a.owner, t0, config); + storage_->report_fault_event("TWO", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", b.owner, t0 + 1, config); + + const auto swept = storage_->check_time_based_confirmation(t0 + 10'000'000'000LL); + ASSERT_EQ(swept.size(), 2u); + EXPECT_EQ(swept[0].fault_code, "TWO"); + EXPECT_NE(swept[0].owner, swept[1].owner) << "one entry per record, not per code"; + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_CONFIRMED); + + // Heal a only: its counter moves, b's does not. + for (int i = 0; i < 4; ++i) { + storage_->report_fault_event("TWO", fd::ReportKind::kPassed, 0, "", a.owner, t0 + 20 + i, config); + } + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_CONFIRMED); + + // The startup pass reclassifies the HEALED record and names it by identity. + const auto reclassified = storage_->reclassify_healed_as_cleared(); + ASSERT_EQ(reclassified.size(), 1u); + EXPECT_EQ(reclassified.front(), a); + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_CONFIRMED); +} + +namespace { + +/// Tables as the previous layout wrote them: the code alone is the key. +constexpr const char * kLegacyPgSchema = R"( + CREATE TABLE faults ( + fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, description TEXT NOT NULL, + first_occurred_ns BIGINT NOT NULL, last_occurred_ns BIGINT NOT NULL, occurrence_count BIGINT NOT NULL, + status TEXT NOT NULL, reporting_sources TEXT NOT NULL, debounce_counter INTEGER NOT NULL DEFAULT 0, + last_failed_ns BIGINT NOT NULL DEFAULT 0, last_passed_ns BIGINT NOT NULL DEFAULT 0, + confirmed_at_ns BIGINT NOT NULL DEFAULT 0); + CREATE TABLE snapshots ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, topic TEXT NOT NULL, + message_type TEXT NOT NULL, data TEXT NOT NULL, captured_at_ns BIGINT NOT NULL, + capture_id BIGINT NOT NULL DEFAULT 0); + CREATE INDEX idx_snapshots_fault_code ON snapshots(fault_code); + CREATE TABLE freeze_frames (fault_code TEXT PRIMARY KEY, data TEXT NOT NULL, captured_at_ns BIGINT NOT NULL); + CREATE TABLE rosbag_files ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + recording_id TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL, format TEXT NOT NULL, + duration_sec DOUBLE PRECISION NOT NULL, size_bytes BIGINT NOT NULL, created_at_ns BIGINT NOT NULL); + CREATE UNIQUE INDEX idx_rosbag_files_fault_path ON rosbag_files(fault_code, file_path); + CREATE INDEX idx_rosbag_files_fault_created ON rosbag_files(fault_code, created_at_ns, id); + )"; + +} // namespace + +TEST_F(PgFaultStorageTest, ATableWrittenByThePreviousLayoutIsMigratedInPlace) { + storage_.reset(); + exec_raw("DROP TABLE faults, snapshots, freeze_frames, near_misses, rosbag_files"); + exec_raw(kLegacyPgSchema); + exec_raw(R"( + -- Two sources on one legacy row: nothing in the old layout splits it, so it folds onto its FIRST source. + INSERT INTO faults VALUES ('SHARED', 2, 'shared', 1, 1, 1, 'CONFIRMED', '["/owner_a","/owner_b"]', -1, 1, 0, 5); + INSERT INTO faults VALUES ('SOLO', 1, 'solo', 2, 2, 1, 'PREFAILED', '["/owner_c"]', -1, 2, 0, 99); + INSERT INTO faults VALUES ('NOSOURCE', 1, 'none', 3, 3, 1, 'CONFIRMED', '[]', -1, 3, 0, 0); + INSERT INTO freeze_frames VALUES ('SHARED', '{"/t":1}', 7); + INSERT INTO freeze_frames VALUES ('GONE', '{"/t":2}', 8); + INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) + VALUES ('SHARED', '/t', 'std_msgs/msg/Float64', '{}', 9); + INSERT INTO rosbag_files (fault_code, file_path, format, duration_sec, size_bytes, created_at_ns) + VALUES ('SHARED', '/bags/fault_SHARED_1', 'mcap', 5.0, 1024, 11); + )"); + + ASSERT_NO_THROW(storage_ = std::make_unique(conn_info_, test_log_sink())); + + const auto scalar = [this](const std::string & sql) { + auto conn = raw_connection(); + pqxx::nontransaction tx(*conn); + return tx.exec(sql)[0][0].as(); + }; + + // The identity is unique over (fault_code, owner), and the old PRIMARY KEY on the code is gone. + EXPECT_EQ(scalar("SELECT COUNT(*) FROM pg_constraint WHERE conrelid = to_regclass('faults') AND contype = 'p'"), "0"); + EXPECT_EQ(scalar("SELECT COUNT(*) FROM pg_constraint WHERE conrelid = to_regclass('freeze_frames') AND " + "contype = 'p'"), + "0"); + EXPECT_NE(scalar("SELECT indexdef FROM pg_indexes WHERE schemaname = current_schema() AND " + "indexname = 'idx_rosbag_files_fault_path'") + .find("(fault_code, owner, file_path)"), + std::string::npos) + << "the rosbag unique index must widen with the identity"; + + // owner is the FIRST legacy reporting source, and reporting_sources is rewritten to it. + EXPECT_EQ(scalar("SELECT owner FROM faults WHERE fault_code = 'SHARED'"), "/owner_a"); + EXPECT_EQ(scalar("SELECT reporting_sources FROM faults WHERE fault_code = 'SHARED'"), "[\"/owner_a\"]"); + EXPECT_EQ(scalar("SELECT owner FROM faults WHERE fault_code = 'NOSOURCE'"), ros2_medkit_fault_manager::kLegacyOwner); + EXPECT_EQ(scalar("SELECT confirmed_at_ns FROM faults WHERE fault_code = 'SOLO'"), "0") + << "a pending record has no confirmation time"; + EXPECT_EQ(scalar("SELECT confirmed_at_ns FROM faults WHERE fault_code = 'SHARED'"), "5"); + + const FaultId shared{"SHARED", "/owner_a"}; + auto migrated = storage_->get_fault(shared); + ASSERT_TRUE(migrated.has_value()); + EXPECT_EQ(migrated->status, Fault::STATUS_CONFIRMED); + EXPECT_FALSE(storage_->get_fault(FaultId{"SHARED", "/owner_b"}).has_value()); + EXPECT_TRUE(storage_->get_fault(FaultId{"SOLO", "/owner_c"}).has_value()); + + // The child rows learned which record they belong to; evidence of a code no record carries stays unassigned. + ASSERT_TRUE(storage_->get_freeze_frame(shared).has_value()); + EXPECT_EQ(storage_->get_freeze_frame(shared)->captured_at_ns, 7); + ASSERT_EQ(storage_->get_snapshots(shared).size(), 1u); + EXPECT_EQ(storage_->get_snapshots(shared).front().owner, "/owner_a"); + ASSERT_TRUE(storage_->get_rosbag_file(shared).has_value()); + EXPECT_EQ(storage_->get_rosbag_file(shared)->owner, "/owner_a"); + EXPECT_EQ(scalar("SELECT COUNT(*) FROM freeze_frames WHERE fault_code = 'GONE' AND owner = ''"), "1"); + + // After the migration a second owner can hold the same code, which the old key refused. + rclcpp::Clock clock; + EXPECT_TRUE(storage_->report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", "/owner_b", + clock.now().nanoseconds(), default_config())); + EXPECT_EQ(storage_->get_faults_by_code("SHARED").size(), 2u); + + // Re-opening a migrated database changes nothing. + storage_.reset(); + storage_ = std::make_unique(conn_info_, test_log_sink()); + EXPECT_EQ(storage_->size(), 4u); + EXPECT_EQ(scalar("SELECT owner FROM faults WHERE fault_code = 'SHARED' AND status = 'CONFIRMED' " + "ORDER BY owner LIMIT 1"), + "/owner_a"); + ASSERT_TRUE(storage_->get_freeze_frame(shared).has_value()); +} + +TEST_F(PgFaultStorageTest, ChildRowsOfASharedCodeAreNotGivenAnArbitraryOwner) { + storage_.reset(); + exec_raw("DROP TABLE faults, snapshots, freeze_frames, near_misses, rosbag_files"); + // faults already keyed by owner with TWO owners of one code, snapshots still without the column. + exec_raw(R"( + CREATE TABLE faults ( + fault_code TEXT NOT NULL, severity INTEGER NOT NULL, description TEXT NOT NULL, + first_occurred_ns BIGINT NOT NULL, last_occurred_ns BIGINT NOT NULL, occurrence_count BIGINT NOT NULL, + status TEXT NOT NULL, reporting_sources TEXT NOT NULL, debounce_counter INTEGER NOT NULL DEFAULT 0, + last_failed_ns BIGINT NOT NULL DEFAULT 0, last_passed_ns BIGINT NOT NULL DEFAULT 0, + confirmed_at_ns BIGINT NOT NULL DEFAULT 0, owner TEXT NOT NULL DEFAULT ''); + CREATE UNIQUE INDEX idx_faults_code_owner ON faults(fault_code, owner); + INSERT INTO faults VALUES ('SHARED', 2, 'a', 1, 1, 1, 'CONFIRMED', '["/owner_a"]', -1, 1, 0, 0, '/owner_a'); + INSERT INTO faults VALUES ('SHARED', 2, 'b', 1, 1, 1, 'CONFIRMED', '["/owner_b"]', -1, 1, 0, 0, '/owner_b'); + CREATE TABLE snapshots ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, topic TEXT NOT NULL, + message_type TEXT NOT NULL, data TEXT NOT NULL, captured_at_ns BIGINT NOT NULL, + capture_id BIGINT NOT NULL DEFAULT 0); + INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) + VALUES ('SHARED', '/t', 'std_msgs/msg/Float64', '{}', 9); + )"); + + ASSERT_NO_THROW(storage_ = std::make_unique(conn_info_, test_log_sink())); + + EXPECT_TRUE(storage_->get_snapshots(FaultId{"SHARED", "/owner_a"}).empty()) + << "A must not be handed unproven evidence"; + EXPECT_TRUE(storage_->get_snapshots(FaultId{"SHARED", "/owner_b"}).empty()) << "and neither must B"; + auto conn = raw_connection(); + pqxx::nontransaction tx(*conn); + EXPECT_EQ(tx.exec("SELECT COUNT(*) FROM snapshots WHERE owner = ''")[0][0].as(), 1) + << "the row is kept, unassigned"; } int main(int argc, char ** argv) { From be32a4e6f8f4e675cbf6ef6040ba4905d81673a3 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 16:27:01 +0100 Subject: [PATCH 03/52] fault_manager: choose per fault code whether each source gets its own record A fault code now has one record shared by every source that reports it, as before the record identity change, unless its entry in fault_thresholds.yaml sets `per_source: true`. That key is a strict YAML boolean: any other value rejects the file like every other malformed value. The shared record is owned by the reserved id `*`, which ReportFault refuses as a source_id. One function, FaultManagerCore::record_id_for, decides the identity of a report. The storage backends receive the decided FaultId and take the reporting source separately, for reporting_sources and the near-miss entries. A single-record request naming a source_id reaches the record it owns, otherwise the record that source reports into; an empty one keeps the unscoped resolution. The shared record has one debounce counter again, so the warning for a code debounced under two policies comes back for shared codes. A per-source record never warns. Stored records keep the identity they were written with when a code is switched between modes. At startup the node logs one warning per code that holds active records of the other identity. A store written when every code was one record opens with each row as the shared record of its code, keeping its reporting_sources. Its snapshots, freeze frame, recording links and near misses attach to it, the near misses of every source included. near_misses gains an owner column and is read and trimmed per record. The owner guessed from the first source, and the code that existed only for that guess, are gone. The PostgreSQL migration does the same. Entity-default snapshot and rosbag topic resolution reads the record's reporting sources again, so a shared record captures the topics of every node that reported it. --- .github/copilot-instructions.md | 2 +- docs/api/messages.rst | 28 +- docs/config/fault-manager.rst | 148 ++- docs/tutorials/docker.rst | 2 +- .../action_status_bridge_node.hpp | 5 +- .../src/action_status_bridge_node.cpp | 11 +- .../test/test_action_status_bridge.cpp | 2 +- .../test/test_integration.test.py | 36 +- src/ros2_medkit_fault_manager/CMakeLists.txt | 6 + src/ros2_medkit_fault_manager/README.md | 160 ++-- .../config/fault_manager.yaml | 4 +- .../config/snapshots.yaml | 9 +- .../design/index.rst | 47 +- .../core/fault_manager_core.hpp | 25 +- .../core/fault_manager_startup.hpp | 9 +- .../fault_storage.hpp | 98 +- .../postgres_fault_storage.hpp | 31 +- .../sqlite_fault_storage.hpp | 60 +- .../threshold_resolver.hpp | 20 +- .../src/core/fault_manager_core.cpp | 106 ++- .../src/core/fault_manager_startup.cpp | 48 +- .../src/fault_manager_node.cpp | 13 +- .../src/fault_storage.cpp | 61 +- .../src/postgres_fault_storage.cpp | 302 +++--- .../src/rosbag_capture.cpp | 10 +- .../src/snapshot_capture.cpp | 25 +- .../src/sqlite_fault_storage.cpp | 630 +++++-------- .../src/threshold_resolver.cpp | 23 + ..._fault_code_thresholds_integration.test.py | 156 +++- .../integration/test_fault_identity.test.py | 240 +++++ .../test_fault_identity_restart.test.py | 372 ++++++++ .../test/integration/test_integration.test.py | 171 ++-- .../test_severity_consistency.test.py | 43 +- .../test/test_entity_thresholds.cpp | 74 +- .../test/test_fault_code_thresholds.cpp | 63 +- .../test/test_fault_manager.cpp | 640 +++++++------ .../test_fault_manager_core_concurrency.cpp | 5 +- .../test/test_fault_severity.cpp | 26 +- .../test/test_fault_thresholds.yaml | 3 +- .../test/test_postgres_storage.cpp | 741 ++++++++------- .../test/test_rosbag_capture.cpp | 89 +- .../test/test_rosbag_storage_parity.cpp | 5 +- .../test/test_snapshot_capture.cpp | 21 +- .../test/test_sqlite_storage.cpp | 864 ++++++++---------- .../test/test_integration.test.py | 43 +- src/ros2_medkit_gateway/README.md | 2 +- src/ros2_medkit_msgs/README.md | 23 +- src/ros2_medkit_msgs/msg/Fault.msg | 18 +- src/ros2_medkit_msgs/msg/FaultEvent.msg | 4 +- src/ros2_medkit_msgs/msg/MutedFaultInfo.msg | 12 +- src/ros2_medkit_msgs/srv/ClearFault.srv | 12 +- src/ros2_medkit_msgs/srv/GetFault.srv | 10 +- src/ros2_medkit_msgs/srv/GetRosbag.srv | 10 +- src/ros2_medkit_msgs/srv/GetSnapshots.srv | 10 +- .../srv/ListFaultsForEntity.srv | 3 +- src/ros2_medkit_msgs/srv/ListRosbags.srv | 2 +- src/ros2_medkit_msgs/srv/ReportFault.srv | 15 +- 57 files changed, 3138 insertions(+), 2460 deletions(-) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 697495548..b8b36b43d 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -13,7 +13,7 @@ Multi-package colcon workspace under `src/`: | Package | Purpose | |---------|---------| | `ros2_medkit_gateway` | HTTP gateway - REST server, discovery, entity management, handlers, plugin framework | -| `ros2_medkit_fault_manager` | Fault records per (code, reporting source) in SQLite, AUTOSAR DEM-style debounce, rosbag/snapshot capture | +| `ros2_medkit_fault_manager` | Fault aggregation with SQLite, AUTOSAR DEM-style debounce, rosbag/snapshot capture | | `ros2_medkit_fault_reporter` | Client library for nodes to report faults | | `ros2_medkit_diagnostic_bridge` | Bridges `/diagnostics` topic to fault manager | | `ros2_medkit_serialization` | Runtime JSON <-> ROS 2 message serialization via dynmsg | diff --git a/docs/api/messages.rst b/docs/api/messages.rst index a4b6b4860..e6c2d6f02 100644 --- a/docs/api/messages.rst +++ b/docs/api/messages.rst @@ -17,10 +17,11 @@ Fault.msg Core fault data model representing one fault record. -A record is identified by the pair (``fault_code``, owning reporting source). The owner is -the ``source_id`` a ``ReportFault`` call carried. Two sources reporting one ``fault_code`` -are two records, each with its own status, debounce counter, ``occurrence_count``, severity -and timestamps, and each cleared on its own. +A record is identified by the pair (``fault_code``, owner). A code listed with +``per_source: true`` in ``fault_thresholds.yaml`` has one record per reporting source, owned by +the ``source_id`` of the ``ReportFault`` call. Every other code has one record shared by all +its sources, owned by the reserved id ``*``. Each record has its own status, debounce counter, +``occurrence_count``, severity and timestamps, and is cleared on its own. .. code-block:: text @@ -54,8 +55,8 @@ and timestamps, and each cleared on its own. # Current fault status (PREFAILED, PREPASSED, CONFIRMED, HEALED, CLEARED) string status - # The reporting source that owns this record, as a one-element list. Together - # with fault_code it identifies the record. + # Every source that reported this record: one for a per-source record, all of + # them for a shared one. string[] reporting_sources **Severity Constants:** @@ -153,7 +154,7 @@ MutedFaultInfo.msg ~~~~~~~~~~~~~~~~~~ Information about a correlated (muted) symptom record. One entry per muted RECORD: -a root cause mutes the symptoms of its own reporting source only, so two sources +a root cause mutes the symptoms of its own record owner only, so two per-source records muted on one ``fault_code`` produce two entries carrying that code, told apart by ``source_id``. @@ -163,7 +164,7 @@ muted on one ``fault_code`` produce two entries carrying that code, told apart b string root_cause_code # Root cause that triggered muting string rule_id # Correlation rule ID that matched uint32 delay_ms # Time delay from root cause [ms] - string source_id # Reporting source that owns the muted record + string source_id # Owner of the muted record (``*`` for a shared record) ClusterInfo.msg ~~~~~~~~~~~~~~~ @@ -200,7 +201,7 @@ Report a fault event to the FaultManager. uint8 severity # Fault.SEVERITY_* constant (for FAILED events) string description # Human-readable description string source_id # Fully qualified node name (e.g., "/powertrain/temp_sensor"). - # Owns the record: (fault_code, source_id) is the record identity. + # Owns the record of a per_source code. "*" is reserved and refused. **Response:** @@ -234,7 +235,7 @@ Clear/acknowledge one fault record. string fault_code # Fault code to clear bool skip_correlation_auto_clear # Opt out of correlation cascade clear - string source_id # Reporting source that owns the record + string source_id # Owner of the record, or a source that reports into it **Response:** @@ -244,9 +245,10 @@ Clear/acknowledge one fault record. string message # Status message or error description string[] auto_cleared_codes # Symptoms auto-cleared with root cause -``source_id`` names the owner of the record to clear. Leaving it empty is unscoped: the -call applies only when exactly one record carries the ``fault_code``, and fails otherwise -with ``success=false`` and a ``message`` beginning ``ambiguous:`` that lists the owners. +``source_id`` names the owner of the record to clear (``*`` for a shared record). When no +record has that owner, it names the record that source reports into. Leaving it empty is +unscoped: the call applies only when exactly one record carries the ``fault_code``, and fails +otherwise with ``success=false`` and a ``message`` beginning ``ambiguous:`` that lists the owners. An ambiguous call clears nothing. ``GetFault``, ``GetSnapshots`` and ``GetRosbag`` carry the same field and resolve the same way. On ``GetRosbag`` it scopes the ``fault_code`` lookup only, and the ``recording_id`` path ignores it. diff --git a/docs/config/fault-manager.rst b/docs/config/fault-manager.rst index d0af73c87..5b27416cb 100644 --- a/docs/config/fault-manager.rst +++ b/docs/config/fault-manager.rst @@ -1,8 +1,9 @@ Fault Manager Configuration =========================== -The ``ros2_medkit_fault_manager`` node keeps and manages one fault record per -(``fault_code``, reporting source) pair. +The ``ros2_medkit_fault_manager`` node aggregates and manages faults from multiple sources. +By default one fault code is one record, shared by all the sources that report it. A code +can be switched to one record per reporting source, see `Record Identity per Fault Code`_. This page documents all configuration parameters. .. contents:: Table of Contents @@ -63,11 +64,12 @@ A build without the option stops at startup when ``storage_type`` is ``postgres` Give each fault manager its own database, or its own schema with ``options=-csearch_path=`` in the connection string. - Tables written before records were keyed by owner (one row per fault code) are migrated in place - on the first connection: ``owner`` is added to ``faults``, ``freeze_frames``, ``snapshots`` and - ``rosbag_files``, the primary keys on the fault code are replaced by unique indexes that include - it, and each row takes the first entry of its ``reporting_sources``. The same rules as for - SQLite apply, see the package README. The migration runs in one transaction. Keep a copy of the - database first: an earlier release cannot use the migrated tables. + on the first connection: ``owner`` is added to ``faults``, ``freeze_frames``, ``snapshots``, + ``near_misses`` and ``rosbag_files``, the primary keys on the fault code are replaced by unique + indexes that include it, and every existing row becomes the shared record of its code (owner + ``*``), keeping its ``reporting_sources``. The same rules as for SQLite apply, see the package + README. The migration runs in one transaction. Keep a copy of the database first: an earlier + release cannot use the migrated tables. - The audit log stays a local SQLite file, see ``audit_log.database_path``. - The node never logs ``database_url``. It logs host, port, database and user, or the service name. Error text from the server is logged and returned with the password from ``database_url`` @@ -208,16 +210,16 @@ The fault manager uses AUTOSAR DEM-style debounce filtering to prevent fault fla Faults with ``SEVERITY_CRITICAL`` always bypass debounce regardless of this setting. The bypass (``critical_immediate_confirm`` in the debounce config, on by default and not a ROS parameter) reads the **stored** severity of the fault record, not the severity of the report. - A record keeps the highest severity its source ever reported, across occurrences, so after a - CRITICAL occurrence is cleared, one WARN report from the same source confirms the next - occurrence at once, and the record shows CRITICAL. The same holds for a FAILED report on a - HEALED CRITICAL record: the latch does not hold it back. Another source's record of the same - code has its own severity and is not affected. + A record keeps the highest severity ever reported for it, across occurrences, so after a + CRITICAL occurrence is cleared, one WARN report confirms the next occurrence at once, and + the record shows CRITICAL. The same holds for a FAILED report on a HEALED CRITICAL record: the + latch does not hold it back. Another record of the same code (a ``per_source`` code has one per + source) has its own severity and is not affected. .. note:: A FAILED report whose severity differs from the stored severity of its record is logged as a - warning naming the code, both severities and the source. One source should report one code + warning naming the code, both severities and the source. A source should report one code with one severity, so the warning points at a misconfigured reporter. It is logged once per code, source and reported severity until the node restarts, so a reporter that repeats the report at a high rate produces one line. @@ -258,18 +260,20 @@ CONFIRMED - the fault nearly happened. PASSED reports move the counter in the he (the fault receding) and are not near misses. The fault manager appends one entry per near miss to the series of the fault record it moved (one -series per fault code and reporting source), holding the timestamp, the counter value after the -report, the confirmation threshold, the severity the report carried (not the stored severity of the -record), the reporting source and the fault status the report left behind. +series per record: per fault code, or per fault code and reporting source for a ``per_source`` +code), holding the timestamp, the counter value after the report, the confirmation threshold, the +severity the report carried (not the stored severity of the record), the reporting source and the +fault status the report left behind. The status matters when reading the series. The HEALED latch holds the status all the way from the healing threshold down to the confirmation threshold, so reports on the way back into a fault that does confirm are also near misses by the definition above. Entries recording ``PREFAILED`` are approaches from a resting state; entries recording ``HEALED`` are a counter walking back down under -the latch. The recorded confirmation threshold and the counter both belong to the record's own -reporting source, so with per-entity thresholds each entry still reads as the record's distance to -confirmation. The series is **retained when the fault is cleared**, because acknowledging one -fault cycle must not erase how often that record approached confirmation across cycles. +the latch. The recorded confirmation threshold belongs to the reporting source, while the counter +is shared by all sources of a shared record, so with per-entity thresholds it is not on its own the +distance to confirmation (a ``per_source`` record has one source, and it is). The series is +**retained when the fault is cleared**, because acknowledging one fault cycle must not erase how +often that record approached confirmation across cycles. .. code-block:: yaml @@ -352,8 +356,8 @@ threshold overrides: - The ``source_id`` is the identifier passed in ``ReportFault`` service requests, typically the fully qualified name of the reporting ROS 2 node (e.g., ``/sensors/lidar/front_node``). - It also owns the record it creates, so you can inspect actual ``source_id`` values in the - one-element ``reporting_sources`` field of existing faults via ``GET /api/v1/faults``. + You can inspect actual ``source_id`` values in the ``reporting_sources`` field of existing + faults via ``GET /api/v1/faults``. - The ``source_id`` from ``ReportFault`` requests is matched against configured prefixes. - The **longest matching prefix** wins. For example, ``/sensors/lidar/front`` matches ``/sensors/lidar`` over ``/sensors``. @@ -363,11 +367,13 @@ threshold overrides: .. note:: - When multiple entities report the same ``fault_code``, each of them owns its own - record and each event applies the thresholds resolved from that event's - ``source_id``. The debounce counter belongs to the same record, so an entity's - configured band governs exactly the counter its own reports move: one entity's - reports can neither confirm nor heal another entity's fault. + When multiple entities report the same ``fault_code``, each event applies the + thresholds resolved from that event's ``source_id``. This means the debounce + behavior follows the reporting entity, not the fault. The debounce counter, + however, belongs to the record, and the record of a code is shared by all its + sources unless the code is ``per_source``, so the two entities share one counter + under two policies - see `Per-Fault-Code Thresholds`_ for what that does and how to + settle it. The node warns when it happens. ``auto_confirm_after_sec`` is global-only and cannot be overridden per-entity. Critical faults skip debounce and confirm on their first occurrence; that is @@ -377,18 +383,18 @@ Per-Fault-Code Thresholds ~~~~~~~~~~~~~~~~~~~~~~~~~ Per-entity thresholds are the right tool when you know which subsystems are noisy but not -which codes they will emit. They have one limit: the override is chosen per ``source_id``, so -two entities reporting one code debounce it under two different policies, each on its own -record's counter: +which codes they will emit. They have one limit: the debounce counter is kept per record, and +a code's record is shared by all its sources, while the override is chosen per ``source_id``. +Two entities reporting one code therefore share a counter and debounce it under two policies, +and the report that happens to arrive decides the transition: .. code-block:: text - motor (confirmation_threshold=-5) reports OVERHEAT -> motor's record: counter=-1, PREFAILED - lidar (confirmation_threshold=-1) reports OVERHEAT -> lidar's record: counter=-1, CONFIRMED + motor (confirmation_threshold=-5) reports OVERHEAT -> counter=-1, PREFAILED + lidar (confirmation_threshold=-1) reports OVERHEAT -> counter=-1, CONFIRMED -When the policy should follow the code and not the reporter, per-fault_code thresholds are -the layer that does it: they are matched on the code itself, so they resolve the same -whoever reports. +The motor's policy was bypassed. Per-fault_code thresholds are the layer that removes +that: they are matched on the code itself, so they resolve the same whoever reports. .. code-block:: yaml @@ -399,7 +405,7 @@ whoever reports. config_file: "/etc/ros2_medkit/fault_thresholds.yaml" The file is a map of fault codes to threshold overrides, the same three fields an entity -override carries: +override carries, plus ``per_source`` (see `Record Identity per Fault Code`_): .. code-block:: yaml @@ -437,14 +443,78 @@ override carries: - Like the entity file, this one is loaded once at node startup. Changes require a restart. - At startup, a HEALED fault is reclassified to CLEARED when its code has healing disabled. The code's entry in this file decides, and the global ``healing_enabled`` applies when the - entry does not set it. Entity overrides are not used here: the pass runs before any source - has reported, so it has no ``source_id`` to match a prefix against. + entry does not set it. Entity overrides are not used here, because a stored fault belongs + to its code and can have several sources. .. note:: An override that names only some fields settles only those. If ``MOTOR_OVERHEAT`` pins ``confirmation_threshold`` but not ``healing_threshold``, the healing direction still - follows whichever entity reported. Pin all three fields to settle a code completely. + follows whichever entity reported, and the node still reports the conflict below. Pin + all three fields to settle a code completely. + +**When one code is debounced two ways** + +The node resolves the policy for every report, and warns the first time two sources resolve +different policies for the shared record of one code: + +.. code-block:: text + + [WARN] Fault code 'OVERHEAT' is debounced two ways: '/powertrain/motor/left' resolves + confirmation=-5 healing_enabled=false healing=10, '/sensors/lidar/front' resolves + confirmation=-1 healing_enabled=true healing=1. The debounce counter belongs to the shared + record of the fault code, so whichever source reports decides the transition and the other + policy is bypassed. Pin confirmation_threshold, healing_enabled and healing_threshold for + this code in fault_thresholds.config_file, or set per_source: true there to give each source + its own record. + +The warning names both sources and both resolved policies, and is emitted **once per fault +code** for the life of the node, so a busy reporter does not turn it into a log storm. It +is a diagnostic, not an error: the configuration is legal and the fault is still debounced, +just not under a policy an operator chose. Giving the code an entry in +``fault_thresholds.config_file`` that pins all three fields, or ``per_source: true``, ends it. +A ``per_source`` code has one source per record and never warns. + +Record Identity per Fault Code +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A fault record is identified by the pair (``fault_code``, owner). By default a code has **one record +shared by all its reporting sources**: a FAILED from any source moves the one debounce counter, a +PASSED from any source moves it back, ``reporting_sources`` lists every source that reported, and a +clear ends the whole record. The shared record is owned by the reserved id ``*``. + +``per_source: true`` on a code's entry in ``fault_thresholds.config_file`` gives each reporting +source **its own record** of that code, owned by the ``source_id`` of its reports. Each record has its +own status, debounce counter, ``occurrence_count``, severity, snapshots, near-miss series and +recordings, and is cleared and healed on its own. + +.. code-block:: yaml + + # fault_thresholds.yaml + BATTERY_LOW: + per_source: true # one record per reporting source + confirmation_threshold: -3 + + ESTOP: + confirmation_threshold: -1 # per_source absent: one record shared by all sources + +- ``per_source`` is a YAML boolean, ``true`` or ``false``. Absent means ``false``, and so does a code + with no entry. Any other value (``yes``, ``3``) rejects the whole file with an error, the same as + any other malformed value in it. Entity overrides do not carry the key. +- ``ReportFault`` refuses ``*`` as a ``source_id``, because it names the shared record. +- The services that act on one record (``ClearFault``, ``GetFault``, ``GetSnapshots``, + ``GetRosbag``) take a ``source_id``. It names the record it owns (``*`` for the shared one). When no + record has that owner, it names the record that source reports into, so a client that knows only a + reporting source still reaches the shared record of a default code. An empty ``source_id`` applies + when exactly one record carries the code, and is refused as ``ambiguous:`` when several do. +- A record belongs to an entity when the entity is one of its ``reporting_sources``. +- The file is read at startup. When a code is switched between the two modes, the records already + stored keep the identity they were written with, and new reports go to the configured one. The + node logs one warning per code that holds active records of the other identity, naming the + code, its configured mode and the owners found. Nothing stored is rewritten. +- A database written when every code was one record opens with each row as the shared record of its + code, keeping its ``reporting_sources``, with its snapshots, freeze frame, recordings and near + misses attached to it. Snapshot Configuration ---------------------- diff --git a/docs/tutorials/docker.rst b/docs/tutorials/docker.rst index 853f2c3f9..4e898f8b9 100644 --- a/docs/tutorials/docker.rst +++ b/docs/tutorials/docker.rst @@ -52,7 +52,7 @@ Images are available for all supported ROS 2 distributions: Each image includes the gateway and all open-core packages: - ``ros2_medkit_gateway`` - HTTP REST server -- ``ros2_medkit_fault_manager`` - Fault record keeping and lifecycle management +- ``ros2_medkit_fault_manager`` - Fault aggregation and management - ``ros2_medkit_fault_reporter`` - Client library for fault reporting - ``ros2_medkit_diagnostic_bridge`` - Bridges ``/diagnostics`` to fault manager - ``ros2_medkit_serialization`` - Runtime JSON/ROS 2 serialization diff --git a/src/ros2_medkit_action_status_bridge/include/ros2_medkit_action_status_bridge/action_status_bridge_node.hpp b/src/ros2_medkit_action_status_bridge/include/ros2_medkit_action_status_bridge/action_status_bridge_node.hpp index b7f23ec02..8fe9f89f6 100644 --- a/src/ros2_medkit_action_status_bridge/include/ros2_medkit_action_status_bridge/action_status_bridge_node.hpp +++ b/src/ros2_medkit_action_status_bridge/include/ros2_medkit_action_status_bridge/action_status_bridge_node.hpp @@ -142,9 +142,8 @@ class ActionStatusBridgeNode : public rclcpp::Node { /// Get (creating on first use) the FaultReporter for an action. The reporter's /// source_id is fixed when first created: the resolved server FQN if discovery /// has settled, otherwise the action name as a fallback so the fault still fires - /// on time. It is never re-attributed afterwards: the fault manager keeps one - /// record per (fault code, source), so a swapped source would open a second - /// record and leave the provisional one raised with nothing left to heal it. + /// on time. It is never re-attributed afterwards (reporting_sources is + /// append-only and the per-entity scope filter is strict-AND). ros2_medkit_fault_reporter::FaultReporter * reporter_for(const std::string & action_name); /// Resolve the action server's node FQN from its status-topic publisher, for diff --git a/src/ros2_medkit_action_status_bridge/src/action_status_bridge_node.cpp b/src/ros2_medkit_action_status_bridge/src/action_status_bridge_node.cpp index 493a36ee1..c3c665686 100644 --- a/src/ros2_medkit_action_status_bridge/src/action_status_bridge_node.cpp +++ b/src/ros2_medkit_action_status_bridge/src/action_status_bridge_node.cpp @@ -535,12 +535,11 @@ ros2_medkit_fault_reporter::FaultReporter * ActionStatusBridgeNode::reporter_for // so the fault still fires on time. Reporting the fault on time takes priority // over entity attribution when discovery is slow. // - // The source is NOT re-attributed later: the fault manager keeps one record per - // (fault code, source), so swapping the provisional action-name source for the - // FQN would open a second record under the FQN and leave the provisional one - // raised, with no reporter left to send its PASSED. Correct attribution for the - // slow-discovery case is a separate concern (it needs a way to supersede a - // provisional source). + // The source is NOT re-attributed later: reporting_sources is append-only on the + // manager side and the per-entity /faults scope filter is strict-AND, so a + // provisional action-name source cannot be swapped for the FQN afterwards. + // Correct attribution for the slow-discovery case is a separate concern (it + // needs a way to supersede a provisional source). const std::string fqn = server_fqn_for_action(action_name); const std::string & source_id = fqn.empty() ? action_name : fqn; auto reporter = std::make_unique(this->shared_from_this(), source_id); diff --git a/src/ros2_medkit_action_status_bridge/test/test_action_status_bridge.cpp b/src/ros2_medkit_action_status_bridge/test/test_action_status_bridge.cpp index fdbbb589b..7bf4aa249 100644 --- a/src/ros2_medkit_action_status_bridge/test/test_action_status_bridge.cpp +++ b/src/ros2_medkit_action_status_bridge/test/test_action_status_bridge.cpp @@ -283,7 +283,7 @@ TEST_F(ActionStatusBridgeTest, ReporterFor_StickyCreatedOnceNeverSwapped) { ActionStatusBridgeTestAccess access(node.get()); // No publisher exists, so the server FQN is unresolved and the reporter falls // back to the action name. It must be created once and reused, never swapped: - // a swapped source would open a second fault record and strand the first. + // reporting_sources is append-only, so a provisional source cannot be undone. const void * first = access.reporter_identity("/nav"); EXPECT_NE(first, nullptr); EXPECT_EQ(access.reporter_identity("/nav"), first); diff --git a/src/ros2_medkit_diagnostic_bridge/test/test_integration.test.py b/src/ros2_medkit_diagnostic_bridge/test/test_integration.test.py index d2b5e521e..64931ef1e 100644 --- a/src/ros2_medkit_diagnostic_bridge/test/test_integration.test.py +++ b/src/ros2_medkit_diagnostic_bridge/test/test_integration.test.py @@ -153,7 +153,7 @@ def publish_diagnostic(self, name, level, message='Test message', hardware_id='t # Give time for message to be processed time.sleep(0.3) - def publish_until(self, name, level, expected_code, *, predicate=None, owner=None, + def publish_until(self, name, level, expected_code, *, predicate=None, message='Test message', hardware_id='test_hw', statuses=None, timeout=25.0): """ Republish a diagnostic until a matching fault satisfies *predicate*. @@ -167,10 +167,6 @@ def publish_until(self, name, level, expected_code, *, predicate=None, owner=Non Re-emission also drives WARN/PASSED past their occurrence threshold within the filter window deterministically. - A record is the pair (fault code, reporting source), so one code can - name several records. With *owner* set, only the record that source - owns is considered. - Returns the matching fault once *predicate* holds; fails at the deadline. """ @@ -183,17 +179,15 @@ def predicate(_fault): self.publish_diagnostic(name, level, message, hardware_id) fault = next( (f for f in self.list_faults(statuses=statuses) - if f.fault_code == expected_code - and (owner is None or list(f.reporting_sources) == [owner])), + if f.fault_code == expected_code), None, ) if fault is not None: last = fault if predicate(fault): return fault - owned = f' owned by {owner}' if owner is not None else '' self.fail( - f'Fault {expected_code}{owned} not satisfied within {timeout}s ' + f'Fault {expected_code} not satisfied within {timeout}s ' f'(last seen: {last})' ) @@ -259,30 +253,26 @@ def test_05_fault_code_generation(self): diag_name, DiagnosticStatus.ERROR, expected_code, ) - def test_06_each_forwarded_hardware_id_owns_its_own_record(self): - """Test that each forwarded hardware_id reports the code as its own record.""" + def test_06_hardware_id_is_forwarded_as_source(self): + """Test that hardware_id is forwarded into fault reporting_sources.""" self.publish_until( 'shared_sensor', DiagnosticStatus.STALE, 'SHARED_SENSOR', message='No data from source A', hardware_id='/my_lidar_driver', - owner='/my_lidar_driver', + predicate=lambda f: '/my_lidar_driver' in f.reporting_sources, ) - self.publish_until( + + fault = self.publish_until( 'shared_sensor', DiagnosticStatus.STALE, 'SHARED_SENSOR', message='No data from source B', hardware_id='/my_camera_driver', - owner='/my_camera_driver', + predicate=lambda f: '/my_lidar_driver' in f.reporting_sources + and '/my_camera_driver' in f.reporting_sources, ) - # One diagnostic name forwarded under two hardware ids is two records - # of one code, each owned by the hardware id it came from. - records = [f for f in self.list_faults() if f.fault_code == 'SHARED_SENSOR'] - self.assertEqual( - sorted(list(f.reporting_sources) for f in records), - [['/my_camera_driver'], ['/my_lidar_driver']], - ) - for record in records: - self.assertEqual(record.severity, Fault.SEVERITY_CRITICAL, record.reporting_sources) + self.assertEqual(fault.severity, Fault.SEVERITY_CRITICAL) + self.assertIn('/my_lidar_driver', fault.reporting_sources) + self.assertIn('/my_camera_driver', fault.reporting_sources) def test_07_empty_hardware_id_falls_back_to_bridge_source(self): """Test that empty hardware_id falls back to the bridge node FQN.""" diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index ad71bde55..0af0a80fa 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -297,6 +297,12 @@ endif() medkit_add_launch_test(test_fault_code_healing_restart test/integration/test_fault_code_healing_restart.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_identity + test/integration/test_fault_identity.test.py TIMEOUT 120 LABELS "integration") + + medkit_add_launch_test(test_fault_identity_restart + test/integration/test_fault_identity_restart.test.py TIMEOUT 180 LABELS "integration") + # Drives healing with the event counts a one-event-per-transition reporter # actually sends: one FAILED per raise, one PASSED per clear. Parametrized over # healing_threshold, so the node launches twice, and one case holds a settled diff --git a/src/ros2_medkit_fault_manager/README.md b/src/ros2_medkit_fault_manager/README.md index 19773f672..ec3339d38 100644 --- a/src/ros2_medkit_fault_manager/README.md +++ b/src/ros2_medkit_fault_manager/README.md @@ -4,67 +4,70 @@ Central fault manager node for the ros2_medkit fault management system. ## Overview -The FaultManager node provides a central point for fault record keeping and lifecycle management. -It receives fault reports from multiple sources, keeps one record per `(fault_code, reporting -source)` pair, and provides query and clearing interfaces. - -A record is identified by its `fault_code` and by the `source_id` the `ReportFault` call carried. -That source is the record's **owner**. Two sources reporting one `fault_code` are two records, each -with its own status, debounce counter, occurrence count, severity, timestamps, freeze frame, -snapshots, near-miss series and rosbag links. A clear or an auto-heal driven by one owner never -touches another owner's record. The services that act on a single record (`~/clear_fault`, -`~/get_fault`, `~/get_snapshots`, `~/get_rosbag`) carry a `source_id` request field naming the -owner. Leaving it empty is unscoped and applies only when exactly one record carries the -`fault_code`, otherwise the call fails with a message beginning `ambiguous:` that lists the -owners and changes nothing. +The FaultManager node provides a central point for fault aggregation and lifecycle management. +It receives fault reports from multiple sources, keeps them as fault records, and provides query +and clearing interfaces. + +A record is identified by its `fault_code` and an **owner**. By default a code has **one record +shared by all its reporting sources**, owned by the reserved id `*`: a FAILED from any source moves +its one debounce counter, a PASSED from any source moves it back, `reporting_sources` lists every +source that reported it, and a clear ends the whole record. A code listed with `per_source: true` in +`fault_thresholds.config_file` instead has **one record per reporting source**, owned by the +`source_id` of its reports. Each record has its own status, debounce counter, occurrence count, +severity, timestamps, freeze frame, snapshots, near-miss series and rosbag links, and is cleared and +healed on its own. `ReportFault` refuses `*` as a `source_id`. + +The services that act on a single record (`~/clear_fault`, `~/get_fault`, `~/get_snapshots`, +`~/get_rosbag`) carry a `source_id` request field. It names the record it owns (`*` for a shared +one). When no record has that owner, it names the record that source reports into, so a client that +knows only a reporting source still reaches the shared record of a default code. Leaving it empty is +unscoped and applies only when exactly one record carries the `fault_code`, otherwise the call fails +with a message beginning `ambiguous:` that lists the owners and changes nothing. + +### Identity per fault code + +```yaml +# fault_thresholds.config_file +BATTERY_LOW: + per_source: true # one record per reporting source + confirmation_threshold: -3 + +ESTOP: + confirmation_threshold: -1 # per_source absent: one record shared by all sources +``` + +`per_source` is a YAML boolean and defaults to `false`; a code with no entry is shared too. Any other +value (`yes`, `3`) rejects the whole file with an error, like every other malformed value in it. +Entity overrides do not carry the key. The file is read at startup. + +Switching a code between the two modes keeps the records already stored as they were written, and +sends new reports to the configured identity. For each code that holds active records of the other +identity the node logs one warning at startup, naming the code, its configured mode and the owners +found. Nothing stored is rewritten. + +A code reported by several sources and debounced under several policies (per-entity thresholds) has +one counter, so the node warns once per code when two sources resolve different policies for the +shared record. Set `per_source: true` or pin the policy in `fault_thresholds.config_file` to end it. ### Upgrading a database written before the owner column -A SQLite store written by an earlier release is keyed by the bare `fault_code` and is rebuilt on -the first open. The owner of each migrated record is the first entry of that row's legacy -`reporting_sources` value. A row that listed several sources folds onto its first one, because the -old schema recorded no per-source counter, status or timestamps to split it by, and inventing them -would put numbers in the store that no report ever produced. `confirmed_at_ns` is copied as it was, -except on a PREFAILED or PREPASSED row, where it becomes 0 (see Storage Backends below). - -The rebuild is one way. A fault manager from an earlier release opened on a migrated store aborts -on the first report of a fault code the store does not hold yet -(`NOT NULL constraint failed: faults.owner`), and on a fault code two sources share it writes one -source's data into the other source's row. If a rollback may be needed, keep a copy of `faults.db` -from before the upgrade and restore that copy instead of pointing the earlier release at the -migrated file. - -The recovery reads that column as text rather than requiring it to be valid JSON, because earlier -builds escaped only the quote, the backslash and `\b \f \n \r \t`, so a `source_id` carrying any -other control byte was written as text no JSON parser accepts. The mapping is exact: - -| stored `reporting_sources` | migrated owner | -|---|---| -| empty | `legacy` | -| `sensor_a` (a bare word, not an array) | `sensor_a` | -| `[]` | `legacy` | -| `["a<0x01>b"]` | `a<0x01>b` (the control byte is kept) | -| anything that is neither a JSON array nor a bare word (an object wrapper, a value behind a byte-order mark) | `legacy` | - -A migrated record never carries an empty owner. Where no source can be read the record gets the -synthetic owner `legacy` and one warning names it, so it stays addressable through `source_id` like -any other record: `ClearFault` and `GetFault` take `legacy` and act on it alone, and it appears -under that name in an `ambiguous:` refusal. New rows are always written as valid JSON, control -bytes escaped as `\u00XX`. - -The empty owner therefore means exactly one thing: an evidence row (freeze frame, snapshot, rosbag -link) not yet assigned to a record. Such a row takes the owner of its fault code when that code has -exactly one owner. When it has several, or none, the row keeps the empty owner and is named in a -warning rather than handed to an owner the database cannot prove it belongs to. That backfill runs -on any open that finds a row it can actually assign, not only on the open that adds the column, so -a store left half-migrated by an interrupted run is healed the next time it is opened, while a row -nothing can resolve is left alone instead of being retried forever. - -A PostgreSQL database written by an earlier release is migrated in place on the first connection, -by the same rules: the owner is the first legacy `reporting_sources` entry (`legacy` when none is -readable), the primary keys on the fault code become unique indexes over `(fault_code, owner)`, and -evidence rows are assigned only where their code has exactly one owner. It runs in one transaction, -so a failure leaves the tables as they were. +A SQLite store written by an earlier release is keyed by the bare `fault_code` and is rebuilt on the +first open. That store holds one record per code, fed by every source, which is exactly what a +default code is now, so every row becomes the shared record of its code (owner `*`) and keeps its +`reporting_sources` as written. Its freeze frame, snapshots, recording links and near misses attach +to that record, the near misses of every source included. `confirmed_at_ns` is copied as it was, +except on a PREFAILED or PREPASSED row, where it becomes 0 (see Storage Backends below). All of it +runs in one transaction. + +The rebuild is one way. A fault manager from an earlier release opened on a migrated store aborts on +the first report of a fault code the store does not hold yet (`NOT NULL constraint failed: +faults.owner`). If a rollback may be needed, keep a copy of `faults.db` from before the upgrade and +restore that copy instead of pointing the earlier release at the migrated file. + +A PostgreSQL database written by an earlier release is migrated in place on the first connection, by +the same rules: the primary keys on the fault code become unique indexes over `(fault_code, owner)` +and every row becomes the shared record of its code. It runs in one transaction, so a failure leaves +the tables as they were. The table names `faults_new` and `freeze_frames_new` are reserved for the rebuild. Anything found under those names is debris from a rebuild that did not finish and is dropped on open, with a @@ -108,11 +111,11 @@ ros2 service call /fault_manager/clear_fault ros2_medkit_msgs/srv/ClearFault \ ## Features -- **Per-source records**: Same `fault_code` from different sources creates one record per source, - each filtered, cleared and healed on its own +- **Multi-source aggregation**: Same `fault_code` from different sources creates a single fault, + unless the code is configured `per_source`, which gives each source its own record - **Occurrence tracking**: Counts outages, not reports - the count starts at one and rises only - when a cleared record is raised again by its own owner -- **Severity escalation**: A record has one severity, the highest its owner has reported, kept + when a cleared fault is raised again - and tracks all reporting sources +- **Severity escalation**: A record has one severity, the highest ever reported for it, kept within an occurrence and across occurrences. A report with a different severity is logged (see [Severity](#severity)) - **Persistent storage**: SQLite (default) or PostgreSQL backend ensures faults survive node restarts @@ -143,7 +146,7 @@ ros2 service call /fault_manager/clear_fault ros2_medkit_msgs/srv/ClearFault \ Snapshots capture topic data when faults are confirmed for post-mortem debugging. -Each confirm also writes a **freeze-frame**: a single compact JSON object mapping every captured topic to its value at confirmation time, keyed by the record. Which topics are captured is decided by the fault CODE (`fault_specific` and `patterns` are configuration about what a code means), while what is written belongs to the record, so two owners confirming one code capture the same topics into two separate frames. It differs from per-topic snapshots in two ways: snapshots are deleted when the record is cleared, while the freeze-frame is retained across `clear_fault` (once the snapshots are gone, `~/get_fault` serves the retained frame so the confirmed-state record stays available after acknowledgement), and a re-confirm that captures nothing (e.g. source publishers down) never overwrites an existing non-empty frame. A fault code with no configured capture set gets no freeze-frame row. A configured capture that samples nothing on its first run records an empty `{}` frame. Freeze-frame storage is bounded by the number of distinct records (one row per record, replaced in place) and rows are never evicted. +Each confirm also writes a **freeze-frame**: a single compact JSON object mapping every captured topic to its value at confirmation time, keyed by the record. Which topics are captured is decided by the fault CODE (`fault_specific` and `patterns` are configuration about what a code means), while what is written belongs to the record, so two `per_source` records of one code capture the same topics into two separate frames. It differs from per-topic snapshots in two ways: snapshots are deleted when the record is cleared, while the freeze-frame is retained across `clear_fault` (once the snapshots are gone, `~/get_fault` serves the retained frame so the confirmed-state record stays available after acknowledgement), and a re-confirm that captures nothing (e.g. source publishers down) never overwrites an existing non-empty frame. A fault code with no configured capture set gets no freeze-frame row. A configured capture that samples nothing on its first run records an empty `{}` frame. Freeze-frame storage is bounded by the number of distinct records (one row per record, replaced in place) and rows are never evicted. Under a fault storm, captures are bounded by a worker pool (`capture_pool_size`) draining a bounded queue (`capture_queue_depth`); excess captures are dropped per `capture_queue_full_policy` and logged (throttled). The pool is shared and is created when snapshots **or** rosbag is enabled, so these parameters bound both. `capture_pool_size` parallelizes freeze-frame snapshot capture only - rosbag stays single-writer regardless of pool size, and correlated faults confirming inside one post-roll window share a single recording. @@ -225,12 +228,12 @@ A wrong configuration stops the node at startup with exit code 1: a connection s ## Severity -A record has one severity. The stored severity is the highest any FAILED report from its owner has +A record has one severity. The stored severity is the highest any FAILED report for it has carried, within an occurrence and across occurrences: `~/clear_fault` ends an occurrence but does not lower the severity, and the SQLite store keeps it across restarts. Two reports of one code from one source with different severities mean a misconfigured reporter, and keeping the maximum is the -conservative reading. Another source's record of the same code keeps its own severity: a CRITICAL -from one owner neither escalates nor confirms another owner's record. +conservative reading. Another record of the same code (a `per_source` code has one per source) keeps +its own severity: a CRITICAL report for one record neither escalates nor confirms another. What each surface carries: @@ -280,9 +283,13 @@ counter walking back down under the latch. Without the field the two cannot be t rows written before the field existed. With per-entity thresholds the recorded `confirmation_threshold` is the one belonging to the -**reporting source**. The counter it describes is that source's own record, so the pair is the -distance to confirmation for the record. The counter is no longer shared between sources of one -fault code: each source's reports move only its own record. +**reporting source**, while the debounce counter is shared by every source of a shared record. It +is therefore not by itself the distance to confirmation for the record as a whole. An entry in +`fault_thresholds.config_file` that sets `confirmation_threshold` makes the two agree again: it +resolves the same for every source, so the recorded threshold is the record's own. An entry with +only healing fields keeps the per-source threshold. A `per_source` record has one source, so its +recorded threshold is already the record's own. The node warns once per code when two sources resolve different +policies for the shared record. Entries are kept and evicted in **arrival order**, not by their timestamps. Reporters carry their own clocks, so a report can arrive carrying a timestamp behind one already stored; ordering the @@ -299,9 +306,9 @@ retained snapshots, because it records the most recent confirmation while the sn to earlier ones. `~/get_snapshots` returns one entry per topic and serves the newest capture of that topic, whichever storage backend is in use. -The bound is **per record, not per database**. Fault codes are unbounded in cardinality and so is -the set of reporting sources, so a reporter emitting a stream of distinct codes still grows the -table. The bound caps what any single record costs, not the total. +The bound is **per record, not per database**. Fault codes are unbounded in cardinality (and so is +the set of reporting sources of a `per_source` code), so a reporter emitting a stream of distinct +codes still grows the table. The bound caps what any single record costs, not the total. Retention is **bounded per record** by `near_miss.max_per_fault` (default 200), evicting the **oldest** entries first. That is the same direction as `snapshots.max_per_fault` and the rosbag @@ -316,7 +323,7 @@ storage API or the database file. ## Advanced: Tamper-Evident Audit Log -An optional append-only, hash-chained audit log records every fault state transition (`occurred`, `confirmed`, `healed`, `cleared`) so the fault history is independently verifiable. Auto-recovery (a record reaching the healing threshold via PASSED events) is recorded as a distinct `healed` row, so the record's END is in the timeline and is not confused with a manual `cleared`. Every row's `source` is the record's owner, the automatic transitions included: the `transition` column already says what moved the row, and with several owners per fault code the source has to answer whose record moved. The manager has no acknowledge action separate from clearing, so `~/clear_fault` is recorded as `cleared` (clear == ack), and there is no `ack` kind. The log also records its own lifecycle with `logging_activated` / `logging_deactivated` markers at start and stop. Every row records the record's stored severity at the transition (see [Severity](#severity)). It is **off by default** because it adds a write and storage cost per transition. +An optional append-only, hash-chained audit log records every fault state transition (`occurred`, `confirmed`, `healed`, `cleared`) so the fault history is independently verifiable. Auto-recovery (a record reaching the healing threshold via PASSED events) is recorded as a distinct `healed` row, so the record's END is in the timeline and is not confused with a manual `cleared`. The `source` of a row a report drove (`occurred`, `confirmed`) is the reporting source. The automatic and clear transitions name the owner of the record that moved, which for a code shared by all its sources is `*`. The manager has no acknowledge action separate from clearing, so `~/clear_fault` is recorded as `cleared` (clear == ack), and there is no `ack` kind. The log also records its own lifecycle with `logging_activated` / `logging_deactivated` markers at start and stop. Every row records the record's stored severity at the transition (see [Severity](#severity)). It is **off by default** because it adds a write and storage cost per transition. Each transition appends one immutable row holding `record_hash = sha256(prev_hash + canonical(event))` (OpenSSL EVP SHA-256), the `prev_hash` it links to, and a monotonic `seq`. The hash is computed once at insert and never recomputed. A persisted chain head lets the chain resume across restarts. The log is stored in its own SQLite database (separate from the fault store) and is treated as append-only: the manager only ever inserts rows, and `BEFORE UPDATE` / `BEFORE DELETE` triggers reject out-of-band edits (the guarded rotation prune excepted). @@ -473,7 +480,7 @@ Event types: `0` = EVENT_FAILED, `1` = EVENT_PASSED A FAILED report on a record whose stored severity is CRITICAL confirms it at once, regardless of threshold and of the HEALED latch. The stored severity decides, not the report: after a CRITICAL -occurrence is cleared, one WARN report from the same source confirms the next occurrence, which +occurrence is cleared, one WARN report for the same record confirms the next occurrence, which shows CRITICAL. This holds under a per-entity threshold override too. ## Advanced: Fault Correlation @@ -571,7 +578,8 @@ Response includes: - `cluster_count`: Number of active fault clusters - `muted_faults[]`: Details of muted records (when `include_muted=true`), one entry per muted record. Muting is per record, so a symptom muted under one owner's root cause never hides - another owner's record of the same code, and each entry names its owner in `source_id`. + another owner's record of the same code, and each entry names its owner in `source_id` (`*` for a + shared record). - `clusters[]`: Details of active clusters (when `include_clusters=true`) ### REST API (via Gateway) @@ -595,7 +603,7 @@ Response fields: "root_cause_code": "ESTOP_001", "rule_id": "estop_cascade", "delay_ms": 50, - "source_id": "/powertrain/motor_controller" + "source_id": "*" } ], "clusters": [ diff --git a/src/ros2_medkit_fault_manager/config/fault_manager.yaml b/src/ros2_medkit_fault_manager/config/fault_manager.yaml index ce1a3243f..88e0f6804 100644 --- a/src/ros2_medkit_fault_manager/config/fault_manager.yaml +++ b/src/ros2_medkit_fault_manager/config/fault_manager.yaml @@ -17,8 +17,8 @@ fault_manager: # Near-miss series: one appended entry per FAILED report that moved the # debounce counter without confirming. Retained across clear_fault, bounded - # per record - one (fault_code, reporting source) pair - oldest evicted - # first. 0 = unlimited (grows with the reporting rate). + # per record, oldest evicted first. 0 = unlimited (grows with the + # reporting rate). # near_miss.max_per_fault: 200 # Healing OFF by default: a recovery signal (e.g. action SUCCEEDED) does not diff --git a/src/ros2_medkit_fault_manager/config/snapshots.yaml b/src/ros2_medkit_fault_manager/config/snapshots.yaml index 0a1e274d8..ef678653f 100644 --- a/src/ros2_medkit_fault_manager/config/snapshots.yaml +++ b/src/ros2_medkit_fault_manager/config/snapshots.yaml @@ -8,12 +8,13 @@ # 2. patterns - regex pattern match on fault code # 3. default_topics - fallback for all other faults # 4. entity-default - zero-config fallback when nothing above matches: -# the owning reporting source node's own published topics are captured +# the reporting source node's own published topics are captured # (parameter snapshots.entity_default, on by default) # # Resolution is by fault CODE: this file says what a code means, not who reported -# it. What is written belongs to the RECORD, so two sources reporting one code -# capture the same topics into two separate freeze frames. +# it. What is written belongs to the RECORD, so two per_source records of one code +# (fault_thresholds.config_file) capture the same topics into two separate freeze +# frames. # # A fault code listed in fault_specific (or matched by a pattern) with an # EMPTY topic list is an explicit per-fault opt-out: nothing is captured and @@ -109,7 +110,7 @@ rosbag: # Options: # "entity" - DEFAULT. Subscribe broadly for pre-roll, but on fault confirmation # write only the faulting source node's topics (+ /tf, /tf_static), - # resolved from the record's owning reporting source. Useful black box with + # resolved from the record's reporting sources. Useful black box with # zero per-topic config. Falls back to the full buffer if the source # cannot be resolved to a live node. # "config" - Use same topics as JSON snapshots (from this config file) diff --git a/src/ros2_medkit_fault_manager/design/index.rst b/src/ros2_medkit_fault_manager/design/index.rst index e73ab81b1..04a1e1207 100644 --- a/src/ros2_medkit_fault_manager/design/index.rst +++ b/src/ros2_medkit_fault_manager/design/index.rst @@ -187,13 +187,13 @@ Main Components 3. **InMemoryFaultStorage** - Thread-safe in-memory implementation of FaultStorage - Uses ``std::map`` keyed by ``FaultId`` for O(log n) lookups, ordered by code first - Protected by ``std::mutex`` for concurrent service request handling - - Keeps one record per reporting source, filtered and cleared independently + - Keeps one record per ``FaultId``: one record shared by every source of a code, or one per source for a code configured ``per_source`` - Implements severity escalation per record (higher severity overwrites lower, across occurrences too) - - Tracks occurrence counts per record + - Tracks occurrence counts and all reporting sources per record 4. **FaultState** - Internal representation of one fault record - Maps directly to ``ros2_medkit_msgs::msg::Fault`` via ``to_msg()`` - - Carries its owner and emits it as the single entry of reporting_sources + - Carries its owner and uses ``std::set`` for reporting_sources to ensure uniqueness - Tracks first and last occurrence timestamps - Manages fault status lifecycle with debounce (PREFAILED -> CONFIRMED -> CLEARED) @@ -247,12 +247,13 @@ Services Reports a new fault or updates an existing one. -- **Input validation**: fault_code and source_id cannot be empty, event_type must be valid +- **Input validation**: fault_code and source_id cannot be empty, source_id cannot be the reserved ``*``, event_type must be valid - **Event types**: FAILED (fault detected) or PASSED (fault condition cleared) - **Debounce**: FAILED events decrement counter, PASSED events increment counter -- **Record identity**: (fault_code, source_id). The same fault_code from different sources - creates one record per source, each debounced, cleared and healed on its own -- **Severity escalation**: A record's severity is updated if its own owner reports a higher one +- **Record identity**: ``FaultManagerCore::record_id_for`` decides it. A code configured ``per_source`` + has one record per source_id, any other code one record shared by all its sources (owner ``*``, + which ``ReportFault`` refuses as a source_id). Each record is debounced, cleared and healed on its own +- **Severity escalation**: A record's severity is updated if a higher one is reported for it - **Severity mismatch**: A FAILED report whose severity differs from the stored severity of its record is logged as a warning, once per code, source and reported severity until restart - **Request queue**: Holds ``report_fault_queue_depth`` requests (default 100) until the executor @@ -296,25 +297,30 @@ callbacks may execute on different threads. Fault Records ~~~~~~~~~~~~~ -A fault record is identified by the pair ``(fault_code, source_id)``, where the -``source_id`` is the one the ``ReportFault`` call carried. That source is the record's -owner. Repeated reports from one source update that source's record. A report from a -source that owns no record for the code opens a new one. This provides: +A fault record is identified by the pair ``(fault_code, owner)``. A code listed with +``per_source: true`` in ``fault_thresholds.config_file`` has one record per reporting source, +owned by the ``source_id`` of its reports. Every other code has one record shared by all its +sources, owned by the reserved id ``*``. Reports are routed to a record by one function, +``record_id_for``, and the storage backends receive the decided identity. This provides: -- **Deduplication**: Prevents fault flooding from one source's repeated reports -- **Source attribution**: Each record names the one source that owns it +- **Deduplication**: Prevents fault flooding from repeated reports +- **Source tracking**: A record lists every source that reported it - **Occurrence counting**: Tracks how many times a record was raised -Keeping the records apart is what makes the per-source state mean anything: the debounce -counter, the status, the severity, the occurrence count and the timestamps all belong to -one reporter, so one reporter recovering cannot walk another reporter's fault back from -confirmation, and one reporter's CRITICAL cannot escalate another's record. +A per-source record keeps the per-reporter state apart: the debounce counter, the status, the +severity, the occurrence count and the timestamps all belong to one reporter, so one reporter +recovering cannot walk another reporter's fault back from confirmation, and one reporter's CRITICAL +cannot escalate another's record. A shared record has one counter fed by every source. + +A request that names a ``source_id`` addresses the record it owns, else the record that source +reports into. Stored records keep the identity they were written with when a code is switched +between the two modes; the node logs a warning per code at startup. Severity Escalation ~~~~~~~~~~~~~~~~~~~ -When a record is re-reported by its owner with a higher severity, the stored severity is -updated. This ensures the record reflects the worst-case condition its own reporter saw. +When a record is re-reported with a higher severity, the stored severity is updated. This +ensures the record reflects the worst-case condition reported for it. Severity levels are ordered: ``INFO(0) < WARN(1) < ERROR(2) < CRITICAL(3)``. @@ -333,8 +339,7 @@ is the conservative choice. - **Mismatch warning**: when a FAILED report carries another severity than the one stored for its record, the node logs the code, both severities and the source. A set of ``(fault_code, source_id, severity)`` keys limits it to one line per key for the process - lifetime, so the log volume does not follow the report rate. The source owns the record, so the - set holds at most four keys per record the storage already keeps. + lifetime, so the log volume does not follow the report rate. Status Lifecycle (Debounce Model) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp index 36704260b..7841f391a 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -202,16 +203,23 @@ class FaultManagerCore { ReportOutcome report(const fd::FaultReport & report); /// Clears one record. An empty @p source_id addresses the only record of @p code and is refused as - /// ambiguous when several owners hold it; a set one addresses exactly that owner's record. + /// ambiguous when several records hold it. A set one addresses the record owned by that id, and + /// otherwise the record that source reports into (see resolve_target). ClearOutcome clear(const std::string & code, const std::string & source_id, bool skip_correlation_auto_clear); ListOutcome list(const ListQuery & query); FaultDetail get(const std::string & code, const std::string & source_id); /// Resolves the record a single-record request addresses, under the core mutex. Empty @p source_id - /// picks the only record of @p fault_code; several records make it ambiguous. On failure returns - /// nullopt and sets @p error. + /// picks the only record of @p fault_code; several records make it ambiguous. A set one names the + /// record it owns (`*` names the shared one); when no record has that owner, it names the record + /// record_id_for gives that source. On failure returns nullopt and sets @p error. std::optional resolve_target(const std::string & fault_code, const std::string & source_id, std::string & error); + /// The record a report from @p source_id for @p fault_code goes into: one record per source when + /// the code is configured per_source, else the one record shared by every source. The only place + /// that builds a record identity from a (code, source) pair. + FaultId record_id_for(const std::string & fault_code, const std::string & source_id) const; + /// Throws FaultStorage::IgnorableConnectionException when the storage is unreachable. std::vector list_for_entity(const std::string & entity_id); @@ -278,7 +286,17 @@ class FaultManagerCore { /// Warns once per (code, source, severity) when a FAILED report disagrees with the stored severity. void warn_severity_mismatch(const fd::FaultReport & report, uint8_t stored_severity); + struct DebouncePolicyWitness { + DebounceConfig config; + std::string source_id; + bool warned{false}; + }; + DebounceConfig resolve_config(const std::string & source_id, const std::string & fault_code) const; + /// Warns once per code when two sources feeding the shared record of @p id resolve different debounce + /// policies. A per-source record has one source and never warns. + void warn_on_conflicting_debounce_policy(const FaultId & id, const std::string & source_id, + const DebounceConfig & resolved); void audit_transition(const char * transition, const fd::FaultRecord & fault, const std::string & source_id, int64_t occurred_at_ns); void publish_event(const char * event_type, const fd::FaultRecord & fault, @@ -300,6 +318,7 @@ class FaultManagerCore { std::mutex mutex_; std::set> severity_mismatch_logged_; + std::unordered_map debounce_policy_witness_; /// Next due time of each tick job, ns since the epoch. int64_t auto_confirm_due_ns_{0}; int64_t cleanup_due_ns_{0}; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_startup.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_startup.hpp index f486e92c0..939a946a9 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_startup.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_startup.hpp @@ -45,7 +45,7 @@ CoreParts build_core_parts(const FaultManagerConfig & config, LogSink & sink); /// Appends the audit event of one reclassified fault. May throw to fail the startup. using ReclassifyAuditFn = - std::function; + std::function; /// Turns stale HEALED rows into CLEARED for codes whose healing is now disabled. Calls @p audit for /// each flipped fault when @p audit_enabled. Storage that is unreachable skips it with a warning. @@ -53,4 +53,11 @@ void reclassify_stale_healed(FaultStorage & storage, const FaultCodeThresholdRes const DebounceConfig & global_debounce, bool audit_enabled, LogSink & sink, const std::string & logger, const ReclassifyAuditFn & audit); +/// Logs one warning per fault code that holds an active (not CLEARED) record stored under the other +/// identity than the code is configured for: a shared record of a per_source code, or a per-source record +/// of a shared one. Stored records keep the identity they were written with; new reports use the +/// configured one. Storage that is unreachable skips it with a warning. +void warn_on_identity_mode_mismatch(FaultStorage & storage, const FaultCodeThresholdResolver * fault_code_resolver, + LogSink & sink, const std::string & logger); + } // namespace ros2_medkit_fault_manager::core diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp index dff17263a..bd38cd133 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp @@ -91,13 +91,20 @@ bool is_near_miss(bool is_failed_event, const std::string & resulting_status); /// defaults (-1 / 3). Returns true if the config was already valid. bool sanitize_debounce_config(DebounceConfig & config); -/// Identity of one fault record: a fault code plus the reporting source that owns it. +/// Owner of a record that every reporting source of its fault code shares. /// -/// The owner is the source_id the ReportFault call carried. Two sources reporting one -/// fault_code are two records, and every piece of per-fault state (status, debounce -/// counter, occurrence count, timestamps, severity, freeze frame, snapshots, near -/// misses, rosbag links) belongs to one record. A clear or a heal driven by one owner -/// never touches another owner's record of the same code. +/// Reserved: ReportFault refuses it as a source_id, so a shared record can never be +/// mistaken for the per-source record of a source that happens to be called `*`. +inline constexpr const char * kSharedOwner = "*"; + +/// Identity of one fault record: a fault code plus the owner of the record. +/// +/// FaultManagerCore::record_id_for decides the owner. A code configured `per_source` gives +/// each reporting source its own record (owner = the report's source_id), any other code has +/// one record shared by every source (owner = kSharedOwner). Every piece of per-record state +/// (status, debounce counter, occurrence count, timestamps, severity, freeze frame, snapshots, +/// near misses, rosbag links) belongs to one record. A clear or a heal of one record never +/// touches another record of the same code. /// /// An aggregate on purpose: FaultId{code, owner} is the only way to build one, so no /// call site can silently pass a bare code where a record identity is required. @@ -110,27 +117,25 @@ struct FaultId { } /// Ordered by code first, so the records of one code are contiguous in a map and - /// get_faults_by_code is a range scan rather than a full sweep. + /// the ids of one code are a range scan rather than a full sweep. bool operator<(const FaultId & other) const { return std::tie(fault_code, owner) < std::tie(other.fault_code, other.owner); } }; -/// Owner given to a record migrated out of a database that recorded no readable -/// reporting source. -/// -/// A migrated record is never left with an empty owner. It would be unreachable -/// through the source_id every single-record service takes, and it would leave the -/// child backfill looking for work on every open. The empty owner means one thing -/// only, a child row (freeze frame, snapshot, rosbag link) not yet assigned to a -/// record, and keeping that meaning unambiguous is what this constant is for. -inline constexpr const char * kLegacyOwner = "legacy"; +/// A stored record together with the identity it is stored under. +struct FaultEntry { + FaultId id; + fd::FaultRecord record; +}; /// Internal fault state of one record, stored in memory. Times are wall-clock ns >= 0; 0 means never. struct FaultState { std::string fault_code; - /// Reporting source that owns this record. Half of the record identity. + /// Owner of this record. Half of the record identity. std::string owner; + /// Every source that reported FAILED for this record. One source for a per-source record. + std::set reporting_sources; uint8_t severity{0}; std::string description; int64_t first_occurred_ns{0}; @@ -153,7 +158,7 @@ struct FaultState { /// one moment and only mean anything together. struct SnapshotData { std::string fault_code; - std::string owner; ///< Reporting source that owns the record this reading belongs to + std::string owner; ///< Owner of the record this reading belongs to std::string topic; std::string message_type; std::string data; ///< JSON-encoded message data @@ -172,7 +177,7 @@ struct SnapshotData { /// (lookup returns nullopt, never an empty {}). struct FreezeFrameData { std::string fault_code; - std::string owner; ///< Reporting source that owns the record this frame belongs to + std::string owner; ///< Owner of the record this frame belongs to std::string data; ///< Compact JSON object: {"": , ...} int64_t captured_at_ns{0}; }; @@ -202,16 +207,16 @@ std::string rosbag_recording_id(const std::string & file_path); /// long-running appliance keeps the recent series rather than freezing it at boot. struct NearMissRecord { std::string fault_code; + std::string owner; ///< Owner of the record the series belongs to int64_t occurred_at_ns{0}; ///< Timestamp of the report that moved the counter int32_t debounce_counter{0}; ///< Counter value AFTER this report - /// Confirmation threshold the report was evaluated against. The counter it belongs to is - /// this record's own, so with per-entity overrides both are the reporting source's and the - /// value is the distance to confirmation for the record. + /// Confirmation threshold the report was evaluated against: the distance to confirmation of + /// the record's counter at that moment. int32_t confirmation_threshold{0}; uint8_t severity{0}; ///< Severity carried by the report - std::string source_id; ///< Reporting source, which is the record's owner + std::string source_id; ///< Source that sent the report; one record can hold reports of several /// Fault status after the report was applied. Never CONFIRMED - that is what makes the report a /// near miss. It separates a counter climbing from a resting state (PREFAILED) from one walking @@ -227,7 +232,7 @@ struct NearMissRecord { /// only when its last row goes. struct RosbagFileInfo { std::string fault_code; - std::string owner; ///< Reporting source that owns the record holding this link + std::string owner; ///< Owner of the record holding this link std::string recording_id; ///< Basename of file_path; shared by every row of a burst std::string file_path; std::string format; ///< "sqlite3" or "mcap" @@ -264,17 +269,18 @@ class FaultStorage { virtual DebounceConfig get_debounce_config() const = 0; /// Report a fault event (FAILED or PASSED) - /// @param fault_code Global fault identifier + /// @param id The record this event applies to, as FaultManagerCore::record_id_for decided it. + /// The event creates that record when none exists and the event is FAILED. /// @param kind Whether the report is a FAILED or a PASSED event /// @param severity Fault severity level (only used for FAILED events) /// @param description Human-readable description (only used for FAILED events) - /// @param source_id Reporting source identifier. Together with fault_code it names the - /// record this event applies to, and it creates that record when none exists. + /// @param source_id Source that sent the report. A FAILED event adds it to the record's + /// reporting sources and the near-miss series keeps it per entry. /// @param timestamp_ns Wall-clock time of the event, ns (>= 0) - /// @param config Debounce configuration to apply for this event (resolved per-entity by the node) + /// @param config Debounce configuration to apply for this event (resolved per entity and code by the core) /// @return true if this is a new occurrence (new record or reactivated CLEARED record), /// false if an existing active record was updated - virtual bool report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, + virtual bool report_fault_event(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) = 0; @@ -283,26 +289,37 @@ class FaultStorage { /// @param severity Severity level to filter (if filter_by_severity is true) /// @param statuses List of statuses to include (empty = CONFIRMED only) /// @return Vector of matching faults - virtual std::vector list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const = 0; + std::vector list_faults(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const { + std::vector records; + for (auto & entry : list_fault_entries(filter_by_severity, severity, statuses)) { + records.push_back(std::move(entry.record)); + } + return records; + } + + /// list_faults with the identity each record is stored under. + virtual std::vector list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const = 0; /// Get a single fault record /// @param id The record to look up /// @return The fault if found, nullopt otherwise virtual std::optional get_fault(const FaultId & id) const = 0; - /// Every record carrying @p fault_code, one per owning reporting source. + /// The identity of every record carrying @p fault_code: one shared record, or one per + /// source for a per-source code, or both after the mode of the code changed. /// /// The resolution step behind an unscoped service call: with exactly one record the /// call applies to it, with several it is ambiguous and the owners are the answer. /// Returns them ordered by owner, so an ambiguity message is stable. /// @param fault_code The fault code to look up - /// @return Every record carrying the code, empty when none does - virtual std::vector get_faults_by_code(const std::string & fault_code) const = 0; + /// @return Every record id carrying the code, empty when none does + virtual std::vector get_ids_by_code(const std::string & fault_code) const = 0; /// Clear one fault record (manual acknowledgment). Drops that record's per-topic snapshots. /// The freeze-frame and the near-miss series are RETAINED, because they outlive a single fault - /// cycle and cannot be reconstructed afterwards. Another owner's record of the same code is + /// cycle and cannot be reconstructed afterwards. Another record of the same code is /// untouched, snapshots included. /// @param id The record to clear /// @return true if the record was found and cleared, false if not found @@ -547,16 +564,15 @@ class InMemoryFaultStorage : public FaultStorage { void set_debounce_config(const DebounceConfig & config) override; DebounceConfig get_debounce_config() const override; - bool report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, - const std::string & description, const std::string & source_id, int64_t timestamp_ns, - const DebounceConfig & config) override; + bool report_fault_event(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, + const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) override; - std::vector list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const override; + std::vector list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const override; std::optional get_fault(const FaultId & id) const override; - std::vector get_faults_by_code(const std::string & fault_code) const override; + std::vector get_ids_by_code(const std::string & fault_code) const override; bool clear_fault(const FaultId & id) override; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/postgres_fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/postgres_fault_storage.hpp index 000997c8e..0b1205750 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/postgres_fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/postgres_fault_storage.hpp @@ -62,16 +62,15 @@ class PgFaultStorage : public FaultStorage { void set_debounce_config(const DebounceConfig & config) override; DebounceConfig get_debounce_config() const override; - bool report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, - const std::string & description, const std::string & source_id, int64_t timestamp_ns, - const DebounceConfig & config) override; + bool report_fault_event(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, + const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) override; - std::vector list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const override; + std::vector list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const override; std::optional get_fault(const FaultId & id) const override; - std::vector get_faults_by_code(const std::string & fault_code) const override; + std::vector get_ids_by_code(const std::string & fault_code) const override; bool clear_fault(const FaultId & id) override; @@ -139,7 +138,8 @@ class PgFaultStorage : public FaultStorage { void create_schema(pqxx::work & tx) const; /// Bring tables written before records were keyed by (fault_code, owner) to the current layout, in @p tx. - /// Does nothing on a fresh database or one already migrated. + /// Each legacy row becomes the shared record of its code, with every child row attached to it. Does nothing + /// on a fresh database or one already migrated. void migrate_to_owner_identity(pqxx::work & tx) const; /// Whether @p table exists in the current schema. @@ -151,12 +151,6 @@ class PgFaultStorage : public FaultStorage { /// Drop the PRIMARY KEY of @p table, whatever it is named. void drop_primary_key(pqxx::work & tx, const char * table) const; - /// Whether a child row with an empty owner belongs to a code that has exactly one owner. - bool has_assignable_child_rows(pqxx::work & tx) const; - - /// Give child rows with an empty owner the owner of their code, where the code has exactly one. - void backfill_child_owners(pqxx::work & tx) const; - /// @p text with every copy of the known password replaced by "***". std::string redact(std::string text) const; @@ -172,24 +166,27 @@ class PgFaultStorage : public FaultStorage { /// report_fault_event body without taking mutex_ or opening a transaction. Caller holds mutex_ /// and supplies the transaction, so the fault row and any near-miss row commit together. - bool report_fault_event_locked(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, + bool report_fault_event_locked(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config, pqxx::work & tx); /// Append one entry to the near-miss series and evict the oldest entries beyond /// max_near_misses_per_fault_. Caller holds mutex_ and has already written the fault row. - /// @param fault_code The fault code that nearly confirmed + /// @param id The record that nearly confirmed; it scopes the series /// @param occurred_at_ns Timestamp of the report /// @param debounce_counter Counter value after the report /// @param config Debounce config the report was evaluated against /// @param severity Severity carried by the report - /// @param source_id Reporting source + /// @param source_id Source that sent the report /// @param resulting_status Fault status after the report was applied /// @param tx Transaction the fault row was written in - void record_near_miss_locked(const std::string & fault_code, int64_t occurred_at_ns, int32_t debounce_counter, + void record_near_miss_locked(const FaultId & id, int64_t occurred_at_ns, int32_t debounce_counter, const DebounceConfig & config, uint8_t severity, const std::string & source_id, const std::string & resulting_status, pqxx::work & tx); + /// One row selected with kFaultColumns, as a record. + fd::FaultRecord read_fault_row(const pqxx::row & r) const; + /// Deserialize JSON array string from PostgreSQL TEXT/JSONB field std::vector parse_json_array(const std::string & json_str) const; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp index daf2a90d6..e107b8851 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp @@ -48,16 +48,15 @@ class SqliteFaultStorage : public FaultStorage { void set_debounce_config(const DebounceConfig & config) override; DebounceConfig get_debounce_config() const override; - bool report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, - const std::string & description, const std::string & source_id, int64_t timestamp_ns, - const DebounceConfig & config) override; + bool report_fault_event(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, + const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) override; - std::vector list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const override; + std::vector list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const override; std::optional get_fault(const FaultId & id) const override; - std::vector get_faults_by_code(const std::string & fault_code) const override; + std::vector get_ids_by_code(const std::string & fault_code) const override; bool clear_fault(const FaultId & id) override; @@ -128,41 +127,12 @@ class SqliteFaultStorage : public FaultStorage { /// Detected per table by PRAGMA table_info lacking `owner`, which is what makes it safe /// to re-run on every open. faults and freeze_frames are REBUILT (their identity is a /// PRIMARY KEY, which ALTER TABLE cannot change and CREATE TABLE IF NOT EXISTS would - /// silently skip), while snapshots and rosbag_files only gain a column. The fault owner - /// comes from the legacy row's first reporting source, empty when none can be read: that - /// is the only per-source fact the old schema holds, so a row with several sources folds - /// onto its first one rather than inventing per-source counters, statuses and timestamps - /// that were never recorded. - /// - /// It never refuses to finish. A row whose content cannot be read leaves an empty owner - /// and a log line, because a migration that aborts on a row takes the whole database with - /// it: the rebuild rolls back and the next open reaches the same row again, forever. + /// silently skip), while snapshots, rosbag_files and near_misses only gain a column. + /// A legacy database holds one record per fault code, fed by every source, so every row + /// becomes the shared record of its code and every child row attaches to it. The rows keep + /// their reporting_sources as written. All of it runs in one transaction. void migrate_faults_add_owner(); - /// Copy every legacy `faults` row into `faults_new`, deriving the owner in C++. - /// - /// In C++ rather than one INSERT ... SELECT because the derivation must not depend on the - /// column being valid JSON, and SQL's json_extract raises rather than returning NULL on - /// text it cannot parse. Caller holds the migration transaction. - void copy_legacy_fault_rows(); - - /// Give every ownerless child row the owner of its fault, where that is unambiguous. - /// - /// Runs on every open, not only when a column was just added: a child table can be left - /// with empty owners by an open interrupted between the faults rebuild and the backfill, - /// or by a database whose child table gained the column while faults was still legacy. - /// Such a row is invisible to every (code, owner) read, so leaving it is losing evidence. - /// A code carrying several owners is NOT resolved - guessing would hand one owner another - /// owner's evidence - and the row keeps its empty owner with a log line naming it. - void backfill_child_owners(); - - /// Whether any child row is waiting for an owner the database can actually prove, so the - /// backfill above has work to do. Deliberately the backfill's own condition and not just - /// "owner is empty": a row whose code has several owners or none can never be assigned, - /// and treating it as pending would re-enter the write transaction on every open forever. - /// Tables without the column yet are skipped: the migration adds it. - bool has_assignable_child_rows() const; - /// Whether @p table exists, asked of sqlite_master. Used to tell a leftover scratch table /// apart from a clean start, which DROP TABLE IF EXISTS on its own cannot report. bool table_exists(const char * table) const; @@ -191,29 +161,27 @@ class SqliteFaultStorage : public FaultStorage { /// report_fault_event body without taking mutex_ or opening a transaction. Caller holds mutex_ /// and wraps the call, so the fault row and any near-miss row commit together. - bool report_fault_event_locked(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, + bool report_fault_event_locked(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config); /// Append one entry to the near-miss series and evict the oldest entries beyond /// max_near_misses_per_fault_. Caller holds mutex_ and has already written the fault row. - /// @param fault_code The fault code that nearly confirmed + /// @param id The record that nearly confirmed; it scopes the series /// @param occurred_at_ns Timestamp of the report /// @param debounce_counter Counter value after the report /// @param config Debounce config the report was evaluated against /// @param severity Severity carried by the report - /// @param source_id Reporting source, which is the record's owner and scopes the series + /// @param source_id Source that sent the report /// @param resulting_status Fault status after the report was applied - void record_near_miss_locked(const std::string & fault_code, int64_t occurred_at_ns, int32_t debounce_counter, + void record_near_miss_locked(const FaultId & id, int64_t occurred_at_ns, int32_t debounce_counter, const DebounceConfig & config, uint8_t severity, const std::string & source_id, const std::string & resulting_status); /// Run a plain SQL statement or throw with the SQLite error. Caller holds mutex_. void exec_or_throw(const char * sql); - /// Serialize vector of strings to JSON array string. The faults table keeps - /// reporting_sources as the serialized form of owner, so a reader of the raw table - /// still sees the field it always saw. + /// Serialize vector of strings to JSON array string, the form of faults.reporting_sources. static std::string serialize_json_array(const std::vector & vec); std::string db_path_; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/threshold_resolver.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/threshold_resolver.hpp index 80508e6d0..48bf09fb2 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/threshold_resolver.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/threshold_resolver.hpp @@ -40,6 +40,9 @@ struct FaultCodeDebounceOverride { std::optional confirmation_threshold; std::optional healing_enabled; std::optional healing_threshold; + /// Whether each reporting source of the code gets its own record. Unset means false: one record + /// per code, shared by every source. Entity overrides do not carry it. + bool per_source{false}; }; /// Resolves per-entity debounce thresholds using longest-prefix matching. @@ -75,11 +78,12 @@ class EntityThresholdResolver { /// Resolves per-fault_code debounce thresholds by exact match on the fault code. /// -/// The debounce counter lives on the fault code while an entity override is -/// selected by the reporting source, so two entities reporting one code debounce -/// it under two policies. A fault-code override is the layer that removes that: -/// it resolves the same whoever reports, and is applied on top of whatever the -/// entity layer produced. +/// A code reported by several sources has one record and one debounce counter, while +/// an entity override is selected by the reporting source, so two entities reporting +/// one code debounce it under two policies. A fault-code override is the layer that +/// removes that: it resolves the same whoever reports, and is applied on top of +/// whatever the entity layer produced. The same entry carries the per-code choice +/// between one shared record and one record per source. class FaultCodeThresholdResolver { public: FaultCodeThresholdResolver() = default; @@ -94,12 +98,16 @@ class FaultCodeThresholdResolver { /// them. A code with no override returns `base` unchanged. DebounceConfig resolve(const std::string & fault_code, const DebounceConfig & base) const; + /// Whether @p fault_code is configured `per_source`. A code with no entry is not. + bool per_source(const std::string & fault_code) const; + /// Number of configured fault-code entries. size_t size() const; /// Load fault-code threshold overrides from a YAML file. /// Returns an empty vector on parse error (logs warning through @p sink). - /// YAML format: map of fault code -> {confirmation_threshold, healing_enabled, healing_threshold} + /// YAML format: map of fault code -> {confirmation_threshold, healing_enabled, healing_threshold, per_source}. + /// per_source must be the YAML boolean true or false; any other value rejects the whole file. static std::vector load_from_yaml(const std::string & path, core::LogSink & sink); private: diff --git a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp index 99dd349db..34a4be867 100644 --- a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp +++ b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp @@ -76,11 +76,6 @@ const char * severity_name(uint8_t severity) { } } -/// The owner a stored record names, empty when it names none. -std::string owner_of(const fd::FaultRecord & record) { - return record.reporting_sources.empty() ? std::string() : record.reporting_sources.front(); -} - std::string extract_topic_name(const std::string & topic_path) { auto pos = topic_path.rfind('/'); if (pos != std::string::npos && pos < topic_path.length() - 1) { @@ -184,7 +179,14 @@ std::optional FaultManagerCore::resolve_target_locked(const std::string try { if (!source_id.empty()) { + // The record the id owns, which is how a shared record (owner `*`) is addressed. Otherwise the + // record this source reports into, so a client that only knows a reporting source reaches the + // shared record of a default code. FaultId id{fault_code, source_id}; + if (storage_->contains(id)) { + return id; + } + id = record_id_for(fault_code, source_id); if (!storage_->contains(id)) { error = "Fault not found: " + fault_code + " (source " + source_id + ")"; return std::nullopt; @@ -193,9 +195,9 @@ std::optional FaultManagerCore::resolve_target_locked(const std::string } // Unscoped. One record: the call means that record. Several: refuse rather than - // pick, because acting on an arbitrary owner's record is the failure this identity - // change exists to remove, and the caller cannot tell which one it got. - const auto candidates = storage_->get_faults_by_code(fault_code); + // pick, because acting on an arbitrary record is the failure the per-source choice + // exists to avoid, and the caller cannot tell which one it got. + const auto candidates = storage_->get_ids_by_code(fault_code); if (candidates.empty()) { error = "Fault not found: " + fault_code; return std::nullopt; @@ -206,13 +208,13 @@ std::optional FaultManagerCore::resolve_target_locked(const std::string if (!owners.empty()) { owners += ", "; } - owners += owner_of(candidate); + owners += candidate.owner; } - error = "ambiguous: " + fault_code + " is reported by " + std::to_string(candidates.size()) + " sources (" + + error = "ambiguous: " + fault_code + " has " + std::to_string(candidates.size()) + " records, owned by (" + owners + "). Set source_id to pick one."; return std::nullopt; } - return FaultId{fault_code, owner_of(candidates.front())}; + return candidates.front(); } catch (const FaultStorage::IgnorableConnectionException & e) { logf(log_, LogLevel::kWarn, config_.logger_name, "Failed to connect with the fault storage server: %s", e.what()); error = storage_unavailable(e); @@ -220,6 +222,13 @@ std::optional FaultManagerCore::resolve_target_locked(const std::string } } +FaultId FaultManagerCore::record_id_for(const std::string & fault_code, const std::string & source_id) const { + if (fault_code_resolver_ && fault_code_resolver_->per_source(fault_code)) { + return FaultId{fault_code, source_id}; + } + return FaultId{fault_code, kSharedOwner}; +} + std::vector FaultManagerCore::list_for_entity(const std::string & entity_id) { std::lock_guard lock(mutex_); std::vector result; @@ -389,13 +398,17 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { return reject("source_id cannot be empty"); } + if (report.source_id == kSharedOwner) { + logf(log_, LogLevel::kWarn, logger, "ReportFault rejected: source_id '%s' is reserved", kSharedOwner); + return reject(std::string("source_id '") + kSharedOwner + "' is reserved"); + } + const auto connection_lost = [&](const FaultStorage::IgnorableConnectionException & e) { logf(log_, LogLevel::kWarn, logger, "Failed to connect with the fault storage server: %s", e.what()); }; - // The record this report addresses. The source_id owns it, so a report from a second source of the - // same code creates a second record instead of merging. - const FaultId id{report.fault_code, report.source_id}; + // The record this report goes into: the source's own for a per_source code, else the code's shared one. + const FaultId id = record_id_for(report.fault_code, report.source_id); // Get status before update (if the record exists) std::optional fault_before; @@ -408,15 +421,14 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { std::string status_before = fault_before ? fault_before->status : ""; // Resolve the debounce config: global, then the entity override matching - // source_id, then the fault code's own override. The record and the config share - // an owner, so the resolved band always governs the counter it is applied to. + // source_id, then the fault code's own override. auto resolved_config = resolve_config(report.source_id, report.fault_code); const int64_t event_time_ns = clock_.now_ns(); bool is_new = false; try { - is_new = storage_->report_fault_event(report.fault_code, report.kind, report.severity, report.description, - report.source_id, event_time_ns, resolved_config); + is_new = storage_->report_fault_event(id, report.kind, report.severity, report.description, report.source_id, + event_time_ns, resolved_config); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); return outcome; @@ -438,6 +450,10 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { } if (fault_after) { + // Only for records the store holds: a PASSED for an unknown record writes nothing, so it must + // not add an entry here either (the diagnostic bridge sends PASSED for every OK status). + warn_on_conflicting_debounce_policy(id, report.source_id, resolved_config); + // Process through correlation engine (if enabled). Only FAILED events correlate. bool should_mute = false; if (correlation_ && failed) { @@ -479,8 +495,7 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { } just_confirmed = true; } else if (!is_new && fault_after->status == fd::fault_status::kConfirmed) { - // Record was already CONFIRMED, data updated (last_occurred, severity). Not - // reporting_sources: it names the one owner and never grows. Not + // Record was already CONFIRMED, data updated (last_occurred, severity, reporting_sources). Not // occurrence_count: a re-report inside one occurrence does not touch it. if (!should_mute) { publish_event(fd::fault_event_type::kUpdated, *fault_after); @@ -666,8 +681,9 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, const std: ListOutcome FaultManagerCore::list_locked(const ListQuery & query) { const std::string & logger = config_.logger_name; ListOutcome outcome; + std::vector entries; try { - outcome.faults = storage_->list_faults(query.filter_by_severity, query.severity, query.statuses); + entries = storage_->list_fault_entries(query.filter_by_severity, query.severity, query.statuses); } catch (const FaultStorage::IgnorableConnectionException & e) { logf(log_, LogLevel::kWarn, logger, "Failed to connect with the fault storage server: %s", e.what()); return outcome; @@ -692,15 +708,13 @@ ListOutcome FaultManagerCore::list_locked(const ListQuery & query) { outcome.muted_faults.push_back(info); } } else { - // Hide the muted RECORDS, never every record of a muted code. A root cause mutes the symptoms of - // its own owner, so another owner's record of that same code was never suppressed and hiding it - // would drop a live fault from the default view. - auto & faults = outcome.faults; - faults.erase(std::remove_if(faults.begin(), faults.end(), - [this](const fd::FaultRecord & fault) { - return correlation_->is_muted(FaultId{fault.fault_code, owner_of(fault)}); - }), - faults.end()); + // Hide the muted RECORDS, never every record of a muted code: a record the engine never muted + // stays in the default view. + entries.erase(std::remove_if(entries.begin(), entries.end(), + [this](const FaultEntry & entry) { + return correlation_->is_muted(entry.id); + }), + entries.end()); } if (query.include_clusters) { @@ -725,6 +739,11 @@ ListOutcome FaultManagerCore::list_locked(const ListQuery & query) { } } + outcome.faults.reserve(entries.size()); + for (auto & entry : entries) { + outcome.faults.push_back(std::move(entry.record)); + } + logf(log_, LogLevel::kDebug, logger, "ListFaults returned %zu faults (muted=%u, clusters=%u)", outcome.faults.size(), outcome.muted_count, outcome.cluster_count); return outcome; @@ -870,12 +889,37 @@ void FaultManagerCore::warn_severity_mismatch(const fd::FaultReport & report, ui } logf(log_, LogLevel::kWarn, config_.logger_name, "Fault '%s' reported with severity %s by '%s', but its record is stored as %s. A record has one " - "severity and keeps the highest its source ever reported. Check the reporter's configuration. Logged " + "severity and keeps the highest ever reported. Check the reporter's configuration. Logged " "once per code, source and severity until restart.", report.fault_code.c_str(), severity_name(report.severity), report.source_id.c_str(), severity_name(stored_severity)); } +void FaultManagerCore::warn_on_conflicting_debounce_policy(const FaultId & id, const std::string & source_id, + const DebounceConfig & resolved) { + if (id.owner != kSharedOwner) { + return; + } + auto [it, inserted] = + debounce_policy_witness_.try_emplace(id.fault_code, DebouncePolicyWitness{resolved, source_id, false}); + if (inserted || it->second.warned || debounce_policy_equal(it->second.config, resolved)) { + return; + } + + // One counter, two policies: the report that arrives decides the transition, so the other + // source's debounce policy is bypassed for as long as both report this code. + it->second.warned = true; + logf(log_, LogLevel::kWarn, config_.logger_name, + "Fault code '%s' is debounced two ways: '%s' resolves confirmation=%d healing_enabled=%s healing=%d, " + "'%s' resolves confirmation=%d healing_enabled=%s healing=%d. The debounce counter belongs to the shared " + "record of the fault code, so whichever source reports decides the transition and the other policy is " + "bypassed. Pin confirmation_threshold, healing_enabled and healing_threshold for this code in " + "fault_thresholds.config_file, or set per_source: true there to give each source its own record.", + id.fault_code.c_str(), it->second.source_id.c_str(), it->second.config.confirmation_threshold, + it->second.config.healing_enabled ? "true" : "false", it->second.config.healing_threshold, source_id.c_str(), + resolved.confirmation_threshold, resolved.healing_enabled ? "true" : "false", resolved.healing_threshold); +} + void FaultManagerCore::audit_transition(const char * transition, const fd::FaultRecord & fault, const std::string & source_id, int64_t occurred_at_ns) { if (!audit_) { diff --git a/src/ros2_medkit_fault_manager/src/core/fault_manager_startup.cpp b/src/ros2_medkit_fault_manager/src/core/fault_manager_startup.cpp index 1c014431e..4d7eb1398 100644 --- a/src/ros2_medkit_fault_manager/src/core/fault_manager_startup.cpp +++ b/src/ros2_medkit_fault_manager/src/core/fault_manager_startup.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -335,7 +336,7 @@ void reclassify_stale_healed(FaultStorage & storage, const FaultCodeThresholdRes for (const auto & id : reclassified) { auto fault = storage.get_fault(id); if (fault) { - audit(*fault, reclassified_at_ns); + audit(id, *fault, reclassified_at_ns); } } } @@ -349,4 +350,49 @@ void reclassify_stale_healed(FaultStorage & storage, const FaultCodeThresholdRes } } +void warn_on_identity_mode_mismatch(FaultStorage & storage, const FaultCodeThresholdResolver * fault_code_resolver, + LogSink & sink, const std::string & logger) { + std::vector entries; + try { + entries = storage.list_fault_entries(false, 0, + {fd::fault_status::kPrefailed, fd::fault_status::kPrepassed, + fd::fault_status::kConfirmed, fd::fault_status::kHealed}); + } catch (const FaultStorage::IgnorableConnectionException & e) { + logf(sink, LogLevel::kWarn, logger, "Startup identity check not done, fault storage unavailable: %s", e.what()); + return; + } + + // Owners of the active records of each code, and whether any of them disagrees with the configured mode. + struct CodeState { + std::vector owners; + bool mismatch{false}; + }; + std::map codes; + for (const auto & entry : entries) { + const bool per_source = fault_code_resolver != nullptr && fault_code_resolver->per_source(entry.id.fault_code); + auto & state = codes[entry.id.fault_code]; + state.owners.push_back(entry.id.owner); + if ((entry.id.owner == kSharedOwner) == per_source) { + state.mismatch = true; + } + } + + for (const auto & [code, state] : codes) { + if (!state.mismatch) { + continue; + } + const bool per_source = fault_code_resolver != nullptr && fault_code_resolver->per_source(code); + std::string owners; + for (const auto & owner : state.owners) { + owners += (owners.empty() ? "" : ", ") + owner; + } + logf(sink, LogLevel::kWarn, logger, + "Fault code '%s' is configured %s but holds active records owned by (%s). Stored records keep the identity " + "they were written with; new reports go into the configured one.", + code.c_str(), + per_source ? "per_source: one record per reporting source" : "shared: one record for all sources", + owners.c_str()); + } +} + } // namespace ros2_medkit_fault_manager::core diff --git a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp index a83aeee96..dee60d96f 100644 --- a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp @@ -136,13 +136,12 @@ FaultManagerNode::FaultManagerNode(const rclcpp::NodeOptions & options) : Node(" // Runs after the override files are loaded, so a code with its own healing_enabled: true keeps // its HEALED status. Each flipped fault is audited, or the audit log's verify() would not see it. - core::reclassify_stale_healed( - *storage_, core_->fault_code_resolver(), core_->global_debounce(), core_->audit_log() != nullptr, log_sink_, - config_.logger_name, [this](const fd::FaultRecord & fault, int64_t reclassified_at_ns) { - audit_transition(kTransitionCleared, fault, - fault.reporting_sources.empty() ? std::string() : fault.reporting_sources.front(), - reclassified_at_ns); - }); + core::reclassify_stale_healed(*storage_, core_->fault_code_resolver(), core_->global_debounce(), + core_->audit_log() != nullptr, log_sink_, config_.logger_name, + [this](const FaultId & id, const fd::FaultRecord & fault, int64_t reclassified_at_ns) { + audit_transition(kTransitionCleared, fault, id.owner, reclassified_at_ns); + }); + core::warn_on_identity_mode_mismatch(*storage_, core_->fault_code_resolver(), log_sink_, config_.logger_name); // Create service servers report_fault_srv_ = create_service( diff --git a/src/ros2_medkit_fault_manager/src/fault_storage.cpp b/src/ros2_medkit_fault_manager/src/fault_storage.cpp index 11f81f2c5..e538aff1e 100644 --- a/src/ros2_medkit_fault_manager/src/fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_storage.cpp @@ -93,11 +93,7 @@ fd::FaultRecord FaultState::to_record() const { msg.last_passed_ns = last_passed_ns; msg.occurrence_count = occurrence_count; msg.status = status; - - // The owner, as a one-element list: the field names the source that owns the record, - // and the services that act on a single record take that value back as their source_id. - msg.reporting_sources.assign(1, owner); - + msg.reporting_sources.assign(reporting_sources.begin(), reporting_sources.end()); return msg; } @@ -117,26 +113,25 @@ void InMemoryFaultStorage::update_status(FaultState & state, const DebounceConfi state.status = compute_debounce_status(state.debounce_counter, state.status, config); } -bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, +bool InMemoryFaultStorage::report_fault_event(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) { std::lock_guard lock(mutex_); const bool is_failed = (kind == fd::ReportKind::kFailed); - const FaultId id{fault_code, source_id}; - auto it = faults_.find(id); if (it == faults_.end()) { - // New record - only create one for FAILED events. A source that has never reported - // this code owns no record yet, so a PASSED from it has nothing to heal. + // New record - only create one for FAILED events. A PASSED event for a record that + // does not exist has nothing to heal. if (!is_failed) { return false; // PASSED event for non-existent record is ignored } FaultState state; - state.fault_code = fault_code; - state.owner = source_id; + state.fault_code = id.fault_code; + state.owner = id.owner; + state.reporting_sources.insert(source_id); state.severity = severity; state.description = description; state.first_occurred_ns = timestamp_ns; @@ -176,6 +171,7 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd state.first_occurred_ns = timestamp_ns; state.last_failed_ns = timestamp_ns; state.last_occurred_ns = timestamp_ns; + state.reporting_sources.insert(source_id); if (state.occurrence_count < std::numeric_limits::max()) { ++state.occurrence_count; } @@ -198,11 +194,11 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd } // Bring a counter left outside this config's band back into range before applying the report. - // A record's counter belongs to one source, but that source's resolved band can change under - // it: an entity-threshold file reloaded with different values, or a global config that moved - // between runs, leaves a stored value clamped to the old ceiling sitting above the new one. - // The SQLite backend clamps on read for the same reason, and the two backends have to agree on - // the counter they record and on the status it produces. + // The band a record resolves to can change under it (per-entity overrides when several sources + // share the counter, a reloaded threshold file, a global config that moved between runs), which + // leaves a stored value clamped to the old ceiling sitting above the new one. The SQLite + // backend clamps on read for the same reason, and the two backends have to agree on the counter + // they record and on the status it produces. state.debounce_counter = clamp_debounce_counter(state.debounce_counter, config); if (is_failed) { @@ -221,8 +217,10 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd // run the counter off to INT32_MIN and delay later healing). state.debounce_counter = clamp_debounce_counter(state.debounce_counter - 1, config); - // Update severity if higher. Per record: a CRITICAL from one owner does not escalate - // another owner's record of the same code. + // Add source if not already present + state.reporting_sources.insert(source_id); + + // Update severity if higher if (severity > state.severity) { state.severity = severity; } @@ -257,8 +255,8 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd return false; } -std::vector InMemoryFaultStorage::list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const { +std::vector InMemoryFaultStorage::list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const { std::lock_guard lock(mutex_); // Determine which statuses to include @@ -280,7 +278,7 @@ std::vector InMemoryFaultStorage::list_faults(bool filter_by_se } } - std::vector result; + std::vector result; result.reserve(faults_.size()); for (const auto & [id, state] : faults_) { @@ -294,7 +292,7 @@ std::vector InMemoryFaultStorage::list_faults(bool filter_by_se continue; } - result.push_back(state.to_record()); + result.push_back(FaultEntry{id, state.to_record()}); } return result; @@ -311,18 +309,18 @@ std::optional InMemoryFaultStorage::get_fault(const FaultId & i return it->second.to_record(); } -std::vector InMemoryFaultStorage::get_faults_by_code(const std::string & fault_code) const { +std::vector InMemoryFaultStorage::get_ids_by_code(const std::string & fault_code) const { std::lock_guard lock(mutex_); // FaultId orders by code first, so the records of one code are one contiguous range // and this is a range scan. Owner order inside it is the map's, which is what makes // an ambiguity message list the owners the same way every time. - std::vector result; + std::vector result; for (auto it = faults_.lower_bound(FaultId{fault_code, ""}); it != faults_.end(); ++it) { if (it->first.fault_code != fault_code) { break; } - result.push_back(it->second.to_record()); + result.push_back(it->first); } return result; } @@ -519,6 +517,7 @@ void InMemoryFaultStorage::record_near_miss(const FaultState & state, const Debo NearMissRecord record; record.fault_code = state.fault_code; + record.owner = state.owner; record.occurred_at_ns = timestamp_ns; record.debounce_counter = state.debounce_counter; record.confirmation_threshold = config.confirmation_threshold; @@ -831,13 +830,13 @@ std::vector InMemoryFaultStorage::list_rosbags_for_entity(const std::vector rows; for (const auto & row : rosbag_files_) { - // The row already carries its record's owner, so the entity match is on the row. - // The faults_ lookup stays as the existence check it always was: a row whose - // record is gone is not served. - if (faults_.find(FaultId{row.info.fault_code, row.info.owner}) == faults_.end()) { + // The entity owns the row when it is among the reporting sources of the row's record. + auto fault_it = faults_.find(FaultId{row.info.fault_code, row.info.owner}); + if (fault_it == faults_.end()) { continue; } - if (row.info.owner == entity_fqn) { + const auto & sources = fault_it->second.reporting_sources; + if (sources.find(entity_fqn) != sources.end()) { rows.push_back(&row); } } diff --git a/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp b/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp index 645d87a08..c86bb48dd 100644 --- a/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp @@ -50,6 +50,11 @@ constexpr const char * kDefaultKeepalivesIdle = "5"; /// Time after a failed connection round before the next one. Requests in between fail at once. constexpr std::chrono::seconds kReconnectBackoff{5}; +/// Columns every fault read selects; PgFaultStorage::read_fault_row decodes them. +constexpr const char * kFaultColumns = + "fault_code, severity, description, first_occurred_ns, last_occurred_ns, occurrence_count, status, " + "reporting_sources, last_passed_ns, owner"; + /// Parsed connection options by keyword, or nullopt when libpq rejects the string. std::optional> parse_conn_info(const std::string & conn_info) { char * err = nullptr; @@ -329,52 +334,6 @@ void PgFaultStorage::drop_primary_key(pqxx::work & tx, const char * table) const } } -bool PgFaultStorage::has_assignable_child_rows(pqxx::work & tx) const { - if (!table_exists(tx, "faults") || !table_has_column(tx, "faults", "owner")) { - return false; - } - for (const char * table : {"freeze_frames", "snapshots", "rosbag_files"}) { - if (!table_exists(tx, table) || !table_has_column(tx, table, "owner")) { - continue; // the column arrives with the migration, which then fills it - } - // The same condition the backfill's UPDATE applies, so "is there work" and "what the work - // does" cannot disagree. - const std::string sql = std::string("SELECT 1 FROM ") + table + - " t WHERE t.owner = '' AND (SELECT COUNT(DISTINCT f.owner) FROM faults f " - "WHERE f.fault_code = t.fault_code) = 1 LIMIT 1"; - if (!execute(tx, sql).empty()) { - return true; - } - } - return false; -} - -void PgFaultStorage::backfill_child_owners(pqxx::work & tx) const { - for (const char * table : {"freeze_frames", "snapshots", "rosbag_files"}) { - if (!table_exists(tx, table) || !table_has_column(tx, table, "owner")) { - continue; - } - - // Correlated on the child row's own fault_code and applied only where that code has exactly - // ONE owner, so a code with two owners never hands one owner's evidence to the other. - execute(tx, std::string("UPDATE ") + table + - " SET owner = (SELECT MIN(f.owner) FROM faults f WHERE f.fault_code = " + table + - ".fault_code) WHERE owner = '' AND (SELECT COUNT(DISTINCT f.owner) FROM faults f WHERE " - "f.fault_code = " + - table + ".fault_code) = 1"); - - // What is left is evidence for a code no fault row carries any more, or for a code several - // owners share. Neither can be assigned without guessing, so it is named and kept. - const auto leftover = execute(tx, std::string("SELECT fault_code, COUNT(*) AS n FROM ") + table + - " WHERE owner = '' GROUP BY fault_code"); - for (const auto & r : leftover) { - core::logf(log_sink_, core::LogLevel::kWarn, "postgres_fault_storage", - "%s: %lld row(s) for fault '%s' keep an empty owner, the code has no single owner to assign them to", - table, static_cast(r["n"].as()), r["fault_code"].as().c_str()); - } - } -} - void PgFaultStorage::migrate_to_owner_identity(pqxx::work & tx) const { // Probed per table: a table that does not exist yet is built in the new layout by create_schema. const auto needs_owner = [&](const char * table) { @@ -384,62 +343,57 @@ void PgFaultStorage::migrate_to_owner_identity(pqxx::work & tx) const { const bool freeze_frames_needs_owner = needs_owner("freeze_frames"); const bool snapshots_needs_owner = needs_owner("snapshots"); const bool rosbag_files_needs_owner = needs_owner("rosbag_files"); + const bool near_misses_needs_owner = needs_owner("near_misses"); if (!faults_needs_owner && !freeze_frames_needs_owner && !snapshots_needs_owner && !rosbag_files_needs_owner && - !has_assignable_child_rows(tx)) { - return; // fresh database, or fully migrated - safe to re-run on every connection + !near_misses_needs_owner) { + return; // fresh database, or already migrated - safe to re-run on every connection } + // Every row of a legacy table is a shared record: the old layout held one row per fault code, + // fed by every source. The row keeps its reporting_sources as written, and every child row + // attaches to that shared record. + const std::string shared = kSharedOwner; + if (faults_needs_owner) { drop_primary_key(tx, "faults"); + // A pending row carries a confirmation time from the occurrence before a reactivation. execute(tx, "ALTER TABLE faults ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); - - // The owner is the first reporting source the row recorded. reporting_sources is rewritten to - // that one owner, as every write keeps it afterwards. - const auto rows = execute(tx, "SELECT fault_code, reporting_sources, status, confirmed_at_ns FROM faults"); - for (const auto & r : rows) { - const std::string fault_code = r["fault_code"].as(); - const auto sources = parse_json_array(r["reporting_sources"].as()); - const std::string owner = - (sources.empty() || sources.front().empty()) ? std::string(kLegacyOwner) : sources.front(); - if (owner == kLegacyOwner) { - core::logf(log_sink_, core::LogLevel::kWarn, "postgres_fault_storage", - "Fault '%s' carried no readable reporting source, migrated under owner '%s'", fault_code.c_str(), - kLegacyOwner); - } - // A pending row carries a confirmation time from the occurrence before a reactivation. - const std::string status = r["status"].as(); - const bool pending = status == fd::fault_status::kPrefailed || status == fd::fault_status::kPrepassed; - execute(tx, "UPDATE faults SET owner = $1, reporting_sources = $2, confirmed_at_ns = $3 WHERE fault_code = $4", - owner, serialize_json_array({owner}), pending ? int64_t{0} : r["confirmed_at_ns"].as(), - fault_code); - } + execute(tx, + "UPDATE faults SET owner = $1, confirmed_at_ns = CASE WHEN status IN ('PREFAILED', 'PREPASSED') " + "THEN 0 ELSE confirmed_at_ns END", + shared); } - if (freeze_frames_needs_owner) { drop_primary_key(tx, "freeze_frames"); execute(tx, "ALTER TABLE freeze_frames ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + execute(tx, "UPDATE freeze_frames SET owner = $1", shared); } if (snapshots_needs_owner) { execute(tx, "ALTER TABLE snapshots ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + execute(tx, "UPDATE snapshots SET owner = $1", shared); } if (rosbag_files_needs_owner) { execute(tx, "ALTER TABLE rosbag_files ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + execute(tx, "UPDATE rosbag_files SET owner = $1", shared); // CREATE INDEX IF NOT EXISTS in create_schema keeps an index that exists, so the two indexes // that carry fault_code are dropped to be rebuilt with owner. execute(tx, "DROP INDEX IF EXISTS idx_rosbag_files_fault_path"); execute(tx, "DROP INDEX IF EXISTS idx_rosbag_files_fault_created"); } - - backfill_child_owners(tx); + if (near_misses_needs_owner) { + execute(tx, "ALTER TABLE near_misses ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + execute(tx, "UPDATE near_misses SET owner = $1", shared); + } } void PgFaultStorage::create_schema(pqxx::work & tx) const { migrate_to_owner_identity(tx); - // A row is one fault RECORD, identified by (fault_code, owner). owner is the reporting - // source; reporting_sources keeps it serialised so a raw-table reader still sees the - // field it always saw. Uniqueness through the index below, the same shape as SQLite. + // A row is one fault RECORD, identified by (fault_code, owner). owner is the shared owner + // for a code every source feeds, or the source_id for a per_source code; reporting_sources + // lists every source that reported the record. Uniqueness through the index below, the + // same shape as SQLite. execute(tx, R"( CREATE TABLE IF NOT EXISTS faults ( fault_code TEXT NOT NULL, @@ -488,12 +442,14 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { // Create near_misses table: append-only series of FAILED reports that moved the debounce // counter without confirming the fault. One row per qualifying report, never updated in // place, and NOT removed on clear_fault - acknowledging a fault cycle must not erase how - // often that code approached confirmation. Bounded per fault code by the caller-supplied - // limit, evicting the oldest rows first. + // often that code approached confirmation. A row belongs to the record named by + // (fault_code, owner) and keeps the source_id that reported it. Bounded per record by the + // caller-supplied limit, evicting the oldest rows first. execute(tx, R"( CREATE TABLE IF NOT EXISTS near_misses ( id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', occurred_at_ns BIGINT NOT NULL, debounce_counter INTEGER NOT NULL, confirmation_threshold INTEGER NOT NULL, @@ -501,7 +457,8 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { source_id TEXT NOT NULL, resulting_status TEXT NOT NULL DEFAULT '' ); - CREATE INDEX IF NOT EXISTS idx_near_misses_fault_code ON near_misses(fault_code, id); + DROP INDEX IF EXISTS idx_near_misses_fault_code; + CREATE INDEX IF NOT EXISTS idx_near_misses_record ON near_misses(fault_code, owner, id); )"); // Create rosbag_files table. One row = one LINK (a fault claiming a recording): @@ -534,7 +491,7 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { FROM faults LIMIT 0; SELECT id, fault_code, owner, topic, message_type, data, captured_at_ns, capture_id FROM snapshots LIMIT 0; SELECT fault_code, owner, data, captured_at_ns FROM freeze_frames LIMIT 0; - SELECT id, fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, severity, source_id, + SELECT id, fault_code, owner, occurred_at_ns, debounce_counter, confirmation_threshold, severity, source_id, resulting_status FROM near_misses LIMIT 0; SELECT id, fault_code, owner, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns @@ -676,7 +633,21 @@ std::string PgFaultStorage::serialize_json_array(const std::vector return oss.str(); } -bool PgFaultStorage::report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, +fd::FaultRecord PgFaultStorage::read_fault_row(const pqxx::row & r) const { + fd::FaultRecord fault; + fault.fault_code = r["fault_code"].as(); + fault.severity = static_cast(r["severity"].as()); + fault.description = r["description"].as(); + fault.first_occurred_ns = r["first_occurred_ns"].as(); + fault.last_occurred_ns = r["last_occurred_ns"].as(); + fault.occurrence_count = static_cast(r["occurrence_count"].as()); + fault.status = r["status"].as(); + fault.reporting_sources = parse_json_array(r["reporting_sources"].as()); + fault.last_passed_ns = r["last_passed_ns"].as(); + return fault; +} + +bool PgFaultStorage::report_fault_event(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) { std::lock_guard lock(mutex_); @@ -687,28 +658,29 @@ bool PgFaultStorage::report_fault_event(const std::string & fault_code, fd::Repo // advance it a second time and the near miss it retried for would still be // missing from the series. run_in_transaction commits both rows together. return run_in_transaction("report_fault_event", [&](pqxx::work & tx) { - return report_fault_event_locked(fault_code, kind, severity, description, source_id, timestamp_ns, config, tx); + return report_fault_event_locked(id, kind, severity, description, source_id, timestamp_ns, config, tx); }); } -bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, +bool PgFaultStorage::report_fault_event_locked(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config, pqxx::work & tx) { const bool is_failed = (kind == fd::ReportKind::kFailed); + const std::string & fault_code = id.fault_code; // No row lock needed: mutex_ serializes every call on this connection, and one database belongs to exactly one // fault manager. - // Check if the record exists. The report is addressed to (fault_code, source_id): - // another source's record of the same code is a different row and is not touched. + // Check if the record exists. Another record of the same code is a different row and is not touched. auto res = execute(tx, - "SELECT severity, occurrence_count, status, debounce_counter, confirmed_at_ns, " - "first_occurred_ns FROM faults WHERE fault_code = $1 AND owner = $2", - fault_code, source_id); + "SELECT severity, occurrence_count, reporting_sources, status, debounce_counter, " + "confirmed_at_ns, first_occurred_ns FROM faults WHERE fault_code = $1 AND owner = $2", + fault_code, id.owner); if (!res.empty()) { // Record exists - update it int existing_severity = res[0]["severity"].as(); int64_t existing_count = res[0]["occurrence_count"].as(); + std::string sources_json = res[0]["reporting_sources"].as(); std::string current_status = res[0]["status"].as(); int32_t debounce_counter = res[0]["debounce_counter"].as(); int64_t confirmed_at_ns = res[0]["confirmed_at_ns"].as(); @@ -738,8 +710,13 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f if (is_failed) { // FAILED event - // Escalate severity if new severity is higher. Per record: a CRITICAL from one - // owner does not escalate another owner's record of the same code. + // Parse existing sources and add new one + std::vector sources = parse_json_array(sources_json); + std::set sources_set(sources.begin(), sources.end()); + sources_set.insert(source_id); + sources.assign(sources_set.begin(), sources_set.end()); + + // Escalate severity if new severity is higher int new_severity = std::max(existing_severity, static_cast(severity)); // Increment count with saturation - only on a genuine new occurrence (reactivation @@ -770,27 +747,24 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f confirmed_at_ns = timestamp_ns; } - // reporting_sources is not touched: it is the one owner, fixed at insert, and this - // row is already that owner's. if (description.empty()) { execute(tx, "UPDATE faults SET severity = $1, last_occurred_ns = $2, last_failed_ns = $3, occurrence_count = $4, " - "status = $5, debounce_counter = $6, confirmed_at_ns = $7, " - "first_occurred_ns = $8 WHERE fault_code = $9 AND owner = $10", - new_severity, timestamp_ns, timestamp_ns, new_count, new_status, debounce_counter, confirmed_at_ns, - first_occurred_ns, fault_code, source_id); + "reporting_sources = $5, status = $6, debounce_counter = $7, confirmed_at_ns = $8, " + "first_occurred_ns = $9 WHERE fault_code = $10 AND owner = $11", + new_severity, timestamp_ns, timestamp_ns, new_count, serialize_json_array(sources), new_status, + debounce_counter, confirmed_at_ns, first_occurred_ns, fault_code, id.owner); } else { execute(tx, "UPDATE faults SET severity = $1, description = $2, last_occurred_ns = $3, last_failed_ns = $4, " - "occurrence_count = $5, status = $6, debounce_counter = $7, " - "confirmed_at_ns = $8, first_occurred_ns = $9 WHERE fault_code = $10 AND owner = $11", - new_severity, description, timestamp_ns, timestamp_ns, new_count, new_status, debounce_counter, - confirmed_at_ns, first_occurred_ns, fault_code, source_id); + "occurrence_count = $5, reporting_sources = $6, status = $7, debounce_counter = $8, " + "confirmed_at_ns = $9, first_occurred_ns = $10 WHERE fault_code = $11 AND owner = $12", + new_severity, description, timestamp_ns, timestamp_ns, new_count, serialize_json_array(sources), + new_status, debounce_counter, confirmed_at_ns, first_occurred_ns, fault_code, id.owner); } if (is_near_miss(true, new_status)) { - record_near_miss_locked(fault_code, timestamp_ns, debounce_counter, config, severity, source_id, new_status, - tx); + record_near_miss_locked(id, timestamp_ns, debounce_counter, config, severity, source_id, new_status, tx); } } else { // PASSED event - increment towards healing, clamped to the thresholds. @@ -804,15 +778,15 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f execute(tx, "UPDATE faults SET last_passed_ns = $1, status = $2, debounce_counter = $3 " "WHERE fault_code = $4 AND owner = $5", - timestamp_ns, new_status, debounce_counter, fault_code, source_id); + timestamp_ns, new_status, debounce_counter, fault_code, id.owner); } return is_reactivation; // Reactivation treated as new occurrence for event publishing } - // New fault - only create for FAILED events + // New record - only create one for FAILED events if (!is_failed) { - return false; // PASSED event for non-existent fault is ignored + return false; // PASSED event for non-existent record is ignored } // Determine initial status based on debounce logic (shared with the in-memory backend). @@ -832,17 +806,17 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", fault_code, static_cast(severity), description, timestamp_ns, timestamp_ns, 1 /* occurrence_count = 1 */, initial_status, serialize_json_array({source_id}), initial_counter, timestamp_ns, 0, - confirmed_now ? timestamp_ns : 0, source_id); + confirmed_now ? timestamp_ns : 0, id.owner); if (is_near_miss(true, initial_status)) { - record_near_miss_locked(fault_code, timestamp_ns, initial_counter, config, severity, source_id, initial_status, tx); + record_near_miss_locked(id, timestamp_ns, initial_counter, config, severity, source_id, initial_status, tx); } return true; // New fault created } -std::vector PgFaultStorage::list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const { +std::vector PgFaultStorage::list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const { std::lock_guard lock(mutex_); // Determine which statuses to include @@ -863,30 +837,19 @@ std::vector PgFaultStorage::list_faults(bool filter_by_severity const std::vector status_filter_vector(status_filter.begin(), status_filter.end()); // Build query - std::string sql = - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, occurrence_count, status, " - "reporting_sources, last_passed_ns FROM faults WHERE status = ANY($1) "; + std::string sql = std::string("SELECT ") + kFaultColumns + " FROM faults WHERE status = ANY($1) "; if (filter_by_severity) { sql += " AND severity = $2"; } - return run_in_transaction("list_faults", [&](pqxx::work & tx) { + return run_in_transaction("list_fault_entries", [&](pqxx::work & tx) { auto res = filter_by_severity ? execute(tx, sql, status_filter_vector, static_cast(severity)) : execute(tx, sql, status_filter_vector); - std::vector result; + std::vector result; for (const auto & r : res) { - fd::FaultRecord fault; - fault.fault_code = r["fault_code"].as(); - fault.severity = static_cast(r["severity"].as()); - fault.description = r["description"].as(); - fault.first_occurred_ns = r["first_occurred_ns"].as(); - fault.last_occurred_ns = r["last_occurred_ns"].as(); - fault.occurrence_count = static_cast(r["occurrence_count"].as()); - fault.status = r["status"].as(); - fault.reporting_sources = parse_json_array(r["reporting_sources"].as()); - fault.last_passed_ns = r["last_passed_ns"].as(); - result.push_back(fault); + result.push_back( + FaultEntry{FaultId{r["fault_code"].as(), r["owner"].as()}, read_fault_row(r)}); } return result; }); @@ -896,51 +859,23 @@ std::optional PgFaultStorage::get_fault(const FaultId & id) con std::lock_guard lock(mutex_); return run_in_transaction("get_fault", [&](pqxx::work & tx) -> std::optional { - auto res = execute(tx, - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, last_passed_ns FROM faults " - "WHERE fault_code = $1 AND owner = $2", + auto res = execute(tx, std::string("SELECT ") + kFaultColumns + " FROM faults WHERE fault_code = $1 AND owner = $2", id.fault_code, id.owner); if (res.empty()) { return std::nullopt; } - auto r = res[0]; - fd::FaultRecord fault; - fault.fault_code = r["fault_code"].as(); - fault.severity = static_cast(r["severity"].as()); - fault.description = r["description"].as(); - fault.first_occurred_ns = r["first_occurred_ns"].as(); - fault.last_occurred_ns = r["last_occurred_ns"].as(); - fault.occurrence_count = static_cast(r["occurrence_count"].as()); - fault.status = r["status"].as(); - fault.reporting_sources = parse_json_array(r["reporting_sources"].as()); - fault.last_passed_ns = r["last_passed_ns"].as(); - return fault; + return read_fault_row(res[0]); }); } -std::vector PgFaultStorage::get_faults_by_code(const std::string & fault_code) const { +std::vector PgFaultStorage::get_ids_by_code(const std::string & fault_code) const { std::lock_guard lock(mutex_); - return run_in_transaction("get_faults_by_code", [&](pqxx::work & tx) { - auto res = execute(tx, - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, last_passed_ns FROM faults " - "WHERE fault_code = $1 ORDER BY owner ASC", - fault_code); - std::vector result; + return run_in_transaction("get_ids_by_code", [&](pqxx::work & tx) { + auto res = execute(tx, "SELECT owner FROM faults WHERE fault_code = $1 ORDER BY owner ASC", fault_code); + std::vector result; for (const auto & r : res) { - fd::FaultRecord fault; - fault.fault_code = r["fault_code"].as(); - fault.severity = static_cast(r["severity"].as()); - fault.description = r["description"].as(); - fault.first_occurred_ns = r["first_occurred_ns"].as(); - fault.last_occurred_ns = r["last_occurred_ns"].as(); - fault.occurrence_count = static_cast(r["occurrence_count"].as()); - fault.status = r["status"].as(); - fault.reporting_sources = parse_json_array(r["reporting_sources"].as()); - fault.last_passed_ns = r["last_passed_ns"].as(); - result.push_back(std::move(fault)); + result.push_back(FaultId{fault_code, r["owner"].as()}); } return result; }); @@ -1215,7 +1150,7 @@ size_t PgFaultStorage::set_max_near_misses_per_fault(size_t max_count) { auto res = execute(tx, "DELETE FROM near_misses WHERE id IN (" "SELECT id FROM (SELECT id, ROW_NUMBER() OVER " - "(PARTITION BY fault_code, source_id ORDER BY id DESC) AS rn FROM near_misses) AS ranked " + "(PARTITION BY fault_code, owner ORDER BY id DESC) AS rn FROM near_misses) AS ranked " "WHERE rn > $1)", static_cast(max_count)); // Returned rather than logged: the storage layer has no logger, and a bound applied by mistake @@ -1224,15 +1159,15 @@ size_t PgFaultStorage::set_max_near_misses_per_fault(size_t max_count) { }); } -void PgFaultStorage::record_near_miss_locked(const std::string & fault_code, int64_t occurred_at_ns, - int32_t debounce_counter, const DebounceConfig & config, uint8_t severity, +void PgFaultStorage::record_near_miss_locked(const FaultId & id, int64_t occurred_at_ns, int32_t debounce_counter, + const DebounceConfig & config, uint8_t severity, const std::string & source_id, const std::string & resulting_status, pqxx::work & tx) { execute(tx, - "INSERT INTO near_misses (fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, severity, " - "source_id, resulting_status) VALUES ($1, $2, $3, $4, $5, $6, $7)", - fault_code, occurred_at_ns, debounce_counter, config.confirmation_threshold, static_cast(severity), - source_id, resulting_status); + "INSERT INTO near_misses (fault_code, owner, occurred_at_ns, debounce_counter, confirmation_threshold, " + "severity, source_id, resulting_status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + id.fault_code, id.owner, occurred_at_ns, debounce_counter, config.confirmation_threshold, + static_cast(severity), source_id, resulting_status); if (max_near_misses_per_fault_ == 0 || max_near_misses_per_fault_ > static_cast(std::numeric_limits::max())) { @@ -1248,27 +1183,26 @@ void PgFaultStorage::record_near_miss_locked(const std::string & fault_code, int // by timestamp would then drop the row that was just appended and make the two backends, which // append in arrival order, disagree on the same input. execute(tx, - "DELETE FROM near_misses WHERE fault_code = $1 AND source_id = $2 AND id NOT IN " - "(SELECT id FROM near_misses WHERE fault_code = $1 AND source_id = $2 ORDER BY id DESC LIMIT $3)", - fault_code, source_id, static_cast(max_near_misses_per_fault_)); + "DELETE FROM near_misses WHERE fault_code = $1 AND owner = $2 AND id NOT IN " + "(SELECT id FROM near_misses WHERE fault_code = $1 AND owner = $2 ORDER BY id DESC LIMIT $3)", + id.fault_code, id.owner, static_cast(max_near_misses_per_fault_)); } std::vector PgFaultStorage::get_near_misses(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("get_near_misses", [&](pqxx::work & tx) { - // source_id already IS the owner on this table, so the series only needs the extra - // predicate, not a new column. auto res = execute(tx, - "SELECT fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, " + "SELECT fault_code, owner, occurred_at_ns, debounce_counter, confirmation_threshold, " "severity, source_id, resulting_status FROM near_misses " - "WHERE fault_code = $1 AND source_id = $2 ORDER BY id ASC", + "WHERE fault_code = $1 AND owner = $2 ORDER BY id ASC", id.fault_code, id.owner); std::vector result; for (const auto & r : res) { NearMissRecord record; record.fault_code = r["fault_code"].as(); + record.owner = r["owner"].as(); record.occurred_at_ns = r["occurred_at_ns"].as(); record.debounce_counter = r["debounce_counter"].as(); record.confirmation_threshold = r["confirmation_threshold"].as(); @@ -1610,16 +1544,14 @@ std::vector PgFaultStorage::list_rosbags_for_entity(const std::s std::lock_guard lock(mutex_); return run_in_transaction("list_rosbags_for_entity", [&](pqxx::work & tx) { - // The row carries its record's owner, so the entity match is an equality on the row - // and the json scan over reporting_sources is gone. The join to faults stays as the - // existence check it always was: a row whose record is gone is not served. + // The rows whose record lists the entity among its reporting sources. auto res = execute(tx, "SELECT r.fault_code, r.owner, r.recording_id, r.file_path, r.format, r.duration_sec, r.size_bytes, " "r.created_at_ns " "FROM rosbag_files r " "JOIN faults f ON r.fault_code = f.fault_code AND r.owner = f.owner " - "WHERE r.owner = $1 " + "JOIN jsonb_array_elements_text(f.reporting_sources::jsonb) AS j(value) ON j.value = $1 " "ORDER BY r.created_at_ns DESC, r.id DESC", entity_fqn); @@ -1635,23 +1567,11 @@ std::vector PgFaultStorage::get_all_faults() const { std::lock_guard lock(mutex_); return run_in_transaction("get_all_faults", [&](pqxx::work & tx) { - auto res = execute(tx, - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, last_passed_ns FROM faults"); + auto res = execute(tx, std::string("SELECT ") + kFaultColumns + " FROM faults"); std::vector result; for (const auto & r : res) { - fd::FaultRecord fault; - fault.fault_code = r["fault_code"].as(); - fault.severity = static_cast(r["severity"].as()); - fault.description = r["description"].as(); - fault.first_occurred_ns = r["first_occurred_ns"].as(); - fault.last_occurred_ns = r["last_occurred_ns"].as(); - fault.occurrence_count = static_cast(r["occurrence_count"].as()); - fault.status = r["status"].as(); - fault.reporting_sources = parse_json_array(r["reporting_sources"].as()); - fault.last_passed_ns = r["last_passed_ns"].as(); - result.push_back(fault); + result.push_back(read_fault_row(r)); } return result; }); diff --git a/src/ros2_medkit_fault_manager/src/rosbag_capture.cpp b/src/ros2_medkit_fault_manager/src/rosbag_capture.cpp index 7a0fd3030..b1168af72 100644 --- a/src/ros2_medkit_fault_manager/src/rosbag_capture.cpp +++ b/src/ros2_medkit_fault_manager/src/rosbag_capture.cpp @@ -1033,12 +1033,12 @@ std::set RosbagCapture::compute_entity_topics(const FaultId & id) { // A throw here would terminate the process - the exact crash the crash-safety work // removed - so any failure degrades to "write everything". try { - if (storage_->contains(id)) { - // The record's owner is the reporting node's FQN (e.g. "/planner_server"). - // Split it into (name, namespace) to match against topic endpoints. + auto fault = storage_->get_fault(id); + if (fault && !fault->reporting_sources.empty()) { + // reporting_sources hold the reporting node's FQN (e.g. "/planner_server"). + // Split each into (name, namespace) to match against topic endpoints. std::set> wanted; - { - const std::string & source = id.owner; + for (const auto & source : fault->reporting_sources) { std::string ns = "/"; std::string name = source; const auto slash = source.rfind('/'); diff --git a/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp b/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp index 6ae8925b7..dfe6a66b4 100644 --- a/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp +++ b/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp @@ -303,7 +303,8 @@ std::vector SnapshotCapture::resolve_entity_topics(const FaultId & // pool worker with no outer catch, and storage_->get_fault() can throw on // the sqlite backend. Any failure degrades to "no entity-default capture". try { - if (!storage_->contains(id)) { + auto fault = storage_->get_fault(id); + if (!fault || fault->reporting_sources.empty()) { return {}; } @@ -313,18 +314,16 @@ std::vector SnapshotCapture::resolve_entity_topics(const FaultId & // same name and freeze-frame its topics. Their zero-config frames come // from the gateway's EntityFreezeFrameCapture instead. // - // One owner, so one (name, namespace) pair: the record names exactly the - // source whose own data belongs in its frame. std::set> wanted; - { - const std::string & source = id.owner; - if (source.size() >= 2 && source.front() == '/') { - const auto slash = source.rfind('/'); - const std::string name = source.substr(slash + 1); - const std::string ns = (slash == 0) ? "/" : source.substr(0, slash); - if (!name.empty()) { - wanted.emplace(name, ns); - } + for (const auto & source : fault->reporting_sources) { + if (source.size() < 2 || source.front() != '/') { + continue; + } + const auto slash = source.rfind('/'); + const std::string name = source.substr(slash + 1); + const std::string ns = (slash == 0) ? "/" : source.substr(0, slash); + if (!name.empty()) { + wanted.emplace(name, ns); } } if (wanted.empty()) { @@ -364,7 +363,7 @@ std::vector SnapshotCapture::resolve_entity_topics(const FaultId & } if (!topics.empty()) { - RCLCPP_INFO(node_->get_logger(), "Entity-default capture for fault '%s': %zu topic(s) from the owning node", + RCLCPP_INFO(node_->get_logger(), "Entity-default capture for fault '%s': %zu topic(s) from the reporting node(s)", fault_code.c_str(), topics.size()); } return {topics.begin(), topics.end()}; diff --git a/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp b/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp index 3016d3467..bea6abf3b 100644 --- a/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp @@ -110,107 +110,109 @@ class SqliteStatement { sqlite3_stmt * stmt_{nullptr}; }; -/// First reporting source of a LEGACY `reporting_sources` value, recovered without -/// requiring the text to be valid JSON. -/// -/// Earlier builds wrote this column with an escape set that covered only the quote, -/// the backslash and \b \f \n \r \t, so a source_id carrying any other control byte -/// was stored as text no JSON parser accepts. Asking SQLite's json_extract to read it -/// raises "malformed JSON", which inside the migration means the rebuild rolls back -/// and the node refuses to start again for as long as the row exists. The recovery -/// therefore reads the text itself and treats JSON as a shape it recognises, not as a -/// precondition. -/// -/// A record always ends up addressable. Where nothing can be read the answer is the -/// synthetic owner `legacy`, never the empty string: an empty owner means one thing -/// only, a child row not yet assigned to a record, and a fault carrying it would be -/// unreachable through source_id and would keep the child backfill looking for work. -/// -/// The mapping, exhaustively: -/// "" -> "legacy" (nothing recorded) -/// "sensor_a" -> "sensor_a" (bare word, never a JSON array) -/// "[]" -> "legacy" (array with no source) -/// "[\"a<0x01>b\"]" -> "a<0x01>b" (control byte kept verbatim) -/// "[...]", "{...}" -> "legacy" (neither an array nor a bare word) -/// anything unterminated -> what was read before the text ran out -std::string legacy_first_source(const std::string & raw) { - std::size_t pos = 0; - while (pos < raw.size() && std::isspace(static_cast(raw[pos])) != 0) { - ++pos; - } - if (pos >= raw.size()) { - return kLegacyOwner; - } - - if (raw[pos] != '[') { - // Not an array. Only a bare word is recoverable here, meaning text carrying no - // JSON punctuation at all: an older writer, or a hand edit, put the source in the - // column unquoted. Anything else (a byte-order mark in front of the array, an - // object wrapper, a quoted scalar) is structure this function does not claim to - // read, and returning it whole would mint an owner out of punctuation. - const bool bare_word = - raw.find_first_of("[]{}\",:", pos) == std::string::npos && static_cast(raw[pos]) < 0x80; - return bare_word ? raw.substr(pos) : std::string(kLegacyOwner); - } - - ++pos; // past '[' - while (pos < raw.size() && std::isspace(static_cast(raw[pos])) != 0) { - ++pos; - } - if (pos >= raw.size() || raw[pos] != '"') { - return kLegacyOwner; // "[]", or a first element that is not a string - } - - ++pos; // past the opening quote - std::string out; - while (pos < raw.size() && raw[pos] != '"') { - if (raw[pos] != '\\' || pos + 1 >= raw.size()) { - out.push_back(raw[pos]); // every other byte is itself, control bytes included +/// The strings of a JSON array written by serialize_json_array. A malformed value is logged and +/// yields what was read before the text went wrong. +std::vector parse_json_array(const std::string & json_str, core::LogSink & sink) { + std::vector result; + + if (json_str.size() < 2 || json_str.front() != '[' || json_str.back() != ']') { + if (!json_str.empty()) { + core::logf(sink, core::LogLevel::kWarn, "sqlite_fault_storage", "Malformed JSON array in database: '%s'", + json_str.c_str()); + } + return result; + } + + const std::string content = json_str.substr(1, json_str.size() - 2); + size_t pos = 0; + while (pos < content.size()) { + while (pos < content.size() && std::isspace(static_cast(content[pos])) != 0) { ++pos; - continue; } - const char escaped = raw[pos + 1]; - pos += 2; - switch (escaped) { - case 'b': - out.push_back('\b'); - break; - case 'f': - out.push_back('\f'); - break; - case 'n': - out.push_back('\n'); - break; - case 'r': - out.push_back('\r'); - break; - case 't': - out.push_back('\t'); - break; - case 'u': { - // Only the \u00XX range this writer ever emits is decoded. Anything wider is - // left as written rather than guessed at, because a wrong transcoding would - // change an identity that other rows are keyed by. - if (pos + 4 <= raw.size() && raw.compare(pos, 2, "00") == 0) { - const std::string digits = raw.substr(pos + 2, 2); - char * end = nullptr; - const auto value = std::strtol(digits.c_str(), &end, 16); - if (end != nullptr && *end == '\0') { - out.push_back(static_cast(value)); - pos += 4; - break; + if (pos >= content.size() || content[pos] != '"') { + break; + } + ++pos; // past the opening quote + + std::string value; + while (pos < content.size() && content[pos] != '"') { + if (content[pos] != '\\' || pos + 1 >= content.size()) { + value.push_back(content[pos]); + ++pos; + continue; + } + const char escaped = content[pos + 1]; + pos += 2; + switch (escaped) { + case 'b': + value.push_back('\b'); + break; + case 'f': + value.push_back('\f'); + break; + case 'n': + value.push_back('\n'); + break; + case 'r': + value.push_back('\r'); + break; + case 't': + value.push_back('\t'); + break; + case 'u': { + // serialize_json_array writes only \u00XX, for control bytes. + if (pos + 4 <= content.size() && content.compare(pos, 2, "00") == 0) { + char * end = nullptr; + const std::string digits = content.substr(pos + 2, 2); + const auto code = std::strtol(digits.c_str(), &end, 16); + if (end != nullptr && *end == '\0') { + value.push_back(static_cast(code)); + pos += 4; + break; + } } + value.push_back('u'); + break; } - out.push_back('\\'); - out.push_back('u'); - break; + default: + value.push_back(escaped); // covers \" \\ and \/ + break; } - default: - out.push_back(escaped); // covers \" and \\, and keeps anything unexpected - break; + } + if (pos < content.size()) { + ++pos; // past the closing quote + } + result.push_back(std::move(value)); + + while (pos < content.size() && + (std::isspace(static_cast(content[pos])) != 0 || content[pos] == ',')) { + ++pos; } } - return out.empty() ? std::string(kLegacyOwner) : out; + return result; +} + +/// Columns every fault read selects, in the order read_fault_row and read_fault_owner decode them. +constexpr const char * kFaultColumns = + "fault_code, severity, description, first_occurred_ns, last_occurred_ns, " + "occurrence_count, status, reporting_sources, last_passed_ns, owner"; + +fd::FaultRecord read_fault_row(SqliteStatement & stmt, core::LogSink & sink) { + fd::FaultRecord fault; + fault.fault_code = stmt.column_text(0); + fault.severity = static_cast(stmt.column_int(1)); + fault.description = stmt.column_text(2); + fault.first_occurred_ns = stmt.column_int64(3); + fault.last_occurred_ns = stmt.column_int64(4); + fault.occurrence_count = static_cast(stmt.column_int64(5)); + fault.status = stmt.column_text(6); + fault.reporting_sources = parse_json_array(stmt.column_text(7), sink); + fault.last_passed_ns = stmt.column_int64(8); + return fault; +} + +std::string read_fault_owner(SqliteStatement & stmt) { + return stmt.column_text(9); } } // namespace @@ -265,9 +267,8 @@ void SqliteFaultStorage::initialize_schema() { // rosbag_files table: a column constraint cannot be dropped with ALTER TABLE and // forces a full rebuild the day the identity changes again. // - // reporting_sources is kept as the serialized form of owner so a reader of the raw - // table still sees the field it always saw. owner is the authoritative one, and - // every write below derives the JSON from it, so the two cannot drift. + // owner is the shared owner for a code every source feeds, or the source_id for a + // per_source code. reporting_sources lists every source that reported the record. const char * create_faults_table_sql = R"( CREATE TABLE IF NOT EXISTS faults ( fault_code TEXT NOT NULL, @@ -394,12 +395,14 @@ void SqliteFaultStorage::initialize_schema() { // Create near_misses table: append-only series of FAILED reports that moved the debounce // counter without confirming the fault. One row per qualifying report, never updated in // place, and NOT removed on clear_fault - acknowledging a fault cycle must not erase how - // often that record approached confirmation. Bounded per record (fault_code, source_id) by - // the caller-supplied limit, evicting the oldest rows first. + // often that record approached confirmation. A row belongs to the record named by + // (fault_code, owner) and keeps the source_id that reported it. Bounded per record by the + // caller-supplied limit, evicting the oldest rows first. const char * create_near_misses_table_sql = R"( CREATE TABLE IF NOT EXISTS near_misses ( id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', occurred_at_ns INTEGER NOT NULL, debounce_counter INTEGER NOT NULL, confirmation_threshold INTEGER NOT NULL, @@ -407,7 +410,6 @@ void SqliteFaultStorage::initialize_schema() { source_id TEXT NOT NULL, resulting_status TEXT NOT NULL DEFAULT '' ); - CREATE INDEX IF NOT EXISTS idx_near_misses_fault_code ON near_misses(fault_code, id); )"; if (sqlite3_exec(db_, create_near_misses_table_sql, nullptr, nullptr, &err_msg) != SQLITE_OK) { @@ -476,6 +478,16 @@ void SqliteFaultStorage::initialize_schema() { // before it. migrate_faults_add_owner(); + // After the migration, which adds near_misses.owner to a database that predates it. + if (sqlite3_exec(db_, + "DROP INDEX IF EXISTS idx_near_misses_fault_code;" + "CREATE INDEX IF NOT EXISTS idx_near_misses_record ON near_misses(fault_code, owner, id);", + nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + throw std::runtime_error("Failed to create near_misses index: " + error); + } + // The record identity, expressed as an index rather than a PRIMARY KEY so the next // identity change is a DROP INDEX instead of another table rebuild. if (sqlite3_exec(db_, "CREATE UNIQUE INDEX IF NOT EXISTS idx_faults_code_owner ON faults(fault_code, owner)", nullptr, @@ -618,26 +630,6 @@ bool SqliteFaultStorage::table_has_owner(const char * table) const { return false; } -bool SqliteFaultStorage::has_assignable_child_rows() const { - for (const char * table : {"freeze_frames", "snapshots", "rosbag_files"}) { - if (!table_has_owner(table)) { - continue; // the column arrives with the migration, which then fills it - } - // The same condition the backfill's UPDATE applies, so "is there work" and "what - // the work does" cannot disagree. Asking only for an empty owner would call a row - // that no open can ever assign (its code has several owners, or none) pending - // work forever, and every open would re-enter the write transaction and repeat - // the warning that says the row cannot be assigned. - const std::string sql = std::string("SELECT 1 FROM ") + table + " t WHERE t.owner = '' AND (SELECT COUNT(" + - "DISTINCT f.owner) FROM faults f WHERE f.fault_code = t.fault_code) = 1 LIMIT 1"; - SqliteStatement stmt(db_, sql.c_str()); - if (stmt.step() == SQLITE_ROW) { - return true; - } - } - return false; -} - bool SqliteFaultStorage::table_exists(const char * table) const { SqliteStatement stmt(db_, "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?"); stmt.bind_text(1, table); @@ -662,154 +654,28 @@ void SqliteFaultStorage::drop_scratch_table(const char * table) { } } -void SqliteFaultStorage::copy_legacy_fault_rows() { - struct LegacyFault { - std::string fault_code; - std::string description; - std::string status; - std::string owner; - int severity{0}; - int64_t first_occurred_ns{0}; - int64_t last_occurred_ns{0}; - int64_t occurrence_count{0}; - int64_t debounce_counter{0}; - int64_t last_failed_ns{0}; - int64_t last_passed_ns{0}; - int64_t confirmed_at_ns{0}; - }; - - // Read the whole table before writing a row of it. The write target is a different - // table, but the old one is dropped immediately afterwards, and holding a read cursor - // across that is a shape worth not having. - std::vector rows; - { - SqliteStatement select(db_, - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, debounce_counter, last_failed_ns, " - "last_passed_ns, confirmed_at_ns FROM faults"); - while (select.step() == SQLITE_ROW) { - LegacyFault row; - row.fault_code = select.column_text(0); - row.severity = select.column_int(1); - row.description = select.column_text(2); - row.first_occurred_ns = select.column_int64(3); - row.last_occurred_ns = select.column_int64(4); - row.occurrence_count = select.column_int64(5); - row.status = select.column_text(6); - row.owner = legacy_first_source(select.column_text(7)); - row.debounce_counter = select.column_int64(8); - row.last_failed_ns = select.column_int64(9); - row.last_passed_ns = select.column_int64(10); - row.confirmed_at_ns = select.column_int64(11); - rows.push_back(std::move(row)); - } - } - - SqliteStatement insert(db_, - "INSERT INTO faults_new " - "(fault_code, severity, description, first_occurred_ns, last_occurred_ns, occurrence_count, " - " status, reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, " - " confirmed_at_ns, owner) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); - - for (const auto & row : rows) { - if (row.owner == kLegacyOwner) { - // Kept, not dropped. The fault state is real and an operator can still see it, - // it simply belongs to no source that this database ever recorded readably. It - // gets a name rather than an empty owner so it stays addressable by source_id. - core::logf(log_sink_, core::LogLevel::kWarn, "sqlite_fault_storage", - "Fault '%s' carried no readable reporting source, migrated under owner '%s'", row.fault_code.c_str(), - kLegacyOwner); - } - insert.reset(); - insert.bind_text(1, row.fault_code); - insert.bind_int(2, row.severity); - insert.bind_text(3, row.description); - insert.bind_int64(4, row.first_occurred_ns); - insert.bind_int64(5, row.last_occurred_ns); - insert.bind_int64(6, row.occurrence_count); - insert.bind_text(7, row.status); - // reporting_sources is rewritten to the one owner at the same time, so the column - // and owner agree from the first open after the migration, exactly as every write - // keeps them agreeing afterwards. Through serialize_json_array, so the result is - // valid JSON even when the value that came in was not. - insert.bind_text(8, serialize_json_array({row.owner})); - insert.bind_int64(9, row.debounce_counter); - insert.bind_int64(10, row.last_failed_ns); - insert.bind_int64(11, row.last_passed_ns); - // A row that confirmed stays CONFIRMED or HEALED until a clear. A pending row with a value - // carries it from the occurrence before a reactivation, which earlier releases did not reset. - const bool pending = row.status == fd::fault_status::kPrefailed || row.status == fd::fault_status::kPrepassed; - insert.bind_int64(12, pending ? 0 : row.confirmed_at_ns); - insert.bind_text(13, row.owner); - if (insert.step() != SQLITE_DONE) { - throw std::runtime_error(std::string("fault owner migration failed (copy fault rows): ") + sqlite3_errmsg(db_)); - } - } -} - -void SqliteFaultStorage::backfill_child_owners() { - char * err_msg = nullptr; - for (const char * table : {"freeze_frames", "snapshots", "rosbag_files"}) { - if (!table_has_owner(table)) { - continue; - } - - // Correlated on the child row's own fault_code, and applied only where that code - // has exactly ONE owner. The uncorrelated form this replaces picked whichever row - // SQLite reached first, which on a code with two owners handed one owner's - // snapshots and recordings to the other. - const std::string sql = std::string("UPDATE ") + table + " SET owner = (SELECT f.owner FROM faults f WHERE " + - "f.fault_code = " + table + ".fault_code) WHERE owner = '' AND (SELECT COUNT(DISTINCT " + - "f.owner) FROM faults f WHERE f.fault_code = " + table + ".fault_code) = 1"; - if (sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &err_msg) != SQLITE_OK) { - std::string error = err_msg ? err_msg : "Unknown error"; - sqlite3_free(err_msg); - err_msg = nullptr; - throw std::runtime_error(std::string("fault owner migration failed (backfill ") + table + ".owner): " + error); - } - - // What is left is either evidence for a code no fault row carries any more, or for - // a code several owners share. Neither can be assigned without guessing, so it is - // named instead. The row stays: no (code, owner) read returns it to an owner, and - // deleting an operator's evidence to tidy a column is the worse trade. - SqliteStatement leftover( - db_, - (std::string("SELECT fault_code, COUNT(*) FROM ") + table + " WHERE owner = '' GROUP BY fault_code").c_str()); - while (leftover.step() == SQLITE_ROW) { - core::logf(log_sink_, core::LogLevel::kWarn, "sqlite_fault_storage", - "%s: %lld row(s) for fault '%s' keep an empty owner, the code has no single owner to " - "assign them to", - table, static_cast(leftover.column_int64(1)), leftover.column_text(0).c_str()); - } - } -} - void SqliteFaultStorage::migrate_faults_add_owner() { - // Probed PER TABLE, not once for the whole migration. A database can hold a legacy - // faults table next to a snapshots table this release just created with owner - // already on it: the four tables come from four independent CREATE TABLE IF NOT - // EXISTS statements, so they do not move as a unit. One gate on faults alone either - // re-ALTERs a column that is already there or skips one that is missing, and both - // throw on the very next open. + // Probed PER TABLE, not once for the whole migration: the tables come from independent + // CREATE TABLE IF NOT EXISTS statements, so a database can hold a legacy faults table next to + // a child table this release just created with owner already on it. const bool faults_needs_owner = !table_has_owner("faults"); const bool freeze_frames_needs_owner = !table_has_owner("freeze_frames"); const bool snapshots_needs_owner = !table_has_owner("snapshots"); const bool rosbag_files_needs_owner = !table_has_owner("rosbag_files"); - const bool schema_work = - faults_needs_owner || freeze_frames_needs_owner || snapshots_needs_owner || rosbag_files_needs_owner; - - // The child backfill is not tied to the ALTER that adds the column, so its own - // condition is asked separately: an open interrupted between the faults rebuild and - // the backfill, or a child table that gained the column while faults was still - // legacy, leaves rows no owner can see. They are healed on the next open. - if (!schema_work && !has_assignable_child_rows()) { - return; // fresh database, or fully migrated - safe to re-run on every open + const bool near_misses_needs_owner = !table_has_owner("near_misses"); + if (!faults_needs_owner && !freeze_frames_needs_owner && !snapshots_needs_owner && !rosbag_files_needs_owner && + !near_misses_needs_owner) { + return; // fresh database, or already migrated - safe to re-run on every open } + // Every row of a legacy database is a shared record: the old schema held one row per fault + // code, fed by every source. The row keeps its reporting_sources as written, and every child + // row (snapshot, freeze frame, rosbag link, near miss) attaches to that shared record. + const std::string shared = std::string("'") + kSharedOwner + "'"; + char * err_msg = nullptr; - const auto exec = [this, &err_msg](const char * sql, const char * what) { - if (sqlite3_exec(db_, sql, nullptr, nullptr, &err_msg) != SQLITE_OK) { + const auto exec = [this, &err_msg](const std::string & sql, const char * what) { + if (sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &err_msg) != SQLITE_OK) { std::string error = err_msg ? err_msg : "Unknown error"; sqlite3_free(err_msg); err_msg = nullptr; @@ -825,10 +691,8 @@ void SqliteFaultStorage::migrate_faults_add_owner() { // documented table-rebuild procedure. // // The DROP first because a rebuild that was interrupted by a crash, or a manual - // recovery with an external tool, can leave faults_new behind. Without the DROP - // that leftover makes CREATE TABLE fail on every open and the schema stays - // legacy forever. It is safe because faults_new is this procedure's scratch - // table and nothing else ever reads it. + // recovery with an external tool, can leave faults_new behind. It is safe because + // faults_new is this procedure's scratch table and nothing else ever reads it. drop_scratch_table("faults_new"); exec( "CREATE TABLE faults_new (" @@ -847,7 +711,17 @@ void SqliteFaultStorage::migrate_faults_add_owner() { " owner TEXT NOT NULL)", "create new faults table"); - copy_legacy_fault_rows(); + // A pending row carries a confirmation time from the occurrence before a reactivation, + // which earlier releases did not reset. + exec( + "INSERT INTO faults_new (fault_code, severity, description, first_occurred_ns, last_occurred_ns, " + " occurrence_count, status, reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, " + " confirmed_at_ns, owner) " + "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, occurrence_count, status, " + " reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, " + " CASE WHEN status IN ('PREFAILED', 'PREPASSED') THEN 0 ELSE confirmed_at_ns END, " + + shared + " FROM faults", + "copy fault rows"); exec("DROP TABLE faults", "drop old faults table"); exec("ALTER TABLE faults_new RENAME TO faults", "rename faults"); @@ -855,8 +729,6 @@ void SqliteFaultStorage::migrate_faults_add_owner() { if (freeze_frames_needs_owner) { // freeze_frames: same problem, same procedure, same reason for the DROP. - // The owner is left empty here and assigned by the shared child backfill below, - // so one rule decides who a frame belongs to no matter which open produced it. drop_scratch_table("freeze_frames_new"); exec( "CREATE TABLE freeze_frames_new (" @@ -865,33 +737,35 @@ void SqliteFaultStorage::migrate_faults_add_owner() { " data TEXT NOT NULL," " captured_at_ns INTEGER NOT NULL)", "create new freeze_frames table"); - exec( - "INSERT INTO freeze_frames_new (fault_code, owner, data, captured_at_ns) " - "SELECT fault_code, '', data, captured_at_ns FROM freeze_frames", - "copy freeze frames"); + exec("INSERT INTO freeze_frames_new (fault_code, owner, data, captured_at_ns) SELECT fault_code, " + shared + + ", data, captured_at_ns FROM freeze_frames", + "copy freeze frames"); exec("DROP TABLE freeze_frames", "drop old freeze_frames table"); exec("ALTER TABLE freeze_frames_new RENAME TO freeze_frames", "rename freeze_frames"); } - // snapshots and rosbag_files only gain a column: their identity is a rowid, not - // the fault code, so ALTER TABLE is enough. The column arrives empty and the - // shared backfill fills it. + // snapshots, rosbag_files and near_misses only gain a column: their identity is a rowid, + // not the fault code, so ALTER TABLE is enough. if (snapshots_needs_owner) { exec("ALTER TABLE snapshots ADD COLUMN owner TEXT NOT NULL DEFAULT ''", "add snapshots.owner"); + exec("UPDATE snapshots SET owner = " + shared, "assign snapshots to shared records"); } if (rosbag_files_needs_owner) { exec("ALTER TABLE rosbag_files ADD COLUMN owner TEXT NOT NULL DEFAULT ''", "add rosbag_files.owner"); + exec("UPDATE rosbag_files SET owner = " + shared, "assign rosbag links to shared records"); // The rosbag indexes that carry fault_code have to widen with it. CREATE INDEX IF // NOT EXISTS in initialize_schema() is a no-op on an index that already exists, so - // without these drops the unique index would keep its two-column shape and refuse - // two owners of one code linking the same bag. + // without these drops the unique index would keep its two-column shape. exec("DROP INDEX IF EXISTS idx_rosbag_files_fault_path", "drop rosbag unique index"); exec("DROP INDEX IF EXISTS idx_rosbag_files_fault_created", "drop rosbag fault_created index"); } - backfill_child_owners(); + if (near_misses_needs_owner) { + exec("ALTER TABLE near_misses ADD COLUMN owner TEXT NOT NULL DEFAULT ''", "add near_misses.owner"); + exec("UPDATE near_misses SET owner = " + shared, "assign near misses to shared records"); + } exec("COMMIT", "commit"); } catch (...) { @@ -1021,7 +895,7 @@ std::string SqliteFaultStorage::serialize_json_array(const std::vector lock(mutex_); @@ -1036,13 +910,13 @@ bool SqliteFaultStorage::report_fault_event(const std::string & fault_code, fd:: // heartbeat - including one that turns out to write nothing at all - contend for that lock and // fail with SQLITE_BUSY where before it could not. if (kind != fd::ReportKind::kFailed) { - return report_fault_event_locked(fault_code, kind, severity, description, source_id, timestamp_ns, config); + return report_fault_event_locked(id, kind, severity, description, source_id, timestamp_ns, config); } exec_or_throw("BEGIN IMMEDIATE"); try { const bool is_new_occurrence = - report_fault_event_locked(fault_code, kind, severity, description, source_id, timestamp_ns, config); + report_fault_event_locked(id, kind, severity, description, source_id, timestamp_ns, config); exec_or_throw("COMMIT"); return is_new_occurrence; } catch (...) { @@ -1051,28 +925,28 @@ bool SqliteFaultStorage::report_fault_event(const std::string & fault_code, fd:: } } -bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_code, fd::ReportKind kind, - uint8_t severity, const std::string & description, - const std::string & source_id, int64_t timestamp_ns, - const DebounceConfig & config) { +bool SqliteFaultStorage::report_fault_event_locked(const FaultId & id, fd::ReportKind kind, uint8_t severity, + const std::string & description, const std::string & source_id, + int64_t timestamp_ns, const DebounceConfig & config) { const bool is_failed = (kind == fd::ReportKind::kFailed); + const std::string & fault_code = id.fault_code; - // Check if the record exists. The report is addressed to (fault_code, source_id): - // another source's record of the same code is a different row and is not touched. + // Check if the record exists. Another record of the same code is a different row and is not touched. SqliteStatement check_stmt(db_, - "SELECT severity, occurrence_count, status, debounce_counter, " + "SELECT severity, occurrence_count, reporting_sources, status, debounce_counter, " "confirmed_at_ns, first_occurred_ns FROM faults WHERE fault_code = ? AND owner = ?"); check_stmt.bind_text(1, fault_code); - check_stmt.bind_text(2, source_id); + check_stmt.bind_text(2, id.owner); if (check_stmt.step() == SQLITE_ROW) { // Record exists - update it int existing_severity = check_stmt.column_int(0); int64_t existing_count = check_stmt.column_int64(1); - std::string current_status = check_stmt.column_text(2); - int32_t debounce_counter = check_stmt.column_int(3); - int64_t confirmed_at_ns = check_stmt.column_int64(4); - int64_t first_occurred_ns = check_stmt.column_int64(5); + std::string sources_json = check_stmt.column_text(2); + std::string current_status = check_stmt.column_text(3); + int32_t debounce_counter = check_stmt.column_int(4); + int64_t confirmed_at_ns = check_stmt.column_int64(5); + int64_t first_occurred_ns = check_stmt.column_int64(6); // Bring a runaway counter persisted by an older build (the bug this fixes) back into range on // first touch; this also keeps the +1/-1 below overflow-safe. The counter is local to this call. @@ -1098,8 +972,13 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod if (is_failed) { // FAILED event - // Escalate severity if new severity is higher. Per record: a CRITICAL from one - // owner does not escalate another owner's record of the same code. + // Parse existing sources and add new one + std::vector sources = parse_json_array(sources_json, log_sink_); + std::set sources_set(sources.begin(), sources.end()); + sources_set.insert(source_id); + sources.assign(sources_set.begin(), sources_set.end()); + + // Escalate severity if new severity is higher int new_severity = std::max(existing_severity, static_cast(severity)); // Increment count with saturation - only on a genuine new occurrence (reactivation @@ -1134,11 +1013,11 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod SqliteStatement update_stmt( db_, description.empty() ? "UPDATE faults SET severity = ?, last_occurred_ns = ?, last_failed_ns = ?, " "occurrence_count = ?, " - "status = ?, debounce_counter = ?, confirmed_at_ns = ?, " + "reporting_sources = ?, status = ?, debounce_counter = ?, confirmed_at_ns = ?, " "first_occurred_ns = ? WHERE fault_code = ? AND owner = ?" : "UPDATE faults SET severity = ?, description = ?, last_occurred_ns = ?, " "last_failed_ns = ?, " - "occurrence_count = ?, status = ?, debounce_counter = ?, " + "occurrence_count = ?, reporting_sources = ?, status = ?, debounce_counter = ?, " "confirmed_at_ns = ?, first_occurred_ns = ? WHERE fault_code = ? AND owner = ?"); if (description.empty()) { @@ -1146,24 +1025,26 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod update_stmt.bind_int64(2, timestamp_ns); update_stmt.bind_int64(3, timestamp_ns); update_stmt.bind_int64(4, new_count); - update_stmt.bind_text(5, new_status); - update_stmt.bind_int(6, debounce_counter); - update_stmt.bind_int64(7, confirmed_at_ns); - update_stmt.bind_int64(8, first_occurred_ns); - update_stmt.bind_text(9, fault_code); - update_stmt.bind_text(10, source_id); + update_stmt.bind_text(5, serialize_json_array(sources)); + update_stmt.bind_text(6, new_status); + update_stmt.bind_int(7, debounce_counter); + update_stmt.bind_int64(8, confirmed_at_ns); + update_stmt.bind_int64(9, first_occurred_ns); + update_stmt.bind_text(10, fault_code); + update_stmt.bind_text(11, id.owner); } else { update_stmt.bind_int(1, new_severity); update_stmt.bind_text(2, description); update_stmt.bind_int64(3, timestamp_ns); update_stmt.bind_int64(4, timestamp_ns); update_stmt.bind_int64(5, new_count); - update_stmt.bind_text(6, new_status); - update_stmt.bind_int(7, debounce_counter); - update_stmt.bind_int64(8, confirmed_at_ns); - update_stmt.bind_int64(9, first_occurred_ns); - update_stmt.bind_text(10, fault_code); - update_stmt.bind_text(11, source_id); + update_stmt.bind_text(6, serialize_json_array(sources)); + update_stmt.bind_text(7, new_status); + update_stmt.bind_int(8, debounce_counter); + update_stmt.bind_int64(9, confirmed_at_ns); + update_stmt.bind_int64(10, first_occurred_ns); + update_stmt.bind_text(11, fault_code); + update_stmt.bind_text(12, id.owner); } if (update_stmt.step() != SQLITE_DONE) { @@ -1171,7 +1052,7 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod } if (is_near_miss(true, new_status)) { - record_near_miss_locked(fault_code, timestamp_ns, debounce_counter, config, severity, source_id, new_status); + record_near_miss_locked(id, timestamp_ns, debounce_counter, config, severity, source_id, new_status); } } else { // PASSED event - increment towards healing, clamped to the thresholds. @@ -1189,7 +1070,7 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod update_stmt.bind_text(2, new_status); update_stmt.bind_int(3, debounce_counter); update_stmt.bind_text(4, fault_code); - update_stmt.bind_text(5, source_id); + update_stmt.bind_text(5, id.owner); if (update_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to update fault: ") + sqlite3_errmsg(db_)); @@ -1199,8 +1080,8 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod return is_reactivation; // Reactivation treated as new occurrence for event publishing } - // New record - only create one for FAILED events. A source that has never reported - // this code owns no record yet, so a PASSED from it has nothing to heal. + // New record - only create one for FAILED events. A PASSED event for a record that + // does not exist has nothing to heal. if (!is_failed) { return false; // PASSED event for non-existent record is ignored } @@ -1234,21 +1115,21 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod insert_stmt.bind_int64(10, timestamp_ns); // last_failed_ns insert_stmt.bind_int64(11, 0); // last_passed_ns (never passed) insert_stmt.bind_int64(12, confirmed_now ? timestamp_ns : 0); - insert_stmt.bind_text(13, source_id); + insert_stmt.bind_text(13, id.owner); if (insert_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to insert fault: ") + sqlite3_errmsg(db_)); } if (is_near_miss(true, initial_status)) { - record_near_miss_locked(fault_code, timestamp_ns, initial_counter, config, severity, source_id, initial_status); + record_near_miss_locked(id, timestamp_ns, initial_counter, config, severity, source_id, initial_status); } return true; // New fault created } -std::vector SqliteFaultStorage::list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const { +std::vector SqliteFaultStorage::list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const { std::lock_guard lock(mutex_); // Determine which statuses to include @@ -1268,9 +1149,7 @@ std::vector SqliteFaultStorage::list_faults(bool filter_by_seve } // Build query - std::string sql = - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, owner, last_passed_ns FROM faults WHERE status IN ("; + std::string sql = std::string("SELECT ") + kFaultColumns + " FROM faults WHERE status IN ("; for (size_t i = 0; i < status_filter.size(); ++i) { if (i > 0) { sql += ", "; @@ -1293,54 +1172,17 @@ std::vector SqliteFaultStorage::list_faults(bool filter_by_seve stmt.bind_int(param_index, static_cast(severity)); } - std::vector result; + std::vector result; while (stmt.step() == SQLITE_ROW) { - fd::FaultRecord fault; - fault.fault_code = stmt.column_text(0); - fault.severity = static_cast(stmt.column_int(1)); - fault.description = stmt.column_text(2); - - int64_t first_ns = stmt.column_int64(3); - int64_t last_ns = stmt.column_int64(4); - fault.first_occurred_ns = first_ns; - fault.last_occurred_ns = last_ns; - - fault.occurrence_count = static_cast(stmt.column_int64(5)); - fault.status = stmt.column_text(6); - // The owner, as a one-element list: the column is the record identity, and the - // services that act on a single record take that value back as their source_id. - fault.reporting_sources.assign(1, stmt.column_text(7)); - fault.last_passed_ns = stmt.column_int64(8); - - result.push_back(fault); + FaultEntry entry; + entry.record = read_fault_row(stmt, log_sink_); + entry.id = FaultId{entry.record.fault_code, read_fault_owner(stmt)}; + result.push_back(std::move(entry)); } return result; } -namespace { - -/// Shared projection so every fault read decodes the same column order. -fd::FaultRecord read_fault_row(SqliteStatement & stmt) { - fd::FaultRecord fault; - fault.fault_code = stmt.column_text(0); - fault.severity = static_cast(stmt.column_int(1)); - fault.description = stmt.column_text(2); - fault.first_occurred_ns = stmt.column_int64(3); - fault.last_occurred_ns = stmt.column_int64(4); - fault.occurrence_count = static_cast(stmt.column_int64(5)); - fault.status = stmt.column_text(6); - fault.reporting_sources.assign(1, stmt.column_text(7)); - fault.last_passed_ns = stmt.column_int64(8); - return fault; -} - -constexpr const char * kFaultColumns = - "fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, owner, last_passed_ns"; - -} // namespace - std::optional SqliteFaultStorage::get_fault(const FaultId & id) const { std::lock_guard lock(mutex_); @@ -1353,21 +1195,20 @@ std::optional SqliteFaultStorage::get_fault(const FaultId & id) return std::nullopt; } - return read_fault_row(stmt); + return read_fault_row(stmt, log_sink_); } -std::vector SqliteFaultStorage::get_faults_by_code(const std::string & fault_code) const { +std::vector SqliteFaultStorage::get_ids_by_code(const std::string & fault_code) const { std::lock_guard lock(mutex_); // Ordered by owner so an ambiguity message lists the owners the same way every time, - // and so the two backends answer in the same order. - SqliteStatement stmt( - db_, (std::string("SELECT ") + kFaultColumns + " FROM faults WHERE fault_code = ? ORDER BY owner ASC").c_str()); + // and so the backends answer in the same order. + SqliteStatement stmt(db_, "SELECT owner FROM faults WHERE fault_code = ? ORDER BY owner ASC"); stmt.bind_text(1, fault_code); - std::vector result; + std::vector result; while (stmt.step() == SQLITE_ROW) { - result.push_back(read_fault_row(stmt)); + result.push_back(FaultId{fault_code, stmt.column_text(0)}); } return result; } @@ -1737,7 +1578,7 @@ size_t SqliteFaultStorage::set_max_near_misses_per_fault(size_t max_count) { SqliteStatement trim_stmt(db_, "DELETE FROM near_misses WHERE id IN (" "SELECT id FROM (SELECT id, ROW_NUMBER() OVER " - "(PARTITION BY fault_code, source_id ORDER BY id DESC) AS rn FROM near_misses) " + "(PARTITION BY fault_code, owner ORDER BY id DESC) AS rn FROM near_misses) " "WHERE rn > ?)"); trim_stmt.bind_int64(1, static_cast(max_count)); @@ -1751,21 +1592,21 @@ size_t SqliteFaultStorage::set_max_near_misses_per_fault(size_t max_count) { return changed > 0 ? static_cast(changed) : 0; } -void SqliteFaultStorage::record_near_miss_locked(const std::string & fault_code, int64_t occurred_at_ns, - int32_t debounce_counter, const DebounceConfig & config, - uint8_t severity, const std::string & source_id, - const std::string & resulting_status) { +void SqliteFaultStorage::record_near_miss_locked(const FaultId & id, int64_t occurred_at_ns, int32_t debounce_counter, + const DebounceConfig & config, uint8_t severity, + const std::string & source_id, const std::string & resulting_status) { SqliteStatement insert_stmt(db_, - "INSERT INTO near_misses (fault_code, occurred_at_ns, debounce_counter, " + "INSERT INTO near_misses (fault_code, owner, occurred_at_ns, debounce_counter, " "confirmation_threshold, severity, source_id, resulting_status) " - "VALUES (?, ?, ?, ?, ?, ?, ?)"); - insert_stmt.bind_text(1, fault_code); - insert_stmt.bind_int64(2, occurred_at_ns); - insert_stmt.bind_int(3, debounce_counter); - insert_stmt.bind_int(4, config.confirmation_threshold); - insert_stmt.bind_int(5, static_cast(severity)); - insert_stmt.bind_text(6, source_id); - insert_stmt.bind_text(7, resulting_status); + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); + insert_stmt.bind_text(1, id.fault_code); + insert_stmt.bind_text(2, id.owner); + insert_stmt.bind_int64(3, occurred_at_ns); + insert_stmt.bind_int(4, debounce_counter); + insert_stmt.bind_int(5, config.confirmation_threshold); + insert_stmt.bind_int(6, static_cast(severity)); + insert_stmt.bind_text(7, source_id); + insert_stmt.bind_text(8, resulting_status); if (insert_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to store near miss: ") + sqlite3_errmsg(db_)); @@ -1785,11 +1626,11 @@ void SqliteFaultStorage::record_near_miss_locked(const std::string & fault_code, // by timestamp would then drop the row that was just appended and make the two backends, which // append in arrival order, disagree on the same input. SqliteStatement trim_stmt(db_, - "DELETE FROM near_misses WHERE fault_code = ?1 AND source_id = ?2 AND id NOT IN " - "(SELECT id FROM near_misses WHERE fault_code = ?1 AND source_id = ?2 " + "DELETE FROM near_misses WHERE fault_code = ?1 AND owner = ?2 AND id NOT IN " + "(SELECT id FROM near_misses WHERE fault_code = ?1 AND owner = ?2 " "ORDER BY id DESC LIMIT ?3)"); - trim_stmt.bind_text(1, fault_code); - trim_stmt.bind_text(2, source_id); + trim_stmt.bind_text(1, id.fault_code); + trim_stmt.bind_text(2, id.owner); trim_stmt.bind_int64(3, static_cast(max_near_misses_per_fault_)); if (trim_stmt.step() != SQLITE_DONE) { @@ -1802,12 +1643,10 @@ std::vector SqliteFaultStorage::get_near_misses(const FaultId & std::vector result; - // source_id already IS the owner on this table, so the series only needs the extra - // predicate, not a new column. SqliteStatement stmt(db_, "SELECT fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, " - "severity, source_id, resulting_status FROM near_misses " - "WHERE fault_code = ? AND source_id = ? ORDER BY id ASC"); + "severity, source_id, resulting_status, owner FROM near_misses " + "WHERE fault_code = ? AND owner = ? ORDER BY id ASC"); stmt.bind_text(1, id.fault_code); stmt.bind_text(2, id.owner); @@ -1820,6 +1659,7 @@ std::vector SqliteFaultStorage::get_near_misses(const FaultId & record.severity = static_cast(stmt.column_int(4)); record.source_id = stmt.column_text(5); record.resulting_status = stmt.column_text(6); + record.owner = stmt.column_text(7); result.push_back(std::move(record)); } @@ -2222,15 +2062,15 @@ std::vector SqliteFaultStorage::list_rosbags_for_entity(const st std::vector result; - // The row carries its record's owner, so the entity match is an equality on the row - // and the json_each() scan over reporting_sources is gone. The join to faults stays - // as the existence check it always was: a row whose record is gone is not served. + // Join rosbag_files with the row's record and keep the rows whose record lists the entity among its + // reporting sources. json_each() rather than LIKE, which treats '_' as a single-char wildcard and + // would produce false positives on ROS names. SqliteStatement stmt(db_, "SELECT r.fault_code, r.owner, r.recording_id, r.file_path, r.format, r.duration_sec, " "r.size_bytes, r.created_at_ns " "FROM rosbag_files r " "JOIN faults f ON r.fault_code = f.fault_code AND r.owner = f.owner " - "WHERE r.owner = ? " + "JOIN json_each(f.reporting_sources) j ON j.value = ? " "ORDER BY r.created_at_ns DESC, r.id DESC"); stmt.bind_text(1, entity_fqn); @@ -2249,7 +2089,7 @@ std::vector SqliteFaultStorage::get_all_faults() const { std::vector result; while (stmt.step() == SQLITE_ROW) { - result.push_back(read_fault_row(stmt)); + result.push_back(read_fault_row(stmt, log_sink_)); } return result; diff --git a/src/ros2_medkit_fault_manager/src/threshold_resolver.cpp b/src/ros2_medkit_fault_manager/src/threshold_resolver.cpp index af3deecd5..1cd5873b9 100644 --- a/src/ros2_medkit_fault_manager/src/threshold_resolver.cpp +++ b/src/ros2_medkit_fault_manager/src/threshold_resolver.cpp @@ -65,6 +65,21 @@ OverrideFields parse_override_fields(const YAML::Node & node, core::LogSink & si return fields; } +/// A YAML boolean, spelled true or false. Throws YAML::BadConversion for any other scalar: yaml-cpp +/// reads "yes", "on" and "y" as booleans too, and a typo must not decide how a code is identified. +bool parse_strict_bool(const YAML::Node & node) { + if (node.IsScalar()) { + const std::string & text = node.Scalar(); + if (text == "true" || text == "True" || text == "TRUE") { + return true; + } + if (text == "false" || text == "False" || text == "FALSE") { + return false; + } + } + throw YAML::BadConversion(node.Mark()); +} + /// Open a threshold config file and hand back its top-level map, or an undefined /// node when the file is missing, unparseable or not a map. Every rejection is /// logged here so both loaders report the same way. @@ -199,6 +214,11 @@ DebounceConfig FaultCodeThresholdResolver::resolve(const std::string & fault_cod return result; } +bool FaultCodeThresholdResolver::per_source(const std::string & fault_code) const { + const auto it = entries_.find(fault_code); + return it != entries_.end() && it->second.per_source; +} + size_t FaultCodeThresholdResolver::size() const { return entries_.size(); } @@ -239,6 +259,9 @@ std::vector FaultCodeThresholdResolver::load_from_yam entry.confirmation_threshold = fields.confirmation_threshold; entry.healing_enabled = fields.healing_enabled; entry.healing_threshold = fields.healing_threshold; + if (item.second["per_source"]) { + entry.per_source = parse_strict_bool(item.second["per_source"]); + } entries.push_back(std::move(entry)); } diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_code_thresholds_integration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_code_thresholds_integration.test.py index 6506aeac5..85206fab3 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_code_thresholds_integration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_code_thresholds_integration.test.py @@ -17,10 +17,11 @@ Integration tests for per-fault_code debounce thresholds. The node is launched with both layers configured, because the point of the -fault-code layer is what it does to the entity layer underneath it: an entity -override is chosen by the reporting source, so without this layer two entities -reporting one code debounce it two ways. Each owner's record keeps its own -counter, and the fault-code override sets the band it is judged against. +fault-code layer is what it does to the entity layer underneath it: the debounce +counter belongs to the fault code while an entity override is chosen by the +reporting source, so without this layer two entities reporting one code debounce +it two ways. The last case here covers the warning that says so when no +fault-code override settles it. """ import os @@ -36,11 +37,27 @@ from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import GetFault, ReportFault +# The node's own wording, so a reworded warning fails here rather than going quiet. +CONFLICT_WARNING = 'is debounced two ways' + # Entities whose debounce policies differ, from test_entity_thresholds.yaml. LIDAR = '/sensors/lidar/front' MOTOR = '/powertrain/motor/left' +def _output_text(proc_output, process): + """ + Return everything the process has written so far, as one string. + + Concatenated with NO separator: proc_output yields raw stream chunks, not + lines, so joining on a newline would splice one into the middle of a log + line and break a substring match on a message that is plainly there. + """ + return ''.join( + output.text.decode(errors='replace') for output in proc_output[process] + ) + + def generate_test_description(): """Launch fault_manager with both the entity and fault-code layers.""" pkg_share = get_package_share_directory('ros2_medkit_fault_manager') @@ -125,11 +142,10 @@ def _report(self, fault_code, source_id): resp = self._call(self.report_client, req) self.assertTrue(resp.accepted) - def _status(self, fault_code, source_id): - """Get the current status of the record one source owns.""" + def _status(self, fault_code): + """Get a fault's current status.""" req = GetFault.Request() req.fault_code = fault_code - req.source_id = source_id resp = self._call(self.get_client, req) self.assertTrue(resp.success, resp.error_message) return resp.fault.status @@ -144,63 +160,131 @@ def test_01_fault_code_beats_the_entity_that_reports_it(self): top of the entity one. """ self._report('SHARED.OVERHEAT', LIDAR) - self.assertEqual(self._status('SHARED.OVERHEAT', LIDAR), Fault.STATUS_PREFAILED) + self.assertEqual(self._status('SHARED.OVERHEAT'), Fault.STATUS_PREFAILED) self._report('SHARED.OVERHEAT', LIDAR) - self.assertEqual(self._status('SHARED.OVERHEAT', LIDAR), Fault.STATUS_PREFAILED) + self.assertEqual(self._status('SHARED.OVERHEAT'), Fault.STATUS_PREFAILED) self._report('SHARED.OVERHEAT', LIDAR) - self.assertEqual(self._status('SHARED.OVERHEAT', LIDAR), Fault.STATUS_CONFIRMED) + self.assertEqual(self._status('SHARED.OVERHEAT'), Fault.STATUS_CONFIRMED) # @verifies REQ_INTEROP_107 - def test_02_one_code_debounces_alike_for_each_entitys_record(self): + def test_02_one_code_debounces_alike_from_two_entities(self): """ - Confirm each owner's SHARED.JAM record on its own fourth event. + Confirm SHARED.JAM on its own fourth event, whoever reported it. - The two sources own two records on their own counters. Under the entity - layer alone lidar's -1 would confirm its record on the first event and - the motor's -5 would take five. + Alternating the two sources is the problem case: under the + entity layer alone lidar's -1 would confirm this on the second event, + bypassing the motor's policy. """ - for source in (MOTOR, LIDAR): - for _ in range(3): - self._report('SHARED.JAM', source) - self.assertEqual( - self._status('SHARED.JAM', source), Fault.STATUS_PREFAILED) + self._report('SHARED.JAM', MOTOR) + self.assertEqual(self._status('SHARED.JAM'), Fault.STATUS_PREFAILED) + + self._report('SHARED.JAM', LIDAR) + self.assertEqual(self._status('SHARED.JAM'), Fault.STATUS_PREFAILED) - self._report('SHARED.JAM', source) - self.assertEqual( - self._status('SHARED.JAM', source), Fault.STATUS_CONFIRMED) + self._report('SHARED.JAM', MOTOR) + self.assertEqual(self._status('SHARED.JAM'), Fault.STATUS_PREFAILED) + + self._report('SHARED.JAM', LIDAR) + self.assertEqual(self._status('SHARED.JAM'), Fault.STATUS_CONFIRMED) # @verifies REQ_INTEROP_107 def test_03_a_code_with_no_override_keeps_the_entity_policy(self): """The new layer is opt-in: an unlisted code still follows its entity.""" self._report('MOTOR.ONLY', MOTOR) - self.assertEqual(self._status('MOTOR.ONLY', MOTOR), Fault.STATUS_PREFAILED) + self.assertEqual(self._status('MOTOR.ONLY'), Fault.STATUS_PREFAILED) self._report('LIDAR.ONLY', LIDAR) - self.assertEqual(self._status('LIDAR.ONLY', LIDAR), Fault.STATUS_CONFIRMED) + self.assertEqual(self._status('LIDAR.ONLY'), Fault.STATUS_CONFIRMED) # @verifies REQ_INTEROP_107 - def test_04_a_half_pinned_code_confirms_on_the_pinned_threshold(self): + def test_04_a_half_pinned_code_is_still_reported(self, proc_output, + fault_manager_node): """ Report SHARED.PARTIAL, which pins confirmation but not healing. - Both owners' records confirm on their own second event, whatever - confirmation threshold their entity carries. + The override settles the direction it names and no more: the two sources + heal this code under thresholds 1 and 5. Warning on the whole policy + rather than on the presence of an override is what keeps that visible. """ - for source in (LIDAR, MOTOR): - self._report('SHARED.PARTIAL', source) - self.assertEqual( - self._status('SHARED.PARTIAL', source), Fault.STATUS_PREFAILED) + self._report('SHARED.PARTIAL', LIDAR) + self.assertEqual(self._status('SHARED.PARTIAL'), Fault.STATUS_PREFAILED) + + self._report('SHARED.PARTIAL', MOTOR) + self.assertEqual(self._status('SHARED.PARTIAL'), Fault.STATUS_CONFIRMED) + + proc_output.assertWaitFor( + f"Fault code 'SHARED.PARTIAL' {CONFLICT_WARNING}", + process=fault_manager_node, + timeout=10.0, + ) - self._report('SHARED.PARTIAL', source) - self.assertEqual( - self._status('SHARED.PARTIAL', source), Fault.STATUS_CONFIRMED) + def test_05_conflicting_policies_are_reported_once(self, proc_output, + fault_manager_node): + """ + Warn on two entities reporting one unlisted code, and warn once. + + Without the warning the bypass is invisible - the fault confirms under + whichever policy happened to report, and nothing in the log distinguishes + that from the policy the operator configured. + """ + self._report('CONFLICTING.CODE', LIDAR) + self._report('CONFLICTING.CODE', MOTOR) + + proc_output.assertWaitFor( + f"Fault code 'CONFLICTING.CODE' {CONFLICT_WARNING}", + process=fault_manager_node, + timeout=10.0, + ) + + # Report again. The node must not warn a second time; that is checked + # in TestShutdown, once the whole output has been read. + self._report('CONFLICTING.CODE', LIDAR) + self._report('CONFLICTING.CODE', MOTOR) @launch_testing.post_shutdown_test() class TestShutdown(unittest.TestCase): - """Verify fault_manager exits cleanly.""" + """Verify fault_manager exits cleanly and stayed quiet where it should.""" + + def test_an_overridden_code_never_warns(self, proc_output, + fault_manager_node): + """ + Stay quiet about a code whose override settles it for both sources. + + SHARED.JAM had two sources and no conflict: a fault-code override + resolves the same for both, so there is nothing to warn about. Checked + after shutdown, when the whole output is in hand - asserting an absence + against a stream still being written proves nothing. + """ + text = _output_text(proc_output, fault_manager_node) + self.assertNotIn("Fault code 'SHARED.JAM'", text) + + # And a code only one source ever reports has nothing to conflict with. + # A witness compared against the global config instead of against the + # first report would warn here, on every ordinary single-reporter robot. + self.assertNotIn("Fault code 'SHARED.OVERHEAT'", text) + self.assertNotIn("Fault code 'MOTOR.ONLY'", text) + self.assertNotIn("Fault code 'LIDAR.ONLY'", text) + + def test_conflict_is_warned_once_with_both_policies(self, proc_output, + fault_manager_node): + """ + Warn once for CONFLICTING.CODE, naming both sources and both policies. + + Counted here, after shutdown: during the run proc_output may not have + read the last warning yet, so a count there can pass on two warnings. + """ + text = _output_text(proc_output, fault_manager_node) + self.assertEqual(text.count("Fault code 'CONFLICTING.CODE'"), 1) + # Lidar reported first, so it is named first. Values come from + # test_entity_thresholds.yaml; motor inherits healing_enabled=false. + self.assertIn( + f"'{LIDAR}' resolves confirmation=-1 healing_enabled=true healing=1, " + f"'{MOTOR}' resolves confirmation=-3 healing_enabled=false healing=5", + text, + ) def test_exit_code(self, proc_info): """Check process exit code.""" diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py new file mode 100644 index 000000000..d0c670ac1 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Record identity per fault code, through a real fault manager. + +A code listed with per_source: true in fault_thresholds.yaml gets one record per +reporting source. Every other code, listed or not, has one record shared by all +its sources, with one debounce counter. The shared record is owned by the +reserved id '*', which no reporter may use. +""" + +import os +import tempfile +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import launch_testing.markers +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault + +TIMEOUT_SEC = 30.0 +ALL_STATUSES = [ + Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, + Fault.STATUS_HEALED, Fault.STATUS_CLEARED, +] + +# Codes with an entry: per_source absent, false and true. A code with no entry is shared too. +THRESHOLDS = """\ +ID.ABSENT: + confirmation_threshold: -3 +ID.FALSE: + per_source: false + confirmation_threshold: -3 +ID.PER: + per_source: true + confirmation_threshold: -1 + healing_enabled: true + healing_threshold: 1 +ID.PER_LIST: + per_source: true + confirmation_threshold: -1 +""" + +THRESHOLDS_FILE = tempfile.NamedTemporaryFile( + 'w', suffix='.yaml', prefix='fault_identity_', delete=False) +THRESHOLDS_FILE.write(THRESHOLDS) +THRESHOLDS_FILE.close() + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'healing_enabled': False, + 'fault_thresholds.config_file': THRESHOLDS_FILE.name, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestFaultIdentity(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_fault_identity_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + cls.get_client = cls.node.create_client(GetFault, '/fault_manager/get_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + for client in (cls.report_client, cls.list_client, cls.get_client, cls.clear_client): + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source, event_type=ReportFault.Request.EVENT_FAILED): + request = ReportFault.Request() + request.fault_code = code + request.event_type = event_type + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + return self._call(self.report_client, request).accepted + + def _get(self, code, source=''): + request = GetFault.Request() + request.fault_code = code + request.source_id = source + return self._call(self.get_client, request) + + def _clear(self, code, source=''): + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + return self._call(self.clear_client, request) + + def _records(self, code): + request = ListFaults.Request() + request.statuses = ALL_STATUSES + return [f for f in self._call(self.list_client, request).faults if f.fault_code == code] + + def test_default_code_has_one_counter_for_all_sources(self): + """Three sources, one FAILED each: one record that confirms on the third.""" + sources = ['/debounce/a', '/debounce/b', '/debounce/c'] + expected = [Fault.STATUS_PREFAILED, Fault.STATUS_PREFAILED, Fault.STATUS_CONFIRMED] + for code in ('ID.ABSENT', 'ID.FALSE'): + with self.subTest(code=code): + for source, status in zip(sources, expected): + self.assertTrue(self._report(code, source)) + records = self._records(code) + self.assertEqual(len(records), 1) + self.assertEqual(records[0].status, status) + self.assertEqual(list(records[0].reporting_sources), sources) + + def test_default_code_is_one_record_that_any_source_reaches(self): + """Any reporting source and the unscoped call reach the one record; a clear ends it.""" + code = 'ID.PLAIN' + sources = ['/plain/a', '/plain/b', '/plain/c'] + for source in sources: + self.assertTrue(self._report(code, source)) + + for source in ['', '*'] + sources: + with self.subTest(get_source=source): + response = self._get(code, source) + self.assertTrue(response.success, response.error_message) + self.assertEqual(response.fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(list(response.fault.reporting_sources), sources) + self.assertEqual(response.fault.occurrence_count, 1) + + cleared = self._clear(code, '/plain/b') + self.assertTrue(cleared.success, cleared.message) + self.assertEqual(self._get(code).fault.status, Fault.STATUS_CLEARED) + self.assertEqual(len(self._records(code)), 1) + + # A later FAILED reactivates the same record as a new occurrence. + self.assertTrue(self._report(code, '/plain/a')) + response = self._get(code) + self.assertEqual(response.fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(response.fault.occurrence_count, 2) + + unscoped = self._clear(code) + self.assertTrue(unscoped.success, unscoped.message) + self.assertEqual(self._get(code).fault.status, Fault.STATUS_CLEARED) + + def test_per_source_code_has_a_record_per_source(self): + """Three records; an unscoped call is refused; clear and heal touch one record.""" + code = 'ID.PER' + sources = ['/per/a', '/per/b', '/per/c'] + for source in sources: + self.assertTrue(self._report(code, source)) + + records = self._records(code) + owners = sorted(list(r.reporting_sources) for r in records) + self.assertEqual(owners, [[s] for s in sources]) + self.assertTrue(all(r.status == Fault.STATUS_CONFIRMED for r in records)) + + ambiguous_get = self._get(code) + self.assertFalse(ambiguous_get.success) + self.assertTrue(ambiguous_get.error_message.startswith('ambiguous:'), + ambiguous_get.error_message) + ambiguous_clear = self._clear(code) + self.assertFalse(ambiguous_clear.success) + self.assertTrue(ambiguous_clear.message.startswith('ambiguous:'), ambiguous_clear.message) + + # A clear of /per/a leaves /per/b and /per/c CONFIRMED. + self.assertTrue(self._clear(code, '/per/a').success) + self.assertEqual(self._get(code, '/per/a').fault.status, Fault.STATUS_CLEARED) + self.assertEqual(self._get(code, '/per/b').fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(self._get(code, '/per/c').fault.status, Fault.STATUS_CONFIRMED) + + # Two PASSED from /per/b heal /per/b only. + for _ in range(2): + self.assertTrue(self._report(code, '/per/b', ReportFault.Request.EVENT_PASSED)) + self.assertEqual(self._get(code, '/per/b').fault.status, Fault.STATUS_HEALED) + self.assertEqual(self._get(code, '/per/c').fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(self._get(code, '/per/a').fault.status, Fault.STATUS_CLEARED) + + def test_default_and_per_source_codes_side_by_side(self): + """The same three sources: one record for a default code, three for a per_source one.""" + sources = ['/side/a', '/side/b', '/side/c'] + for source in sources: + self.assertTrue(self._report('ID.PLAIN_LIST', source)) + self.assertTrue(self._report('ID.PER_LIST', source)) + self.assertEqual(len(self._records('ID.PLAIN_LIST')), 1) + self.assertEqual(len(self._records('ID.PER_LIST')), 3) + + def test_the_shared_owner_is_not_a_source_id(self): + """A report claiming the reserved owner is refused and creates nothing.""" + for code in ('ID.RESERVED_SHARED', 'ID.PER'): + with self.subTest(code=code): + before = len(self._records(code)) + self.assertFalse(self._report(code, '*')) + self.assertEqual(len(self._records(code)), before) + + +@launch_testing.post_shutdown_test() +class TestFaultIdentityShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE.name) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py new file mode 100644 index 000000000..a42277456 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Record identity across restarts on one SQLite file. + +- Changing a code between shared and per_source keeps the stored records as they + were written and sends new reports to the configured identity. The fault + manager says so once per code at startup. +- A database written when every code was one shared record opens as shared + records: each row keeps its reporting sources and every child row (near + misses, snapshots, freeze frame, recording link) attaches to it. + +The test starts the fault manager itself, so a restart is a process exit and a +new process, not a respawn. +""" + +import os +import queue +import signal +import sqlite3 +import subprocess +import tempfile +import threading +import time +import unittest + +from ament_index_python.packages import get_package_prefix +import launch +from launch import LaunchDescription +import launch_testing.actions +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault + +TIMEOUT_SEC = 30.0 +ALL_STATUSES = [ + Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, + Fault.STATUS_HEALED, Fault.STATUS_CLEARED, +] +SERVICE = '/fault_manager/report_fault' + +# The fault manager's tables as the release before record identity created them: one row per +# fault code, no owner column anywhere. +MAIN_SCHEMA = """ +CREATE TABLE faults ( + fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, description TEXT NOT NULL, + first_occurred_ns INTEGER NOT NULL, last_occurred_ns INTEGER NOT NULL, + occurrence_count INTEGER NOT NULL, status TEXT NOT NULL, reporting_sources TEXT NOT NULL, + debounce_counter INTEGER NOT NULL DEFAULT 0, last_failed_ns INTEGER NOT NULL DEFAULT 0, + last_passed_ns INTEGER NOT NULL DEFAULT 0, confirmed_at_ns INTEGER NOT NULL DEFAULT 0); +CREATE TABLE snapshots ( + id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, topic TEXT NOT NULL, + message_type TEXT NOT NULL, data TEXT NOT NULL, captured_at_ns INTEGER NOT NULL, + capture_id INTEGER NOT NULL DEFAULT 0); +CREATE INDEX idx_snapshots_fault_code ON snapshots(fault_code); +CREATE INDEX idx_snapshots_fault_topic ON snapshots(fault_code, topic); +CREATE TABLE freeze_frames ( + fault_code TEXT PRIMARY KEY, data TEXT NOT NULL, captured_at_ns INTEGER NOT NULL); +CREATE TABLE near_misses ( + id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, occurred_at_ns INTEGER NOT NULL, + debounce_counter INTEGER NOT NULL, confirmation_threshold INTEGER NOT NULL, + severity INTEGER NOT NULL, source_id TEXT NOT NULL, resulting_status TEXT NOT NULL DEFAULT ''); +CREATE INDEX idx_near_misses_fault_code ON near_misses(fault_code, id); +CREATE TABLE rosbag_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, + recording_id TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL, format TEXT NOT NULL, + duration_sec REAL NOT NULL, size_bytes INTEGER NOT NULL, created_at_ns INTEGER NOT NULL); +CREATE INDEX idx_rosbag_files_fault_code ON rosbag_files(fault_code); +CREATE INDEX idx_rosbag_files_created_at ON rosbag_files(created_at_ns); +CREATE INDEX idx_rosbag_files_fault_created ON rosbag_files(fault_code, created_at_ns, id); +CREATE INDEX idx_rosbag_files_recording ON rosbag_files(recording_id); +CREATE INDEX idx_rosbag_files_path ON rosbag_files(file_path); +CREATE UNIQUE INDEX idx_rosbag_files_fault_path ON rosbag_files(fault_code, file_path); +""" + + +def generate_test_description(): + """Start only a placeholder process, so launch keeps the test running.""" + keep_alive = launch.actions.ExecuteProcess(cmd=['sleep', '600']) + return LaunchDescription([keep_alive, launch_testing.actions.ReadyToTest()]) + + +class FaultManagerProcess: + """One fault_manager_node process with its output readable line by line.""" + + def __init__(self, exe, params_file): + self.proc = subprocess.Popen( + [exe, '--ros-args', '-r', '__node:=fault_manager', '--params-file', params_file], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, errors='replace') + self.lines = queue.Queue() + self.seen = [] + threading.Thread(target=self._pump, daemon=True).start() + + def _pump(self): + for line in self.proc.stdout: + self.lines.put(line) + self.lines.put(None) + + def wait_for_line(self, text): + """Return the first output line containing text, or None at the end or the timeout.""" + for line in self.seen: + if text in line: + return line + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + try: + line = self.lines.get(timeout=1.0) + except queue.Empty: + continue + if line is None: + return None + self.seen.append(line) + if text in line: + return line + return None + + def stop(self): + if self.proc.poll() is None: + self.proc.send_signal(signal.SIGINT) + return self.proc.wait(timeout=TIMEOUT_SEC) + + def kill(self): + if self.proc.poll() is None: + self.proc.kill() + self.proc.wait() + + +class TestFaultIdentityRestart(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + cls.exe = os.path.join( + get_package_prefix('ros2_medkit_fault_manager'), + 'lib', 'ros2_medkit_fault_manager', 'fault_manager_node') + rclpy.init() + cls.node = Node('test_fault_identity_restart_client') + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + cls.tmp.cleanup() + + def _path(self, name): + return os.path.join(self.tmp.name, name) + + def _write(self, name, text): + path = self._path(name) + with open(path, 'w') as f: + f.write(text) + return path + + def _wait_until_service_is_gone(self): + """Wait until the graph no longer lists the previous process's services.""" + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + names = [name for name, _ in self.node.get_service_names_and_types()] + if SERVICE not in names: + return + rclpy.spin_once(self.node, timeout_sec=0.1) + self.fail('the stopped fault manager is still listed in the graph') + + def _start(self, db_path, thresholds): + """Start the fault manager on db_path with a thresholds file, and wait until it answers.""" + self._wait_until_service_is_gone() + thresholds_file = self._write('thresholds.yaml', thresholds) + params = self._write('params.yaml', ( + 'fault_manager:\n' + ' ros__parameters:\n' + ' storage_type: sqlite\n' + f' database_path: {db_path}\n' + ' confirmation_threshold: -1\n' + ' snapshots.enabled: false\n' + ' snapshots.rosbag.enabled: false\n' + f' fault_thresholds.config_file: {thresholds_file}\n' + )) + process = FaultManagerProcess(self.exe, params) + self.addCleanup(process.kill) + self.report = self.node.create_client(ReportFault, SERVICE) + self.get = self.node.create_client(GetFault, '/fault_manager/get_fault') + self.list = self.node.create_client(ListFaults, '/fault_manager/list_faults') + self.clear = self.node.create_client(ClearFault, '/fault_manager/clear_fault') + for client in (self.report, self.get, self.list, self.clear): + self.assertTrue(client.wait_for_service(timeout_sec=TIMEOUT_SEC)) + self.assertIsNotNone( + process.wait_for_line('FaultManager node started'), 'the node never finished starting') + return process + + def _stop(self, process): + for client in (self.report, self.get, self.list, self.clear): + self.node.destroy_client(client) + self.assertEqual(process.stop(), 0) + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report, request).accepted) + + def _get(self, code, source=''): + request = GetFault.Request() + request.fault_code = code + request.source_id = source + return self._call(self.get, request) + + def _records(self, code): + request = ListFaults.Request() + request.statuses = ALL_STATUSES + return [f for f in self._call(self.list, request).faults if f.fault_code == code] + + def _sources_of_records(self, code): + return sorted(list(f.reporting_sources) for f in self._records(code)) + + def test_changing_the_mode_keeps_stored_records_and_routes_new_reports(self): + db_path = self._path('mode_change.db') + shared_then_per_source = 'M.SHARED:\n confirmation_threshold: -1\n' \ + 'N.PER:\n per_source: true\n confirmation_threshold: -1\n' + per_source_then_shared = 'M.SHARED:\n per_source: true\n confirmation_threshold: -1\n' \ + 'N.PER:\n confirmation_threshold: -1\n' + + process = self._start(db_path, shared_then_per_source) + try: + for code in ('M.SHARED', 'N.PER'): + for source in ('/a', '/b'): + self._report(code, source) + self.assertEqual(self._sources_of_records('M.SHARED'), [['/a', '/b']]) + self.assertEqual(self._sources_of_records('N.PER'), [['/a'], ['/b']]) + # The startup check runs before the node says it started, and _start waited for that. + self.assertFalse( + [line for line in process.seen if 'is configured' in line], + 'a store that matches its configuration must not warn') + finally: + self._stop(process) + + process = self._start(db_path, per_source_then_shared) + try: + self.assertIsNotNone( + process.wait_for_line("Fault code 'M.SHARED' is configured per_source"), + 'no startup warning for the shared record of a per_source code') + self.assertIsNotNone( + process.wait_for_line("Fault code 'N.PER' is configured shared"), + 'no startup warning for the per-source records of a shared code') + + # The stored records are as they were written, and addressable by their owner. + self.assertEqual(self._sources_of_records('M.SHARED'), [['/a', '/b']]) + self.assertEqual(self._sources_of_records('N.PER'), [['/a'], ['/b']]) + shared = self._get('M.SHARED', '*') + self.assertTrue(shared.success, shared.error_message) + self.assertEqual(list(shared.fault.reporting_sources), ['/a', '/b']) + for source in ('/a', '/b'): + owned = self._get('N.PER', source) + self.assertTrue(owned.success, owned.error_message) + self.assertEqual(list(owned.fault.reporting_sources), [source]) + + # New reports go to the configured identity. + self._report('M.SHARED', '/c') + self._report('N.PER', '/c') + self.assertEqual(self._sources_of_records('M.SHARED'), [['/a', '/b'], ['/c']]) + self.assertEqual(self._sources_of_records('N.PER'), [['/a'], ['/b'], ['/c']]) + self.assertEqual( + list(self._get('M.SHARED', '*').fault.reporting_sources), ['/a', '/b']) + self.assertEqual(list(self._get('N.PER', '*').fault.reporting_sources), ['/c']) + finally: + self._stop(process) + + with sqlite3.connect(db_path) as db: + rows = db.execute( + 'SELECT fault_code, owner FROM faults ORDER BY fault_code, owner').fetchall() + self.assertEqual(rows, [ + ('M.SHARED', '*'), ('M.SHARED', '/c'), + ('N.PER', '*'), ('N.PER', '/a'), ('N.PER', '/b'), + ]) + + def test_a_store_written_with_one_record_per_code_opens_as_shared_records(self): + db_path = self._path('legacy.db') + bag_path = self._path('fault_MIG.CODE_1') + os.makedirs(bag_path) + with sqlite3.connect(db_path) as db: + db.executescript(MAIN_SCHEMA) + db.execute( + "INSERT INTO faults VALUES ('MIG.CODE', 2, 'from the old release', 1, 1, 1, " + "'CONFIRMED', '[\"/s1\",\"/s2\"]', -1, 1, 0, 0)") + db.execute( + "INSERT INTO faults VALUES ('MIG.OLD', 1, 'cleared long ago', 1, 1, 1, " + "'CLEARED', '[\"/s3\"]', 0, 1, 0, 0)") + db.execute( + "INSERT INTO faults VALUES ('MIG.TWO', 2, 'second code', 1, 1, 1, " + "'CONFIRMED', '[\"/s5\"]', -1, 1, 0, 0)") + for second, source in ((5, '/s1'), (6, '/s1'), (7, '/s2')): + db.execute( + 'INSERT INTO near_misses (fault_code, occurred_at_ns, debounce_counter, ' + 'confirmation_threshold, severity, source_id, resulting_status) ' + "VALUES ('MIG.CODE', ?, -1, -3, 2, ?, 'PREFAILED')", (second, source)) + db.execute( + 'INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) ' + "VALUES ('MIG.CODE', '/t', 'std_msgs/msg/Float64', '{\"data\": 1.5}', 9)") + db.execute("INSERT INTO freeze_frames VALUES ('MIG.CODE', '{\"/t\": 1.5}', 9)") + db.execute( + 'INSERT INTO rosbag_files (fault_code, recording_id, file_path, format, ' + 'duration_sec, size_bytes, created_at_ns) ' + "VALUES ('MIG.CODE', 'fault_MIG.CODE_1', ?, 'mcap', 5.0, 100, 11)", (bag_path,)) + + process = self._start(db_path, 'UNRELATED.CODE:\n confirmation_threshold: -1\n') + try: + # Each code is one record, with every source it recorded. + self.assertEqual(self._sources_of_records('MIG.CODE'), [['/s1', '/s2']]) + self.assertEqual(self._sources_of_records('MIG.OLD'), [['/s3']]) + for source in ('', '*', '/s1', '/s2'): + record = self._get('MIG.CODE', source) + self.assertTrue(record.success, f'source {source!r}: {record.error_message}') + self.assertEqual(record.fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(list(record.fault.reporting_sources), ['/s1', '/s2']) + detail = self._get('MIG.CODE').environment_data.snapshots + self.assertEqual(sorted(s.type for s in detail), ['freeze_frame', 'rosbag']) + + # It then behaves as any shared record: a new source joins it, a clear ends it, a later + # FAILED starts a new occurrence. + self._report('MIG.CODE', '/s4') + self.assertEqual(self._sources_of_records('MIG.CODE'), [['/s1', '/s2', '/s4']]) + self.assertEqual(self._get('MIG.CODE').fault.occurrence_count, 1) + request = ClearFault.Request() + request.fault_code = 'MIG.TWO' + self.assertTrue(self._call(self.clear, request).success) + self.assertEqual(self._get('MIG.TWO').fault.status, Fault.STATUS_CLEARED) + self._report('MIG.TWO', '/s1') + self.assertEqual(self._get('MIG.TWO').fault.occurrence_count, 2) + finally: + self._stop(process) + + with sqlite3.connect(db_path) as db: + self.assertEqual( + db.execute('SELECT fault_code, owner FROM faults ORDER BY fault_code').fetchall(), + [('MIG.CODE', '*'), ('MIG.OLD', '*'), ('MIG.TWO', '*')]) + for table in ('snapshots', 'freeze_frames', 'rosbag_files'): + self.assertEqual( + db.execute(f'SELECT owner FROM {table}').fetchall(), [('*',)], table) + # Every near miss of both sources belongs to the one shared record. + self.assertEqual( + db.execute( + "SELECT source_id FROM near_misses WHERE fault_code = 'MIG.CODE' " + "AND owner = '*' ORDER BY id").fetchall(), + [('/s1',), ('/s1',), ('/s2',)]) + self.assertEqual( + db.execute( + 'SELECT COUNT(*) FROM near_misses n WHERE NOT EXISTS ' + '(SELECT 1 FROM faults f WHERE f.fault_code = n.fault_code ' + 'AND f.owner = n.owner)').fetchone(), + (0,)) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py index d45916824..7ebcef92b 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py @@ -15,8 +15,10 @@ """Integration tests for ros2_medkit_fault_manager services.""" +import atexit import json import os +import tempfile import threading import time import unittest @@ -72,6 +74,13 @@ def get_coverage_env(): def generate_test_description(): """Generate launch description with fault_manager node.""" + # One code gets a record per source, for the scoped-clear test. Every other code is shared. + per_source_codes = tempfile.NamedTemporaryFile( + 'w', suffix='.yaml', prefix='per_source_codes_', delete=False) + per_source_codes.write('TEST_SCOPED_CLEAR:\n per_source: true\n') + per_source_codes.close() + atexit.register(os.unlink, per_source_codes.name) + # Get path to test snapshot config pkg_share = get_package_share_directory('ros2_medkit_fault_manager') snapshot_config = os.path.join(pkg_share, 'test', 'test_snapshots.yaml') @@ -96,6 +105,7 @@ def generate_test_description(): 'snapshots.capture_queue_depth': 2, 'snapshots.capture_queue_full_policy': 'reject_newest', 'correlation.config_file': correlation_config, # Enable correlation + 'fault_thresholds.config_file': per_source_codes.name, }], # Give the node room to flush coverage data at shutdown before SIGKILL. sigterm_timeout='30', @@ -322,8 +332,8 @@ def test_09_clear_fault_empty_code_fails(self): self.assertIn('empty', response.message.lower()) print(f'Clear empty code response: {response.message}') - def test_10_two_sources_are_two_records(self): - """Two sources reporting one fault_code own two independent records.""" + def test_10_report_updates_existing_fault(self): + """Test that reporting same fault_code updates existing fault.""" fault_code = 'TEST_FAULT_UPDATE' # Report first time @@ -337,17 +347,18 @@ def test_10_two_sources_are_two_records(self): response1 = self._call_service(self.report_fault_client, request1) self.assertTrue(response1.accepted) - # Report second time from a different source, at a higher severity + # Report second time from different source request2 = ReportFault.Request() request2.fault_code = fault_code request2.event_type = ReportFault.Request.EVENT_FAILED - request2.severity = Fault.SEVERITY_ERROR + request2.severity = Fault.SEVERITY_ERROR # Higher severity request2.description = 'Second report' request2.source_id = '/node2' response2 = self._call_service(self.report_fault_client, request2) self.assertTrue(response2.accepted) + # Verify fault was updated get_request = ListFaults.Request() get_request.filter_by_severity = False get_request.severity = 0 @@ -355,37 +366,34 @@ def test_10_two_sources_are_two_records(self): get_response = self._call_service(self.list_faults_client, get_request) - by_owner = {} + updated_fault = None for f in get_response.faults: if f.fault_code == fault_code: - self.assertEqual(len(f.reporting_sources), 1) - by_owner[f.reporting_sources[0]] = f - - self.assertEqual(sorted(by_owner), ['/node1', '/node2']) - # Severity is the record's own maximum, so one source reporting ERROR does - # not escalate the other source's record. - self.assertEqual(by_owner['/node1'].severity, Fault.SEVERITY_WARN) - self.assertEqual(by_owner['/node2'].severity, Fault.SEVERITY_ERROR) - self.assertEqual(by_owner['/node1'].description, 'First report') - self.assertEqual(by_owner['/node2'].description, 'Second report') - # Each record counts its own occurrences. - self.assertEqual(by_owner['/node1'].occurrence_count, 1) - self.assertEqual(by_owner['/node2'].occurrence_count, 1) - print(f'Two records for {fault_code}: owners={sorted(by_owner)}') + updated_fault = f + break + + self.assertIsNotNone(updated_fault) + self.assertEqual(updated_fault.severity, Fault.SEVERITY_ERROR) + # Edge-counting: a re-report of a still-active fault is the same + # continuous occurrence, so the count stays at 1 (it only increments + # on reactivation after CLEARED). + self.assertEqual(updated_fault.occurrence_count, 1) + self.assertEqual(len(updated_fault.reporting_sources), 2) + print(f'Updated fault: occurrence_count={updated_fault.occurrence_count}, ' + f'sources={updated_fault.reporting_sources}') def test_11_confirmation_workflow(self): """Test fault auto-confirms after reaching threshold (default=-3).""" fault_code = 'TEST_FAULT_CONFIRM' - # Report 3 times from ONE source: the debounce counter belongs to that - # source's record, so depth is what one reporter has said. + # Report fault 3 times (default threshold) for i in range(3): request = ReportFault.Request() request.fault_code = fault_code request.event_type = ReportFault.Request.EVENT_FAILED request.severity = Fault.SEVERITY_ERROR request.description = f'Report {i + 1}' - request.source_id = '/node1' + request.source_id = f'/node{i + 1}' self._call_service(self.report_fault_client, request) # Query CONFIRMED faults (should include our fault now) @@ -446,22 +454,19 @@ def test_12_prefailed_excluded_from_default_query(self): print('PREFAILED query found the fault as expected') def test_13_multi_source_confirmation(self): - """Each source reporting one fault_code confirms its own record.""" + """Test fault confirms when multiple sources report same fault.""" fault_code = 'TEST_FAULT_MULTI_SRC' - # Each of 3 sources reports enough times to confirm its OWN record. Under a - # shared counter three single reports confirmed one fault between them. Now - # each reporter has to reach its own threshold. + # Report from 3 different sources sources = ['/sensor1', '/sensor2', '/sensor3'] for source in sources: - for _ in range(3): # launch threshold is -3 - request = ReportFault.Request() - request.fault_code = fault_code - request.event_type = ReportFault.Request.EVENT_FAILED - request.severity = Fault.SEVERITY_ERROR - request.description = 'Multi-source fault' - request.source_id = source - self._call_service(self.report_fault_client, request) + request = ReportFault.Request() + request.fault_code = fault_code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = 'Multi-source fault' + request.source_id = source + self._call_service(self.report_fault_client, request) # Query and verify get_request = ListFaults.Request() @@ -471,17 +476,14 @@ def test_13_multi_source_confirmation(self): response = self._call_service(self.list_faults_client, get_request) - matching = [f for f in response.faults if f.fault_code == fault_code] - self.assertEqual(len(matching), 3) - owners = [] - for fault in matching: - self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) - self.assertEqual(len(fault.reporting_sources), 1) - # Edge-counting is per record: one raise edge each. - self.assertEqual(fault.occurrence_count, 1) - owners.append(fault.reporting_sources[0]) - self.assertEqual(sorted(owners), sorted(sources)) - print(f'Three records confirmed: owners={sorted(owners)}') + fault = next((f for f in response.faults if f.fault_code == fault_code), None) + self.assertIsNotNone(fault) + self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(len(fault.reporting_sources), 3) + # Edge-counting: three sources reporting the same active fault are one + # continuous occurrence; the count tracks raise edges, not reports. + self.assertEqual(fault.occurrence_count, 1) + print(f'Multi-source fault confirmed: sources={fault.reporting_sources}') def test_14_critical_severity_immediate_confirmation(self): """Test CRITICAL severity bypasses debounce and confirms immediately.""" @@ -514,14 +516,14 @@ def test_15_passed_event_increments_counter(self): """Test PASSED events increment debounce counter towards healing.""" fault_code = 'TEST_FAULT_PASSED' - # Report 2 FAILED events from one source (its counter = -2) + # Report 2 FAILED events (counter = -2) for i in range(2): request = ReportFault.Request() request.fault_code = fault_code request.event_type = ReportFault.Request.EVENT_FAILED request.severity = Fault.SEVERITY_ERROR request.description = f'Failed report {i + 1}' - request.source_id = '/node1' + request.source_id = f'/node{i + 1}' self._call_service(self.report_fault_client, request) # Verify fault is PREFAILED @@ -542,9 +544,7 @@ def test_15_passed_event_increments_counter(self): request.event_type = ReportFault.Request.EVENT_PASSED request.severity = 0 # Ignored for PASSED request.description = '' # Ignored for PASSED - # The same source: a PASSED moves the counter of the record its own - # source owns, and a source that owns no record has nothing to heal. - request.source_id = '/node1' + request.source_id = '/test_node' self._call_service(self.report_fault_client, request) # Verify fault is PREPASSED (counter > 0) @@ -811,9 +811,7 @@ def test_22_correlation_symptoms_muted(self): request.event_type = ReportFault.Request.EVENT_FAILED request.severity = Fault.SEVERITY_CRITICAL request.description = f'Symptom: {symptom}' - # Same owner as the root cause: correlation relates records of one owner, - # so a root cause only explains faults its own reporter raised. - request.source_id = '/estop_node' + request.source_id = '/motor_node' response = self._call_service(self.report_fault_client, request) self.assertTrue(response.accepted) @@ -849,10 +847,9 @@ def test_23_correlation_auto_clear_with_root(self): When ESTOP_001 is cleared, all its correlated symptoms should also be cleared (auto_clear_with_root=true). """ - # Clear the root cause record + # Clear the root cause clear_request = ClearFault.Request() clear_request.fault_code = 'ESTOP_001' - clear_request.source_id = '/estop_node' clear_response = self._call_service(self.clear_fault_client, clear_request) @@ -879,57 +876,6 @@ def test_23_correlation_auto_clear_with_root(self): self.assertEqual(len(remaining_muted), 0) print('All symptoms auto-cleared with root cause') - def test_23b_auto_clear_stays_inside_one_owner(self): - """A root cause auto-clears only its own reporter's symptom record.""" - # Two owners raise the same root cause and the same symptom. - for owner in ['/cascade_a', '/cascade_b']: - for code in ['ESTOP_001', 'MOTOR_COMM_CASCADE']: - request = ReportFault.Request() - request.fault_code = code - request.event_type = ReportFault.Request.EVENT_FAILED - request.severity = Fault.SEVERITY_CRITICAL - request.description = f'{code} from {owner}' - request.source_id = owner - self.assertTrue(self._call_service(self.report_fault_client, request).accepted) - - clear_request = ClearFault.Request() - clear_request.fault_code = 'ESTOP_001' - clear_request.source_id = '/cascade_a' - clear_response = self._call_service(self.clear_fault_client, clear_request) - self.assertTrue(clear_response.success) - self.assertIn('MOTOR_COMM_CASCADE', clear_response.auto_cleared_codes) - - # B's identical symptom is untouched: it is muted by B's own root cause, - # which nobody cleared. - get_request = ListFaults.Request() - get_request.filter_by_severity = False - get_request.statuses = [Fault.STATUS_CONFIRMED] - get_request.include_muted = True - faults = self._call_service(self.list_faults_client, get_request).faults - - symptom_owners = sorted(f.reporting_sources[0] for f in faults - if f.fault_code == 'MOTOR_COMM_CASCADE') - self.assertEqual(symptom_owners, ['/cascade_b']) - - # include_muted left at its default on purpose: muting is per record, so the - # default query hides B's muted record alone and keeps A's cleared record, - # which carries the same code. - cleared_request = ListFaults.Request() - cleared_request.statuses = [Fault.STATUS_CLEARED] - cleared = [f for f in self._call_service(self.list_faults_client, cleared_request).faults - if f.fault_code == 'MOTOR_COMM_CASCADE'] - self.assertEqual([f.reporting_sources[0] for f in cleared], ['/cascade_a']) - - # The same default query still hides the record that IS muted: B's symptom is - # CONFIRMED under B's own root cause and must not appear. - default_request = ListFaults.Request() - default_request.statuses = [Fault.STATUS_CONFIRMED] - default_confirmed = [f for f in - self._call_service(self.list_faults_client, default_request).faults - if f.fault_code == 'MOTOR_COMM_CASCADE'] - self.assertEqual(default_confirmed, []) - print('Auto-clear cascade stayed inside one owner') - def test_24_correlation_auto_cluster(self): """ Test auto-cluster correlation: sensor errors are grouped. @@ -1012,24 +958,23 @@ def test_26_cleared_fault_can_be_reactivated(self): response = self._call_service(self.report_fault_client, request) self.assertTrue(response.accepted) - # Clear that owner's record + # Clear the fault clear_request = ClearFault.Request() clear_request.fault_code = fault_code - clear_request.source_id = '/node1' clear_response = self._call_service(self.clear_fault_client, clear_request) self.assertTrue(clear_response.success) - # The same owner reports again - reactivates its own record + # Report again - should reactivate request = ReportFault.Request() request.fault_code = fault_code request.event_type = ReportFault.Request.EVENT_FAILED request.severity = Fault.SEVERITY_CRITICAL request.description = 'Reactivated' - request.source_id = '/node1' + request.source_id = '/node2' response = self._call_service(self.report_fault_client, request) self.assertTrue(response.accepted) - # Verify the record is CONFIRMED again, on its second occurrence + # Verify fault is CONFIRMED again get_request = ListFaults.Request() get_request.statuses = [Fault.STATUS_CONFIRMED] get_response = self._call_service(self.list_faults_client, get_request) @@ -1038,8 +983,8 @@ def test_26_cleared_fault_can_be_reactivated(self): self.assertIsNotNone(fault) self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) self.assertEqual(fault.occurrence_count, 2) - self.assertEqual(fault.reporting_sources, ['/node1']) - print(f'Cleared record reactivated: occurrence_count={fault.occurrence_count}') + self.assertEqual(len(fault.reporting_sources), 2) + print(f'Cleared fault reactivated: occurrence_count={fault.occurrence_count}') def test_26b_scoped_clear_leaves_the_other_owners_record(self): """A scoped clear takes one record, an unscoped clear of two is refused.""" diff --git a/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py b/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py index 6d2e9c1d9..619368133 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py @@ -18,8 +18,8 @@ The database starts as one written before records had an owner, so the first start migrates it. The fault manager is then restarted twice on it. Every -record here has one owner. A record keeps the highest severity its owner ever -reported. A report on a CRITICAL record confirms it. A reactivated occurrence, +record here is the shared record of its code. A record keeps the highest +severity ever reported for it. A report on a CRITICAL record confirms it. A reactivated occurrence, migrated or not, has no confirmation time until it confirms. A report whose severity differs from the stored one is logged once per process. """ @@ -46,6 +46,7 @@ DB_PATH = os.path.join(DB_DIR, 'faults.db') EVENT_TOPIC = '/fault_manager/events' SOURCE = '/robot/reporter' +SHARED_OWNER = '*' ALL_STATUSES = [ Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, Fault.STATUS_HEALED, Fault.STATUS_CLEARED, @@ -132,6 +133,7 @@ def setUpClass(cls): assert cls.list_client.wait_for_service(timeout_sec=TIMEOUT_SEC) assert cls.clear_client.wait_for_service(timeout_sec=TIMEOUT_SEC) cls.primes = 0 + cls.markers = 0 @classmethod def tearDownClass(cls): @@ -187,7 +189,7 @@ def _confirmed_at(self, code): with sqlite3.connect(DB_PATH, timeout=10.0) as db: row = db.execute( 'SELECT confirmed_at_ns FROM faults WHERE fault_code = ? AND owner = ?', - (code, SOURCE)).fetchone() + (code, SHARED_OWNER)).fetchone() self.assertIsNotNone(row, f'{code} has no row in the database') return row[0] @@ -235,17 +237,26 @@ def _restart(self, fault_manager_node): return self.fail('the restarted fault manager never answered list_faults') - def _mismatch_lines(self, proc_output, expected): - """Count SEV_CRIT mismatch warnings once ``expected`` arrived, or at the timeout.""" - def count(): - text = ''.join(io.text.decode(errors='replace') for io in proc_output) - return text.count(MISMATCH_LINE) - deadline = time.monotonic() + 5.0 - while time.monotonic() < deadline and count() < expected: - time.sleep(0.2) - # Anything logged late would show here as more than expected. - time.sleep(0.5) - return count() + def _mismatch_lines(self, proc_output): + """ + Count the SEV_CRIT mismatch warnings logged so far. + + A marker code mismatches once, after everything reported before it. The output is one + ordered stream, so once the marker's line has arrived every earlier line has too. + """ + type(self).markers += 1 + marker = f'SEV_MARK_{self.markers}' + self._report(marker, Fault.SEVERITY_CRITICAL) + self._report(marker, Fault.SEVERITY_WARN) + marker_line = f"Fault '{marker}' reported with severity WARN" + + def text(): + return ''.join(io.text.decode(errors='replace') for io in proc_output) + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline and marker_line not in text(): + time.sleep(0.1) + self.assertIn(marker_line, text(), 'the marker mismatch was never logged') + return text().count(MISMATCH_LINE) def test_severity_contract_across_restarts(self, fault_manager_node, proc_output): self._prime_events() @@ -324,7 +335,7 @@ def test_severity_contract_across_restarts(self, fault_manager_node, proc_output self.assertEqual(self._confirmed_at('CONF_AT'), 0) with self.subTest(case='mismatch logged once in this process'): - self.assertEqual(self._mismatch_lines(proc_output, 1), 1) + self.assertEqual(self._mismatch_lines(proc_output), 1) self._restart(fault_manager_node) @@ -354,7 +365,7 @@ def test_severity_contract_across_restarts(self, fault_manager_node, proc_output for _ in range(10): self._report('SEV_CRIT', Fault.SEVERITY_WARN) with self.subTest(case='the bound restarts with the process'): - self.assertEqual(self._mismatch_lines(proc_output, 2), 2) + self.assertEqual(self._mismatch_lines(proc_output), 2) @launch_testing.post_shutdown_test() diff --git a/src/ros2_medkit_fault_manager/test/test_entity_thresholds.cpp b/src/ros2_medkit_fault_manager/test/test_entity_thresholds.cpp index 1e96a1071..ba1e1ccde 100644 --- a/src/ros2_medkit_fault_manager/test/test_entity_thresholds.cpp +++ b/src/ros2_medkit_fault_manager/test/test_entity_thresholds.cpp @@ -30,6 +30,7 @@ namespace fs = std::filesystem; using ros2_medkit_fault_manager::DebounceConfig; using ros2_medkit_fault_manager::EntityDebounceOverride; using ros2_medkit_fault_manager::EntityThresholdResolver; +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::InMemoryFaultStorage; using ros2_medkit_msgs::msg::Fault; using ros2_medkit_msgs::srv::ReportFault; @@ -286,8 +287,8 @@ class PerEntityStorageTest : public ::testing::Test { // @verifies REQ_INTEROP_095 TEST_F(PerEntityStorageTest, LidarFaultConfirmsImmediately) { auto ts = clock_.now(); - storage_.report_fault_event("LIDAR_FAIL", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Lidar failure", - "/sensors/lidar", ts.nanoseconds(), lidar_config_); + storage_.report_fault_event(FaultId{"LIDAR_FAIL", "/sensors/lidar"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Lidar failure", "/sensors/lidar", ts.nanoseconds(), lidar_config_); auto fault = storage_.get_fault({"LIDAR_FAIL", "/sensors/lidar"}); ASSERT_TRUE(fault.has_value()); @@ -299,8 +300,9 @@ TEST_F(PerEntityStorageTest, MotorFaultNeedsDebouncing) { auto ts = clock_.now(); // First event: counter = -1, threshold = -5, so PREFAILED - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor hot", - "/powertrain/motor_left", ts.nanoseconds(), motor_config_); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor_left"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Motor hot", "/powertrain/motor_left", ts.nanoseconds(), + motor_config_); auto fault = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor_left"}); ASSERT_TRUE(fault.has_value()); @@ -308,15 +310,17 @@ TEST_F(PerEntityStorageTest, MotorFaultNeedsDebouncing) { // Events 2-4: still PREFAILED (counter = -2, -3, -4) for (int i = 0; i < 3; ++i) { - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", - "/powertrain/motor_left", clock_.now().nanoseconds(), motor_config_); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor_left"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "", "/powertrain/motor_left", clock_.now().nanoseconds(), + motor_config_); } fault = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor_left"}); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Event 5: counter = -5, meets threshold -> CONFIRMED - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", - "/powertrain/motor_left", clock_.now().nanoseconds(), motor_config_); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor_left"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "", "/powertrain/motor_left", clock_.now().nanoseconds(), + motor_config_); fault = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor_left"}); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -324,8 +328,8 @@ TEST_F(PerEntityStorageTest, MotorFaultNeedsDebouncing) { // @verifies REQ_INTEROP_095 TEST_F(PerEntityStorageTest, UnknownEntityUsesGlobalConfig) { auto ts = clock_.now(); - storage_.report_fault_event("UNKNOWN_FAULT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Unknown", - "/some/unknown/entity", ts.nanoseconds(), global_); + storage_.report_fault_event(FaultId{"UNKNOWN_FAULT", "/some/unknown/entity"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Unknown", "/some/unknown/entity", ts.nanoseconds(), global_); auto fault = storage_.get_fault({"UNKNOWN_FAULT", "/some/unknown/entity"}); ASSERT_TRUE(fault.has_value()); @@ -333,20 +337,40 @@ TEST_F(PerEntityStorageTest, UnknownEntityUsesGlobalConfig) { } // @verifies REQ_INTEROP_095 -// Two entities reporting one code are two records, each evaluated against its own -// entity's band. This used to be one row with one shared counter, so the lidar's -// single report inherited the motor's decrement and confirmed a fault the motor's -// own policy said was still debouncing. -TEST_F(PerEntityStorageTest, DifferentEntitiesSameFaultCodeDebounceIndependently) { +TEST_F(PerEntityStorageTest, DifferentEntitiesSameFaultCode) { auto ts = clock_.now(); + // One record shared by every source of the code. + const FaultId shared{"OVERHEAT", ros2_medkit_fault_manager::kSharedOwner}; - // Motor reports with motor config (debounced, threshold -5) - storage_.report_fault_event("OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheat", + // Motor reports with motor config (debounced) + storage_.report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheat", "/powertrain/motor_left", ts.nanoseconds(), motor_config_); + auto fault = storage_.get_fault(shared); + EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Motor needs 5 events - // Lidar reports the same fault_code with lidar config (threshold -1) - storage_.report_fault_event("OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Lidar overheat", + // Lidar reports same fault_code with lidar config (threshold=-1) + // Counter is now -2, lidar threshold is -1, -2 <= -1 -> CONFIRMED + storage_.report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Lidar overheat", "/sensors/lidar", clock_.now().nanoseconds(), lidar_config_); + fault = storage_.get_fault(shared); + EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Lidar config confirms at -1, counter is -2 + EXPECT_EQ(storage_.size(), 1u); +} + +// Under per-source identity two entities reporting one code are two records, each evaluated against +// its own entity's band. +// @verifies REQ_INTEROP_095 +TEST_F(PerEntityStorageTest, DifferentEntitiesSameFaultCodeDebounceIndependentlyWhenPerSource) { + auto ts = clock_.now(); + + // Motor reports with motor config (debounced, threshold -5) + storage_.report_fault_event(FaultId{"OVERHEAT", "/powertrain/motor_left"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Motor overheat", "/powertrain/motor_left", ts.nanoseconds(), + motor_config_); + + // Lidar reports the same fault_code with lidar config (threshold -1) + storage_.report_fault_event(FaultId{"OVERHEAT", "/sensors/lidar"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Lidar overheat", "/sensors/lidar", clock_.now().nanoseconds(), lidar_config_); auto motor = storage_.get_fault({"OVERHEAT", "/powertrain/motor_left"}); ASSERT_TRUE(motor.has_value()); @@ -366,16 +390,16 @@ TEST_F(PerEntityStorageTest, LidarHealingWithThreshold1) { auto ts = clock_.now(); // Confirm fault - storage_.report_fault_event("LIDAR_FAIL", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Lidar failure", - "/sensors/lidar", ts.nanoseconds(), lidar_config_); + storage_.report_fault_event(FaultId{"LIDAR_FAIL", "/sensors/lidar"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Lidar failure", "/sensors/lidar", ts.nanoseconds(), lidar_config_); auto fault = storage_.get_fault({"LIDAR_FAIL", "/sensors/lidar"}); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // One PASSED event should heal (healing_threshold=1 for lidar) // Counter goes from -1 to 0 (no status change at 0) then +1 -> healed - storage_.report_fault_event("LIDAR_FAIL", fd::ReportKind::kPassed, 0, "", "/sensors/lidar", + storage_.report_fault_event(FaultId{"LIDAR_FAIL", "/sensors/lidar"}, fd::ReportKind::kPassed, 0, "", "/sensors/lidar", clock_.now().nanoseconds(), lidar_config_); - storage_.report_fault_event("LIDAR_FAIL", fd::ReportKind::kPassed, 0, "", "/sensors/lidar", + storage_.report_fault_event(FaultId{"LIDAR_FAIL", "/sensors/lidar"}, fd::ReportKind::kPassed, 0, "", "/sensors/lidar", clock_.now().nanoseconds(), lidar_config_); fault = storage_.get_fault({"LIDAR_FAIL", "/sensors/lidar"}); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); @@ -385,8 +409,8 @@ TEST_F(PerEntityStorageTest, LidarHealingWithThreshold1) { TEST_F(PerEntityStorageTest, BackwardCompatibleWithoutResolver) { // When no per-entity config is used, passing global config gives same behavior as before auto ts = clock_.now(); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - ts.nanoseconds(), global_); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", ts.nanoseconds(), global_); auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); diff --git a/src/ros2_medkit_fault_manager/test/test_fault_code_thresholds.cpp b/src/ros2_medkit_fault_manager/test/test_fault_code_thresholds.cpp index a031abc9c..7bb34d60e 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_code_thresholds.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_code_thresholds.cpp @@ -351,6 +351,26 @@ TEST_F(FaultCodeYamlLoadTest, BadValueReturnsEmpty) { EXPECT_TRUE(entries.empty()); } +// A per_source that is not the YAML boolean true or false would decide how a code is identified by +// accident, so it rejects the whole file like any other malformed value. yaml-cpp itself reads "yes" as true. +// @verifies REQ_INTEROP_107 +TEST_F(FaultCodeYamlLoadTest, PerSourceNonBoolRejectsTheFile) { + for (const char * bad : {"\"yes\"", "3"}) { + auto path = tmpdir_ / "bad_per_source.yaml"; + { + std::ofstream f(path); + f << "GOOD.CODE:\n" + << " per_source: true\n" + << "BAD.CODE:\n" + << " per_source: " << bad << "\n"; + } + + std::vector entries; + ASSERT_NO_THROW(entries = FaultCodeThresholdResolver::load_from_yaml(path.string(), test_log_sink())) << bad; + EXPECT_TRUE(entries.empty()) << "per_source: " << bad; + } +} + // yaml-cpp accepts a key twice. The first entry wins, the same as the resolver. // @verifies REQ_INTEROP_107 TEST_F(FaultCodeYamlLoadTest, RepeatedCodeKeepsTheFirstEntry) { @@ -449,28 +469,46 @@ class FaultCodeStorageTest : public ::testing::Test { codes_ = FaultCodeThresholdResolver({overheat}); } - void report(const std::string & fault_code, const std::string & source_id) { - storage_.report_fault_event(fault_code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", source_id, - clock_.now().nanoseconds(), + /// Reports into the shared record of the code unless @p owner names another record. + void report(const std::string & fault_code, const std::string & source_id, + const std::string & owner = ros2_medkit_fault_manager::kSharedOwner) { + storage_.report_fault_event(FaultId{fault_code, owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", + source_id, clock_.now().nanoseconds(), resolve_layered(entities_, codes_, source_id, fault_code, global_)); } }; // @verifies REQ_INTEROP_107 -TEST_F(FaultCodeStorageTest, TheCodeOverrideGovernsEachOwnersRecord) { +TEST_F(FaultCodeStorageTest, LidarNoLongerConfirmsTheMotorsCodeEarly) { + // Without the fault-code override, lidar's threshold of -1 would confirm this + // on the second event (the counter is shared). With it, both sources debounce + // at -3. + const FaultId shared{"OVERHEAT", ros2_medkit_fault_manager::kSharedOwner}; + report("OVERHEAT", "/powertrain/motor/left"); + EXPECT_EQ(storage_.get_fault(shared)->status, Fault::STATUS_PREFAILED); + + report("OVERHEAT", "/sensors/lidar/front"); + EXPECT_EQ(storage_.get_fault(shared)->status, Fault::STATUS_PREFAILED); + + report("OVERHEAT", "/sensors/lidar/front"); + EXPECT_EQ(storage_.get_fault(shared)->status, Fault::STATUS_CONFIRMED); +} + +// @verifies REQ_INTEROP_107 +TEST_F(FaultCodeStorageTest, TheCodeOverrideGovernsEachOwnersRecordWhenPerSource) { // Without the fault-code override, lidar's threshold of -1 would confirm its record on // its first event. With it, the record of each owner debounces at -3, on its own counter. const FaultId motor{"OVERHEAT", "/powertrain/motor/left"}; const FaultId lidar{"OVERHEAT", "/sensors/lidar/front"}; - report(motor.fault_code, motor.owner); + report(motor.fault_code, motor.owner, motor.owner); EXPECT_EQ(storage_.get_fault(motor)->status, Fault::STATUS_PREFAILED); - report(lidar.fault_code, lidar.owner); + report(lidar.fault_code, lidar.owner, lidar.owner); EXPECT_EQ(storage_.get_fault(lidar)->status, Fault::STATUS_PREFAILED); - report(lidar.fault_code, lidar.owner); + report(lidar.fault_code, lidar.owner, lidar.owner); EXPECT_EQ(storage_.get_fault(lidar)->status, Fault::STATUS_PREFAILED); - report(lidar.fault_code, lidar.owner); + report(lidar.fault_code, lidar.owner, lidar.owner); EXPECT_EQ(storage_.get_fault(lidar)->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(storage_.get_fault(motor)->status, Fault::STATUS_PREFAILED) << "the motor's own counter is at 1 of 3"; @@ -479,10 +517,10 @@ TEST_F(FaultCodeStorageTest, TheCodeOverrideGovernsEachOwnersRecord) { // @verifies REQ_INTEROP_107 TEST_F(FaultCodeStorageTest, ACodeWithoutAnOverrideKeepsEntityBehaviour) { report("MOTOR_STALL", "/powertrain/motor/left"); - EXPECT_EQ(storage_.get_fault({"MOTOR_STALL", "/powertrain/motor/left"})->status, Fault::STATUS_PREFAILED); + EXPECT_EQ(storage_.get_fault({"MOTOR_STALL", "*"})->status, Fault::STATUS_PREFAILED); report("LIDAR_BLOCKED", "/sensors/lidar/front"); - EXPECT_EQ(storage_.get_fault({"LIDAR_BLOCKED", "/sensors/lidar/front"})->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_.get_fault({"LIDAR_BLOCKED", "*"})->status, Fault::STATUS_CONFIRMED); } // The startup pass flips HEALED to CLEARED only for codes whose healing is off. @@ -492,10 +530,11 @@ TEST_F(FaultCodeStorageTest, ReclassifyKeepsCodesThatStillHeal) { healing.healing_enabled = true; healing.healing_threshold = 1; for (const std::string code : {"KEEPS.HEALING", "STOPPED.HEALING"}) { - storage_.report_fault_event(code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", "/src", + storage_.report_fault_event(FaultId{code, "/src"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", "/src", clock_.now().nanoseconds(), healing); for (int i = 0; i < 3 && storage_.get_fault({code, "/src"})->status != Fault::STATUS_HEALED; ++i) { - storage_.report_fault_event(code, fd::ReportKind::kPassed, 0, "", "/src", clock_.now().nanoseconds(), healing); + storage_.report_fault_event(FaultId{code, "/src"}, fd::ReportKind::kPassed, 0, "", "/src", + clock_.now().nanoseconds(), healing); } ASSERT_EQ(storage_.get_fault({code, "/src"})->status, Fault::STATUS_HEALED); } diff --git a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp index e76cdd547..e8b4096d2 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp @@ -60,6 +60,7 @@ using ros2_medkit_fault_manager::clamp_debounce_counter; using ros2_medkit_fault_manager::compute_debounce_status; using ros2_medkit_fault_manager::DebounceConfig; +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::FaultManagerNode; using ros2_medkit_fault_manager::InMemoryFaultStorage; using ros2_medkit_fault_manager::sanitize_debounce_config; @@ -100,9 +101,9 @@ TEST_F(FaultStorageTest, ReportNewFaultEvent) { rclcpp::Clock clock; auto timestamp = clock.now(); - bool is_new = storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "Motor temperature exceeded threshold", "/powertrain/motor", - timestamp.nanoseconds(), default_config()); + bool is_new = storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Motor temperature exceeded threshold", + "/powertrain/motor", timestamp.nanoseconds(), default_config()); EXPECT_TRUE(is_new); EXPECT_EQ(storage_.size(), 1u); @@ -113,8 +114,9 @@ TEST_F(FaultStorageTest, PassedEventForNonExistentFaultIgnored) { rclcpp::Clock clock; auto timestamp = clock.now(); - bool is_new = storage_.report_fault_event("NON_EXISTENT", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "Test", - "/node1", timestamp.nanoseconds(), default_config()); + bool is_new = + storage_.report_fault_event(FaultId{"NON_EXISTENT", "/node1"}, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, + "Test", "/node1", timestamp.nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_.size(), 0u); @@ -125,12 +127,13 @@ TEST_F(FaultStorageTest, ReportExistingFaultEventUpdates) { auto timestamp1 = clock.now(); auto timestamp2 = clock.now(); - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Initial report", - "/powertrain/motor1", timestamp1.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "Initial report", "/powertrain/motor1", timestamp1.nanoseconds(), + default_config()); - bool is_new = - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Second report", - "/powertrain/motor1", timestamp2.nanoseconds(), default_config()); + bool is_new = storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Second report", "/powertrain/motor1", + timestamp2.nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_.size(), 1u); @@ -156,12 +159,14 @@ TEST_F(FaultStorageTest, TwoSourcesReportingOneCodeAreTwoRecords) { const rclcpp::Time b_time(2000, 0, RCL_SYSTEM_TIME); // A reports twice and reaches its threshold. B reports once and does not. - EXPECT_TRUE(storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "from a", - "/powertrain/motor1", a_time.nanoseconds(), config)); - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "from a", - "/powertrain/motor1", a_time.nanoseconds(), config); - EXPECT_TRUE(storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "from b", - "/powertrain/motor2", b_time.nanoseconds(), config)) + EXPECT_TRUE(storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "from a", "/powertrain/motor1", a_time.nanoseconds(), + config)); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "from a", "/powertrain/motor1", a_time.nanoseconds(), config); + EXPECT_TRUE(storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor2"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "from b", "/powertrain/motor2", b_time.nanoseconds(), + config)) << "a source that owns no record yet gets a new one, not an update"; EXPECT_EQ(storage_.size(), 2u); @@ -188,25 +193,23 @@ TEST_F(FaultStorageTest, TwoSourcesReportingOneCodeAreTwoRecords) { } // The resolution step behind an unscoped service call. -TEST_F(FaultStorageTest, GetFaultsByCodeReturnsEveryOwnerAndGetFaultExactlyOne) { +TEST_F(FaultStorageTest, GetIdsByCodeReturnsEveryOwnerAndGetFaultExactlyOne) { rclcpp::Clock clock; - storage_.report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_b", - clock.now().nanoseconds(), default_config()); - storage_.report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_a", - clock.now().nanoseconds(), default_config()); - storage_.report_fault_event("OTHER", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_a", - clock.now().nanoseconds(), default_config()); - - const auto shared = storage_.get_faults_by_code("SHARED"); + storage_.report_fault_event(FaultId{"SHARED", "/owner_b"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_b", clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"SHARED", "/owner_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_a", clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"OTHER", "/owner_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_a", clock.now().nanoseconds(), default_config()); + + const auto shared = storage_.get_ids_by_code("SHARED"); ASSERT_EQ(shared.size(), 2u); // Ordered by owner, so an ambiguity message names them the same way every time. - ASSERT_EQ(shared[0].reporting_sources.size(), 1u); - ASSERT_EQ(shared[1].reporting_sources.size(), 1u); - EXPECT_EQ(shared[0].reporting_sources.front(), "/owner_a"); - EXPECT_EQ(shared[1].reporting_sources.front(), "/owner_b"); + EXPECT_EQ(shared[0].owner, "/owner_a"); + EXPECT_EQ(shared[1].owner, "/owner_b"); - EXPECT_EQ(storage_.get_faults_by_code("OTHER").size(), 1u); - EXPECT_TRUE(storage_.get_faults_by_code("NEVER_REPORTED").empty()); + EXPECT_EQ(storage_.get_ids_by_code("OTHER").size(), 1u); + EXPECT_TRUE(storage_.get_ids_by_code("NEVER_REPORTED").empty()); auto one = storage_.get_fault({"SHARED", "/owner_a"}); ASSERT_TRUE(one.has_value()); @@ -220,8 +223,8 @@ TEST_F(FaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCount) { // Simulate a level-triggered poller re-reporting the same still-true condition // every cycle (issue #11): occurrence_count must stay at 1, not grow per poll. for (int i = 0; i < 25; ++i) { - storage_.report_fault_event("TANK_OVERFILL", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "level = 95 > 80", - "/tank", clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"TANK_OVERFILL", "/tank"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "level = 95 > 80", "/tank", clock.now().nanoseconds(), default_config()); } auto fault = storage_.get_fault({"TANK_OVERFILL", "/tank"}); @@ -235,8 +238,8 @@ TEST_F(FaultStorageTest, ListFaultsDefaultReturnsConfirmedOnly) { auto timestamp = clock.now(); // With default threshold=-1, single report confirms immediately - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); // Default query should return the CONFIRMED fault auto faults = storage_.list_faults(false, 0, {}); @@ -253,8 +256,8 @@ TEST_F(FaultStorageTest, ListFaultsWithPrefailedStatus) { config.confirmation_threshold = -3; storage_.set_debounce_config(config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), config); // Query with PREFAILED status auto faults = storage_.list_faults(false, 0, {Fault::STATUS_PREFAILED}); @@ -268,10 +271,10 @@ TEST_F(FaultStorageTest, ListFaultsFilterBySeverity) { auto timestamp = clock.now(); // With default threshold=-1, faults are immediately CONFIRMED - storage_.report_fault_event("FAULT_INFO", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Info", "/node1", - timestamp.nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_ERROR", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Error", "/node1", - timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_INFO", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Info", + "/node1", timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_ERROR", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Error", + "/node1", timestamp.nanoseconds(), default_config()); // Filter by ERROR severity (query CONFIRMED since that's the default status now) auto faults = storage_.list_faults(true, Fault::SEVERITY_ERROR, {Fault::STATUS_CONFIRMED}); @@ -283,8 +286,8 @@ TEST_F(FaultStorageTest, ClearFault) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Test", "/node1", timestamp.nanoseconds(), default_config()); bool cleared = storage_.clear_fault({"MOTOR_OVERHEAT", "/node1"}); EXPECT_TRUE(cleared); @@ -303,8 +306,8 @@ TEST_F(FaultStorageTest, GetClearedFaults) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); storage_.clear_fault({"FAULT_1", "/node1"}); // Query cleared faults @@ -318,8 +321,8 @@ TEST_F(FaultStorageTest, InvalidStatusDefaultsToConfirmed) { auto timestamp = clock.now(); // With default threshold=-1, fault is immediately CONFIRMED - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); // Query with invalid status - defaults to CONFIRMED, which now matches our fault auto faults = storage_.list_faults(false, 0, {"INVALID_STATUS"}); @@ -364,10 +367,10 @@ TEST_F(FaultStorageTest, FaultStaysPrefailedAboveThreshold) { // Both reports from one source: the counter belongs to that source's record, so // debounce depth is what one reporter has said, not what several have said between them. - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -384,8 +387,8 @@ TEST_F(FaultStorageTest, FaultConfirmsAtThreshold) { storage_.set_debounce_config(config); for (int i = 0; i < 3; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); } auto fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -399,8 +402,8 @@ TEST_F(FaultStorageTest, ConfirmedFaultStaysConfirmed) { // One source reports 4 times: occurrence_count counts outages, not reports. for (int i = 0; i < 4; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); } auto fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -414,8 +417,8 @@ TEST_F(FaultStorageTest, MultiSourceConfirmsFault) { // Report the same fault code from 3 different sources for (const auto * source : {"/sensor1", "/sensor2", "/sensor3"}) { - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", source, - clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", source}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Test", source, clock.now().nanoseconds(), default_config()); } // Three records, each confirmed on its own report and naming only its own source. @@ -433,12 +436,12 @@ TEST_F(FaultStorageTest, SameSourceMultipleReportsConfirms) { rclcpp::Clock clock; // Same source reports 3 times - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -454,8 +457,8 @@ TEST_F(FaultStorageTest, ImmediateConfirmationWithThresholdZero) { storage_.set_debounce_config(config); // Single report should confirm immediately - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -467,8 +470,8 @@ TEST_F(FaultStorageTest, CriticalSeverityBypassesDebounce) { rclcpp::Clock clock; // CRITICAL severity should confirm immediately - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "Critical test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, + "Critical test", "/node1", clock.now().nanoseconds(), default_config()); auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -484,8 +487,8 @@ TEST_F(FaultStorageTest, CriticalBypassCanBeDisabled) { storage_.set_debounce_config(config); // CRITICAL should NOT confirm immediately when disabled - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "Critical test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, + "Critical test", "/node1", clock.now().nanoseconds(), config); auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -497,8 +500,9 @@ TEST_F(FaultStorageTest, ClearedFaultCanBeReactivated) { // Report to confirm (with default threshold=-1, single report confirms) auto first_ts = clock.now(); - bool is_new = storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Initial", - "/node1", first_ts.nanoseconds(), default_config()); + bool is_new = + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Initial", "/node1", first_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); auto fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -515,8 +519,8 @@ TEST_F(FaultStorageTest, ClearedFaultCanBeReactivated) { // The same owner reports again after a gap - reactivates its own record as a new cycle auto second_ts = rclcpp::Time(first_ts.nanoseconds() + 1'000'000'000LL); // +1s - is_new = storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Reactivated", - "/node1", second_ts.nanoseconds(), default_config()); + is_new = storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Reactivated", "/node1", second_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); // Should return true like a new fault fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -533,7 +537,7 @@ TEST_F(FaultStorageTest, ClearedFaultCanBeReactivated) { // rather than reactivating another owner's. It used to reactivate the shared row, // which is what made occurrence_count and reporting_sources grow across owners. const rclcpp::Time third_ts(first_ts.nanoseconds() + 2'000'000'000LL, RCL_SYSTEM_TIME); - EXPECT_TRUE(storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + EXPECT_TRUE(storage_.report_fault_event(FaultId{"FAULT_1", "/node2"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "From the other source", "/node2", third_ts.nanoseconds(), default_config())); auto other = storage_.get_fault({"FAULT_1", "/node2"}); ASSERT_TRUE(other.has_value()); @@ -547,14 +551,14 @@ TEST_F(FaultStorageTest, PassedEventForClearedFaultIgnored) { rclcpp::Clock clock; // Report and confirm - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); // Clear the fault storage_.clear_fault({"FAULT_1", "/node1"}); // PASSED event should be ignored for CLEARED fault - bool result = storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", + bool result = storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), default_config()); EXPECT_FALSE(result); @@ -573,8 +577,8 @@ TEST_F(FaultStorageTest, ClearedFaultReactivationRestartsDebounce) { // One source reports 3 times to confirm its own record for (int i = 0; i < 3; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); } auto fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -585,8 +589,8 @@ TEST_F(FaultStorageTest, ClearedFaultReactivationRestartsDebounce) { storage_.clear_fault({"FAULT_1", "/node1"}); // Reactivate - should start in PREFAILED with counter=-1 - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -594,8 +598,8 @@ TEST_F(FaultStorageTest, ClearedFaultReactivationRestartsDebounce) { // Report 2 more times to re-confirm for (int i = 0; i < 2; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); } fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -613,13 +617,14 @@ TEST_F(FaultStorageTest, PassedEventIncrementsCounter) { // Report 2 FAILED events from one source (its counter = -2, PREFAILED) for (int i = 0; i < 2; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); } // Report 3 PASSED events (counter = -2 + 3 = +1) for (int i = 0; i < 3; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } auto fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -637,15 +642,15 @@ TEST_F(FaultStorageTest, PassedFromOneSourceMovesOnlyThatSourcesRecord) { config.healing_enabled = true; config.healing_threshold = 1; - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node_a", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node_b", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node_a", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node_b"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node_b", clock.now().nanoseconds(), config); // Two PASSED from A only: -1 -> 0 -> +1, which is A's healing threshold. for (int i = 0; i < 2; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node_a", clock.now().nanoseconds(), - config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node_a"}, fd::ReportKind::kPassed, 0, "", "/node_a", + clock.now().nanoseconds(), config); } auto a = storage_.get_fault({"FAULT_1", "/node_a"}); @@ -660,11 +665,11 @@ TEST_F(FaultStorageTest, PassedFromOneSourceMovesOnlyThatSourcesRecord) { // A PASSED from a source that owns no record for this code has nothing to heal. TEST_F(FaultStorageTest, PassedFromASourceWithNoRecordIsIgnored) { rclcpp::Clock clock; - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node_a", - clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node_a", clock.now().nanoseconds(), default_config()); - const bool is_new = storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node_b", - clock.now().nanoseconds(), default_config()); + const bool is_new = storage_.report_fault_event(FaultId{"FAULT_1", "/node_b"}, fd::ReportKind::kPassed, 0, "", + "/node_b", clock.now().nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_.size(), 1u) << "a PASSED must never create a record"; @@ -677,10 +682,10 @@ TEST_F(FaultStorageTest, PassedFromASourceWithNoRecordIsIgnored) { // Severity is per record: a CRITICAL from one source does not escalate another's. TEST_F(FaultStorageTest, CriticalFromOneSourceDoesNotEscalateAnothersRecord) { rclcpp::Clock clock; - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "warn", "/node_b", - clock.now().nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "critical", "/node_a", - clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node_b"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "warn", + "/node_b", clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, + "critical", "/node_a", clock.now().nanoseconds(), default_config()); auto a = storage_.get_fault({"FAULT_1", "/node_a"}); auto b = storage_.get_fault({"FAULT_1", "/node_b"}); @@ -698,10 +703,10 @@ TEST_F(FaultStorageTest, PassedEventDoesNotAdvanceLastOccurred) { const rclcpp::Time failed_at(1000, 0, RCL_SYSTEM_TIME); const rclcpp::Time passed_at(9000, 0, RCL_SYSTEM_TIME); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - failed_at.nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", passed_at.nanoseconds(), - default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", failed_at.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + passed_at.nanoseconds(), default_config()); auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -715,14 +720,14 @@ TEST_F(FaultStorageTest, HealingDisabledByDefault) { rclcpp::Clock clock; // Report 1 FAILED event -> confirmed (threshold -1) - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); // With healing disabled, a heal heartbeat must not downgrade a confirmed fault. // The counter clamps at healing_threshold and the status latches CONFIRMED. for (int i = 0; i < 10; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), default_config()); } auto fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -738,12 +743,13 @@ TEST_F(FaultStorageTest, HealingWhenEnabled) { storage_.set_debounce_config(config); // Report 1 FAILED event (counter = -1) - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); // Report 4 PASSED events (counter = -1 + 4 = +3, reaches healing threshold) for (int i = 0; i < 4; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } auto fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -759,8 +765,8 @@ TEST_F(FaultStorageTest, TimeBasedConfirmationDisabledByDefault) { config.confirmation_threshold = -3; storage_.set_debounce_config(config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); // Advance time and check - should not auto-confirm (auto_confirm_after_sec = 0) auto future_time = rclcpp::Time(clock.now().nanoseconds() + static_cast(20e9)); @@ -781,8 +787,8 @@ TEST_F(FaultStorageTest, TimeBasedConfirmationWhenEnabled) { storage_.set_debounce_config(config); auto now = clock.now(); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - now.nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", now.nanoseconds(), config); // Check before timeout - should not confirm auto before_timeout = rclcpp::Time(now.nanoseconds() + static_cast(5e9)); @@ -809,8 +815,9 @@ TEST_F(FaultStorageTest, ConfirmedFaultCanHealWithPassedEvents) { // Report 3 FAILED events to confirm (counter = -3) for (int i = 0; i < 3; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", - "/node" + std::to_string(i), clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node" + std::to_string(i)}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node" + std::to_string(i), clock.now().nanoseconds(), + config); } auto fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -819,7 +826,8 @@ TEST_F(FaultStorageTest, ConfirmedFaultCanHealWithPassedEvents) { // Report 6 PASSED events (counter = -3 + 6 = +3, reaches healing threshold) for (int i = 0; i < 6; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -835,12 +843,13 @@ TEST_F(FaultStorageTest, HealedFaultCanRecurWithFailedEvents) { storage_.set_debounce_config(config); // Report 1 FAILED event (counter = -1) - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); // Report 4 PASSED events to heal (counter = -1 + 4 = +3) for (int i = 0; i < 4; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } auto fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -849,8 +858,8 @@ TEST_F(FaultStorageTest, HealedFaultCanRecurWithFailedEvents) { // Report 3 FAILED events - fault should recur and confirm (counter = +3 - 3 = 0, then -3) for (int i = 0; i < 6; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Recurrence", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Recurrence", "/node1", clock.now().nanoseconds(), config); } fault = storage_.get_fault({"FAULT_1", "/node1"}); @@ -868,26 +877,27 @@ TEST_F(FaultStorageTest, HeartbeatHealClampedAndStatusLatched) { config.healing_enabled = true; config.healing_threshold = 3; - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); ASSERT_EQ(storage_.get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); // Long heal heartbeat: counter clamps at healing_threshold, then heals. for (int i = 0; i < 100; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } EXPECT_EQ(storage_.get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_HEALED); // Re-confirmation is bounded and the healed status latches until the counter // walks all the way back down to the confirmation threshold. for (int i = 0; i < 3; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); EXPECT_EQ(storage_.get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); } - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); EXPECT_EQ(storage_.get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); } @@ -900,15 +910,16 @@ TEST_F(FaultStorageTest, ConfirmedLatchSurvivesFailedAtPositiveCounter) { config.healing_threshold = 3; config.critical_immediate_confirm = true; - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), - config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + clock.now().nanoseconds(), config); EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); } @@ -920,18 +931,20 @@ TEST_F(FaultStorageTest, HealedLatchSurvivesPassedAtNegativeCounter) { config.healing_enabled = true; config.healing_threshold = 3; - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), - config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); for (int i = 0; i < 5; ++i) { // +3 -> -2, latched HEALED - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); } - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } @@ -943,23 +956,26 @@ TEST_F(FaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePassed) { config.healing_threshold = 3; config.critical_immediate_confirm = true; - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { // -1 -> +3 -> HEALED - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); // CRITICAL confirms immediately from HEALED; counter must not stay at +2 where one PASSED heals it. - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { // -1 -> +2, still CONFIRMED - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); } - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } @@ -969,10 +985,11 @@ TEST_F(FaultStorageTest, ReclassifyHealedAsCleared) { config.healing_enabled = true; config.healing_threshold = 3; - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), - config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); @@ -1020,10 +1037,11 @@ TEST_F(FaultStorageTest, HealingThresholdZeroHealsOnSinglePassed) { config.healing_enabled = true; config.healing_threshold = 0; - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), - config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + clock.now().nanoseconds(), config); ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } @@ -1043,12 +1061,12 @@ TEST_F(FaultStorageTest, GetAllFaultsReturnsAllFaults) { auto timestamp = clock.now(); // Add faults with different statuses - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test 1", "/node1", - timestamp.nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_2", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Test 2", "/node2", - timestamp.nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_3", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Test 3", "/node3", - timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test 1", + "/node1", timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_2", "/node2"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Test 2", + "/node2", timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_3", "/node3"}, fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Test 3", + "/node3", timestamp.nanoseconds(), default_config()); auto all_faults = storage_.get_all_faults(); EXPECT_EQ(all_faults.size(), 3u); @@ -1165,8 +1183,9 @@ static void drive_near_misses(ros2_medkit_fault_manager::FaultStorage & storage, constexpr int64_t kBaseNs = 1700000000000000000LL; for (int i = 0; i < count; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", kBaseNs + static_cast(i) * 1000000LL, config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + kBaseNs + static_cast(i) * 1000000LL, config); } } @@ -1297,6 +1316,25 @@ class FaultEventPublishingTest : public ::testing::Test { test_node_.reset(); fault_manager_.reset(); received_events_.clear(); + if (!thresholds_path_.empty()) { + std::error_code ec; + std::filesystem::remove(thresholds_path_, ec); + } + } + + /// The override that makes every code in @p codes per_source. The file is written now, because the node + /// reads it at construction, and removed in TearDown. + rclcpp::Parameter per_source_codes(const std::vector & codes) { + std::random_device rd; + std::mt19937_64 gen(rd()); + std::uniform_int_distribution dist; + thresholds_path_ = + std::filesystem::temp_directory_path() / ("test_per_source_" + std::to_string(dist(gen)) + ".yaml"); + std::ofstream out(thresholds_path_); + for (const auto & code : codes) { + out << code << ":\n per_source: true\n"; + } + return rclcpp::Parameter("fault_thresholds.config_file", thresholds_path_.string()); } void spin_for(std::chrono::milliseconds duration) { @@ -1412,6 +1450,7 @@ class FaultEventPublishingTest : public ::testing::Test { } std::string ns_; + std::filesystem::path thresholds_path_; std::shared_ptr fault_manager_; std::shared_ptr test_node_; rclcpp::Subscription::SharedPtr event_subscription_; @@ -1464,9 +1503,19 @@ TEST_F(FaultEventPublishingTest, UpdateExistingFaultPublishesUpdatedEvent) { EXPECT_EQ(received_events_[0].fault.reporting_sources.front(), "/test_node1"); } -// A second source reporting a code someone else already reported opens its OWN record, -// so the event is a confirmation of a new record rather than an update of a shared one. -TEST_F(FaultEventPublishingTest, ASecondSourceConfirmsItsOwnRecord) { +/// A node where the codes the tests below report are configured per_source, so each source gets its own record. +class PerSourceNodeTest : public FaultEventPublishingTest { + protected: + std::vector fault_manager_overrides() override { + auto overrides = FaultEventPublishingTest::fault_manager_overrides(); + overrides.push_back(per_source_codes({"TWO_OWNERS", "SHARED_CODE", "ONLY_ONE"})); + return overrides; + } +}; + +// With a per_source code, a second source reporting a code someone else already reported opens its OWN +// record, so the event is a confirmation of a new record rather than an update of a shared one. +TEST_F(PerSourceNodeTest, ASecondSourceConfirmsItsOwnRecord) { ASSERT_TRUE(call_report_fault("TWO_OWNERS", Fault::SEVERITY_WARN, "/test_node1")); ASSERT_TRUE(spin_until([this]() { return !received_events_.empty(); @@ -1646,6 +1695,10 @@ class SeverityConsistencyNodeTest : public FaultEventPublishingTest, public ::te std::uniform_int_distribution dist; dir_ = std::filesystem::temp_directory_path() / ("test_severity_node_" + std::to_string(dist(gen))); std::filesystem::create_directories(dir_); + // The codes two sources report each get a record per source, which is what the owner-scoped + // assertions below are about. + std::ofstream(dir_ / "fault_thresholds.yaml") + << "MISMATCH:\n per_source: true\nENTITY_MIXED:\n per_source: true\n"; } ~SeverityConsistencyNodeTest() override { @@ -1665,6 +1718,7 @@ class SeverityConsistencyNodeTest : public FaultEventPublishingTest, public ::te rclcpp::Parameter("confirmation_threshold", -3), rclcpp::Parameter("audit_log.enabled", true), rclcpp::Parameter("audit_log.database_path", audit_path()), + rclcpp::Parameter("fault_thresholds.config_file", (dir_ / "fault_thresholds.yaml").string()), }; } @@ -1777,7 +1831,7 @@ TEST_P(SeverityConsistencyNodeTest, AuditRowsCarryTheStoredSeverity) { // The near-miss rows of the same reports keep what was reported. std::vector near_miss_severities; - for (const auto & record : fault_manager_->get_storage().get_near_misses({"AUDIT_REACT", "/a"})) { + for (const auto & record : fault_manager_->get_storage().get_near_misses({"AUDIT_REACT", "*"})) { near_miss_severities.push_back(record.severity); } EXPECT_EQ(near_miss_severities, (std::vector{error, error, warn, warn})); @@ -1796,6 +1850,7 @@ class SeverityEntityThresholdNodeTest : public SeverityConsistencyNodeTest { rclcpp::Parameter("database_path", (dir_ / "faults.db").string()), rclcpp::Parameter("confirmation_threshold", -1), rclcpp::Parameter("entity_thresholds.config_file", (dir_ / "thresholds.yaml").string()), + rclcpp::Parameter("fault_thresholds.config_file", (dir_ / "fault_thresholds.yaml").string()), }; } }; @@ -2038,7 +2093,7 @@ TEST_F(FaultEventPublishingTest, GetFaultReturnsExtendedDataRecords) { // --- Record resolution on the single-record services --- -TEST_F(FaultEventPublishingTest, ScopedClearClearsOneRecordAndPublishesItsOwner) { +TEST_F(PerSourceNodeTest, ScopedClearClearsOneRecordAndPublishesItsOwner) { ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_a")); ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_b")); ASSERT_TRUE(spin_until([this]() { @@ -2063,7 +2118,7 @@ TEST_F(FaultEventPublishingTest, ScopedClearClearsOneRecordAndPublishesItsOwner) EXPECT_EQ(other->fault.status, Fault::STATUS_CONFIRMED) << "clearing one owner must not clear another"; } -TEST_F(FaultEventPublishingTest, UnscopedClearOfTwoRecordsIsRefusedAndClearsNothing) { +TEST_F(PerSourceNodeTest, UnscopedClearOfTwoRecordsIsRefusedAndClearsNothing) { ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_a")); ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_b")); ASSERT_TRUE(spin_until([this]() { @@ -2093,7 +2148,7 @@ TEST_F(FaultEventPublishingTest, UnscopedClearOfTwoRecordsIsRefusedAndClearsNoth } } -TEST_F(FaultEventPublishingTest, UnscopedClearOfOneRecordSucceeds) { +TEST_F(PerSourceNodeTest, UnscopedClearOfOneRecordSucceeds) { ASSERT_TRUE(call_report_fault("ONLY_ONE", Fault::SEVERITY_ERROR, "/owner_a")); ASSERT_TRUE(spin_until([this]() { return !received_events_.empty(); @@ -2107,7 +2162,7 @@ TEST_F(FaultEventPublishingTest, UnscopedClearOfOneRecordSucceeds) { EXPECT_EQ(response->fault.status, Fault::STATUS_CLEARED); } -TEST_F(FaultEventPublishingTest, GetFaultResolvesTheSameWayAsClear) { +TEST_F(PerSourceNodeTest, GetFaultResolvesTheSameWayAsClear) { ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_WARN, "/owner_a")); ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_CRITICAL, "/owner_b")); ASSERT_TRUE(call_report_fault("ONLY_ONE", Fault::SEVERITY_ERROR, "/owner_a")); @@ -2172,6 +2227,7 @@ class MutedFaultListingTest : public FaultEventPublishingTest { auto overrides = FaultEventPublishingTest::fault_manager_overrides(); overrides.emplace_back("correlation.config_file", config_path_.string()); + overrides.push_back(per_source_codes({"ROOT_CAUSE", "SYMPTOM_SPEED"})); return overrides; } @@ -2281,9 +2337,6 @@ class LegacyOwnedRecordTest : public FaultEventPublishingTest { std::uniform_int_distribution dist; db_path_ = std::filesystem::temp_directory_path() / ("test_legacy_owner_" + std::to_string(dist(gen)) + ".db"); - // One legacy row whose reporting_sources records nothing readable. The migration - // has to give it a name, because a record nobody can address is a record an - // operator cannot clear. sqlite3 * raw = nullptr; EXPECT_EQ(sqlite3_open(db_path_.string().c_str(), &raw), SQLITE_OK); EXPECT_EQ(sqlite3_exec(raw, @@ -2295,7 +2348,7 @@ class LegacyOwnedRecordTest : public FaultEventPublishingTest { "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " "confirmed_at_ns INTEGER NOT NULL DEFAULT 0);" "INSERT INTO faults VALUES ('SHARED_CODE', 2, 'from an older release', 1, 1, 1, " - "'CONFIRMED', '', -1, 1, 0, 0);", + "'CONFIRMED', '[\"/old_source\"]', -1, 1, 0, 0);", nullptr, nullptr, nullptr), SQLITE_OK); sqlite3_close(raw); @@ -2317,43 +2370,27 @@ class LegacyOwnedRecordTest : public FaultEventPublishingTest { std::filesystem::path db_path_; }; -// The migrated record is addressable like any other. A live source reporting the same -// code creates a second record beside it, and from there the ordinary rules apply: the -// unscoped call is refused and names both owners, and a scoped call reaches exactly one. -// An empty owner would fail all of this, because no source_id could ever name it. -TEST_F(LegacyOwnedRecordTest, AMigratedRecordIsAddressableBesideALiveSource) { - const std::string legacy = ros2_medkit_fault_manager::kLegacyOwner; - +// The migrated record is the shared record of its code, so it behaves as a record always did: +// a live source reporting the code joins it, any reporting source and the unscoped call reach it, +// and a clear ends it. +TEST_F(LegacyOwnedRecordTest, AMigratedRecordIsTheSharedRecordOfItsCode) { ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_CRITICAL, "/live_source")); - ASSERT_TRUE(spin_until([this]() { - return !received_events_.empty(); - })); - // Scoped read reaches the migrated record, not the live one. - auto migrated = call_get_fault("SHARED_CODE", legacy); - ASSERT_TRUE(migrated.has_value()); - ASSERT_TRUE(migrated->success) << migrated->error_message; - ASSERT_EQ(migrated->fault.reporting_sources.size(), 1u); - EXPECT_EQ(migrated->fault.reporting_sources.front(), legacy); - EXPECT_EQ(migrated->fault.description, "from an older release"); - - // Unscoped is ambiguous, and the refusal names the migrated owner rather than - // leaving a blank where an owner should be. - const auto refused = clear_fault_response("SHARED_CODE"); - ASSERT_TRUE(refused.has_value()); - EXPECT_FALSE(refused->success); - EXPECT_EQ(refused->message.rfind("ambiguous:", 0), 0u) << "got: " << refused->message; - EXPECT_NE(refused->message.find(legacy), std::string::npos) << "got: " << refused->message; - EXPECT_NE(refused->message.find("/live_source"), std::string::npos) << "got: " << refused->message; - EXPECT_EQ(refused->message.find("()"), std::string::npos) - << "an empty owner would show as a blank entry: " << refused->message; - - // Scoped clear takes the migrated record alone. - ASSERT_TRUE(call_clear_fault("SHARED_CODE", legacy)); - auto live = call_get_fault("SHARED_CODE", "/live_source"); - ASSERT_TRUE(live.has_value()); - ASSERT_TRUE(live->success) << live->error_message; - EXPECT_EQ(live->fault.status, Fault::STATUS_CONFIRMED) << "the live source's record must be untouched"; + const std::vector both = {"/live_source", "/old_source"}; + for (const std::string & source_id : + {std::string("*"), std::string("/old_source"), std::string("/live_source"), std::string()}) { + auto record = call_get_fault("SHARED_CODE", source_id); + ASSERT_TRUE(record.has_value()); + ASSERT_TRUE(record->success) << "source_id '" << source_id << "': " << record->error_message; + EXPECT_EQ(record->fault.reporting_sources, both) << "source_id '" << source_id << "'"; + EXPECT_EQ(record->fault.occurrence_count, 1u) << "the live source joined the record, it did not open a new one"; + } + + ASSERT_TRUE(call_clear_fault("SHARED_CODE", "")); + auto cleared = call_get_fault("SHARED_CODE", ""); + ASSERT_TRUE(cleared.has_value()); + ASSERT_TRUE(cleared->success) << cleared->error_message; + EXPECT_EQ(cleared->fault.status, Fault::STATUS_CLEARED); } /// GetRosbag over the wire, with the links written straight into the node's store: the @@ -2361,6 +2398,12 @@ TEST_F(LegacyOwnedRecordTest, AMigratedRecordIsAddressableBesideALiveSource) { /// is under test is how the handler reports the links, not how they got there. class GetRosbagAttachedCodesTest : public FaultEventPublishingTest { protected: + std::vector fault_manager_overrides() override { + auto overrides = FaultEventPublishingTest::fault_manager_overrides(); + overrides.push_back(per_source_codes({"BURST_CODE", "BURST_FIRST", "BURST_SECOND"})); + return overrides; + } + void SetUp() override { FaultEventPublishingTest::SetUp(); get_rosbag_client_ = test_node_->create_client(ns_ + "/fault_manager/get_rosbag"); @@ -2519,7 +2562,7 @@ TEST_F(FreezeFrameRetentionTest, GetFaultServesRetainedFreezeFrameAfterClear) { // Capture runs asynchronously on the pool; wait for a non-empty freeze-frame. ASSERT_TRUE(spin_until( [this]() { - auto frame = fault_manager_->get_storage().get_freeze_frame({"FF_FAULT", "/test_node"}); + auto frame = fault_manager_->get_storage().get_freeze_frame({"FF_FAULT", "*"}); return frame.has_value() && frame->data != "{}"; }, std::chrono::milliseconds(10000))); @@ -2586,7 +2629,7 @@ struct ReportedBag { ros2_medkit_fault_manager::RosbagFileInfo row_for(const std::string & fault_code) const { ros2_medkit_fault_manager::RosbagFileInfo info; info.fault_code = fault_code; - info.owner = "/test_node"; + info.owner = "*"; info.file_path = dir.string(); info.recording_id = ros2_medkit_fault_manager::rosbag_recording_id(info.file_path); info.format = "sqlite3"; @@ -2634,7 +2677,7 @@ TEST_F(FaultEventPublishingTest, GetFaultReportsARecordingsServedBytesNotItsFoot EXPECT_NE(rosbag_snapshot->size_bytes, bag.footprint) << "the directory total is the quota's figure, not the API's"; // The row itself is untouched: the quota still sees the whole recording. - auto row = fault_manager_->get_storage().get_rosbag_file({"SERVED_BYTES_FAULT", "/test_node"}); + auto row = fault_manager_->get_storage().get_rosbag_file({"SERVED_BYTES_FAULT", "*"}); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->size_bytes, bag.footprint) << "reporting must not have rewritten what the quota counts"; } @@ -2771,13 +2814,13 @@ TEST_F(UnlimitedSnapshotRetentionTest, RetentionAppliesWithNoPerFaultCap) { ASSERT_TRUE(spin_until( [this]() { - return !fault_manager_->get_storage().get_snapshots({"UNCAPPED_FAULT", "/test_node"}).empty(); + return !fault_manager_->get_storage().get_snapshots({"UNCAPPED_FAULT", "*"}).empty(); }, std::chrono::milliseconds(10000))); ASSERT_TRUE(call_clear_fault("UNCAPPED_FAULT")); - EXPECT_FALSE(fault_manager_->get_storage().get_snapshots({"UNCAPPED_FAULT", "/test_node"}).empty()) + EXPECT_FALSE(fault_manager_->get_storage().get_snapshots({"UNCAPPED_FAULT", "*"}).empty()) << "acknowledgement deleted readings the operator asked to keep"; } @@ -2820,8 +2863,9 @@ TEST(MatchesEntityTest, EmptySources) { TEST(InMemorySnapshotRetentionTest, ClearKeepsSnapshotsWhenEvidenceIsRetained) { InMemoryFaultStorage storage; rclcpp::Clock clock; - storage.report_fault_event("KEEP", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "keep", - "/n", clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage.report_fault_event(FaultId{"KEEP", "/n"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "keep", "/n", clock.now().nanoseconds(), + ros2_medkit_fault_manager::DebounceConfig{}); storage.set_retain_snapshots_on_clear(true); ros2_medkit_fault_manager::SnapshotData row; @@ -2844,8 +2888,9 @@ TEST(InMemorySnapshotRetentionTest, ClearKeepsSnapshotsWhenEvidenceIsRetained) { TEST(InMemorySnapshotRetentionTest, ClearStillDropsSnapshotsByDefault) { InMemoryFaultStorage storage; rclcpp::Clock clock; - storage.report_fault_event("DROP", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "drop", - "/n", clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage.report_fault_event(FaultId{"DROP", "/n"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "drop", "/n", clock.now().nanoseconds(), + ros2_medkit_fault_manager::DebounceConfig{}); ros2_medkit_fault_manager::SnapshotData row; row.fault_code = "DROP"; @@ -2960,8 +3005,8 @@ TEST(InMemoryFreezeFrameTest, SurvivesClearFault) { InMemoryFaultStorage storage; rclcpp::Clock clock; - storage.report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", - "/plc_node", clock.now().nanoseconds(), DebounceConfig{}); + storage.report_fault_event(FaultId{"PLC_PRESSURE_HIGH", "/plc_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Pressure high", "/plc_node", clock.now().nanoseconds(), DebounceConfig{}); ros2_medkit_fault_manager::SnapshotData snap; snap.fault_code = "PLC_PRESSURE_HIGH"; @@ -3284,13 +3329,11 @@ TEST_F(FaultAuditIntegrationTest, TransitionsAppendVerifiableChain) { EXPECT_EQ(records[2].event.transition, ros2_medkit_fault_manager::kTransitionConfirmed); EXPECT_EQ(records[3].event.transition, ros2_medkit_fault_manager::kTransitionCleared); EXPECT_EQ(records[1].event.fault_code, "AUDIT_FAULT"); - // Every transition row names the record's owner, the clear included: the - // clear_service and auto_heal literals said which mechanism moved the row, not - // whose fault moved, and with two owners per code that is the question the log - // has to answer. + // The rows a report drove name the reporting source. The clear row names the owner of the record, which + // for a code shared by every source is the shared owner. EXPECT_EQ(records[1].event.source_id, "/plc/pump"); EXPECT_EQ(records[2].event.source_id, "/plc/pump"); - EXPECT_EQ(records[3].event.source_id, "/plc/pump"); + EXPECT_EQ(records[3].event.source_id, ros2_medkit_fault_manager::kSharedOwner); auto result = audit.verify(); EXPECT_TRUE(result.ok) << result.error; @@ -3300,8 +3343,7 @@ TEST_F(FaultAuditIntegrationTest, TransitionsAppendVerifiableChain) { // Completeness: an auto-healed record must record its END. One FAILED confirms the // record (occurred + confirmed), two PASSED drive the debounce counter to the // healing threshold, which must append a distinct "healed" row. That row names the -// record's OWNER: the transition column already says the heal was automatic, so the -// source is free to answer the question the transition cannot, whose record ended. +// record's owner, which for a code shared by every source is the shared owner. // The full occurred -> confirmed -> healed chain must verify. TEST_F(FaultAuditIntegrationTest, AutoHealAppendsHealedRow) { auto send_report = [&](uint8_t event_type) { @@ -3341,7 +3383,7 @@ TEST_F(FaultAuditIntegrationTest, AutoHealAppendsHealedRow) { EXPECT_EQ(transitions[0], ros2_medkit_fault_manager::kTransitionOccurred); EXPECT_EQ(transitions[1], ros2_medkit_fault_manager::kTransitionConfirmed); EXPECT_EQ(transitions[2], ros2_medkit_fault_manager::kTransitionHealed); - EXPECT_EQ(heal_source, "/robot/sensor") << "every audit row names the record's owner, not the mechanism"; + EXPECT_EQ(heal_source, ros2_medkit_fault_manager::kSharedOwner); auto result = audit.verify(); EXPECT_TRUE(result.ok) << result.error; @@ -3460,8 +3502,9 @@ TEST(FaultAuditTimerTest, TimerConfirmationAppendsConfirmedAuditRow) { config.confirmation_threshold = -3; config.auto_confirm_after_sec = 0.2; rclcpp::Clock clock(RCL_SYSTEM_TIME); - node->get_storage_for_test().report_fault_event("AUTO_CONF_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "stuck", "/robot/src", clock.now().nanoseconds(), config); + node->get_storage_for_test().report_fault_event(FaultId{"AUTO_CONF_1", "/robot/src"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "stuck", "/robot/src", + clock.now().nanoseconds(), config); ASSERT_EQ(node->get_storage().get_fault({"AUTO_CONF_1", "/robot/src"})->status, Fault::STATUS_PREFAILED); // Spin until a confirmed audit row appears or the budget expires (the wall @@ -3497,11 +3540,11 @@ void seed_healed_fault(const std::string & db_path, const std::string & fault_co config.healing_enabled = true; config.healing_threshold = 3; rclcpp::Clock clock; - seed.report_fault_event(fault_code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "stale", "/robot/src", - clock.now().nanoseconds(), config); + seed.report_fault_event(FaultId{fault_code, "/robot/src"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "stale", + "/robot/src", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - seed.report_fault_event(fault_code, fd::ReportKind::kPassed, 0, "", "/robot/src", clock.now().nanoseconds(), - config); + seed.report_fault_event(FaultId{fault_code, "/robot/src"}, fd::ReportKind::kPassed, 0, "", "/robot/src", + clock.now().nanoseconds(), config); } ASSERT_EQ(seed.get_fault({fault_code, "/robot/src"})->status, Fault::STATUS_HEALED); } @@ -3700,8 +3743,8 @@ TEST(InMemorySnapshotRetentionTest, RetainedOnClearWhenConfigured) { InMemoryFaultStorage storage; storage.set_retain_snapshots_on_clear(true); - storage.report_fault_event("SNAPSHOT_RETAIN_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "fault with evidence", "/test_node", 1000, default_config()); + storage.report_fault_event(FaultId{"SNAPSHOT_RETAIN_TEST", "/test_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "fault with evidence", "/test_node", 1000, default_config()); store_two_snapshots(storage, "SNAPSHOT_RETAIN_TEST"); ASSERT_EQ(storage.get_snapshots({"SNAPSHOT_RETAIN_TEST", "/test_node"}).size(), 2u); @@ -3713,8 +3756,8 @@ TEST(InMemorySnapshotRetentionTest, RetainedOnClearWhenConfigured) { TEST(InMemorySnapshotRetentionTest, DeletedOnClearByDefault) { InMemoryFaultStorage storage; - storage.report_fault_event("SNAPSHOT_DROP_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "fault with evidence", "/test_node", 1000, default_config()); + storage.report_fault_event(FaultId{"SNAPSHOT_DROP_TEST", "/test_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "fault with evidence", "/test_node", 1000, default_config()); store_two_snapshots(storage, "SNAPSHOT_DROP_TEST"); ASSERT_TRUE(storage.clear_fault({"SNAPSHOT_DROP_TEST", "/test_node"})); @@ -3730,12 +3773,12 @@ TEST(InMemorySnapshotRetentionTest, HealedReclassificationDropsSnapshotsByDefaul config.healing_enabled = true; config.healing_threshold = 1; - storage.report_fault_event("SNAPSHOT_DROP_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "fault with evidence", "/test_node", 1000, config); + storage.report_fault_event(FaultId{"SNAPSHOT_DROP_TEST", "/test_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "fault with evidence", "/test_node", 1000, config); store_two_snapshots(storage, "SNAPSHOT_DROP_TEST"); for (int i = 1; i <= 2; ++i) { - storage.report_fault_event("SNAPSHOT_DROP_TEST", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "/test_node", - 1000 + i, config); + storage.report_fault_event(FaultId{"SNAPSHOT_DROP_TEST", "/test_node"}, fd::ReportKind::kPassed, + Fault::SEVERITY_ERROR, "", "/test_node", 1000 + i, config); } auto healed = storage.get_fault({"SNAPSHOT_DROP_TEST", "/test_node"}); ASSERT_TRUE(healed.has_value()); @@ -3753,12 +3796,12 @@ TEST(InMemorySnapshotRetentionTest, HealedReclassificationKeepsSnapshotsWhenConf config.healing_enabled = true; config.healing_threshold = 1; - storage.report_fault_event("SNAPSHOT_RETAIN_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "fault with evidence", "/test_node", 1000, config); + storage.report_fault_event(FaultId{"SNAPSHOT_RETAIN_TEST", "/test_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "fault with evidence", "/test_node", 1000, config); store_two_snapshots(storage, "SNAPSHOT_RETAIN_TEST"); for (int i = 1; i <= 2; ++i) { - storage.report_fault_event("SNAPSHOT_RETAIN_TEST", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "/test_node", - 1000 + i, config); + storage.report_fault_event(FaultId{"SNAPSHOT_RETAIN_TEST", "/test_node"}, fd::ReportKind::kPassed, + Fault::SEVERITY_ERROR, "", "/test_node", 1000 + i, config); } ASSERT_EQ(storage.reclassify_healed_as_cleared().size(), 1u); @@ -3771,16 +3814,16 @@ TEST(InMemorySnapshotRetentionTest, HealedReclassificationLeavesOtherFaultsSnaps healing.healing_enabled = true; healing.healing_threshold = 1; - storage.report_fault_event("HEALED_FAULT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "/test_node", - 1000, healing); + storage.report_fault_event(FaultId{"HEALED_FAULT", "/test_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "fault", "/test_node", 1000, healing); store_two_snapshots(storage, "HEALED_FAULT"); for (int i = 1; i <= 2; ++i) { - storage.report_fault_event("HEALED_FAULT", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "/test_node", - 1000 + i, healing); + storage.report_fault_event(FaultId{"HEALED_FAULT", "/test_node"}, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, + "", "/test_node", 1000 + i, healing); } - storage.report_fault_event("ACTIVE_FAULT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "/test_node", - 1000, default_config()); + storage.report_fault_event(FaultId{"ACTIVE_FAULT", "/test_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "fault", "/test_node", 1000, default_config()); store_two_snapshots(storage, "ACTIVE_FAULT"); ASSERT_EQ(storage.reclassify_healed_as_cleared().size(), 1u); @@ -3795,8 +3838,8 @@ TEST(InMemorySnapshotRetentionTest, RetentionIsPerFaultCodeNotGlobal) { storage.set_retain_snapshots_on_clear(true); for (const auto & code : {"FAULT_A", "FAULT_B"}) { - storage.report_fault_event(code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "/test_node", 1000, - default_config()); + storage.report_fault_event(FaultId{code, "/test_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", + "/test_node", 1000, default_config()); store_two_snapshots(storage, code); } @@ -3829,8 +3872,9 @@ TEST(InMemoryNearMissTest, SeriesIsAppendedAndSurvivesClear) { const auto config = near_miss_config(); for (int i = 0; i < 3; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); } auto series = storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); @@ -3848,8 +3892,9 @@ TEST(InMemoryNearMissTest, ConfirmingReportIsNotANearMiss) { const auto config = near_miss_config(); for (int i = 0; i < 4; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); } auto fault = storage.get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); @@ -3865,8 +3910,9 @@ TEST(InMemoryNearMissTest, SeriesBoundedKeepingNewest) { storage.set_max_near_misses_per_fault(3); for (int i = 0; i < 5; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); } auto series = storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); @@ -3880,11 +3926,12 @@ TEST(InMemoryNearMissTest, PassedReportIsNotANearMiss) { const auto config = near_miss_config(); for (int i = 0; i < 2; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); } - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", - near_miss_time(2).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", near_miss_time(2).nanoseconds(), config); EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); } @@ -3895,11 +3942,12 @@ TEST(InMemoryNearMissTest, SeriesSurvivesHealedReclassification) { config.healing_enabled = true; config.healing_threshold = 1; - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(0).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(0).nanoseconds(), config); for (int i = 1; i <= 2; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); } auto healed = storage.get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); @@ -3918,10 +3966,12 @@ TEST(InMemoryNearMissTest, BoundIsPerFaultCode) { storage.set_max_near_misses_per_fault(2); for (int i = 0; i < 3; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); - storage.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + near_miss_time(i).nanoseconds(), config); } EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); @@ -3935,8 +3985,9 @@ TEST(InMemoryNearMissTest, BoundZeroIsUnlimited) { storage.set_max_near_misses_per_fault(0); for (int i = 0; i < 150; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); } EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 150u); @@ -3949,8 +4000,9 @@ TEST(InMemoryNearMissTest, ApplyingSmallerBoundTrimsExistingSeries) { storage.set_max_near_misses_per_fault(0); for (int i = 0; i < 5; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); } storage.set_max_near_misses_per_fault(2); @@ -3968,10 +4020,12 @@ TEST(InMemoryNearMissTest, ApplyingBoundReportsHowManyEntriesItDropped) { storage.set_max_near_misses_per_fault(0); for (int i = 0; i < 5; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); - storage.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + near_miss_time(i).nanoseconds(), config); } EXPECT_EQ(storage.set_max_near_misses_per_fault(2), 6u); @@ -3984,18 +4038,20 @@ TEST(InMemoryNearMissTest, EntriesCarryTheResultingStatus) { config.healing_enabled = true; config.healing_threshold = 2; - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(0).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(0).nanoseconds(), config); for (int i = 1; i <= 3; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); } auto healed = storage.get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(4).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(4).nanoseconds(), config); auto series = storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); @@ -4029,14 +4085,14 @@ TEST(StorageBackendParityTest, AChangedBandProducesTheSameSeriesInBothBackends) const ros2_medkit_fault_manager::FaultId id{"SHARED_CODE", "/one_source"}; auto drive = [&](ros2_medkit_fault_manager::FaultStorage & storage) { - storage.report_fault_event("SHARED_CODE", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "dip", "/one_source", 1000, - wide); + storage.report_fault_event(FaultId{"SHARED_CODE", "/one_source"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, + "dip", "/one_source", 1000, wide); for (int i = 1; i <= 7; ++i) { - storage.report_fault_event("SHARED_CODE", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/one_source", - 1000 + i, wide); + storage.report_fault_event(FaultId{"SHARED_CODE", "/one_source"}, fd::ReportKind::kPassed, Fault::SEVERITY_WARN, + "", "/one_source", 1000 + i, wide); } - storage.report_fault_event("SHARED_CODE", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "dip", "/one_source", 2000, - narrow); + storage.report_fault_event(FaultId{"SHARED_CODE", "/one_source"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, + "dip", "/one_source", 2000, narrow); }; InMemoryFaultStorage memory; @@ -4156,8 +4212,8 @@ class SnapshotReadPathTest : public ::testing::Test { TEST_F(SnapshotReadPathTest, NewestSnapshotOfATopicWins) { auto & storage = fault_manager_->get_storage_for_test(); - storage.report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure high", - "/plc", 1000, DebounceConfig{}); + storage.report_fault_event(FaultId{"PLC_PRESSURE_HIGH", "/plc"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "pressure high", "/plc", 1000, DebounceConfig{}); store_snapshot("PLC_PRESSURE_HIGH", "/plc/pressure", R"({"data":"old"})", 1000); store_snapshot("PLC_PRESSURE_HIGH", "/plc/pressure", R"({"data":"new"})", 5000); @@ -4177,8 +4233,8 @@ TEST_F(SnapshotReadPathTest, NewestSnapshotOfATopicWins) { TEST_F(SnapshotReadPathTest, FreezeFrameStaysVisibleBehindRetainedSnapshots) { auto & storage = fault_manager_->get_storage_for_test(); - storage.report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure high", - "/plc", 1000, DebounceConfig{}); + storage.report_fault_event(FaultId{"PLC_PRESSURE_HIGH", "/plc"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "pressure high", "/plc", 1000, DebounceConfig{}); store_snapshot("PLC_PRESSURE_HIGH", "/plc/pressure", R"({"data":"from an earlier occurrence"})", 1000); ros2_medkit_fault_manager::FreezeFrameData frame; diff --git a/src/ros2_medkit_fault_manager/test/test_fault_manager_core_concurrency.cpp b/src/ros2_medkit_fault_manager/test/test_fault_manager_core_concurrency.cpp index 4ad5fc721..708c97318 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_manager_core_concurrency.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_manager_core_concurrency.cpp @@ -88,12 +88,11 @@ class FixedClock : public core::WallClock { /// overlaps with others on every first report of a code. class YieldingStorage : public ros2_medkit_fault_manager::InMemoryFaultStorage { public: - bool report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, + bool report_fault_event(const ros2_medkit_fault_manager::FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const ros2_medkit_fault_manager::DebounceConfig & config) override { pause(); - return InMemoryFaultStorage::report_fault_event(fault_code, kind, severity, description, source_id, timestamp_ns, - config); + return InMemoryFaultStorage::report_fault_event(id, kind, severity, description, source_id, timestamp_ns, config); } std::optional get_fault(const ros2_medkit_fault_manager::FaultId & id) const override { auto result = InMemoryFaultStorage::get_fault(id); diff --git a/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp b/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp index c366a37b8..27efbae7e 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp @@ -108,12 +108,13 @@ class FaultSeverityTest : public ::testing::Test { bool failed(const std::string & code, uint8_t severity, int64_t second, const DebounceConfig & config, const std::string & owner = kOwner) { - return storage_->report_fault_event(code, fd::ReportKind::kFailed, severity, "report", owner, at(second), config); + return storage_->report_fault_event(FaultId{code, owner}, fd::ReportKind::kFailed, severity, "report", owner, + at(second), config); } void passed(const std::string & code, int64_t second, const DebounceConfig & config, const std::string & owner = kOwner) { - storage_->report_fault_event(code, fd::ReportKind::kPassed, 0, "", owner, at(second), config); + storage_->report_fault_event(FaultId{code, owner}, fd::ReportKind::kPassed, 0, "", owner, at(second), config); } bool clear(const std::string & code, const std::string & owner = kOwner) { @@ -403,6 +404,7 @@ class SqliteSeverityTest : public ::testing::Test { } EXPECT_EQ(sqlite3_exec(raw, sql.str().c_str(), nullptr, nullptr, nullptr), SQLITE_OK) << sqlite3_errmsg(raw); sqlite3_close(raw); + record_owner_ = ros2_medkit_fault_manager::kSharedOwner; // a migrated row is the shared record of its code reopen(); } @@ -412,19 +414,21 @@ class SqliteSeverityTest : public ::testing::Test { } void failed(const std::string & code, uint8_t severity, int64_t second, const DebounceConfig & config) { - storage_->report_fault_event(code, fd::ReportKind::kFailed, severity, "report", kOwner, at(second), config); + storage_->report_fault_event(FaultId{code, record_owner_}, fd::ReportKind::kFailed, severity, "report", kOwner, + at(second), config); } void passed(const std::string & code, int64_t second, const DebounceConfig & config) { - storage_->report_fault_event(code, fd::ReportKind::kPassed, 0, "", kOwner, at(second), config); + storage_->report_fault_event(FaultId{code, record_owner_}, fd::ReportKind::kPassed, 0, "", kOwner, at(second), + config); } bool clear(const std::string & code) { - return storage_->clear_fault(FaultId{code, kOwner}); + return storage_->clear_fault(FaultId{code, record_owner_}); } fd::FaultRecord fault(const std::string & code) { - auto f = storage_->get_fault(FaultId{code, kOwner}); + auto f = storage_->get_fault(FaultId{code, record_owner_}); EXPECT_TRUE(f.has_value()) << code; return f.value_or(fd::FaultRecord{}); } @@ -437,7 +441,7 @@ class SqliteSeverityTest : public ::testing::Test { &stmt, nullptr), SQLITE_OK); sqlite3_bind_text(stmt, 1, code.c_str(), -1, SQLITE_TRANSIENT); - sqlite3_bind_text(stmt, 2, kOwner, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, record_owner_.c_str(), -1, SQLITE_TRANSIENT); int64_t value = -1; if (sqlite3_step(stmt) == SQLITE_ROW) { value = sqlite3_column_int64(stmt, 0); @@ -449,6 +453,8 @@ class SqliteSeverityTest : public ::testing::Test { std::filesystem::path dir_; std::unique_ptr storage_; + /// Owner of the records the helpers address: the test's reporter, or the shared owner after a legacy open. + std::string record_owner_{kOwner}; }; TEST_F(SqliteSeverityTest, CriticalClearRestartWarnConfirms) { @@ -820,8 +826,10 @@ void run_sweep(const DebounceConfig & config, const std::string & tag, int lengt sqlite.clear_fault(FaultId{code, e.owner}); } else { const fd::ReportKind type = e.kind == Kind::kFailed ? fd::ReportKind::kFailed : fd::ReportKind::kPassed; - memory.report_fault_event(code, type, e.severity, "step " + std::to_string(step), e.owner, t, config); - sqlite.report_fault_event(code, type, e.severity, "step " + std::to_string(step), e.owner, t, config); + memory.report_fault_event(FaultId{code, e.owner}, type, e.severity, "step " + std::to_string(step), e.owner, t, + config); + sqlite.report_fault_event(FaultId{code, e.owner}, type, e.severity, "step " + std::to_string(step), e.owner, t, + config); } std::string diff; diff --git a/src/ros2_medkit_fault_manager/test/test_fault_thresholds.yaml b/src/ros2_medkit_fault_manager/test/test_fault_thresholds.yaml index f27761052..9753b5889 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_thresholds.yaml +++ b/src/ros2_medkit_fault_manager/test/test_fault_thresholds.yaml @@ -13,6 +13,7 @@ SHARED.JAM: healing_threshold: 8 # Pins only the confirmation direction. The healing fields still come from -# whichever entity reported. +# whichever entity reported, so this code is only half settled - and the node +# still says so. SHARED.PARTIAL: confirmation_threshold: -2 diff --git a/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp b/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp index cebfa86c4..34b1dc6a6 100644 --- a/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp +++ b/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp @@ -155,9 +155,9 @@ TEST_F(PgFaultStorageTest, ReportNewFaultEvent) { rclcpp::Clock clock; auto timestamp = clock.now(); - bool is_new = storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "Motor temperature exceeded threshold", "/powertrain/motor", - timestamp.nanoseconds(), default_config()); + bool is_new = storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Motor temperature exceeded threshold", + "/powertrain/motor", timestamp.nanoseconds(), default_config()); EXPECT_TRUE(is_new); EXPECT_EQ(storage_->size(), 1u); @@ -168,8 +168,9 @@ TEST_F(PgFaultStorageTest, PassedEventForNonExistentFaultIgnored) { rclcpp::Clock clock; auto timestamp = clock.now(); - bool is_new = storage_->report_fault_event("NON_EXISTENT", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "Test", - "/node1", timestamp.nanoseconds(), default_config()); + bool is_new = + storage_->report_fault_event(FaultId{"NON_EXISTENT", "/node1"}, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, + "Test", "/node1", timestamp.nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_->size(), 0u); @@ -180,20 +181,20 @@ TEST_F(PgFaultStorageTest, SameCodeDifferentSourcesAreSeparateRecords) { auto timestamp1 = clock.now(); auto timestamp2 = clock.now(); - bool first_new = - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Initial report", - "/powertrain/motor1", timestamp1.nanoseconds(), default_config()); + bool first_new = storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, + fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Initial report", + "/powertrain/motor1", timestamp1.nanoseconds(), default_config()); // A second source reporting the same code owns a NEW record, not an update of the first. - bool second_new = - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Second report", - "/powertrain/motor2", timestamp2.nanoseconds(), default_config()); + bool second_new = storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor2"}, + fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Second report", + "/powertrain/motor2", timestamp2.nanoseconds(), default_config()); EXPECT_TRUE(first_new); EXPECT_TRUE(second_new); EXPECT_EQ(storage_->size(), 2u); - auto by_code = storage_->get_faults_by_code("MOTOR_OVERHEAT"); + auto by_code = storage_->get_ids_by_code("MOTOR_OVERHEAT"); ASSERT_EQ(by_code.size(), 2u); auto first = storage_->get_fault(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}); @@ -215,8 +216,8 @@ TEST_F(PgFaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCount) // Simulate a level-triggered poller re-reporting the same still-true condition // every cycle (issue #11): occurrence_count must stay at 1, not grow per poll. for (int i = 0; i < 25; ++i) { - storage_->report_fault_event("TANK_OVERFILL", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "level = 95 > 80", - "/tank", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"TANK_OVERFILL", "/tank"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "level = 95 > 80", "/tank", clock.now().nanoseconds(), default_config()); } auto fault = storage_->get_fault(FaultId{"TANK_OVERFILL", "/tank"}); @@ -233,10 +234,10 @@ TEST_F(PgFaultStorageTest, ClearingOneOwnerLeavesTheOtherOwnersRecord) { rclcpp::Clock clock; const FaultId a{"SHARED_CODE", "/node_a"}; const FaultId b{"SHARED_CODE", "/node_b"}; - storage_->report_fault_event("SHARED_CODE", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("SHARED_CODE", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"SHARED_CODE", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + a.owner, clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"SHARED_CODE", b.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", + b.owner, clock.now().nanoseconds(), default_config()); ASSERT_EQ(storage_->size(), 2u); EXPECT_TRUE(storage_->clear_fault(a)); @@ -251,15 +252,15 @@ TEST_F(PgFaultStorageTest, RecordsAreDistinctPerOwnerIncludingUnscoped) { const FaultId a{"DUP", "/owner_a"}; const FaultId b{"DUP", "/owner_b"}; const FaultId unscoped{"DUP", ""}; - storage_->report_fault_event("DUP", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "a", a.owner, - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("DUP", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, + storage_->report_fault_event(FaultId{"DUP", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "a", a.owner, clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("DUP", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "u", unscoped.owner, + storage_->report_fault_event(FaultId{"DUP", b.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"DUP", unscoped.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "u", + unscoped.owner, clock.now().nanoseconds(), default_config()); EXPECT_EQ(storage_->size(), 3u); - EXPECT_EQ(storage_->get_faults_by_code("DUP").size(), 3u); + EXPECT_EQ(storage_->get_ids_by_code("DUP").size(), 3u); EXPECT_EQ(storage_->get_fault(a)->severity, Fault::SEVERITY_WARN); EXPECT_EQ(storage_->get_fault(b)->severity, Fault::SEVERITY_ERROR); EXPECT_EQ(storage_->get_fault(unscoped)->severity, Fault::SEVERITY_CRITICAL); @@ -273,18 +274,18 @@ TEST_F(PgFaultStorageTest, DebounceCountersAreIndependentPerOwner) { const FaultId a{"GRIND", "/node_a"}; const FaultId b{"GRIND", "/node_b"}; - storage_->report_fault_event("GRIND", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + storage_->report_fault_event(FaultId{"GRIND", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, clock.now().nanoseconds(), config); - storage_->report_fault_event("GRIND", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + storage_->report_fault_event(FaultId{"GRIND", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, clock.now().nanoseconds(), config); EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_PREFAILED); - storage_->report_fault_event("GRIND", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, + storage_->report_fault_event(FaultId{"GRIND", b.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, clock.now().nanoseconds(), config); EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_PREFAILED); EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_PREFAILED); - storage_->report_fault_event("GRIND", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + storage_->report_fault_event(FaultId{"GRIND", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, clock.now().nanoseconds(), config); EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_PREFAILED); @@ -296,9 +297,9 @@ TEST_F(PgFaultStorageTest, EvidenceAttachesToOneOwnerAndIsInvisibleUnderAnother) rclcpp::Clock clock; const FaultId a{"EVID", "/node_a"}; const FaultId b{"EVID", "/node_b"}; - storage_->report_fault_event("EVID", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + storage_->report_fault_event(FaultId{"EVID", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("EVID", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, + storage_->report_fault_event(FaultId{"EVID", b.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, clock.now().nanoseconds(), default_config()); SnapshotData snap; @@ -344,8 +345,8 @@ TEST_F(PgFaultStorageTest, ListFaultsDefaultReturnsConfirmedOnly) { auto timestamp = clock.now(); // With default threshold=-1, single report confirms immediately - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); // Default query should return the CONFIRMED fault auto faults = storage_->list_faults(false, 0, {}); @@ -362,8 +363,8 @@ TEST_F(PgFaultStorageTest, ListFaultsWithPrefailedStatus) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), config); // Query with PREFAILED status auto faults = storage_->list_faults(false, 0, {Fault::STATUS_PREFAILED}); @@ -377,10 +378,10 @@ TEST_F(PgFaultStorageTest, ListFaultsFilterBySeverity) { auto timestamp = clock.now(); // With default threshold=-1, faults are immediately CONFIRMED - storage_->report_fault_event("FAULT_INFO", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Info", "/node1", - timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_ERROR", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Error", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_INFO", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Info", + "/node1", timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_ERROR", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Error", "/node1", timestamp.nanoseconds(), default_config()); // Filter by ERROR severity (query CONFIRMED since that's the default status now) auto faults = storage_->list_faults(true, Fault::SEVERITY_ERROR, {Fault::STATUS_CONFIRMED}); @@ -392,8 +393,8 @@ TEST_F(PgFaultStorageTest, ClearFault) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Test", "/node1", timestamp.nanoseconds(), default_config()); bool cleared = storage_->clear_fault(FaultId{"MOTOR_OVERHEAT", "/node1"}); EXPECT_TRUE(cleared); @@ -415,10 +416,10 @@ TEST_F(PgFaultStorageTest, PassedEventDoesNotAdvanceLastOccurred) { const rclcpp::Time failed_at(1000, 0, RCL_SYSTEM_TIME); const rclcpp::Time passed_at(9000, 0, RCL_SYSTEM_TIME); - storage_->report_fault_event("FAULT_LO", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - failed_at.nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_LO", fd::ReportKind::kPassed, 0, "", "/node1", passed_at.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"FAULT_LO", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", failed_at.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_LO", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + passed_at.nanoseconds(), default_config()); auto fault = storage_->get_fault(FaultId{"FAULT_LO", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -432,8 +433,8 @@ TEST_F(PgFaultStorageTest, GetClearedFaults) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); storage_->clear_fault(FaultId{"FAULT_1", "/node1"}); // Query cleared faults @@ -447,8 +448,8 @@ TEST_F(PgFaultStorageTest, InvalidStatusDefaultsToConfirmed) { auto timestamp = clock.now(); // With default threshold=-1, fault is immediately CONFIRMED - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); // Query with invalid status - defaults to CONFIRMED, which now matches our fault auto faults = storage_->list_faults(false, 0, {"INVALID_STATUS"}); @@ -463,10 +464,10 @@ TEST_F(PgFaultStorageTest, PersistenceAcrossRestarts) { // With default threshold=-1, faults are immediately CONFIRMED // Report some faults - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Persistent fault 1", - "/node1", timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_2", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Persistent fault 2", "/node2", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Persistent fault 1", "/node1", timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_2", "/node2"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, + "Persistent fault 2", "/node2", timestamp.nanoseconds(), default_config()); storage_->clear_fault(FaultId{"FAULT_2", "/node2"}); // Verify initial state @@ -500,8 +501,8 @@ TEST_F(PgFaultStorageTest, TimestampPrecision) { int64_t test_ns = 1735312456123456789LL; // Specific nanosecond timestamp rclcpp::Time timestamp(test_ns, RCL_SYSTEM_TIME); - storage_->report_fault_event("FAULT_TS", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Timestamp test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_TS", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_INFO, + "Timestamp test", "/node1", timestamp.nanoseconds(), default_config()); auto fault = storage_->get_fault(FaultId{"FAULT_TS", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -583,8 +584,8 @@ TEST(PgFaultStorageConnectionTest, CreatesTheSchemaWhenTheDatabaseAppears) { const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(15); while (!stored && std::chrono::steady_clock::now() < deadline) { try { - storage.report_fault_event("LATE", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "late", "/node", - clock.now().nanoseconds(), default_config()); + storage.report_fault_event(FaultId{"LATE", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "late", + "/node", clock.now().nanoseconds(), default_config()); stored = true; } catch (const ros2_medkit_fault_manager::FaultStorage::IgnorableConnectionException &) { std::this_thread::sleep_for(std::chrono::milliseconds(250)); @@ -1166,7 +1167,7 @@ TEST(PgFaultStorageConnectionTest, ALostConnectionIsOpenedAgainWithTheSchema) { " application_name=" + application + " options='-csearch_path=" + schema + "'"), test_log_sink()); ASSERT_TRUE(storage.connected()); - storage.report_fault_event("LOST_A", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/node", + storage.report_fault_event(FaultId{"LOST_A", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/node", clock.now().nanoseconds(), default_config()); EXPECT_EQ(storage.size(), 1u); @@ -1179,7 +1180,7 @@ TEST(PgFaultStorageConnectionTest, ALostConnectionIsOpenedAgainWithTheSchema) { ntx.quote(application)); } EXPECT_EQ(storage.size(), 0u); - storage.report_fault_event("LOST_B", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", "/node", + storage.report_fault_event(FaultId{"LOST_B", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", "/node", clock.now().nanoseconds(), default_config()); EXPECT_EQ(storage.size(), 1u); } @@ -1196,12 +1197,12 @@ TEST_F(PgFaultStorageTest, ReportingSourcesJsonHandling) { // reporting_sources column, including slashes and an embedded quote. const std::vector owners = {"/node/path/with/slashes", "/another/node", "/special\"chars"}; for (const auto & owner : owners) { - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", owner, - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MULTI_SRC", owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Multi-source", owner, timestamp.nanoseconds(), default_config()); } EXPECT_EQ(storage_->size(), owners.size()); - EXPECT_EQ(storage_->get_faults_by_code("MULTI_SRC").size(), owners.size()); + EXPECT_EQ(storage_->get_ids_by_code("MULTI_SRC").size(), owners.size()); for (const auto & owner : owners) { auto fault = storage_->get_fault(FaultId{"MULTI_SRC", owner}); ASSERT_TRUE(fault.has_value()) << owner; @@ -1248,10 +1249,10 @@ TEST_F(PgFaultStorageTest, FaultStaysPrefailedAboveThreshold) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -1267,12 +1268,12 @@ TEST_F(PgFaultStorageTest, FaultConfirmsAtThreshold) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -1287,8 +1288,8 @@ TEST_F(PgFaultStorageTest, ImmediateConfirmationWithThresholdZero) { storage_->set_debounce_config(config); // Single report should confirm immediately - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -1300,8 +1301,8 @@ TEST_F(PgFaultStorageTest, CriticalSeverityBypassesDebounce) { rclcpp::Clock clock; // CRITICAL severity should confirm immediately - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "Critical test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, + "Critical test", "/node1", clock.now().nanoseconds(), default_config()); auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -1315,8 +1316,9 @@ TEST_F(PgFaultStorageTest, ClearedFaultCanBeReactivated) { // Report to confirm (with default threshold=-1, single report confirms) const FaultId id{"FAULT_1", "/node1"}; auto first_ts = clock.now(); - bool is_new = storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Initial", - "/node1", first_ts.nanoseconds(), default_config()); + bool is_new = + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Initial", "/node1", first_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); auto fault = storage_->get_fault(id); @@ -1333,8 +1335,8 @@ TEST_F(PgFaultStorageTest, ClearedFaultCanBeReactivated) { // The SAME owner re-raising after a clear reopens its record as a new cycle. rclcpp::Time second_ts(first_ts.nanoseconds() + 1'000'000'000LL); // +1s - is_new = storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Reactivated", - "/node1", second_ts.nanoseconds(), default_config()); + is_new = storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Reactivated", "/node1", second_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); // Reactivation returns true like a new fault fault = storage_->get_fault(id); @@ -1352,14 +1354,14 @@ TEST_F(PgFaultStorageTest, PassedEventForClearedFaultIgnored) { rclcpp::Clock clock; // Report and confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); // Clear the fault storage_->clear_fault(FaultId{"FAULT_1", "/node1"}); // PASSED event should be ignored for CLEARED fault - bool result = storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", + bool result = storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), default_config()); EXPECT_FALSE(result); @@ -1377,12 +1379,12 @@ TEST_F(PgFaultStorageTest, ClearedFaultReactivationRestartsDebounce) { storage_->set_debounce_config(config); // Report 3 times to confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -1392,18 +1394,18 @@ TEST_F(PgFaultStorageTest, ClearedFaultReactivationRestartsDebounce) { storage_->clear_fault(FaultId{"FAULT_1", "/node1"}); // Reactivate - should start in PREFAILED with counter=-1 - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Not yet confirmed, needs 2 more FAILED // Report 2 more times to re-confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -1414,12 +1416,12 @@ TEST_F(PgFaultStorageTest, ConfirmationPersistsAfterReopen) { rclcpp::Clock clock; // Report 3 times to confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); // Close and reopen storage storage_.reset(); @@ -1439,15 +1441,15 @@ TEST_F(PgFaultStorageTest, PassedEventIncrementsCounter) { config.confirmation_threshold = -3; // Report 2 FAILED events (counter -2, PREFAILED) - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); // Report 3 PASSED events (counter -2 -> +1) for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); @@ -1466,14 +1468,14 @@ TEST_F(PgFaultStorageTest, HeartbeatHealClampedAndStatusLatched) { config.healing_enabled = true; config.healing_threshold = 3; - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); ASSERT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); // Long heal heartbeat: counter clamps at healing_threshold instead of running off. for (int i = 0; i < 100; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } auto healed = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(healed.has_value()); @@ -1482,13 +1484,13 @@ TEST_F(PgFaultStorageTest, HeartbeatHealClampedAndStatusLatched) { // Re-confirmation is now bounded to (healing_threshold - confirmation_threshold) // reports, and the healed status latches until the counter walks all the way down. for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); EXPECT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); } - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); auto reconfirmed = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(reconfirmed.has_value()); EXPECT_EQ(reconfirmed->status, Fault::STATUS_CONFIRMED); @@ -1503,21 +1505,21 @@ TEST_F(PgFaultStorageTest, ConfirmedFaultSurvivesHealHeartbeat) { config.healing_enabled = false; config.healing_threshold = 3; // upper clamp on the counter, even with healing disabled - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); ASSERT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 20; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); EXPECT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED) << "un-confirmed after " << (i + 1); } // The counter is now positive (clamped at +3). One FAILED must keep it CONFIRMED, not flip it to // PREPASSED via the `counter > 0` branch - that was the asymmetric-latch bug. - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); EXPECT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); } @@ -1531,16 +1533,17 @@ TEST_F(PgFaultStorageTest, ConfirmedLatchSurvivesFailedAtPositiveCounter) { config.critical_immediate_confirm = true; // CRITICAL confirms immediately at counter -1. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); // PASSED events raise the counter into positive territory; latch keeps it CONFIRMED. for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); // One normal FAILED: counter still positive, must NOT become PREPASSED. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); } @@ -1553,20 +1556,22 @@ TEST_F(PgFaultStorageTest, HealedLatchSurvivesPassedAtNegativeCounter) { config.healing_enabled = true; config.healing_threshold = 3; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { // counter -1 -> +3 -> HEALED - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); // FAILED events drive the counter negative while HEALED is latched (not yet at confirmation -3). for (int i = 0; i < 5; ++i) { // +3 -> -2 - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); } // One PASSED: counter goes -2 -> -1, still negative; must stay HEALED, not flip to PREFAILED. - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); } @@ -1578,23 +1583,26 @@ TEST_F(PgFaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePassed) { config.healing_threshold = 3; config.critical_immediate_confirm = true; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { // -1 -> +3 -> HEALED - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); // CRITICAL confirms immediately from HEALED; counter must not stay at +2 where one PASSED heals it. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { // -1 -> +2, still CONFIRMED - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); } - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); } @@ -1606,7 +1614,7 @@ TEST_F(PgFaultStorageTest, RunawayCounterFromOldRowRecovers) { config.confirmation_threshold = -1; config.healing_threshold = 3; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); // Simulate the old bug: poke a huge counter directly into the table behind the storage's back. exec_raw("UPDATE faults SET debounce_counter = 100000, status = 'PREPASSED'"); @@ -1614,7 +1622,7 @@ TEST_F(PgFaultStorageTest, RunawayCounterFromOldRowRecovers) { // The counter is clamped back into range on first touch, so a bounded number of FAILED events // re-confirms (healing_threshold - confirmation_threshold = 4), not the ~100000 the runaway implied. for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); } EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); @@ -1626,10 +1634,11 @@ TEST_F(PgFaultStorageTest, ReclassifyHealedAsCleared) { config.healing_enabled = true; config.healing_threshold = 3; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); @@ -1647,10 +1656,11 @@ TEST_F(PgFaultStorageTest, ReclassifyKeepsCodesTheCallerKeepsHealed) { config.healing_threshold = 3; for (const std::string code : {"KEEP", "FLIP"}) { - storage_->report_fault_event(code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{code, "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event(code, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{code, "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_->get_fault(FaultId{code, "/n"})->status, Fault::STATUS_HEALED); } @@ -1672,13 +1682,13 @@ TEST_F(PgFaultStorageTest, HealingWhenEnabled) { storage_->set_debounce_config(config); // Report 1 FAILED event - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); // Report 4 PASSED events (counter = -1 + 4 = +3, reaches healing threshold) for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); @@ -1694,8 +1704,8 @@ TEST_F(PgFaultStorageTest, TimeBasedConfirmationWhenEnabled) { storage_->set_debounce_config(config); auto now = clock.now(); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - now.nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", now.nanoseconds(), config); // Check before timeout - should not confirm auto before_timeout = rclcpp::Time(now.nanoseconds() + static_cast(5e9)); @@ -1722,10 +1732,10 @@ TEST_F(PgFaultStorageTest, TimeBasedConfirmationConfirmsOnlyReturnedFaults) { storage_->set_debounce_config(config); auto now = clock.now(); - storage_->report_fault_event("TIMED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - now.nanoseconds(), config); - storage_->report_fault_event("NO_FAILURE_TIME", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - now.nanoseconds(), config); + storage_->report_fault_event(FaultId{"TIMED", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", now.nanoseconds(), config); + storage_->report_fault_event(FaultId{"NO_FAILURE_TIME", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Test", "/node1", now.nanoseconds(), config); // A PREFAILED row without a failure time is not eligible. exec_raw("UPDATE faults SET last_failed_ns = 0 WHERE fault_code = 'NO_FAILURE_TIME'"); @@ -1743,15 +1753,15 @@ TEST_F(PgFaultStorageTest, ConfirmedAtRecordedOnImmediateConfirmation) { rclcpp::Clock clock; auto t = clock.now(); // Default config confirms on the first FAILED event. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t.nanoseconds(), default_config()); EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(read_confirmed_at("F"), t.nanoseconds()); // A later FAILED on an already-confirmed fault must NOT move the timestamp. auto t2 = rclcpp::Time(t.nanoseconds() + static_cast(5e9)); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t2.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t2.nanoseconds(), default_config()); EXPECT_EQ(read_confirmed_at("F"), t.nanoseconds()); } @@ -1761,14 +1771,14 @@ TEST_F(PgFaultStorageTest, ConfirmedAtRecordedOnDebouncedConfirmation) { config.confirmation_threshold = -2; // second FAILED event confirms auto t1 = clock.now(); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t1.nanoseconds(), - config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t1.nanoseconds(), config); EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_PREFAILED); EXPECT_EQ(read_confirmed_at("F"), 0) << "not confirmed yet: no confirmation timestamp"; auto t2 = rclcpp::Time(t1.nanoseconds() + static_cast(1e9)); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t2.nanoseconds(), - config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t2.nanoseconds(), config); EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(read_confirmed_at("F"), t2.nanoseconds()); } @@ -1781,8 +1791,8 @@ TEST_F(PgFaultStorageTest, ConfirmedAtRecordedOnTimeBasedConfirmation) { storage_->set_debounce_config(config); auto now = clock.now(); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", now.nanoseconds(), - config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + now.nanoseconds(), config); auto after_timeout = rclcpp::Time(now.nanoseconds() + static_cast(15e9)); auto confirmed = storage_->check_time_based_confirmation(after_timeout.nanoseconds()); @@ -1802,8 +1812,8 @@ TEST_F(PgFaultStorageTest, ConfirmedAtDefaultsToZeroForExternallyInsertedRow) { rclcpp::Clock clock; auto t = clock.now(); - storage_->report_fault_event("NEW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"NEW", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t.nanoseconds(), default_config()); EXPECT_EQ(read_confirmed_at("NEW"), t.nanoseconds()); } @@ -1923,8 +1933,8 @@ TEST_F(PgFaultStorageTest, StoreAndRetrieveSnapshot) { // First, create a fault to associate the snapshot with rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); // Store a snapshot SnapshotData snapshot; @@ -1952,8 +1962,8 @@ TEST_F(PgFaultStorageTest, MultipleSnapshotsForSameFault) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); // Store multiple snapshots for the same fault SnapshotData snapshot1; @@ -1982,8 +1992,8 @@ TEST_F(PgFaultStorageTest, FilterSnapshotsByTopic) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); SnapshotData snapshot1; snapshot1.fault_code = "MOTOR_OVERHEAT"; @@ -2020,7 +2030,7 @@ TEST_F(PgFaultStorageTest, ClearFaultDeletesAssociatedSnapshots) { rclcpp::Clock clock; // Create a fault using report_fault_event - storage_->report_fault_event("SNAPSHOT_CLEAR_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + storage_->report_fault_event(FaultId{"SNAPSHOT_CLEAR_TEST", ""}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test fault for snapshot cleanup", "", clock.now().nanoseconds(), default_config()); // Store snapshots for this fault @@ -2059,7 +2069,7 @@ TEST_F(PgFaultStorageTest, ClearFaultKeepsSnapshotsWhenEvidenceIsRetained) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("KEEP", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "keep", "", + storage_->report_fault_event(FaultId{"KEEP", ""}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "keep", "", clock.now().nanoseconds(), default_config()); storage_->set_retain_snapshots_on_clear(true); EXPECT_TRUE(storage_->retains_snapshots_on_clear()); @@ -2085,8 +2095,8 @@ TEST_F(PgFaultStorageTest, StoreAndRetrieveFreezeFrame) { using ros2_medkit_fault_manager::FreezeFrameData; rclcpp::Clock clock; - storage_->report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", "", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"PLC_PRESSURE_HIGH", ""}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Pressure high", "", clock.now().nanoseconds(), default_config()); FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; @@ -2135,8 +2145,8 @@ TEST_F(PgFaultStorageTest, FreezeFrameSurvivesClearFault) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", "", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"PLC_PRESSURE_HIGH", ""}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Pressure high", "", clock.now().nanoseconds(), default_config()); // A per-topic snapshot (removed on clear) plus a freeze-frame (retained on clear). SnapshotData snapshot; @@ -2190,13 +2200,14 @@ TEST_F(PgFaultStorageTest, ListRosbagsForEntityFiltersCorrectly) { rclcpp::Clock clock; // Create fault with reporting source for entity - storage_->report_fault_event("ENTITY_FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Fault from entity", - "/powertrain/motor", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"ENTITY_FAULT_1", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Fault from entity", "/powertrain/motor", + clock.now().nanoseconds(), default_config()); // Create another fault with different reporting source - storage_->report_fault_event("ENTITY_FAULT_2", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, - "Fault from other entity", "/chassis/brake", clock.now().nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"ENTITY_FAULT_2", "/chassis/brake"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "Fault from other entity", "/chassis/brake", + clock.now().nanoseconds(), default_config()); // Store rosbags for both faults RosbagFileInfo info1; @@ -2476,8 +2487,8 @@ TEST_F(PgFaultStorageTest, SnapshotCapKeepsTheNewestCaptureWholeAndDropsTheOldes using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); // Two topics per capture, room for two captures. storage_->set_max_snapshots_per_fault(4); @@ -2527,8 +2538,8 @@ TEST_F(PgFaultStorageTest, ACaptureLargerThanTheCapIsKeptWholeRatherThanTorn) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("WIDE", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "many topics", "/n", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"WIDE", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "many topics", + "/n", clock.now().nanoseconds(), default_config()); storage_->set_max_snapshots_per_fault(2); std::vector rows; @@ -2554,8 +2565,8 @@ TEST_F(PgFaultStorageTest, SnapshotLimitZeroMeansUnlimited) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("FAULT_A", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", "/node", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_A", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", + "/node", clock.now().nanoseconds(), default_config()); // Default (0) = unlimited storage_->set_max_snapshots_per_fault(0); @@ -2578,10 +2589,10 @@ TEST_F(PgFaultStorageTest, SnapshotLimitPerFaultNotGlobal) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("FAULT_A", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", "/node", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_B", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", "/node", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_A", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", + "/node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_B", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", + "/node", clock.now().nanoseconds(), default_config()); storage_->set_max_snapshots_per_fault(1); @@ -2627,8 +2638,9 @@ TEST_F(PgFaultStorageTest, NearMissSeriesIsAppendedNotOverwritten) { const auto config = four_strike_config(); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } auto fault = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); @@ -2652,8 +2664,9 @@ TEST_F(PgFaultStorageTest, ConfirmingReportIsNotANearMiss) { const auto config = four_strike_config(); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } auto fault = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); @@ -2668,8 +2681,9 @@ TEST_F(PgFaultStorageTest, NearMissSeriesSurvivesClearFault) { const auto config = four_strike_config(); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); @@ -2685,14 +2699,16 @@ TEST_F(PgFaultStorageTest, NearMissSeriesContinuesAcrossReactivation) { const auto config = four_strike_config(); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } ASSERT_TRUE(storage_->clear_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); // A new outage cycle starts: the reactivating report resets the counter to -1 without confirming. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, - "pressure dipping again", "/hydraulics/pump", nth_report_time(10).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping again", "/hydraulics/pump", + nth_report_time(10).nanoseconds(), config); auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 4u) << "the series must span fault cycles, one entry per occurrence"; @@ -2704,8 +2720,9 @@ TEST_F(PgFaultStorageTest, NearMissSeriesSurvivesReopen) { const auto config = four_strike_config(); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } ASSERT_TRUE(storage_->clear_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); @@ -2722,14 +2739,15 @@ TEST_F(PgFaultStorageTest, PassedReportIsNotANearMiss) { const auto config = four_strike_config(); for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); // A PASSED report moves the counter in the healing direction: the fault receding, not nearing. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); } @@ -2738,8 +2756,9 @@ TEST_F(PgFaultStorageTest, CriticalImmediateConfirmIsNotANearMiss) { DebounceConfig config = four_strike_config(); config.critical_immediate_confirm = true; - storage_->report_fault_event("BATTERY_THERMAL_RUNAWAY", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, - "cell over temperature", "/power/bms", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"BATTERY_THERMAL_RUNAWAY", "/power/bms"}, fd::ReportKind::kFailed, + Fault::SEVERITY_CRITICAL, "cell over temperature", "/power/bms", + nth_report_time(0).nanoseconds(), config); auto fault = storage_->get_fault(FaultId{"BATTERY_THERMAL_RUNAWAY", "/power/bms"}); ASSERT_TRUE(fault.has_value()); @@ -2752,8 +2771,9 @@ TEST_F(PgFaultStorageTest, ImmediateConfirmThresholdRecordsNoNearMiss) { // so a fault under this config never has a near miss to record. const auto config = default_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure lost", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "pressure lost", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); auto fault = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); @@ -2766,21 +2786,24 @@ TEST_F(PgFaultStorageTest, FailedReportUnderHealedLatchIsNearMiss) { config.healing_enabled = true; config.healing_threshold = 1; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); for (int i = 1; i <= 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), + config); } auto healed = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; // The latch keeps the status at HEALED, but the counter moved toward confirmation: a near miss. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(3).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(3).nanoseconds(), config); auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); @@ -2793,8 +2816,9 @@ TEST_F(PgFaultStorageTest, NearMissSeriesBoundedKeepingNewest) { storage_->set_max_near_misses_per_fault(3); for (int i = 0; i < 5; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); @@ -2811,8 +2835,9 @@ TEST_F(PgFaultStorageTest, NearMissBoundOfOneKeepsLatestOnly) { storage_->set_max_near_misses_per_fault(1); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); @@ -2826,10 +2851,12 @@ TEST_F(PgFaultStorageTest, NearMissBoundIsPerFaultCode) { storage_->set_max_near_misses_per_fault(2); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + nth_report_time(i).nanoseconds(), config); } EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); @@ -2842,8 +2869,9 @@ TEST_F(PgFaultStorageTest, NearMissBoundZeroIsUnlimited) { storage_->set_max_near_misses_per_fault(0); for (int i = 0; i < 150; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 150u); @@ -2856,11 +2884,13 @@ TEST_F(PgFaultStorageTest, NearMissSeriesSurvivesHealedReclassification) { config.healing_enabled = true; config.healing_threshold = 1; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); for (int i = 1; i <= 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), + config); } auto healed = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); @@ -2880,12 +2910,14 @@ TEST_F(PgFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { storage_->set_max_near_misses_per_fault(2); for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(10 + i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(10 + i).nanoseconds(), config); } // Arrives third, but carries the earliest timestamp of the three. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); @@ -2902,13 +2934,14 @@ TEST_F(PgFaultStorageTest, NearMissEntriesDescribeTheirOwnReport) { const FaultId backup{"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}; DebounceConfig first = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - pump.owner, nth_report_time(0).nanoseconds(), first); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", pump.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, + "pressure dipping", pump.owner, nth_report_time(0).nanoseconds(), first); DebounceConfig second = four_strike_config(); second.confirmation_threshold = -8; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure dipping", - backup.owner, nth_report_time(1).nanoseconds(), second); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", backup.owner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "pressure dipping", backup.owner, + nth_report_time(1).nanoseconds(), second); // Two sources of one code own separate records, so each near-miss series holds only its // owner's reports and is not visible under the other owner. @@ -2929,15 +2962,17 @@ TEST_F(PgFaultStorageTest, NearMissSeriesContinuesAfterReopen) { const auto config = four_strike_config(); for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } storage_.reset(); storage_ = std::make_unique(conn_info_, test_log_sink()); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(2).nanoseconds(), config); auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "a restart must extend the series, not restart or reorder it"; @@ -2956,10 +2991,12 @@ TEST_F(PgFaultStorageTest, ApplyingSmallerBoundTrimsExistingSeries) { storage_->set_max_near_misses_per_fault(0); for (int i = 0; i < 5; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + nth_report_time(i).nanoseconds(), config); } ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 5u); @@ -2979,8 +3016,9 @@ TEST_F(PgFaultStorageTest, ApplyingUnlimitedBoundKeepsExistingSeries) { storage_->set_max_near_misses_per_fault(4); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } storage_->set_max_near_misses_per_fault(0); @@ -2995,15 +3033,17 @@ TEST_F(PgFaultStorageTest, UnlimitedBoundSpeltAsSizeMaxKeepsTheSeries) { config.confirmation_threshold = -20; for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } EXPECT_EQ(storage_->set_max_near_misses_per_fault(std::numeric_limits::max()), 0u); EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(3).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(3).nanoseconds(), config); EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 4u); } @@ -3013,10 +3053,12 @@ TEST_F(PgFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { storage_->set_max_near_misses_per_fault(0); for (int i = 0; i < 5; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + nth_report_time(i).nanoseconds(), config); } // Two codes, five entries each, bound of 2: three dropped per code. @@ -3027,8 +3069,9 @@ TEST_F(PgFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { TEST_F(PgFaultStorageTest, PassedReportOnUnknownFaultWritesNothing) { // A heal heartbeat for a fault that does not exist must stay a read: it writes no row. - EXPECT_FALSE(storage_->report_fault_event("NEVER_REPORTED", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/test_node", nth_report_time(0).nanoseconds(), default_config())); + EXPECT_FALSE(storage_->report_fault_event(FaultId{"NEVER_REPORTED", "/test_node"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/test_node", nth_report_time(0).nanoseconds(), + default_config())); EXPECT_EQ(storage_->size(), 0u); EXPECT_TRUE(storage_->get_near_misses(FaultId{"NEVER_REPORTED", "/test_node"}).empty()); } @@ -3042,11 +3085,13 @@ TEST_F(PgFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) { config.healing_threshold = 2; // An approach that recedes: counter moves, nothing is latched. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); for (int i = 1; i <= 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), + config); } auto healed = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); @@ -3056,8 +3101,9 @@ TEST_F(PgFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) { // From the healing threshold (+2) it takes six FAILED reports to reach the confirmation // threshold (-4). for (int i = 4; i <= 9; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } auto confirmed = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(confirmed.has_value()); @@ -3079,8 +3125,9 @@ TEST_F(PgFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) { TEST_F(PgFaultStorageTest, NearMissResultingStatusSurvivesReopen) { const auto config = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); storage_.reset(); storage_ = std::make_unique(conn_info_, test_log_sink()); @@ -3119,12 +3166,12 @@ static void drive_to_healed(ros2_medkit_fault_manager::FaultStorage & storage, c config.healing_enabled = true; config.healing_threshold = 1; - storage.report_fault_event(fault_code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "", + storage.report_fault_event(FaultId{fault_code, ""}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "", nth_report_time(0).nanoseconds(), config); store_snapshots_for(storage, fault_code, 2); // Two PASSED reports: the first only lifts the counter to 0, which the CONFIRMED latch holds. for (int i = 1; i <= 2; ++i) { - storage.report_fault_event(fault_code, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "", + storage.report_fault_event(FaultId{fault_code, ""}, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "", nth_report_time(i).nanoseconds(), config); } } @@ -3153,31 +3200,31 @@ TEST_F(PgFaultStorageTest, SnapshotsDroppedByHealedReclassificationByDefault) { // --- Record identity: the same per-record contract as the SQLite backend --- -TEST_F(PgFaultStorageTest, GetFaultsByCodeReturnsEveryOwnerAndGetFaultExactlyOne) { +TEST_F(PgFaultStorageTest, GetIdsByCodeReturnsEveryOwnerAndGetFaultExactlyOne) { rclcpp::Clock clock; - storage_->report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_b", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_a", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"SHARED", "/owner_b"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_b", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"SHARED", "/owner_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_a", clock.now().nanoseconds(), default_config()); - const auto shared = storage_->get_faults_by_code("SHARED"); + const auto shared = storage_->get_ids_by_code("SHARED"); ASSERT_EQ(shared.size(), 2u); - ASSERT_EQ(shared[0].reporting_sources.size(), 1u); - ASSERT_EQ(shared[1].reporting_sources.size(), 1u); - EXPECT_EQ(shared[0].reporting_sources.front(), "/owner_a") << "ordered by owner, like the other backends"; - EXPECT_EQ(shared[1].reporting_sources.front(), "/owner_b"); + EXPECT_EQ(shared[0].owner, "/owner_a") << "ordered by owner, like the other backends"; + EXPECT_EQ(shared[1].owner, "/owner_b"); - EXPECT_TRUE(storage_->get_faults_by_code("NEVER_REPORTED").empty()); + EXPECT_TRUE(storage_->get_ids_by_code("NEVER_REPORTED").empty()); ASSERT_TRUE(storage_->get_fault(FaultId{"SHARED", "/owner_a"}).has_value()); EXPECT_FALSE(storage_->get_fault(FaultId{"SHARED", "/nobody"}).has_value()); } TEST_F(PgFaultStorageTest, NearMissSeriesIsPerRecord) { const auto config = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "dip", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "dip", - "/hydraulics/backup_pump", nth_report_time(1).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "dip", "/hydraulics/pump", nth_report_time(0).nanoseconds(), + config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "dip", "/hydraulics/backup_pump", + nth_report_time(1).nanoseconds(), config); const auto pump = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); const auto backup = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}); @@ -3197,17 +3244,17 @@ TEST_F(PgFaultStorageTest, ACriticalRecordConfirmsOnAnyFailedReportAndAnotherOwn const FaultId a{"CRIT", "/a"}; const FaultId b{"CRIT", "/b"}; - storage_->report_fault_event("CRIT", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "", a.owner, + storage_->report_fault_event(FaultId{"CRIT", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "", a.owner, clock.now().nanoseconds(), config); ASSERT_TRUE(storage_->clear_fault(a)); - EXPECT_TRUE(storage_->report_fault_event("CRIT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "", a.owner, - clock.now().nanoseconds(), config)) + EXPECT_TRUE(storage_->report_fault_event(FaultId{"CRIT", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "", + a.owner, clock.now().nanoseconds(), config)) << "a report on a CLEARED record is an occurrence"; EXPECT_EQ(storage_->get_fault(a)->severity, Fault::SEVERITY_CRITICAL) << "the record keeps its highest severity"; EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_CONFIRMED) << "the stored severity decides, not the report's"; - storage_->report_fault_event("CRIT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "", b.owner, + storage_->report_fault_event(FaultId{"CRIT", b.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "", b.owner, clock.now().nanoseconds(), config); EXPECT_EQ(storage_->get_fault(b)->severity, Fault::SEVERITY_WARN) << "severity is per record"; EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_PREFAILED) << "another owner's CRITICAL confirms nothing"; @@ -3225,8 +3272,10 @@ TEST_F(PgFaultStorageTest, HealingAndTimeBasedConfirmationActOnOneRecord) { const FaultId b{"TWO", "/b"}; const int64_t t0 = 1'000'000'000'000LL; - storage_->report_fault_event("TWO", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", a.owner, t0, config); - storage_->report_fault_event("TWO", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", b.owner, t0 + 1, config); + storage_->report_fault_event(FaultId{"TWO", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", a.owner, t0, + config); + storage_->report_fault_event(FaultId{"TWO", b.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", b.owner, + t0 + 1, config); const auto swept = storage_->check_time_based_confirmation(t0 + 10'000'000'000LL); ASSERT_EQ(swept.size(), 2u); @@ -3236,7 +3285,7 @@ TEST_F(PgFaultStorageTest, HealingAndTimeBasedConfirmationActOnOneRecord) { // Heal a only: its counter moves, b's does not. for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("TWO", fd::ReportKind::kPassed, 0, "", a.owner, t0 + 20 + i, config); + storage_->report_fault_event(FaultId{"TWO", a.owner}, fd::ReportKind::kPassed, 0, "", a.owner, t0 + 20 + i, config); } EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_HEALED); EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_CONFIRMED); @@ -3274,19 +3323,26 @@ constexpr const char * kLegacyPgSchema = R"( } // namespace +// Every record of a table written when a fault code was one record, fed by every source, becomes the +// shared record of its code, and every child row attaches to it. TEST_F(PgFaultStorageTest, ATableWrittenByThePreviousLayoutIsMigratedInPlace) { storage_.reset(); exec_raw("DROP TABLE faults, snapshots, freeze_frames, near_misses, rosbag_files"); exec_raw(kLegacyPgSchema); exec_raw(R"( - -- Two sources on one legacy row: nothing in the old layout splits it, so it folds onto its FIRST source. + CREATE TABLE near_misses ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + occurred_at_ns BIGINT NOT NULL, debounce_counter INTEGER NOT NULL, confirmation_threshold INTEGER NOT NULL, + severity INTEGER NOT NULL, source_id TEXT NOT NULL, resulting_status TEXT NOT NULL DEFAULT ''); + CREATE INDEX idx_near_misses_fault_code ON near_misses(fault_code, id); INSERT INTO faults VALUES ('SHARED', 2, 'shared', 1, 1, 1, 'CONFIRMED', '["/owner_a","/owner_b"]', -1, 1, 0, 5); INSERT INTO faults VALUES ('SOLO', 1, 'solo', 2, 2, 1, 'PREFAILED', '["/owner_c"]', -1, 2, 0, 99); - INSERT INTO faults VALUES ('NOSOURCE', 1, 'none', 3, 3, 1, 'CONFIRMED', '[]', -1, 3, 0, 0); INSERT INTO freeze_frames VALUES ('SHARED', '{"/t":1}', 7); - INSERT INTO freeze_frames VALUES ('GONE', '{"/t":2}', 8); INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) VALUES ('SHARED', '/t', 'std_msgs/msg/Float64', '{}', 9); + INSERT INTO near_misses (fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, severity, + source_id, resulting_status) + VALUES ('SHARED', 5, -1, -3, 2, '/owner_a', 'PREFAILED'), ('SHARED', 6, -2, -3, 2, '/owner_b', 'PREFAILED'); INSERT INTO rosbag_files (fault_code, file_path, format, duration_sec, size_bytes, created_at_ns) VALUES ('SHARED', '/bags/fault_SHARED_1', 'mcap', 5.0, 1024, 11); )"); @@ -3310,77 +3366,88 @@ TEST_F(PgFaultStorageTest, ATableWrittenByThePreviousLayoutIsMigratedInPlace) { std::string::npos) << "the rosbag unique index must widen with the identity"; - // owner is the FIRST legacy reporting source, and reporting_sources is rewritten to it. - EXPECT_EQ(scalar("SELECT owner FROM faults WHERE fault_code = 'SHARED'"), "/owner_a"); - EXPECT_EQ(scalar("SELECT reporting_sources FROM faults WHERE fault_code = 'SHARED'"), "[\"/owner_a\"]"); - EXPECT_EQ(scalar("SELECT owner FROM faults WHERE fault_code = 'NOSOURCE'"), ros2_medkit_fault_manager::kLegacyOwner); + // Every row is the shared record of its code and keeps its reporting_sources as written. + EXPECT_EQ(scalar("SELECT COUNT(*) FROM faults WHERE owner = '*'"), "2"); + EXPECT_EQ(scalar("SELECT reporting_sources FROM faults WHERE fault_code = 'SHARED'"), "[\"/owner_a\",\"/owner_b\"]"); EXPECT_EQ(scalar("SELECT confirmed_at_ns FROM faults WHERE fault_code = 'SOLO'"), "0") << "a pending record has no confirmation time"; EXPECT_EQ(scalar("SELECT confirmed_at_ns FROM faults WHERE fault_code = 'SHARED'"), "5"); - const FaultId shared{"SHARED", "/owner_a"}; + const FaultId shared{"SHARED", ros2_medkit_fault_manager::kSharedOwner}; auto migrated = storage_->get_fault(shared); ASSERT_TRUE(migrated.has_value()); EXPECT_EQ(migrated->status, Fault::STATUS_CONFIRMED); - EXPECT_FALSE(storage_->get_fault(FaultId{"SHARED", "/owner_b"}).has_value()); - EXPECT_TRUE(storage_->get_fault(FaultId{"SOLO", "/owner_c"}).has_value()); + EXPECT_EQ(migrated->reporting_sources, (std::vector{"/owner_a", "/owner_b"})); + EXPECT_EQ(storage_->get_ids_by_code("SHARED").size(), 1u); + EXPECT_TRUE(storage_->get_fault(FaultId{"SOLO", "*"}).has_value()); - // The child rows learned which record they belong to; evidence of a code no record carries stays unassigned. + // Every child row attached to the shared record, none lost. ASSERT_TRUE(storage_->get_freeze_frame(shared).has_value()); EXPECT_EQ(storage_->get_freeze_frame(shared)->captured_at_ns, 7); ASSERT_EQ(storage_->get_snapshots(shared).size(), 1u); - EXPECT_EQ(storage_->get_snapshots(shared).front().owner, "/owner_a"); + EXPECT_EQ(storage_->get_snapshots(shared).front().owner, "*"); ASSERT_TRUE(storage_->get_rosbag_file(shared).has_value()); - EXPECT_EQ(storage_->get_rosbag_file(shared)->owner, "/owner_a"); - EXPECT_EQ(scalar("SELECT COUNT(*) FROM freeze_frames WHERE fault_code = 'GONE' AND owner = ''"), "1"); + EXPECT_EQ(storage_->get_rosbag_file(shared)->owner, "*"); + const auto near_misses = storage_->get_near_misses(shared); + ASSERT_EQ(near_misses.size(), 2u) << "the near misses of both sources belong to the one shared record"; + EXPECT_EQ(near_misses[0].source_id, "/owner_a"); + EXPECT_EQ(near_misses[1].source_id, "/owner_b"); + EXPECT_EQ(scalar("SELECT COUNT(*) FROM near_misses n WHERE NOT EXISTS " + "(SELECT 1 FROM faults f WHERE f.fault_code = n.fault_code AND f.owner = n.owner)"), + "0"); - // After the migration a second owner can hold the same code, which the old key refused. + // The migrated record behaves as any shared record: another source joins it. rclcpp::Clock clock; - EXPECT_TRUE(storage_->report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", "/owner_b", - clock.now().nanoseconds(), default_config())); - EXPECT_EQ(storage_->get_faults_by_code("SHARED").size(), 2u); + EXPECT_FALSE(storage_->report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "c", "/owner_c", + clock.now().nanoseconds(), default_config())); + EXPECT_EQ(storage_->get_fault(shared)->reporting_sources, + (std::vector{"/owner_a", "/owner_b", "/owner_c"})); // Re-opening a migrated database changes nothing. storage_.reset(); storage_ = std::make_unique(conn_info_, test_log_sink()); - EXPECT_EQ(storage_->size(), 4u); - EXPECT_EQ(scalar("SELECT owner FROM faults WHERE fault_code = 'SHARED' AND status = 'CONFIRMED' " - "ORDER BY owner LIMIT 1"), - "/owner_a"); + EXPECT_EQ(storage_->size(), 2u); + EXPECT_EQ(scalar("SELECT COUNT(*) FROM faults WHERE owner = '*'"), "2"); ASSERT_TRUE(storage_->get_freeze_frame(shared).has_value()); + EXPECT_EQ(storage_->get_near_misses(shared).size(), 2u); } -TEST_F(PgFaultStorageTest, ChildRowsOfASharedCodeAreNotGivenAnArbitraryOwner) { - storage_.reset(); - exec_raw("DROP TABLE faults, snapshots, freeze_frames, near_misses, rosbag_files"); - // faults already keyed by owner with TWO owners of one code, snapshots still without the column. - exec_raw(R"( - CREATE TABLE faults ( - fault_code TEXT NOT NULL, severity INTEGER NOT NULL, description TEXT NOT NULL, - first_occurred_ns BIGINT NOT NULL, last_occurred_ns BIGINT NOT NULL, occurrence_count BIGINT NOT NULL, - status TEXT NOT NULL, reporting_sources TEXT NOT NULL, debounce_counter INTEGER NOT NULL DEFAULT 0, - last_failed_ns BIGINT NOT NULL DEFAULT 0, last_passed_ns BIGINT NOT NULL DEFAULT 0, - confirmed_at_ns BIGINT NOT NULL DEFAULT 0, owner TEXT NOT NULL DEFAULT ''); - CREATE UNIQUE INDEX idx_faults_code_owner ON faults(fault_code, owner); - INSERT INTO faults VALUES ('SHARED', 2, 'a', 1, 1, 1, 'CONFIRMED', '["/owner_a"]', -1, 1, 0, 0, '/owner_a'); - INSERT INTO faults VALUES ('SHARED', 2, 'b', 1, 1, 1, 'CONFIRMED', '["/owner_b"]', -1, 1, 0, 0, '/owner_b'); - CREATE TABLE snapshots ( - id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, topic TEXT NOT NULL, - message_type TEXT NOT NULL, data TEXT NOT NULL, captured_at_ns BIGINT NOT NULL, - capture_id BIGINT NOT NULL DEFAULT 0); - INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) - VALUES ('SHARED', '/t', 'std_msgs/msg/Float64', '{}', 9); - )"); +// One record shared by every source of a code: one debounce counter, every source listed. +TEST_F(PgFaultStorageTest, ASharedRecordHasOneCounterAndListsEverySource) { + rclcpp::Clock clock; + const FaultId shared{"ONE_COUNTER", ros2_medkit_fault_manager::kSharedOwner}; + auto config = default_config(); + config.confirmation_threshold = -3; - ASSERT_NO_THROW(storage_ = std::make_unique(conn_info_, test_log_sink())); + EXPECT_TRUE(storage_->report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/src_a", + clock.now().nanoseconds(), config)); + EXPECT_FALSE(storage_->report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", "/src_b", + clock.now().nanoseconds(), config)); + EXPECT_EQ(storage_->get_fault(shared)->status, Fault::STATUS_PREFAILED); + EXPECT_FALSE(storage_->report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "c", "/src_c", + clock.now().nanoseconds(), config)); - EXPECT_TRUE(storage_->get_snapshots(FaultId{"SHARED", "/owner_a"}).empty()) - << "A must not be handed unproven evidence"; - EXPECT_TRUE(storage_->get_snapshots(FaultId{"SHARED", "/owner_b"}).empty()) << "and neither must B"; - auto conn = raw_connection(); - pqxx::nontransaction tx(*conn); - EXPECT_EQ(tx.exec("SELECT COUNT(*) FROM snapshots WHERE owner = ''")[0][0].as(), 1) - << "the row is kept, unassigned"; + EXPECT_EQ(storage_->size(), 1u); + auto record = storage_->get_fault(shared); + ASSERT_TRUE(record.has_value()); + EXPECT_EQ(record->status, Fault::STATUS_CONFIRMED) << "three sources, one counter"; + EXPECT_EQ(record->reporting_sources, (std::vector{"/src_a", "/src_b", "/src_c"})); + EXPECT_EQ(record->occurrence_count, 1u); + ASSERT_EQ(storage_->list_faults(false, 0, {}).size(), 1u); + EXPECT_EQ(storage_->list_fault_entries(false, 0, {}).front().id, shared); + + // The sources of the record are the entities that own its recordings. + RosbagFileInfo link; + link.fault_code = shared.fault_code; + link.owner = shared.owner; + link.file_path = "/bags/fault_ONE_COUNTER_1"; + link.format = "mcap"; + link.created_at_ns = 1; + storage_->store_rosbag_file(link); + for (const char * entity : {"/src_a", "/src_b", "/src_c"}) { + EXPECT_EQ(storage_->list_rosbags_for_entity(entity).size(), 1u) << entity; + } + EXPECT_TRUE(storage_->list_rosbags_for_entity("/src_d").empty()); } int main(int argc, char ** argv) { diff --git a/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp b/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp index 73635ec97..2d6917206 100644 --- a/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp +++ b/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp @@ -1651,9 +1651,9 @@ TEST_F(RosbagCaptureIntegrationTest, AcknowledgingDuringThePostRollKeepsAConfigu storage.set_max_rosbags_per_fault(rosbag_config.max_bags_per_fault); rclcpp::Clock clock; - storage.report_fault_event("ACK_MID_ROLL", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "mid roll", "/node", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage.report_fault_event(FaultId{"ACK_MID_ROLL", "/node"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "mid roll", "/node", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), &storage, rosbag_config, snapshot_config); @@ -1786,9 +1786,9 @@ TEST_F(RosbagCaptureIntegrationTest, AClearWhileTheRowIsStoredTakesTheRowWithIt) rosbag_config.auto_cleanup = true; rosbag_config.max_bags_per_fault = 1; rclcpp::Clock clock; - gated.report_fault_event("CLEARED_IN_STORE", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "store race", kOwner, clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + gated.report_fault_event(FaultId{"CLEARED_IN_STORE", kOwner}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "store race", kOwner, + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), &gated, rosbag_config, snapshot_config); @@ -2104,12 +2104,12 @@ TEST_F(RosbagCaptureIntegrationTest, AttachedFaultTopicsJoinTheEntityScopedRecor auto pub_b = node_b->create_publisher("/scope_topic_b", 10); rclcpp::Clock clock; - storage_->report_fault_event("FAULT_A", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "fault a", "/scope_source_a", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); - storage_->report_fault_event("FAULT_B", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "fault b", "/scope_source_b", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"FAULT_A", "/scope_source_a"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "fault a", "/scope_source_a", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"FAULT_B", "/scope_source_b"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "fault b", "/scope_source_b", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); @@ -2476,12 +2476,12 @@ TEST_F(RosbagCaptureIntegrationTest, EntityScopedPostFaultOnlyBagCarriesOnlyTheF auto pub_b = node_b->create_publisher("/boundary_scope_b", 10); rclcpp::Clock clock; - storage_->report_fault_event("SCOPED_A", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "scoped a", "/boundary_scope_source_a", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); - storage_->report_fault_event("SCOPED_B", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "scoped b", "/boundary_scope_source_b", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"SCOPED_A", "/boundary_scope_source_a"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "scoped a", "/boundary_scope_source_a", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"SCOPED_B", "/boundary_scope_source_b"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "scoped b", "/boundary_scope_source_b", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); @@ -2542,12 +2542,12 @@ TEST_F(RosbagCaptureIntegrationTest, ABoundaryRecordingKeepsItsScopeWhileTheOldF auto pub_b = node_b->create_publisher("/scope_race_b", 10); rclcpp::Clock clock; - storage_->report_fault_event("SCOPE_RACE_A", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "scope race a", "/scope_race_source_a", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); - storage_->report_fault_event("SCOPE_RACE_B", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "scope race b", "/scope_race_source_b", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"SCOPE_RACE_A", "/scope_race_source_a"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "scope race a", "/scope_race_source_a", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"SCOPE_RACE_B", "/scope_race_source_b"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "scope race b", "/scope_race_source_b", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); @@ -3047,18 +3047,18 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultAttachingToABoundaryRecordingWidensIt auto pub_bystander = node_bystander->create_publisher("/widen_bystander", 10); rclcpp::Clock clock; - storage_->report_fault_event("WIDEN_C", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "widen c", "/widen_source_c", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); - storage_->report_fault_event("WIDEN_D", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "widen d", "/widen_source_d", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"WIDEN_C", "/widen_source_c"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "widen c", "/widen_source_c", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"WIDEN_D", "/widen_source_d"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "widen d", "/widen_source_d", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); // The fault that opens the boundary recording needs a reported source too, or its // scope resolves to nothing and the recording degrades to writing every topic - // which is what the bystander assertion below catches. - storage_->report_fault_event("WIDEN_C2", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "widen c2", "/widen_source_c", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"WIDEN_C2", "/widen_source_c"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "widen c2", "/widen_source_c", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); @@ -3206,7 +3206,7 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultClearedWhileItsBagWasOpeningGetsNoRow auto snapshot_config = create_snapshot_config(); rclcpp::Clock clock; - storage_->report_fault_event("CLEARED_WHILE_OPENING", fd::ReportKind::kFailed, + storage_->report_fault_event(FaultId{"CLEARED_WHILE_OPENING", "/test_source"}, fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "cleared while opening", "/test_source", clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); @@ -3242,10 +3242,12 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultPastTheAttachmentCapStillGetsItsTopic rclcpp::Clock clock; const ros2_medkit_fault_manager::DebounceConfig debounce{}; - storage_->report_fault_event("CAP_PRIMARY", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "cap primary", "/cap_primary_source", clock.now().nanoseconds(), debounce); - storage_->report_fault_event("CAP_LATE", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "cap late", "/cap_late_source", clock.now().nanoseconds(), debounce); + storage_->report_fault_event(FaultId{"CAP_PRIMARY", "/cap_primary_source"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "cap primary", "/cap_primary_source", + clock.now().nanoseconds(), debounce); + storage_->report_fault_event(FaultId{"CAP_LATE", "/cap_late_source"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "cap late", "/cap_late_source", + clock.now().nanoseconds(), debounce); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); @@ -3265,8 +3267,9 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultPastTheAttachmentCapStillGetsItsTopic // late fault's topic into the bag for a reason that has nothing to do with the cap. for (int i = 0; i < 40; ++i) { const std::string filler = "CAP_FILLER_" + std::string(i < 10 ? "0" : "") + std::to_string(i); - storage_->report_fault_event(filler, fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "cap filler", "/cap_primary_source", clock.now().nanoseconds(), debounce); + storage_->report_fault_event(FaultId{filler, "/cap_primary_source"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "cap filler", "/cap_primary_source", + clock.now().nanoseconds(), debounce); capture.on_fault_confirmed({filler, "/cap_primary_source"}); } capture.on_fault_confirmed({"CAP_LATE", "/cap_late_source"}); @@ -3327,9 +3330,9 @@ TEST_F(RosbagCaptureIntegrationTest, ReconfirmingTheRecordingsOwnFaultResolvesNo auto pub = source->create_publisher("/reconfirm_probe", 10); rclcpp::Clock clock; - storage_ptr->report_fault_event("RECONFIRMED", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "reconfirmed", "/reconfirm_source", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage_ptr->report_fault_event(FaultId{"RECONFIRMED", "/reconfirm_source"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "reconfirmed", "/reconfirm_source", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); RosbagCapture capture(node_.get(), storage_ptr, rosbag_config, snapshot_config); capture.start(); diff --git a/src/ros2_medkit_fault_manager/test/test_rosbag_storage_parity.cpp b/src/ros2_medkit_fault_manager/test/test_rosbag_storage_parity.cpp index ecc6028a0..07b358370 100644 --- a/src/ros2_medkit_fault_manager/test/test_rosbag_storage_parity.cpp +++ b/src/ros2_medkit_fault_manager/test/test_rosbag_storage_parity.cpp @@ -449,8 +449,9 @@ class SnapshotRetentionParityTest : public ::testing::Test { void confirm(const std::string & code) { rclcpp::Clock clock; - storage_->report_fault_event(code, fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "parity", - kOwner, clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{code, kOwner}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "parity", kOwner, + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); } std::filesystem::path dir_; diff --git a/src/ros2_medkit_fault_manager/test/test_snapshot_capture.cpp b/src/ros2_medkit_fault_manager/test/test_snapshot_capture.cpp index 70b32d099..80c7283fa 100644 --- a/src/ros2_medkit_fault_manager/test/test_snapshot_capture.cpp +++ b/src/ros2_medkit_fault_manager/test/test_snapshot_capture.cpp @@ -527,9 +527,9 @@ class EntityDefaultCaptureTest : public SnapshotCaptureTest { /// Store a CONFIRMED fault whose reporting source is this test node. void store_fault_from_this_node(const std::string & fault_code) { ros2_medkit_fault_manager::DebounceConfig debounce; // threshold -1: first FAILED confirms - storage_->report_fault_event(fault_code, fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "test fault", node_->get_fully_qualified_name(), rclcpp::Clock().now().nanoseconds(), - debounce); + storage_->report_fault_event(ros2_medkit_fault_manager::FaultId{fault_code, node_->get_fully_qualified_name()}, + fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "test fault", + node_->get_fully_qualified_name(), rclcpp::Clock().now().nanoseconds(), debounce); } /// The record this node owns: the entity-default scope is resolved from the owner. @@ -678,8 +678,9 @@ TEST_F(EntityDefaultCaptureTest, UnresolvableSourceWritesNoRow) { // Plugin-style bare entity id: not a live node, resolves to no topics. ros2_medkit_fault_manager::DebounceConfig debounce; - storage_->report_fault_event("PLC_FAULT", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "plc fault", "plc_app", rclcpp::Clock().now().nanoseconds(), debounce); + storage_->report_fault_event(ros2_medkit_fault_manager::FaultId{"PLC_FAULT", "plc_app"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "plc fault", "plc_app", + rclcpp::Clock().now().nanoseconds(), debounce); capture.capture({"PLC_FAULT", "plc_app"}); @@ -700,8 +701,9 @@ TEST_F(EntityDefaultCaptureTest, BarePluginIdNeverMatchesSameNamedNode) { SnapshotCapture capture(node_.get(), storage_.get(), config); ros2_medkit_fault_manager::DebounceConfig debounce; - storage_->report_fault_event("BARE_ID_FAULT", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "plugin fault", node_->get_name(), rclcpp::Clock().now().nanoseconds(), debounce); + storage_->report_fault_event(ros2_medkit_fault_manager::FaultId{"BARE_ID_FAULT", node_->get_name()}, + fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "plugin fault", + node_->get_name(), rclcpp::Clock().now().nanoseconds(), debounce); ScopedPublisherThread pub_thread([&pub](std::atomic & stop) { while (!stop.load()) { @@ -733,8 +735,9 @@ void await_publisher(const std::shared_ptr & node, const std::stri /// Put the fault in the store as CONFIRMED, the way a real confirmation would. void confirm_fault(ros2_medkit_fault_manager::FaultStorage * storage, const std::string & code) { rclcpp::Clock clock; - storage->report_fault_event(code, fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "test", - "/node", clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage->report_fault_event(ros2_medkit_fault_manager::FaultId{code, "/node"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "test", "/node", clock.now().nanoseconds(), + ros2_medkit_fault_manager::DebounceConfig{}); } } // namespace diff --git a/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp b/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp index 0bfb24958..4a7bdf261 100644 --- a/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp +++ b/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp @@ -39,6 +39,7 @@ #include "test_log_sink.hpp" using ros2_medkit_fault_manager::DebounceConfig; +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::RosbagFileInfo; using ros2_medkit_fault_manager::SqliteFaultStorage; using ros2_medkit_msgs::msg::Fault; @@ -77,9 +78,9 @@ TEST_F(SqliteFaultStorageTest, ReportNewFaultEvent) { rclcpp::Clock clock; auto timestamp = clock.now(); - bool is_new = storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "Motor temperature exceeded threshold", "/powertrain/motor", - timestamp.nanoseconds(), default_config()); + bool is_new = storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Motor temperature exceeded threshold", + "/powertrain/motor", timestamp.nanoseconds(), default_config()); EXPECT_TRUE(is_new); EXPECT_EQ(storage_->size(), 1u); @@ -90,8 +91,9 @@ TEST_F(SqliteFaultStorageTest, PassedEventForNonExistentFaultIgnored) { rclcpp::Clock clock; auto timestamp = clock.now(); - bool is_new = storage_->report_fault_event("NON_EXISTENT", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "Test", - "/node1", timestamp.nanoseconds(), default_config()); + bool is_new = + storage_->report_fault_event(FaultId{"NON_EXISTENT", "/node1"}, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, + "Test", "/node1", timestamp.nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_->size(), 0u); @@ -102,12 +104,13 @@ TEST_F(SqliteFaultStorageTest, ReportExistingFaultEventUpdates) { auto timestamp1 = clock.now(); auto timestamp2 = clock.now(); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Initial report", - "/powertrain/motor1", timestamp1.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "Initial report", "/powertrain/motor1", timestamp1.nanoseconds(), + default_config()); - bool is_new = - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Second report", - "/powertrain/motor1", timestamp2.nanoseconds(), default_config()); + bool is_new = storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Second report", "/powertrain/motor1", + timestamp2.nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_->size(), 1u); @@ -131,12 +134,14 @@ TEST_F(SqliteFaultStorageTest, TwoSourcesReportingOneCodeAreTwoRecords) { const rclcpp::Time a_time(1000, 0, RCL_SYSTEM_TIME); const rclcpp::Time b_time(2000, 0, RCL_SYSTEM_TIME); - EXPECT_TRUE(storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "from a", - "/powertrain/motor1", a_time.nanoseconds(), config)); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "from a", - "/powertrain/motor1", a_time.nanoseconds(), config); - EXPECT_TRUE(storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "from b", - "/powertrain/motor2", b_time.nanoseconds(), config)); + EXPECT_TRUE(storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "from a", "/powertrain/motor1", a_time.nanoseconds(), + config)); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "from a", "/powertrain/motor1", a_time.nanoseconds(), config); + EXPECT_TRUE(storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor2"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "from b", "/powertrain/motor2", b_time.nanoseconds(), + config)); EXPECT_EQ(storage_->size(), 2u); @@ -154,21 +159,19 @@ TEST_F(SqliteFaultStorageTest, TwoSourcesReportingOneCodeAreTwoRecords) { EXPECT_EQ(b->first_occurred_ns, b_time.nanoseconds()); } -TEST_F(SqliteFaultStorageTest, GetFaultsByCodeReturnsEveryOwnerAndGetFaultExactlyOne) { +TEST_F(SqliteFaultStorageTest, GetIdsByCodeReturnsEveryOwnerAndGetFaultExactlyOne) { rclcpp::Clock clock; - storage_->report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_b", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("SHARED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/owner_a", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"SHARED", "/owner_b"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_b", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"SHARED", "/owner_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_a", clock.now().nanoseconds(), default_config()); - const auto shared = storage_->get_faults_by_code("SHARED"); + const auto shared = storage_->get_ids_by_code("SHARED"); ASSERT_EQ(shared.size(), 2u); - ASSERT_EQ(shared[0].reporting_sources.size(), 1u); - ASSERT_EQ(shared[1].reporting_sources.size(), 1u); - EXPECT_EQ(shared[0].reporting_sources.front(), "/owner_a") << "ordered by owner, like the in-memory backend"; - EXPECT_EQ(shared[1].reporting_sources.front(), "/owner_b"); + EXPECT_EQ(shared[0].owner, "/owner_a") << "ordered by owner, like the in-memory backend"; + EXPECT_EQ(shared[1].owner, "/owner_b"); - EXPECT_TRUE(storage_->get_faults_by_code("NEVER_REPORTED").empty()); + EXPECT_TRUE(storage_->get_ids_by_code("NEVER_REPORTED").empty()); ASSERT_TRUE(storage_->get_fault({"SHARED", "/owner_a"}).has_value()); EXPECT_FALSE(storage_->get_fault({"SHARED", "/nobody"}).has_value()); } @@ -179,8 +182,8 @@ TEST_F(SqliteFaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCo // Simulate a level-triggered poller re-reporting the same still-true condition // every cycle (issue #11): occurrence_count must stay at 1, not grow per poll. for (int i = 0; i < 25; ++i) { - storage_->report_fault_event("TANK_OVERFILL", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "level = 95 > 80", - "/tank", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"TANK_OVERFILL", "/tank"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "level = 95 > 80", "/tank", clock.now().nanoseconds(), default_config()); } auto fault = storage_->get_fault({"TANK_OVERFILL", "/tank"}); @@ -194,8 +197,8 @@ TEST_F(SqliteFaultStorageTest, ListFaultsDefaultReturnsConfirmedOnly) { auto timestamp = clock.now(); // With default threshold=-1, single report confirms immediately - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); // Default query should return the CONFIRMED fault auto faults = storage_->list_faults(false, 0, {}); @@ -212,8 +215,8 @@ TEST_F(SqliteFaultStorageTest, ListFaultsWithPrefailedStatus) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), config); // Query with PREFAILED status auto faults = storage_->list_faults(false, 0, {Fault::STATUS_PREFAILED}); @@ -227,10 +230,10 @@ TEST_F(SqliteFaultStorageTest, ListFaultsFilterBySeverity) { auto timestamp = clock.now(); // With default threshold=-1, faults are immediately CONFIRMED - storage_->report_fault_event("FAULT_INFO", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Info", "/node1", - timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_ERROR", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Error", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_INFO", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Info", + "/node1", timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_ERROR", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Error", "/node1", timestamp.nanoseconds(), default_config()); // Filter by ERROR severity (query CONFIRMED since that's the default status now) auto faults = storage_->list_faults(true, Fault::SEVERITY_ERROR, {Fault::STATUS_CONFIRMED}); @@ -242,8 +245,8 @@ TEST_F(SqliteFaultStorageTest, ClearFault) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Test", "/node1", timestamp.nanoseconds(), default_config()); bool cleared = storage_->clear_fault({"MOTOR_OVERHEAT", "/node1"}); EXPECT_TRUE(cleared); @@ -265,10 +268,10 @@ TEST_F(SqliteFaultStorageTest, PassedEventDoesNotAdvanceLastOccurred) { const rclcpp::Time failed_at(1000, 0, RCL_SYSTEM_TIME); const rclcpp::Time passed_at(9000, 0, RCL_SYSTEM_TIME); - storage_->report_fault_event("FAULT_LO", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - failed_at.nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_LO", fd::ReportKind::kPassed, 0, "", "/node1", passed_at.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"FAULT_LO", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", failed_at.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_LO", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + passed_at.nanoseconds(), default_config()); auto fault = storage_->get_fault({"FAULT_LO", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -284,8 +287,8 @@ TEST_F(SqliteFaultStorageTest, ReopenRepairsLastOccurredInflatedByOldPassedBug) // again would keep the wrong timestamp forever. Opening the storage must // repair them from last_failed_ns. const rclcpp::Time failed_at(1000, 0, RCL_SYSTEM_TIME); - storage_->report_fault_event("FAULT_MIG", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - failed_at.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_MIG", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", failed_at.nanoseconds(), default_config()); storage_.reset(); // Simulate the old bug: a PASSED event at t=9000s re-dated the row. @@ -307,8 +310,8 @@ TEST_F(SqliteFaultStorageTest, GetClearedFaults) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); storage_->clear_fault({"FAULT_1", "/node1"}); // Query cleared faults @@ -322,8 +325,8 @@ TEST_F(SqliteFaultStorageTest, InvalidStatusDefaultsToConfirmed) { auto timestamp = clock.now(); // With default threshold=-1, fault is immediately CONFIRMED - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); // Query with invalid status - defaults to CONFIRMED, which now matches our fault auto faults = storage_->list_faults(false, 0, {"INVALID_STATUS"}); @@ -338,10 +341,10 @@ TEST_F(SqliteFaultStorageTest, PersistenceAcrossRestarts) { // With default threshold=-1, faults are immediately CONFIRMED // Report some faults - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Persistent fault 1", - "/node1", timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_2", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Persistent fault 2", "/node2", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Persistent fault 1", "/node1", timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_2", "/node2"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, + "Persistent fault 2", "/node2", timestamp.nanoseconds(), default_config()); storage_->clear_fault({"FAULT_2", "/node2"}); // Verify initial state @@ -375,8 +378,8 @@ TEST_F(SqliteFaultStorageTest, TimestampPrecision) { int64_t test_ns = 1735312456123456789LL; // Specific nanosecond timestamp rclcpp::Time timestamp(test_ns, RCL_SYSTEM_TIME); - storage_->report_fault_event("FAULT_TS", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Timestamp test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_TS", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_INFO, + "Timestamp test", "/node1", timestamp.nanoseconds(), default_config()); auto fault = storage_->get_fault({"FAULT_TS", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -396,8 +399,8 @@ TEST(SqliteInMemoryTest, InMemoryDatabase) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage.report_fault_event("MEM_FAULT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "In-memory test", "/test", - timestamp.nanoseconds(), default_config()); + storage.report_fault_event(FaultId{"MEM_FAULT", "/test"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, + "In-memory test", "/test", timestamp.nanoseconds(), default_config()); EXPECT_EQ(storage.size(), 1u); EXPECT_TRUE(storage.contains({"MEM_FAULT", "/test"})); @@ -413,8 +416,8 @@ TEST_F(SqliteFaultStorageTest, ReportingSourcesJsonHandling) { // the key rather than through a JSON array. const std::vector owners = {"/node/path/with/slashes", "/another/node", "/special\"chars"}; for (const auto & owner : owners) { - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", owner, - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MULTI_SRC", owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Multi-source", owner, timestamp.nanoseconds(), default_config()); } EXPECT_EQ(storage_->size(), owners.size()); @@ -470,8 +473,8 @@ TEST_F(SqliteFaultStorageTest, FaultStaysPrefailedAboveThreshold) { // Both reports from one source: the counter belongs to that source's record. for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); } auto fault = storage_->get_fault({"FAULT_1", "/node1"}); @@ -489,8 +492,8 @@ TEST_F(SqliteFaultStorageTest, FaultConfirmsAtThreshold) { storage_->set_debounce_config(config); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); } auto fault = storage_->get_fault({"FAULT_1", "/node1"}); @@ -506,8 +509,8 @@ TEST_F(SqliteFaultStorageTest, ImmediateConfirmationWithThresholdZero) { storage_->set_debounce_config(config); // Single report should confirm immediately - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -519,8 +522,8 @@ TEST_F(SqliteFaultStorageTest, CriticalSeverityBypassesDebounce) { rclcpp::Clock clock; // CRITICAL severity should confirm immediately - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "Critical test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, + "Critical test", "/node1", clock.now().nanoseconds(), default_config()); auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -533,8 +536,9 @@ TEST_F(SqliteFaultStorageTest, ClearedFaultCanBeReactivated) { // Report to confirm (with default threshold=-1, single report confirms) auto first_ts = clock.now(); - bool is_new = storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Initial", - "/node1", first_ts.nanoseconds(), default_config()); + bool is_new = + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Initial", "/node1", first_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); auto fault = storage_->get_fault({"FAULT_1", "/node1"}); @@ -551,8 +555,8 @@ TEST_F(SqliteFaultStorageTest, ClearedFaultCanBeReactivated) { // The same owner reports again after a gap - reactivates its own record rclcpp::Time second_ts(first_ts.nanoseconds() + 1'000'000'000LL); // +1s - is_new = storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Reactivated", - "/node1", second_ts.nanoseconds(), default_config()); + is_new = storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Reactivated", "/node1", second_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); // Should return true like a new fault fault = storage_->get_fault({"FAULT_1", "/node1"}); @@ -570,14 +574,14 @@ TEST_F(SqliteFaultStorageTest, PassedEventForClearedFaultIgnored) { rclcpp::Clock clock; // Report and confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); // Clear the fault storage_->clear_fault({"FAULT_1", "/node1"}); // PASSED event should be ignored for CLEARED fault - bool result = storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", + bool result = storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), default_config()); EXPECT_FALSE(result); @@ -596,8 +600,8 @@ TEST_F(SqliteFaultStorageTest, ClearedFaultReactivationRestartsDebounce) { // One source reports 3 times to confirm its own record for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); } auto fault = storage_->get_fault({"FAULT_1", "/node1"}); @@ -608,8 +612,8 @@ TEST_F(SqliteFaultStorageTest, ClearedFaultReactivationRestartsDebounce) { storage_->clear_fault({"FAULT_1", "/node1"}); // Reactivate - should start in PREFAILED with counter=-1 - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); @@ -617,8 +621,8 @@ TEST_F(SqliteFaultStorageTest, ClearedFaultReactivationRestartsDebounce) { // Report 2 more times to re-confirm for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); } fault = storage_->get_fault({"FAULT_1", "/node1"}); @@ -631,8 +635,8 @@ TEST_F(SqliteFaultStorageTest, ConfirmationPersistsAfterReopen) { // One source reports 3 times to confirm its own record for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); } // Close and reopen storage @@ -654,14 +658,14 @@ TEST_F(SqliteFaultStorageTest, PassedEventIncrementsCounter) { // Report 2 FAILED events from one source (its counter -2, PREFAILED) for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); } // Report 3 PASSED events (counter -2 -> +1) for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } auto fault = storage_->get_fault({"FAULT_1", "/node1"}); @@ -680,14 +684,14 @@ TEST_F(SqliteFaultStorageTest, HeartbeatHealClampedAndStatusLatched) { config.healing_enabled = true; config.healing_threshold = 3; - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); ASSERT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); // Long heal heartbeat: counter clamps at healing_threshold instead of running off. for (int i = 0; i < 100; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } auto healed = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(healed.has_value()); @@ -696,13 +700,13 @@ TEST_F(SqliteFaultStorageTest, HeartbeatHealClampedAndStatusLatched) { // Re-confirmation is now bounded to (healing_threshold - confirmation_threshold) // reports, and the healed status latches until the counter walks all the way down. for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); EXPECT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); } - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); auto reconfirmed = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(reconfirmed.has_value()); EXPECT_EQ(reconfirmed->status, Fault::STATUS_CONFIRMED); @@ -717,21 +721,21 @@ TEST_F(SqliteFaultStorageTest, ConfirmedFaultSurvivesHealHeartbeat) { config.healing_enabled = false; config.healing_threshold = 3; // upper clamp on the counter, even with healing disabled - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); ASSERT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 20; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); EXPECT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED) << "un-confirmed after " << (i + 1); } // The counter is now positive (clamped at +3). One FAILED must keep it CONFIRMED, not flip it to // PREPASSED via the `counter > 0` branch - that was the asymmetric-latch bug. - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); EXPECT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); } @@ -745,16 +749,17 @@ TEST_F(SqliteFaultStorageTest, ConfirmedLatchSurvivesFailedAtPositiveCounter) { config.critical_immediate_confirm = true; // CRITICAL confirms immediately at counter -1. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); // PASSED events raise the counter into positive territory; latch keeps it CONFIRMED. for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); // One normal FAILED: counter still positive, must NOT become PREPASSED. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); } @@ -767,20 +772,22 @@ TEST_F(SqliteFaultStorageTest, HealedLatchSurvivesPassedAtNegativeCounter) { config.healing_enabled = true; config.healing_threshold = 3; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { // counter -1 -> +3 -> HEALED - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); // FAILED events drive the counter negative while HEALED is latched (not yet at confirmation -3). for (int i = 0; i < 5; ++i) { // +3 -> -2 - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); } // One PASSED: counter goes -2 -> -1, still negative; must stay HEALED, not flip to PREFAILED. - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } @@ -792,23 +799,26 @@ TEST_F(SqliteFaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePasse config.healing_threshold = 3; config.critical_immediate_confirm = true; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { // -1 -> +3 -> HEALED - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); // CRITICAL confirms immediately from HEALED; counter must not stay at +2 where one PASSED heals it. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { // -1 -> +2, still CONFIRMED - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); } - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } @@ -820,7 +830,7 @@ TEST_F(SqliteFaultStorageTest, RunawayCounterFromOldDbRecovers) { config.confirmation_threshold = -1; config.healing_threshold = 3; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); // Simulate the old bug: poke a huge counter directly into the DB behind the storage's back. { @@ -834,7 +844,7 @@ TEST_F(SqliteFaultStorageTest, RunawayCounterFromOldDbRecovers) { // The counter is clamped back into range on first touch, so a bounded number of FAILED events // re-confirms (healing_threshold - confirmation_threshold = 4), not the ~100000 the runaway implied. for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); } EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); @@ -846,10 +856,11 @@ TEST_F(SqliteFaultStorageTest, ReclassifyHealedAsCleared) { config.healing_enabled = true; config.healing_threshold = 3; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); @@ -867,10 +878,11 @@ TEST_F(SqliteFaultStorageTest, ReclassifyKeepsCodesTheCallerKeepsHealed) { config.healing_threshold = 3; for (const std::string code : {"KEEP", "FLIP"}) { - storage_->report_fault_event(code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{code, "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event(code, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{code, "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } ASSERT_EQ(storage_->get_fault({code, "/n"})->status, Fault::STATUS_HEALED); } @@ -892,13 +904,13 @@ TEST_F(SqliteFaultStorageTest, HealingWhenEnabled) { storage_->set_debounce_config(config); // Report 1 FAILED event - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); // Report 4 PASSED events (counter = -1 + 4 = +3, reaches healing threshold) for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } auto fault = storage_->get_fault({"FAULT_1", "/node1"}); @@ -914,8 +926,8 @@ TEST_F(SqliteFaultStorageTest, TimeBasedConfirmationWhenEnabled) { storage_->set_debounce_config(config); auto now = clock.now(); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - now.nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", now.nanoseconds(), config); // Check before timeout - should not confirm auto before_timeout = rclcpp::Time(now.nanoseconds() + static_cast(5e9)); @@ -957,15 +969,15 @@ TEST_F(SqliteFaultStorageTest, ConfirmedAtRecordedOnImmediateConfirmation) { rclcpp::Clock clock; auto t = clock.now(); // Default config confirms on the first FAILED event. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t.nanoseconds(), default_config()); EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(read_confirmed_at(temp_db_path_, "F"), t.nanoseconds()); // A later FAILED on an already-confirmed fault must NOT move the timestamp. auto t2 = rclcpp::Time(t.nanoseconds() + static_cast(5e9)); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t2.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t2.nanoseconds(), default_config()); EXPECT_EQ(read_confirmed_at(temp_db_path_, "F"), t.nanoseconds()); } @@ -975,14 +987,14 @@ TEST_F(SqliteFaultStorageTest, ConfirmedAtRecordedOnDebouncedConfirmation) { config.confirmation_threshold = -2; // second FAILED event confirms auto t1 = clock.now(); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t1.nanoseconds(), - config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t1.nanoseconds(), config); EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_PREFAILED); EXPECT_EQ(read_confirmed_at(temp_db_path_, "F"), 0) << "not confirmed yet: no confirmation timestamp"; auto t2 = rclcpp::Time(t1.nanoseconds() + static_cast(1e9)); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t2.nanoseconds(), - config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t2.nanoseconds(), config); EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(read_confirmed_at(temp_db_path_, "F"), t2.nanoseconds()); } @@ -995,8 +1007,8 @@ TEST_F(SqliteFaultStorageTest, ConfirmedAtRecordedOnTimeBasedConfirmation) { storage_->set_debounce_config(config); auto now = clock.now(); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", now.nanoseconds(), - config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + now.nanoseconds(), config); auto after_timeout = rclcpp::Time(now.nanoseconds() + static_cast(15e9)); auto confirmed = storage_->check_time_based_confirmation(after_timeout.nanoseconds()); @@ -1025,13 +1037,14 @@ TEST_F(SqliteFaultStorageTest, ConfirmedAtColumnMigratedIntoOldDatabase) { sqlite3_close(raw); } storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - EXPECT_TRUE(storage_->contains({"OLD", "/n"})) << "the owner is backfilled from the legacy first source"; + EXPECT_TRUE(storage_->contains({"OLD", ros2_medkit_fault_manager::kSharedOwner})) + << "a legacy row becomes the shared record of its code"; EXPECT_EQ(read_confirmed_at(temp_db_path_, "OLD"), 0) << "pre-migration rows carry no confirmation time"; rclcpp::Clock clock; auto t = clock.now(); - storage_->report_fault_event("NEW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"NEW", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t.nanoseconds(), default_config()); EXPECT_EQ(read_confirmed_at(temp_db_path_, "NEW"), t.nanoseconds()); } @@ -1096,11 +1109,11 @@ std::string read_text(const std::filesystem::path & db_path, const std::string & } // namespace -// The identity of a faults row changed, and SQLite cannot change a PRIMARY KEY in -// place while CREATE TABLE IF NOT EXISTS silently skips an existing table. A database -// written by an earlier release therefore has to be rebuilt on open, and its child -// rows have to learn which record they belong to. -TEST_F(SqliteFaultStorageTest, LegacyDatabaseGainsTheOwnerColumnOnOpen) { +// A database written when every fault code was one record, fed by every source. SQLite cannot +// change a PRIMARY KEY in place while CREATE TABLE IF NOT EXISTS silently skips an existing +// table, so the database is rebuilt on open: every row becomes the shared record of its code and +// every child row attaches to it. +TEST_F(SqliteFaultStorageTest, LegacyDatabaseBecomesSharedRecordsOnOpen) { storage_.reset(); std::filesystem::remove(temp_db_path_); { @@ -1115,9 +1128,6 @@ TEST_F(SqliteFaultStorageTest, LegacyDatabaseGainsTheOwnerColumnOnOpen) { "debounce_counter INTEGER NOT NULL DEFAULT 0, " "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " "confirmed_at_ns INTEGER NOT NULL DEFAULT 0);" - // Two sources on one legacy row: the old schema held no per-source counter, - // status or timestamps, so there is nothing to split it by and it folds onto - // its FIRST source. "INSERT INTO faults VALUES ('SHARED', 2, 'shared', 1, 1, 1, 'CONFIRMED', " "'[\"/owner_a\",\"/owner_b\"]', -1, 1, 0, 0);" "INSERT INTO faults VALUES ('SOLO', 1, 'solo', 2, 2, 1, 'CONFIRMED', '[\"/owner_c\"]', -1, 2, 0, 0);" @@ -1129,6 +1139,14 @@ TEST_F(SqliteFaultStorageTest, LegacyDatabaseGainsTheOwnerColumnOnOpen) { "captured_at_ns INTEGER NOT NULL);" "INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) " "VALUES ('SHARED', '/t', 'std_msgs/msg/Float64', '{}', 9);" + "CREATE TABLE near_misses (id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, " + "occurred_at_ns INTEGER NOT NULL, debounce_counter INTEGER NOT NULL, " + "confirmation_threshold INTEGER NOT NULL, severity INTEGER NOT NULL, source_id TEXT NOT NULL, " + "resulting_status TEXT NOT NULL DEFAULT '');" + "INSERT INTO near_misses (fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, " + "severity, source_id, resulting_status) VALUES ('SHARED', 5, -1, -3, 2, '/owner_a', 'PREFAILED');" + "INSERT INTO near_misses (fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, " + "severity, source_id, resulting_status) VALUES ('SHARED', 6, -2, -3, 2, '/owner_b', 'PREFAILED');" "CREATE TABLE rosbag_files (id INTEGER PRIMARY KEY AUTOINCREMENT, " "fault_code TEXT NOT NULL UNIQUE, file_path TEXT NOT NULL, format TEXT NOT NULL, " "duration_sec REAL NOT NULL, size_bytes INTEGER NOT NULL, created_at_ns INTEGER NOT NULL);" @@ -1147,44 +1165,48 @@ TEST_F(SqliteFaultStorageTest, LegacyDatabaseGainsTheOwnerColumnOnOpen) { EXPECT_TRUE(unique); ASSERT_TRUE(has_index_over(temp_db_path_, "freeze_frames", {"fault_code", "owner"}, &unique)); EXPECT_TRUE(unique); - // The rosbag unique index widened with the identity. Without the DROP INDEX in the - // migration, CREATE INDEX IF NOT EXISTS would leave the old two-column shape here. ASSERT_TRUE(has_index_over(temp_db_path_, "rosbag_files", {"fault_code", "owner", "file_path"}, &unique)); EXPECT_TRUE(unique); - // owner is backfilled with the FIRST legacy reporting source, and reporting_sources - // is rewritten to match so the column and the identity cannot disagree. - EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = 'SHARED'"), "/owner_a"); + // Every row is the shared record of its code and keeps its reporting_sources as written. + EXPECT_EQ(read_text(temp_db_path_, "SELECT COUNT(*) FROM faults WHERE owner = '*'"), "2"); EXPECT_EQ(read_text(temp_db_path_, "SELECT reporting_sources FROM faults WHERE fault_code = 'SHARED'"), - "[\"/owner_a\"]"); - EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = 'SOLO'"), "/owner_c"); + "[\"/owner_a\",\"/owner_b\"]"); - const ros2_medkit_fault_manager::FaultId shared{"SHARED", "/owner_a"}; + const ros2_medkit_fault_manager::FaultId shared{"SHARED", ros2_medkit_fault_manager::kSharedOwner}; auto migrated = storage_->get_fault(shared); ASSERT_TRUE(migrated.has_value()); EXPECT_EQ(migrated->status, Fault::STATUS_CONFIRMED); - EXPECT_EQ(migrated->reporting_sources, std::vector{"/owner_a"}); - EXPECT_FALSE(storage_->get_fault({"SHARED", "/owner_b"}).has_value()) << "the second source had no state to migrate"; - EXPECT_TRUE(storage_->get_fault({"SOLO", "/owner_c"}).has_value()); + EXPECT_EQ(migrated->reporting_sources, (std::vector{"/owner_a", "/owner_b"})); + EXPECT_EQ(storage_->get_ids_by_code("SHARED").size(), 1u); + EXPECT_TRUE(storage_->get_fault({"SOLO", "*"}).has_value()); - // The child rows learned which record they belong to. + // Every child row attached to the shared record, none lost. ASSERT_TRUE(storage_->get_freeze_frame(shared).has_value()); EXPECT_EQ(storage_->get_freeze_frame(shared)->captured_at_ns, 7); ASSERT_EQ(storage_->get_snapshots(shared).size(), 1u); - EXPECT_EQ(storage_->get_snapshots(shared).front().owner, "/owner_a"); + EXPECT_EQ(storage_->get_snapshots(shared).front().owner, "*"); ASSERT_TRUE(storage_->get_rosbag_file(shared).has_value()); - EXPECT_EQ(storage_->get_rosbag_file(shared)->owner, "/owner_a"); + EXPECT_EQ(storage_->get_rosbag_file(shared)->owner, "*"); + const auto near_misses = storage_->get_near_misses(shared); + ASSERT_EQ(near_misses.size(), 2u) << "the near misses of both sources belong to the one shared record"; + EXPECT_EQ(near_misses[0].source_id, "/owner_a"); + EXPECT_EQ(near_misses[1].source_id, "/owner_b"); + EXPECT_EQ(read_text(temp_db_path_, + "SELECT COUNT(*) FROM near_misses n WHERE NOT EXISTS " + "(SELECT 1 FROM faults f WHERE f.fault_code = n.fault_code AND f.owner = n.owner)"), + "0"); - // Re-opening a migrated database changes nothing: every probe is per table, so the - // second open neither re-ALTERs a column that is there nor rebuilds a table again. + // Re-opening a migrated database changes nothing. storage_.reset(); storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); EXPECT_EQ(storage_->size(), 2u); - EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = 'SHARED'"), "/owner_a"); + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = 'SHARED'"), "*"); ASSERT_TRUE(storage_->get_freeze_frame(shared).has_value()); EXPECT_EQ(storage_->get_snapshots(shared).size(), 1u); ASSERT_TRUE(storage_->get_rosbag_file(shared).has_value()); + EXPECT_EQ(storage_->get_near_misses(shared).size(), 2u); EXPECT_TRUE(has_index_over(temp_db_path_, "faults", {"fault_code", "owner"})); } @@ -1279,17 +1301,6 @@ namespace { /// "\x01" followed by a hex digit would be read as one wider escape. constexpr char kCtrl = '\x01'; -/// Run one statement on a raw handle, binding @p text as the single parameter. Legacy -/// fixtures carry bytes that cannot survive being pasted into a SQL literal, so the -/// values that matter go in through a binding. -void exec_bound(sqlite3 * raw, const char * sql, const std::string & text) { - sqlite3_stmt * stmt = nullptr; - ASSERT_EQ(sqlite3_prepare_v2(raw, sql, -1, &stmt, nullptr), SQLITE_OK) << sqlite3_errmsg(raw); - ASSERT_EQ(sqlite3_bind_text(stmt, 1, text.c_str(), static_cast(text.size()), SQLITE_TRANSIENT), SQLITE_OK); - ASSERT_EQ(sqlite3_step(stmt), SQLITE_DONE) << sqlite3_errmsg(raw); - sqlite3_finalize(stmt); -} - /// Captures rcutils log output while alive and restores the console handler on every /// exit path, so a live capture never swallows the output of later cases in this /// binary. Same shape as the one the rosbag suite uses: the handler is a plain C @@ -1359,129 +1370,6 @@ constexpr const char * kLegacyFaultsDdl = } // namespace -// A migration that refuses a row refuses the database. The rebuild runs inside one -// transaction, so a row it cannot read rolls the whole thing back, and because the -// migration is re-attempted on every open the node then fails to start for as long as -// that row exists. The column it reads was written by a serializer that escaped only -// the quote, the backslash and \b \f \n \r \t, so any other control byte in a -// source_id produced text no JSON parser accepts. Recovery therefore reads the text -// itself instead of requiring it to be valid JSON, and a row from which nothing can be -// read is migrated with an empty owner rather than taking the database down. -TEST_F(SqliteFaultStorageTest, LegacyReportingSourcesThatAreNotValidJsonStillMigrate) { - storage_.reset(); - std::filesystem::remove(temp_db_path_); - - // The five shapes, with the owner each must produce. - const std::string ctrl_pair = std::string("a") + kCtrl + "b"; - const std::string device = std::string("/dev/ttyUSB0") + kCtrl + "reader"; - const std::string kLegacy = ros2_medkit_fault_manager::kLegacyOwner; - const std::vector> cases = { - {"", kLegacy}, // nothing recorded - {"sensor_a", "sensor_a"}, // bare word, never an array - {"[]", kLegacy}, // array with no source - {"[\"" + ctrl_pair + "\"]", ctrl_pair}, // control byte inside the array - {"[\"" + device + "\"]", device}, // the reported real-world case - {"\xEF\xBB\xBF[\"/bom_source\"]", kLegacy}, // array behind a byte-order mark - {"{\"source\": \"/wrapped\"}", kLegacy}, // an object, not the array this reads - }; - - { - sqlite3 * raw = nullptr; - ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); - ASSERT_EQ(sqlite3_exec(raw, kLegacyFaultsDdl, nullptr, nullptr, nullptr), SQLITE_OK); - for (size_t i = 0; i < cases.size(); ++i) { - const std::string insert = "INSERT INTO faults VALUES ('CODE_" + std::to_string(i) + - "', 2, 'legacy', 1, 1, 1, 'CONFIRMED', ?, -1, 1, 0, 0);"; - exec_bound(raw, insert.c_str(), cases[i].first); - } - sqlite3_close(raw); - } - - // Opening must succeed. On the json_extract path this throws and the process exits. - ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())); - - for (size_t i = 0; i < cases.size(); ++i) { - const std::string code = "CODE_" + std::to_string(i); - EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = '" + code + "'"), cases[i].second) - << "owner recovered from " << cases[i].first; - EXPECT_TRUE(storage_->contains({code, cases[i].second})) << code << " must be reachable by its migrated owner"; - } - // No migrated record is left unaddressable. The empty owner belongs to child rows - // alone, and a fault carrying it could not be named by any source_id. - EXPECT_EQ(read_text(temp_db_path_, "SELECT COUNT(*) FROM faults WHERE owner = ''"), "0"); - EXPECT_EQ(storage_->size(), cases.size()); - - // Whatever came in, what goes back out is valid JSON, so the next release's reader - // is not handed the same problem. - EXPECT_EQ(read_text(temp_db_path_, "SELECT MIN(json_valid(reporting_sources)) FROM faults"), "1"); - - // Idempotent: a second open neither rebuilds nor re-derives. - storage_.reset(); - ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())); - EXPECT_EQ(storage_->size(), cases.size()); - EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = 'CODE_4'"), device); -} - -// End to end: a record whose source cannot be read is still a record. It gets the -// synthetic owner, its evidence is filed under that owner like any other record's, and -// the store then reaches a resting state. The resting state is the point: while a fault -// carried an empty owner its child rows could never be assigned to anyone, yet they -// still looked like pending work, so every single open re-entered the migration write -// transaction and logged a warning that nothing would ever resolve. -TEST_F(SqliteFaultStorageTest, AnUnreadableSourceYieldsALegacyRecordAndThenSettles) { - storage_.reset(); - std::filesystem::remove(temp_db_path_); - { - sqlite3 * raw = nullptr; - ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); - ASSERT_EQ(sqlite3_exec(raw, kLegacyFaultsDdl, nullptr, nullptr, nullptr), SQLITE_OK); - ASSERT_EQ(sqlite3_exec(raw, - "INSERT INTO faults VALUES ('ORPHANED', 2, 'legacy', 1, 1, 1, 'CONFIRMED', " - "'', -1, 1, 0, 0);" - "CREATE TABLE snapshots (id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, " - "topic TEXT NOT NULL, message_type TEXT NOT NULL, data TEXT NOT NULL, " - "captured_at_ns INTEGER NOT NULL);" - "INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) " - "VALUES ('ORPHANED', '/t', 'std_msgs/msg/Float64', '{}', 9);", - nullptr, nullptr, nullptr), - SQLITE_OK); - sqlite3_close(raw); - } - - ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())); - - const std::string legacy = ros2_medkit_fault_manager::kLegacyOwner; - EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = 'ORPHANED'"), legacy); - EXPECT_EQ(read_text(temp_db_path_, "SELECT reporting_sources FROM faults WHERE fault_code = 'ORPHANED'"), - "[\"" + legacy + "\"]") - << "the column and the owner must agree, so a raw reader sees the same name"; - EXPECT_TRUE(storage_->contains({"ORPHANED", legacy})) << "the record must be addressable by source_id"; - EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM snapshots WHERE fault_code = 'ORPHANED'"), legacy); - ASSERT_EQ(storage_->get_snapshots({"ORPHANED", legacy}).size(), 1u); - - // A tripwire that fires on any write to the child table, armed only now, so it sees - // the SECOND open and nothing the first one did. - { - sqlite3 * raw = nullptr; - ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); - ASSERT_EQ(sqlite3_exec(raw, - "CREATE TABLE tripwire (note TEXT);" - "CREATE TRIGGER tripwire_snapshots AFTER UPDATE ON snapshots " - "BEGIN INSERT INTO tripwire VALUES ('backfill ran'); END;", - nullptr, nullptr, nullptr), - SQLITE_OK) - << sqlite3_errmsg(raw); - sqlite3_close(raw); - } - - storage_.reset(); - ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())); - - EXPECT_EQ(read_text(temp_db_path_, "SELECT COUNT(*) FROM tripwire"), "0") - << "a settled store must not rewrite child rows on every open"; - EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM snapshots WHERE fault_code = 'ORPHANED'"), legacy); -} - // A rebuild that was interrupted, or a manual recovery with an external tool, can leave // the scratch table behind. CREATE TABLE (no IF NOT EXISTS, deliberately, so a real // collision is never silently reused) then fails on every open and the schema stays @@ -1519,7 +1407,7 @@ TEST_F(SqliteFaultStorageTest, LeftoverScratchTablesDoNotBlockTheMigration) { log.reset(); EXPECT_EQ(named, 1) << "the dropped table must be named in the log: " << leftover; - const ros2_medkit_fault_manager::FaultId id{"LEFTOVER", "/owner_a"}; + const ros2_medkit_fault_manager::FaultId id{"LEFTOVER", ros2_medkit_fault_manager::kSharedOwner}; EXPECT_TRUE(storage_->contains(id)) << "with leftover " << leftover; EXPECT_TRUE(has_index_over(temp_db_path_, "faults", {"fault_code", "owner"})); ASSERT_TRUE(storage_->get_freeze_frame(id).has_value()) << "with leftover " << leftover; @@ -1527,96 +1415,14 @@ TEST_F(SqliteFaultStorageTest, LeftoverScratchTablesDoNotBlockTheMigration) { } } -// Evidence is never assigned to an owner the database cannot prove. A code carrying two -// owners has no single answer, so the child rows keep the empty owner they came with, -// and neither owner is handed the other's snapshots. -TEST_F(SqliteFaultStorageTest, ChildRowsOfASharedCodeAreNotGivenAnArbitraryOwner) { - storage_.reset(); - std::filesystem::remove(temp_db_path_); - { - // faults already on the new schema with TWO owners of one code, snapshots still - // legacy. Reachable by an open that was interrupted, and by a database an operator - // repaired by hand. - sqlite3 * raw = nullptr; - ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); - ASSERT_EQ(sqlite3_exec(raw, - "CREATE TABLE faults (fault_code TEXT NOT NULL, severity INTEGER NOT NULL, " - "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " - "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " - "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " - "debounce_counter INTEGER NOT NULL DEFAULT 0, " - "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " - "confirmed_at_ns INTEGER NOT NULL DEFAULT 0, owner TEXT NOT NULL);" - "INSERT INTO faults VALUES ('SHARED', 2, 'a', 1, 1, 1, 'CONFIRMED', " - "'[\"/owner_a\"]', -1, 1, 0, 0, '/owner_a');" - "INSERT INTO faults VALUES ('SHARED', 2, 'b', 1, 1, 1, 'CONFIRMED', " - "'[\"/owner_b\"]', -1, 1, 0, 0, '/owner_b');" - "CREATE TABLE snapshots (id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, " - "topic TEXT NOT NULL, message_type TEXT NOT NULL, data TEXT NOT NULL, " - "captured_at_ns INTEGER NOT NULL);" - "INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) " - "VALUES ('SHARED', '/t', 'std_msgs/msg/Float64', '{}', 9);", - nullptr, nullptr, nullptr), - SQLITE_OK); - sqlite3_close(raw); - } - - ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())); - - EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM snapshots WHERE fault_code = 'SHARED'"), "") - << "an ambiguous code must leave the row unassigned, not pick a winner"; - EXPECT_TRUE(storage_->get_snapshots({"SHARED", "/owner_a"}).empty()) << "A must not be handed unproven evidence"; - EXPECT_TRUE(storage_->get_snapshots({"SHARED", "/owner_b"}).empty()) << "and neither must B"; - // The row is kept, not deleted: it is still an operator's evidence. - EXPECT_EQ(read_text(temp_db_path_, "SELECT COUNT(*) FROM snapshots"), "1"); -} - -// The mirror case: a child table that already carries the column but whose rows were -// never filled, with exactly one owner to attribute them to. The backfill is not tied -// to the ALTER that adds the column, so the next open heals them. -TEST_F(SqliteFaultStorageTest, OwnerlessChildRowsAreHealedOnTheNextOpen) { - storage_.reset(); - std::filesystem::remove(temp_db_path_); - { - sqlite3 * raw = nullptr; - ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); - ASSERT_EQ(sqlite3_exec(raw, - "CREATE TABLE faults (fault_code TEXT NOT NULL, severity INTEGER NOT NULL, " - "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " - "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " - "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " - "debounce_counter INTEGER NOT NULL DEFAULT 0, " - "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " - "confirmed_at_ns INTEGER NOT NULL DEFAULT 0, owner TEXT NOT NULL);" - "INSERT INTO faults VALUES ('SOLO', 2, 'a', 1, 1, 1, 'CONFIRMED', " - "'[\"/owner_a\"]', -1, 1, 0, 0, '/owner_a');" - // snapshots is AHEAD of the old migration: the column is there, - // the value never arrived. - "CREATE TABLE snapshots (id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, " - "owner TEXT NOT NULL DEFAULT '', topic TEXT NOT NULL, message_type TEXT NOT NULL, " - "data TEXT NOT NULL, captured_at_ns INTEGER NOT NULL);" - "INSERT INTO snapshots (fault_code, owner, topic, message_type, data, captured_at_ns) " - "VALUES ('SOLO', '', '/t', 'std_msgs/msg/Float64', '{}', 9);", - nullptr, nullptr, nullptr), - SQLITE_OK); - sqlite3_close(raw); - } - - ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())); - - EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM snapshots WHERE fault_code = 'SOLO'"), "/owner_a"); - ASSERT_EQ(storage_->get_snapshots({"SOLO", "/owner_a"}).size(), 1u) << "the owner must now see its own evidence"; - EXPECT_EQ(storage_->get_snapshots({"SOLO", "/owner_a"}).front().owner, "/owner_a"); -} - // New rows must never reproduce the problem the migration had to recover from: a // source_id carrying a control byte round-trips, and what lands in the column is JSON // every reader can parse. TEST_F(SqliteFaultStorageTest, AControlByteInASourceIdStoresAsValidJson) { const std::string source = std::string("/dev/ttyUSB0") + kCtrl + "reader"; rclcpp::Clock clock; - storage_->report_fault_event("CTRL_BYTE", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "d", source, - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"CTRL_BYTE", source}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "d", + source, clock.now().nanoseconds(), default_config()); EXPECT_EQ(read_text(temp_db_path_, "SELECT json_valid(reporting_sources) FROM faults WHERE fault_code='CTRL_BYTE'"), "1") @@ -1691,9 +1497,9 @@ TEST_F(SqliteFaultStorageTest, PreviousReleaseRosbagIndexesWidenWithTheRecordIde << "the narrow unique index must be gone, not left beside the wide one"; EXPECT_TRUE(has_index_over(temp_db_path_, "rosbag_files", {"fault_code", "owner", "created_at_ns", "id"})) << "the per-record read order index widened too"; - EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM rosbag_files WHERE fault_code = 'BURST'"), "/owner_a"); + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM rosbag_files WHERE fault_code = 'BURST'"), "*"); - // What the widened index buys: two owners of one code link the same bag. Under the + // What the widened index buys: two records of one code link the same bag. Under the // previous release's index the second store would collide, and the upsert cannot // even be prepared without a unique constraint over (fault_code, owner, file_path). RosbagFileInfo link; @@ -1703,14 +1509,14 @@ TEST_F(SqliteFaultStorageTest, PreviousReleaseRosbagIndexesWidenWithTheRecordIde link.duration_sec = 6.0; link.size_bytes = 512; link.created_at_ns = 11; - link.owner = "/owner_a"; + link.owner = "*"; storage_->store_rosbag_file(link); link.owner = "/owner_b"; storage_->store_rosbag_file(link); const auto rows = storage_->get_rosbag_files_by_recording("fault_BURST_1"); ASSERT_EQ(rows.size(), 2u) << "one link per record, both pointing at the one bag"; - EXPECT_EQ(rows[0].owner, "/owner_a"); + EXPECT_EQ(rows[0].owner, "*"); EXPECT_EQ(rows[1].owner, "/owner_b"); } @@ -1832,8 +1638,8 @@ TEST_F(SqliteFaultStorageTest, StoreAndRetrieveSnapshot) { // First, create a fault to associate the snapshot with rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); // Store a snapshot SnapshotData snapshot; @@ -1862,8 +1668,8 @@ TEST_F(SqliteFaultStorageTest, MultipleSnapshotsForSameFault) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); // Store multiple snapshots for the same fault SnapshotData snapshot1; @@ -1894,8 +1700,8 @@ TEST_F(SqliteFaultStorageTest, FilterSnapshotsByTopic) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); SnapshotData snapshot1; snapshot1.fault_code = "MOTOR_OVERHEAT"; @@ -1934,9 +1740,9 @@ TEST_F(SqliteFaultStorageTest, ClearFaultDeletesAssociatedSnapshots) { rclcpp::Clock clock; // Create a fault using report_fault_event - storage_->report_fault_event("SNAPSHOT_CLEAR_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "Test fault for snapshot cleanup", "/test_node", clock.now().nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"SNAPSHOT_CLEAR_TEST", "/test_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test fault for snapshot cleanup", "/test_node", + clock.now().nanoseconds(), default_config()); // Store snapshots for this fault SnapshotData snapshot1; @@ -1976,7 +1782,7 @@ TEST_F(SqliteFaultStorageTest, ClearFaultKeepsSnapshotsWhenEvidenceIsRetained) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("KEEP", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "keep", "/n", + storage_->report_fault_event(FaultId{"KEEP", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "keep", "/n", clock.now().nanoseconds(), default_config()); storage_->set_retain_snapshots_on_clear(true); @@ -2002,8 +1808,9 @@ TEST_F(SqliteFaultStorageTest, StoreAndRetrieveFreezeFrame) { using ros2_medkit_fault_manager::FreezeFrameData; rclcpp::Clock clock; - storage_->report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", - "/plc_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"PLC_PRESSURE_HIGH", "/plc_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Pressure high", "/plc_node", clock.now().nanoseconds(), + default_config()); FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; @@ -2056,8 +1863,9 @@ TEST_F(SqliteFaultStorageTest, FreezeFrameSurvivesClearFault) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", - "/plc_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"PLC_PRESSURE_HIGH", "/plc_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Pressure high", "/plc_node", clock.now().nanoseconds(), + default_config()); // A per-topic snapshot (removed on clear) plus a freeze-frame (retained on clear). SnapshotData snapshot; @@ -2114,13 +1922,14 @@ TEST_F(SqliteFaultStorageTest, ListRosbagsForEntityFiltersCorrectly) { rclcpp::Clock clock; // Create fault with reporting source for entity - storage_->report_fault_event("ENTITY_FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Fault from entity", - "/powertrain/motor", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"ENTITY_FAULT_1", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Fault from entity", "/powertrain/motor", + clock.now().nanoseconds(), default_config()); // Create another fault with different reporting source - storage_->report_fault_event("ENTITY_FAULT_2", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, - "Fault from other entity", "/chassis/brake", clock.now().nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"ENTITY_FAULT_2", "/chassis/brake"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "Fault from other entity", "/chassis/brake", + clock.now().nanoseconds(), default_config()); // Store rosbags for both faults RosbagFileInfo info1; @@ -2425,8 +2234,8 @@ TEST_F(SqliteFaultStorageTest, SnapshotCapKeepsTheNewestCaptureWholeAndDropsTheO using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); // Two topics per capture, room for two captures. storage_->set_max_snapshots_per_fault(4); @@ -2477,8 +2286,8 @@ TEST_F(SqliteFaultStorageTest, ACaptureLargerThanTheCapIsKeptWholeRatherThanTorn using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("WIDE", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "many topics", "/n", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"WIDE", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "many topics", + "/n", clock.now().nanoseconds(), default_config()); storage_->set_max_snapshots_per_fault(2); std::vector rows; @@ -2505,8 +2314,8 @@ TEST_F(SqliteFaultStorageTest, SnapshotLimitZeroMeansUnlimited) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("FAULT_A", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", "/node", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_A", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", + "/node", clock.now().nanoseconds(), default_config()); // Default (0) = unlimited storage_->set_max_snapshots_per_fault(0); @@ -2530,10 +2339,10 @@ TEST_F(SqliteFaultStorageTest, SnapshotLimitPerFaultNotGlobal) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("FAULT_A", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", "/node", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_B", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", "/node", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_A", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", + "/node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_B", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", + "/node", clock.now().nanoseconds(), default_config()); storage_->set_max_snapshots_per_fault(1); @@ -2581,8 +2390,9 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesIsAppendedNotOverwritten) { const auto config = four_strike_config(); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } auto fault = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); @@ -2606,8 +2416,9 @@ TEST_F(SqliteFaultStorageTest, ConfirmingReportIsNotANearMiss) { const auto config = four_strike_config(); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } auto fault = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); @@ -2622,8 +2433,9 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesSurvivesClearFault) { const auto config = four_strike_config(); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); @@ -2639,14 +2451,16 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesContinuesAcrossReactivation) { const auto config = four_strike_config(); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } ASSERT_TRUE(storage_->clear_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); // A new outage cycle starts: the reactivating report resets the counter to -1 without confirming. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, - "pressure dipping again", "/hydraulics/pump", nth_report_time(10).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping again", "/hydraulics/pump", + nth_report_time(10).nanoseconds(), config); auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 4u) << "the series must span fault cycles, one entry per occurrence"; @@ -2658,8 +2472,9 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesSurvivesReopen) { const auto config = four_strike_config(); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } ASSERT_TRUE(storage_->clear_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); @@ -2676,14 +2491,15 @@ TEST_F(SqliteFaultStorageTest, PassedReportIsNotANearMiss) { const auto config = four_strike_config(); for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); // A PASSED report moves the counter in the healing direction: the fault receding, not nearing. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); } @@ -2692,8 +2508,9 @@ TEST_F(SqliteFaultStorageTest, CriticalImmediateConfirmIsNotANearMiss) { DebounceConfig config = four_strike_config(); config.critical_immediate_confirm = true; - storage_->report_fault_event("BATTERY_THERMAL_RUNAWAY", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, - "cell over temperature", "/power/bms", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"BATTERY_THERMAL_RUNAWAY", "/power/bms"}, fd::ReportKind::kFailed, + Fault::SEVERITY_CRITICAL, "cell over temperature", "/power/bms", + nth_report_time(0).nanoseconds(), config); auto fault = storage_->get_fault({"BATTERY_THERMAL_RUNAWAY", "/power/bms"}); ASSERT_TRUE(fault.has_value()); @@ -2706,8 +2523,9 @@ TEST_F(SqliteFaultStorageTest, ImmediateConfirmThresholdRecordsNoNearMiss) { // so a fault under this config never has a near miss to record. const auto config = default_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure lost", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "pressure lost", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); auto fault = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); @@ -2720,21 +2538,24 @@ TEST_F(SqliteFaultStorageTest, FailedReportUnderHealedLatchIsNearMiss) { config.healing_enabled = true; config.healing_threshold = 1; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); for (int i = 1; i <= 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), + config); } auto healed = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; // The latch keeps the status at HEALED, but the counter moved toward confirmation: a near miss. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(3).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(3).nanoseconds(), config); auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); @@ -2747,8 +2568,9 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesBoundedKeepingNewest) { storage_->set_max_near_misses_per_fault(3); for (int i = 0; i < 5; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); @@ -2765,8 +2587,9 @@ TEST_F(SqliteFaultStorageTest, NearMissBoundOfOneKeepsLatestOnly) { storage_->set_max_near_misses_per_fault(1); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); @@ -2780,10 +2603,12 @@ TEST_F(SqliteFaultStorageTest, NearMissBoundIsPerFaultCode) { storage_->set_max_near_misses_per_fault(2); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + nth_report_time(i).nanoseconds(), config); } EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); @@ -2796,8 +2621,9 @@ TEST_F(SqliteFaultStorageTest, NearMissBoundZeroIsUnlimited) { storage_->set_max_near_misses_per_fault(0); for (int i = 0; i < 150; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 150u); @@ -2807,8 +2633,9 @@ TEST_F(SqliteFaultStorageTest, NearMissTableCreatedOnDatabaseFromOlderBuild) { // The appliances this matters for already have a faults.db written by a build with no // near_misses table. Opening one must add the table, not fail and not skip recording. const auto config = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); storage_.reset(); { @@ -2821,8 +2648,9 @@ TEST_F(SqliteFaultStorageTest, NearMissTableCreatedOnDatabaseFromOlderBuild) { storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); EXPECT_TRUE(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).empty()); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(1).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(1).nanoseconds(), config); EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); } @@ -2833,11 +2661,13 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesSurvivesHealedReclassification) { config.healing_enabled = true; config.healing_threshold = 1; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); for (int i = 1; i <= 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), + config); } auto healed = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); @@ -2857,12 +2687,14 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { storage_->set_max_near_misses_per_fault(2); for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(10 + i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(10 + i).nanoseconds(), config); } // Arrives third, but carries the earliest timestamp of the three. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); @@ -2876,13 +2708,15 @@ TEST_F(SqliteFaultStorageTest, NearMissEntriesDescribeTheirOwnReport) { // Each entry must describe the report that produced it, not the record's current state, or a // series spanning a threshold change reads as if nothing changed. DebounceConfig first = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), first); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), first); DebounceConfig second = four_strike_config(); second.confirmation_threshold = -8; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure dipping", - "/hydraulics/pump", nth_report_time(1).nanoseconds(), second); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "pressure dipping", "/hydraulics/pump", + nth_report_time(1).nanoseconds(), second); auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); @@ -2898,10 +2732,12 @@ TEST_F(SqliteFaultStorageTest, NearMissEntriesDescribeTheirOwnReport) { // so a plot of one reporter's approaches to confirmation is not diluted by another's. TEST_F(SqliteFaultStorageTest, NearMissSeriesIsPerRecord) { const auto config = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "dip", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "dip", - "/hydraulics/backup_pump", nth_report_time(1).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "dip", "/hydraulics/pump", nth_report_time(0).nanoseconds(), + config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "dip", "/hydraulics/backup_pump", + nth_report_time(1).nanoseconds(), config); const auto pump = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); const auto backup = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}); @@ -2917,15 +2753,17 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesContinuesAfterReopen) { const auto config = four_strike_config(); for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } storage_.reset(); storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(2).nanoseconds(), config); auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "a restart must extend the series, not restart or reorder it"; @@ -2944,10 +2782,12 @@ TEST_F(SqliteFaultStorageTest, ApplyingSmallerBoundTrimsExistingSeries) { storage_->set_max_near_misses_per_fault(0); for (int i = 0; i < 5; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + nth_report_time(i).nanoseconds(), config); } ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 5u); @@ -2967,8 +2807,9 @@ TEST_F(SqliteFaultStorageTest, ApplyingUnlimitedBoundKeepsExistingSeries) { storage_->set_max_near_misses_per_fault(4); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } storage_->set_max_near_misses_per_fault(0); @@ -2983,15 +2824,17 @@ TEST_F(SqliteFaultStorageTest, UnlimitedBoundSpeltAsSizeMaxKeepsTheSeries) { config.confirmation_threshold = -20; for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } EXPECT_EQ(storage_->set_max_near_misses_per_fault(std::numeric_limits::max()), 0u); EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(3).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(3).nanoseconds(), config); EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 4u); } @@ -3001,10 +2844,12 @@ TEST_F(SqliteFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { storage_->set_max_near_misses_per_fault(0); for (int i = 0; i < 5; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + nth_report_time(i).nanoseconds(), config); } // Two records, five entries each, bound of 2: three dropped per record. @@ -3016,8 +2861,9 @@ TEST_F(SqliteFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { TEST_F(SqliteFaultStorageTest, PassedReportOnUnknownFaultWritesNothing) { // A heal heartbeat for a fault that does not exist must stay a read: it writes no row and must // not take the writer lock on the way to finding that out. - EXPECT_FALSE(storage_->report_fault_event("NEVER_REPORTED", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/test_node", nth_report_time(0).nanoseconds(), default_config())); + EXPECT_FALSE(storage_->report_fault_event(FaultId{"NEVER_REPORTED", "/test_node"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/test_node", nth_report_time(0).nanoseconds(), + default_config())); EXPECT_EQ(storage_->size(), 0u); EXPECT_TRUE(storage_->get_near_misses({"NEVER_REPORTED", "/test_node"}).empty()); } @@ -3031,11 +2877,13 @@ TEST_F(SqliteFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) config.healing_threshold = 2; // An approach that recedes: counter moves, nothing is latched. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); for (int i = 1; i <= 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), + config); } auto healed = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); @@ -3045,8 +2893,9 @@ TEST_F(SqliteFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) // From the healing threshold (+2) it takes six FAILED reports to reach the confirmation // threshold (-4). for (int i = 4; i <= 9; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } auto confirmed = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(confirmed.has_value()); @@ -3068,8 +2917,9 @@ TEST_F(SqliteFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) TEST_F(SqliteFaultStorageTest, NearMissResultingStatusSurvivesReopen) { const auto config = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); storage_.reset(); storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); @@ -3082,8 +2932,9 @@ TEST_F(SqliteFaultStorageTest, NearMissResultingStatusSurvivesReopen) { TEST_F(SqliteFaultStorageTest, NearMissResultingStatusColumnAddedToOlderTable) { // A database written by an earlier build of this branch has the table without the column. const auto config = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); storage_.reset(); { @@ -3100,8 +2951,9 @@ TEST_F(SqliteFaultStorageTest, NearMissResultingStatusColumnAddedToOlderTable) { ASSERT_EQ(series.size(), 1u) << "the migration must keep the rows it already had"; EXPECT_TRUE(series[0].resulting_status.empty()) << "an unrecorded latch state reads as empty, not as a status"; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(1).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(1).nanoseconds(), config); auto extended = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(extended.size(), 2u); EXPECT_EQ(extended[1].resulting_status, Fault::STATUS_PREFAILED); @@ -3137,13 +2989,13 @@ static void drive_to_healed(ros2_medkit_fault_manager::FaultStorage & storage, c config.healing_enabled = true; config.healing_threshold = 1; - storage.report_fault_event(fault_code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "/test_node", - nth_report_time(0).nanoseconds(), config); + storage.report_fault_event(FaultId{fault_code, "/test_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", + "/test_node", nth_report_time(0).nanoseconds(), config); store_snapshots_for(storage, fault_code, 2); // Two PASSED reports: the first only lifts the counter to 0, which the CONFIRMED latch holds. for (int i = 1; i <= 2; ++i) { - storage.report_fault_event(fault_code, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "/test_node", - nth_report_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{fault_code, "/test_node"}, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", + "/test_node", nth_report_time(i).nanoseconds(), config); } } diff --git a/src/ros2_medkit_fault_reporter/test/test_integration.test.py b/src/ros2_medkit_fault_reporter/test/test_integration.test.py index c0816b178..fa6233ab8 100644 --- a/src/ros2_medkit_fault_reporter/test/test_integration.test.py +++ b/src/ros2_medkit_fault_reporter/test/test_integration.test.py @@ -162,26 +162,25 @@ def test_02_report_fault_is_stored(self): fault_codes = [f.fault_code for f in faults_response.faults] self.assertIn('REPORTER_TEST_001', fault_codes) - def test_03_two_sources_of_one_code_keep_one_record_each(self): - """Two sources reporting one code get a record each, counted and escalated apart.""" - # The default confirmation threshold confirms on the first FAILED - # report, so every record below is CONFIRMED after its first report. + def test_03_multiple_reports_aggregate(self): + """Test that multiple reports of same fault aggregate.""" + # First report - use CRITICAL to bypass debounce self._report_fault( fault_code='REPORTER_TEST_002', - severity=Fault.SEVERITY_ERROR, + severity=Fault.SEVERITY_CRITICAL, description='First occurrence', source_id='/node_a' ) - # Second report, from a different source + # Second report from different source self._report_fault( fault_code='REPORTER_TEST_002', - severity=Fault.SEVERITY_WARN, + severity=Fault.SEVERITY_CRITICAL, description='Second occurrence', source_id='/node_b' ) - # Third report, /node_a again and more severe + # Third report self._report_fault( fault_code='REPORTER_TEST_002', severity=Fault.SEVERITY_CRITICAL, @@ -189,24 +188,20 @@ def test_03_two_sources_of_one_code_keep_one_record_each(self): source_id='/node_a' ) + # Verify aggregation faults_response = self._list_faults(statuses=['CONFIRMED']) - records = [f for f in faults_response.faults if f.fault_code == 'REPORTER_TEST_002'] - - # One record per reporting source, each naming only its own owner. - self.assertEqual( - sorted(list(f.reporting_sources) for f in records), - [['/node_a'], ['/node_b']], + fault = next( + (f for f in faults_response.faults if f.fault_code == 'REPORTER_TEST_002'), + None ) - by_owner = {f.reporting_sources[0]: f for f in records} - - # Edge counting per record: the third report re-reports /node_a's - # still-active record, so neither count moves past the first edge. - self.assertEqual(by_owner['/node_a'].occurrence_count, 1) - self.assertEqual(by_owner['/node_b'].occurrence_count, 1) - # Severity escalates inside /node_a's record only. The CRITICAL report - # from /node_a leaves /node_b's record at the severity /node_b sent. - self.assertEqual(by_owner['/node_a'].severity, Fault.SEVERITY_CRITICAL) - self.assertEqual(by_owner['/node_b'].severity, Fault.SEVERITY_WARN) + + self.assertIsNotNone(fault) + # Edge-counting: re-reports of a still-active fault do not bump the + # count; severity escalation and source aggregation still apply. + self.assertEqual(fault.occurrence_count, 1) + self.assertEqual(fault.severity, Fault.SEVERITY_CRITICAL) + self.assertIn('/node_a', fault.reporting_sources) + self.assertIn('/node_b', fault.reporting_sources) def test_04_source_id_tracked(self): """Test that source_id is correctly tracked.""" diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index 2f1e129bf..34ba4ce23 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -1047,7 +1047,7 @@ Real-time fault event stream using Server-Sent Events (SSE). Clients receive ins **Event Types:** - `fault_confirmed` - Fault transitioned to CONFIRMED status -- `fault_updated` - Fault data changed without a status transition (`last_occurred`, severity). Not the reporting sources, which name the record's one owner and never change, and not `occurrence_count`, which only changes on an occurrence edge +- `fault_updated` - Fault data changed without a status transition (`last_occurred`, severity, reporting sources). Not `occurrence_count`, which only changes on an occurrence edge - `fault_cleared` - Fault ended: cleared via ClearFault service, or auto-healed by PASSED events (check `fault.status` for `CLEARED` vs `HEALED`) **Example:** diff --git a/src/ros2_medkit_msgs/README.md b/src/ros2_medkit_msgs/README.md index 9ad70535e..1492dc8eb 100644 --- a/src/ros2_medkit_msgs/README.md +++ b/src/ros2_medkit_msgs/README.md @@ -6,7 +6,7 @@ ROS 2 message and service definitions for the ros2_medkit fault management syste This package provides the interface definitions used by the fault management components: -- **FaultManager** (`ros2_medkit_fault_manager`) - Central fault record store and lifecycle management +- **FaultManager** (`ros2_medkit_fault_manager`) - Central fault aggregation and lifecycle management - **FaultReporter** (`ros2_medkit_fault_reporter`) - Client library for fault reporting - **Gateway** (`ros2_medkit_gateway`) - REST API endpoints for fault access @@ -16,13 +16,16 @@ This package provides the interface definitions used by the fault management com Core fault data model representing one fault record with AUTOSAR DEM-style debounce filtering. -A record is identified by the pair (`fault_code`, owning reporting source). The owner is the -`source_id` a `ReportFault` call carried. Two sources reporting one `fault_code` are two records, -each with its own status, debounce counter, `occurrence_count`, severity and timestamps, and each -cleared on its own. The services that act on a single record (`ClearFault`, `GetFault`, -`GetSnapshots`, `GetRosbag`) carry a `source_id` request field naming the owner. An empty -`source_id` is unscoped: the call applies only when exactly one record carries the `fault_code`, -and fails as ambiguous when several do. +A record is identified by the pair (`fault_code`, owner). A code listed with `per_source: true` +in `fault_thresholds.yaml` has one record per reporting source, owned by the `source_id` of the +`ReportFault` call. Every other code has one record shared by all its sources, owned by the +reserved id `*`, which `ReportFault` refuses as a `source_id`. Each record has its own status, +debounce counter, `occurrence_count`, severity and timestamps, and is cleared on its own. The +services that act on a single record (`ClearFault`, `GetFault`, `GetSnapshots`, `GetRosbag`) carry +a `source_id` request field. It names the owner (`*` for a shared record). A reporting source that +owns no record reaches the record it reports into. An empty `source_id` is unscoped: the call +applies only when exactly one record carries the `fault_code`, and fails as ambiguous when +several do. | Field | Type | Description | |-------|------|-------------| @@ -34,7 +37,7 @@ and fails as ambiguous when several do. | `last_passed` | builtin_interfaces/Time | When fault last reported PASSED (zero = never) | | `occurrence_count` | uint32 | Times this fault has occurred, counted on edges (first FAILED, then each FAILED that arrives while CLEARED). Repeats within one occurrence do not increment it | | `status` | string | Current status (see STATUS_* constants) | -| `reporting_sources` | string[] | The reporting source that owns this record, as a one-element list | +| `reporting_sources` | string[] | Every source that reported this record: one for a per-source record, all of them for a shared one | **Severity Levels:** | Constant | Value | Description | @@ -136,7 +139,7 @@ Clear/acknowledge one fault record. Cleared records are retained and queryable w | Field | Type | Description | |-------|------|-------------| | `fault_code` | string | The fault code to clear | -| `source_id` | string | Reporting source that owns the record. Empty means unscoped: the call applies only when exactly one record carries the `fault_code`, otherwise it fails with a message beginning `ambiguous:` and clears nothing | +| `source_id` | string | Owner of the record (`*` for a shared record), or a source that reports into it. Empty means unscoped: the call applies only when exactly one record carries the `fault_code`, otherwise it fails with a message beginning `ambiguous:` and clears nothing | | `skip_correlation_auto_clear` | bool | When `true`, only the requested fault_code is cleared; symptom faults that the correlation engine would normally auto-clear via `auto_clear_with_root` rules are left untouched. Default `false` (cascade clear). The gateway sets this to `true` on per-entity `DELETE /{entity-path}/faults/{fault_code}` so that an operator with access to one entity cannot cascade-clear correlated symptoms reported by apps in other entities. | **Response:** diff --git a/src/ros2_medkit_msgs/msg/Fault.msg b/src/ros2_medkit_msgs/msg/Fault.msg index 99227411b..c0f2e4250 100644 --- a/src/ros2_medkit_msgs/msg/Fault.msg +++ b/src/ros2_medkit_msgs/msg/Fault.msg @@ -14,11 +14,11 @@ # # Fault.msg - Core fault data model for ros2_medkit fault management system. # -# A Fault is one fault record, identified by the pair (fault_code, reporting source). -# The reporting source is the source_id a ReportFault call carried, and it is the -# owner of the record. Two sources reporting one fault_code are two records, each -# with its own status, debounce counter, occurrence_count, severity and timestamps, -# and each cleared on its own. reporting_sources holds the one owner. +# A Fault is one fault record, identified by the pair (fault_code, owner). A code listed +# with per_source: true in fault_thresholds.yaml has one record per reporting source, +# owned by the source_id of the ReportFault call. Every other code has one record shared +# by all its sources, owned by the reserved id "*". Each record has its own status, +# debounce counter, occurrence_count, severity and timestamps, and is cleared on its own. # # Debounce model (AUTOSAR DEM-style): # - FAILED events decrement internal counter (towards confirmation) @@ -26,7 +26,7 @@ # - Status reflects the current debounce state # Global fault identifier (e.g., "MOTOR_OVERHEAT", "SENSOR_FAILURE_001"). -# Half of the record identity: the other half is the owning reporting source. +# Half of the record identity: the other half is the owner. string fault_code # Fault severity level (use SEVERITY_* constants) @@ -56,9 +56,9 @@ uint32 occurrence_count # Current fault status (use STATUS_* constants) string status -# The reporting source that owns this record, as a one-element list. Together with -# fault_code it identifies the record. Services that act on a single record take -# this value as their source_id request field. +# Every source that reported this record: one for a per-source record, all of them for a +# shared one. Services that act on a single record take a source_id request field naming +# the owner, or any source listed here. string[] reporting_sources # Severity level constants diff --git a/src/ros2_medkit_msgs/msg/FaultEvent.msg b/src/ros2_medkit_msgs/msg/FaultEvent.msg index 7fa119289..c1afbe37d 100644 --- a/src/ros2_medkit_msgs/msg/FaultEvent.msg +++ b/src/ros2_medkit_msgs/msg/FaultEvent.msg @@ -35,8 +35,8 @@ string EVENT_CONFIRMED = "fault_confirmed" # the two apart. string EVENT_CLEARED = "fault_cleared" # Emitted when fault data changes without status transition (e.g., last_occurred -# advanced, severity escalated). A report from a source that owns no record yet -# is not one of them: it creates a record of its own. +# advanced, severity escalated, new source added to reporting_sources). A report from a +# source of a per_source code that owns no record yet creates a record of its own instead. # occurrence_count is not one of them: it only changes on an occurrence edge # (the first FAILED, or a FAILED that arrives while the fault is CLEARED). string EVENT_UPDATED = "fault_updated" diff --git a/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg b/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg index c87d67d61..d109ba634 100644 --- a/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg +++ b/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg @@ -18,9 +18,9 @@ # as events) but still stored. This message provides information about # muted records when queried with include_muted=true. # -# One entry per muted RECORD. A root cause mutes the symptoms of its own -# reporting source only, so two sources muted on one fault_code produce two -# entries carrying that code, told apart by source_id. +# One entry per muted RECORD. A root cause mutes the symptoms of its own record owner +# only, so two per-source records muted on one fault_code produce two entries carrying +# that code, told apart by source_id. # The fault code of the muted symptom string fault_code @@ -34,7 +34,7 @@ string rule_id # Time delay from root cause to this symptom [ms] uint32 delay_ms -# Reporting source that owns the muted record (the source_id used in ReportFault). -# Together with fault_code it identifies the one record this entry describes, and -# it is the value the single-record services take to address it. +# Owner of the muted record: the reporting source for a per_source code, "*" for the +# shared record. Together with fault_code it identifies the one record this entry +# describes, and it is the value the single-record services take to address it. string source_id diff --git a/src/ros2_medkit_msgs/srv/ClearFault.srv b/src/ros2_medkit_msgs/srv/ClearFault.srv index 340c58ace..d5a52f5f9 100644 --- a/src/ros2_medkit_msgs/srv/ClearFault.srv +++ b/src/ros2_medkit_msgs/srv/ClearFault.srv @@ -14,8 +14,8 @@ # # ClearFault.srv - Clear/acknowledge a fault in the FaultManager. # -# Marks one fault record as CLEARED. A record is identified by (fault_code, source_id), -# so clearing one owner's record never touches another owner's record of the same code. +# Marks one fault record as CLEARED. A record is identified by (fault_code, owner), so +# clearing one record never touches another record of the same code. # The record is retained for historical purposes and can be retrieved by calling # ListFaults with statuses=["CLEARED"]. # Cleared faults are retained indefinitely unless storage cleanup is configured. @@ -32,9 +32,11 @@ string fault_code # The fault_code to clear # cannot cascade-clear symptom faults reported by apps in other entities. bool skip_correlation_auto_clear -# Reporting source that owns the fault record (the source_id used in ReportFault). -# Together with fault_code it identifies one record. Empty: the call applies only -# when exactly one record carries fault_code, otherwise it fails as ambiguous. +# Which record. Set it to the owner of the record: the reporting source for a per_source +# code, or "*" for the record that every source of any other code shares. A reporting source +# that owns no record reaches the record it reports into, so a client that knows only the +# source it reports as still finds the shared record. Empty: the call applies only when +# exactly one record carries fault_code, otherwise it fails as ambiguous. string source_id --- # Response fields diff --git a/src/ros2_medkit_msgs/srv/GetFault.srv b/src/ros2_medkit_msgs/srv/GetFault.srv index 427af3f7f..d0e869e04 100644 --- a/src/ros2_medkit_msgs/srv/GetFault.srv +++ b/src/ros2_medkit_msgs/srv/GetFault.srv @@ -17,16 +17,18 @@ # Unlike ListFaults.srv (plural) which returns a list of faults for filtering, # this service returns a single fault record with full environment data including # snapshots and extended data records for SOVD-compliant fault responses. -# A record is identified by (fault_code, source_id). +# A record is identified by (fault_code, owner). # Request fields # Fault code to retrieve (e.g., "MOTOR_OVERHEAT") string fault_code -# Reporting source that owns the fault record (the source_id used in ReportFault). -# Together with fault_code it identifies one record. Empty: the call applies only -# when exactly one record carries fault_code, otherwise it fails as ambiguous. +# Which record. Set it to the owner of the record: the reporting source for a per_source +# code, or "*" for the record that every source of any other code shares. A reporting source +# that owns no record reaches the record it reports into, so a client that knows only the +# source it reports as still finds the shared record. Empty: the call applies only when +# exactly one record carries fault_code, otherwise it fails as ambiguous. string source_id --- # Response fields diff --git a/src/ros2_medkit_msgs/srv/GetRosbag.srv b/src/ros2_medkit_msgs/srv/GetRosbag.srv index 5079f1b9f..3b02caceb 100644 --- a/src/ros2_medkit_msgs/srv/GetRosbag.srv +++ b/src/ros2_medkit_msgs/srv/GetRosbag.srv @@ -15,7 +15,7 @@ # GetRosbag.srv - Retrieve rosbag file information for a fault. # # Returns the path to the rosbag file that was recorded when a fault record was -# confirmed. A recording is linked per record, identified by (fault_code, source_id). +# confirmed. A recording is linked per record, identified by (fault_code, owner). # The rosbag contains time-window recording of configured topics before and after # the fault confirmation event. @@ -31,9 +31,11 @@ string fault_code # fault". string recording_id -# Reporting source that owns the fault record (the source_id used in ReportFault). -# Together with fault_code it identifies one record. Empty: the call applies only -# when exactly one record carries fault_code, otherwise it fails as ambiguous. +# Which record. Set it to the owner of the record: the reporting source for a per_source +# code, or "*" for the record that every source of any other code shares. A reporting source +# that owns no record reaches the record it reports into, so a client that knows only the +# source it reports as still finds the shared record. Empty: the call applies only when +# exactly one record carries fault_code, otherwise it fails as ambiguous. # It scopes the fault_code lookup only. The recording_id path ignores it. string source_id --- diff --git a/src/ros2_medkit_msgs/srv/GetSnapshots.srv b/src/ros2_medkit_msgs/srv/GetSnapshots.srv index fcd8dfe91..e39089497 100644 --- a/src/ros2_medkit_msgs/srv/GetSnapshots.srv +++ b/src/ros2_medkit_msgs/srv/GetSnapshots.srv @@ -15,7 +15,7 @@ # GetSnapshots.srv - Retrieve topic snapshots captured when a fault was confirmed. # # Returns topic data that was captured at the moment a fault record transitioned to -# CONFIRMED. Snapshots belong to one record, identified by (fault_code, source_id). +# CONFIRMED. Snapshots belong to one record, identified by (fault_code, owner). # Snapshots provide debugging context by preserving system state at fault time. # Request fields @@ -27,9 +27,11 @@ string fault_code # If specified, returns only the snapshot for that specific topic. string topic -# Reporting source that owns the fault record (the source_id used in ReportFault). -# Together with fault_code it identifies one record. Empty: the call applies only -# when exactly one record carries fault_code, otherwise it fails as ambiguous. +# Which record. Set it to the owner of the record: the reporting source for a per_source +# code, or "*" for the record that every source of any other code shares. A reporting source +# that owns no record reaches the record it reports into, so a client that knows only the +# source it reports as still finds the shared record. Empty: the call applies only when +# exactly one record carries fault_code, otherwise it fails as ambiguous. string source_id --- # Response fields diff --git a/src/ros2_medkit_msgs/srv/ListFaultsForEntity.srv b/src/ros2_medkit_msgs/srv/ListFaultsForEntity.srv index 5ae09449a..c58bdcc14 100644 --- a/src/ros2_medkit_msgs/srv/ListFaultsForEntity.srv +++ b/src/ros2_medkit_msgs/srv/ListFaultsForEntity.srv @@ -14,8 +14,7 @@ # # ListFaultsForEntity.srv - Request list of faults for a specific entity. # -# Filters fault records by their owning reporting source: the entity FQN must match -# the single entry of fault.reporting_sources[], exactly or as an FQN suffix. +# Filters faults by checking if entity FQN appears in fault.reporting_sources[]. # Used by the gateway to serve entity-specific fault lists. # Request fields diff --git a/src/ros2_medkit_msgs/srv/ListRosbags.srv b/src/ros2_medkit_msgs/srv/ListRosbags.srv index c520bb82e..5595816d0 100644 --- a/src/ros2_medkit_msgs/srv/ListRosbags.srv +++ b/src/ros2_medkit_msgs/srv/ListRosbags.srv @@ -20,7 +20,7 @@ # Request fields # Entity FQN (fully qualified name) to filter rosbags by. -# Matches the owning reporting source of each recording's fault record, exactly. +# Matches against fault reporting_sources using prefix matching. string entity_fqn --- # Response fields diff --git a/src/ros2_medkit_msgs/srv/ReportFault.srv b/src/ros2_medkit_msgs/srv/ReportFault.srv index e26fda5d9..b38a77e7f 100644 --- a/src/ros2_medkit_msgs/srv/ReportFault.srv +++ b/src/ros2_medkit_msgs/srv/ReportFault.srv @@ -15,14 +15,16 @@ # ReportFault.srv - Report a fault event to the FaultManager. # # Called by FaultReporter clients or directly by nodes to report fault conditions. -# The FaultManager keeps one record per (fault_code, source_id) pair and applies -# debounce filtering per record, based on FAILED/PASSED event counts. +# The FaultManager keeps one record per fault_code, shared by all sources, or one record per +# (fault_code, source_id) pair for a code configured per_source in fault_thresholds.yaml. It +# applies debounce filtering per record, based on FAILED/PASSED event counts. # Request fields # Global fault identifier. Convention: UPPER_SNAKE_CASE (e.g., "MOTOR_OVERHEAT"). # Recommended max length: 64 characters. Allowed: A-Z, 0-9, underscore. -# Same fault_code from different sources = one record each, filtered independently. +# Same fault_code from different sources = one shared record, or one record each (filtered +# independently) for a per_source code. string fault_code # Event type indicating whether the fault condition is detected or cleared. @@ -47,9 +49,10 @@ string description # Identifier of the reporting node/entity. Recommended: fully qualified ROS 2 node name # including namespace (e.g., "/powertrain/engine/temp_sensor"). -# This value owns the record: (fault_code, source_id) is the record identity, and the -# services that read or clear one record take it back as their source_id field. -# Required: a report with an empty source_id is rejected. +# It is added to the reporting_sources of the record the report goes into, and it owns that +# record when the code is configured per_source. +# Required: a report with an empty source_id is rejected, and so is the reserved id "*", +# which names the record that every source of a code shares. string source_id --- # Response fields From 18a20e0840932d29bc670716a1d5e144f1b6216c Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 17:17:44 +0100 Subject: [PATCH 04/52] fault_manager: shared records confirm CRITICAL on the report's severity A shared record (default code) keeps the original rule: the severity of the report decides critical_immediate_confirm. A per-source record decides on its stored severity. The severity-mismatch warning is logged only for per-source records. --- docs/config/fault-manager.rst | 16 ++--- src/ros2_medkit_fault_manager/README.md | 20 +++--- .../design/index.rst | 11 ++- .../core/fault_manager_core.hpp | 3 +- .../fault_storage.hpp | 5 +- .../src/core/fault_manager_core.cpp | 3 +- .../src/fault_storage.cpp | 9 +-- .../src/postgres_fault_storage.cpp | 7 +- .../src/sqlite_fault_storage.cpp | 7 +- .../integration/test_fault_identity.test.py | 67 ++++++++++++++++++- .../test_severity_consistency.test.py | 23 +++++-- .../test/test_fault_manager.cpp | 13 ++-- .../test/test_fault_severity.cpp | 23 +++++-- 13 files changed, 150 insertions(+), 57 deletions(-) diff --git a/docs/config/fault-manager.rst b/docs/config/fault-manager.rst index 5b27416cb..e0b6e7254 100644 --- a/docs/config/fault-manager.rst +++ b/docs/config/fault-manager.rst @@ -209,17 +209,17 @@ The fault manager uses AUTOSAR DEM-style debounce filtering to prevent fault fla negative, and the node falls back to ``-1`` with a warning. Faults with ``SEVERITY_CRITICAL`` always bypass debounce regardless of this setting. The bypass (``critical_immediate_confirm`` in the debounce config, on by default and not a ROS - parameter) reads the **stored** severity of the fault record, not the severity of the report. - A record keeps the highest severity ever reported for it, across occurrences, so after a - CRITICAL occurrence is cleared, one WARN report confirms the next occurrence at once, and - the record shows CRITICAL. The same holds for a FAILED report on a HEALED CRITICAL record: the - latch does not hold it back. Another record of the same code (a ``per_source`` code has one per - source) has its own severity and is not affected. + parameter) depends on the record's identity. A record shared by all sources of a code (the + default) bypasses debounce on the severity of the **report**. A ``per_source`` record bypasses it + on its **stored** severity, the highest ever reported for that record, so after a CRITICAL + occurrence is cleared, one WARN report from the same source confirms the next occurrence at once, + and the record shows CRITICAL. The same holds for a FAILED report on a HEALED CRITICAL + ``per_source`` record: the latch does not hold it back. .. note:: - A FAILED report whose severity differs from the stored severity of its record is logged as a - warning naming the code, both severities and the source. A source should report one code + A FAILED report whose severity differs from the stored severity of its ``per_source`` record is + logged as a warning naming the code, both severities and the source. A source should report one code with one severity, so the warning points at a misconfigured reporter. It is logged once per code, source and reported severity until the node restarts, so a reporter that repeats the report at a high rate produces one line. diff --git a/src/ros2_medkit_fault_manager/README.md b/src/ros2_medkit_fault_manager/README.md index ec3339d38..db523967d 100644 --- a/src/ros2_medkit_fault_manager/README.md +++ b/src/ros2_medkit_fault_manager/README.md @@ -247,13 +247,13 @@ What each surface carries: The near-miss series is per-report history, so it keeps what the reporter sent. A lower report stays visible there and in the mismatch warning below. -`critical_immediate_confirm` reads the stored severity of the record. A FAILED report on a record -stored as `CRITICAL` confirms it at once, whatever severity the report carries, from a cleared -record and from the HEALED latch alike. So no report leaves a record `CRITICAL` in `PREFAILED` or -`PREPASSED`. A row an earlier release stored that way keeps its status through the migration until -its next FAILED report. +`critical_immediate_confirm` depends on the identity of the record. On a record shared by all sources +of a code (the default) the severity of the report decides. On a `per_source` record the stored +severity decides: a FAILED report on a record stored as `CRITICAL` confirms it at once, whatever +severity the report carries, from a cleared record and from the HEALED latch alike. So no report +leaves a `per_source` record `CRITICAL` in `PREFAILED` or `PREPASSED`. -When a FAILED report carries a severity other than the one stored for its record, the node logs a +When a FAILED report on a `per_source` record carries a severity other than the one stored for its record, the node logs a warning naming the code, the reported and the stored severity and the source. It is logged once per code, source and reported severity until the node restarts, so a reporter that repeats the same report many times a second produces one line. The number of lines is bounded by the distinct @@ -427,8 +427,8 @@ never confirms again, and `occurrence_count` does not move either. Prefer leavin `confirmation_threshold` at `-1` for such a reporter, and filter noisy samples in the reporter itself, where the samples are. -Two things ignore the counter. A FAILED report on a record stored as `SEVERITY_CRITICAL` confirms -at once unless `critical_immediate_confirm` is turned off in the debounce config, and that field is +Two things ignore the counter. A CRITICAL report on a shared record, or a FAILED report on a +`per_source` record stored as `SEVERITY_CRITICAL`, confirms at once unless `critical_immediate_confirm` is turned off in the debounce config, and that field is not exposed as a ROS parameter. `auto_confirm_after_sec` promotes on elapsed time since the last FAILED, which is not the same as observing that the condition is still there. @@ -478,8 +478,8 @@ Event types: `0` = EVENT_FAILED, `1` = EVENT_PASSED ### Immediate Confirmation -A FAILED report on a record whose stored severity is CRITICAL confirms it at once, regardless of -threshold and of the HEALED latch. The stored severity decides, not the report: after a CRITICAL +A CRITICAL FAILED report on a shared record confirms it at once, regardless of threshold and of the +HEALED latch. On a `per_source` record the stored severity decides, not the report: after a CRITICAL occurrence is cleared, one WARN report for the same record confirms the next occurrence, which shows CRITICAL. This holds under a per-entity threshold override too. diff --git a/src/ros2_medkit_fault_manager/design/index.rst b/src/ros2_medkit_fault_manager/design/index.rst index 04a1e1207..c631ea165 100644 --- a/src/ros2_medkit_fault_manager/design/index.rst +++ b/src/ros2_medkit_fault_manager/design/index.rst @@ -255,7 +255,7 @@ Reports a new fault or updates an existing one. which ``ReportFault`` refuses as a source_id). Each record is debounced, cleared and healed on its own - **Severity escalation**: A record's severity is updated if a higher one is reported for it - **Severity mismatch**: A FAILED report whose severity differs from the stored severity of its - record is logged as a warning, once per code, source and reported severity until restart + per-source record is logged as a warning, once per code, source and reported severity until restart - **Request queue**: Holds ``report_fault_queue_depth`` requests (default 100) until the executor reads them. The default service QoS holds 10. A reporter has one request in flight per fault code, so one reporter can fill the queue alone. Past the depth the middleware drops requests without an @@ -354,11 +354,10 @@ Faults follow an AUTOSAR DEM-style debounce lifecycle: FAILED events decrement the debounce counter (towards confirmation). PASSED events increment the debounce counter (towards healing). -CRITICAL severity bypasses debounce and confirms immediately. The check reads the stored severity -of the record after the report is applied, so any FAILED report on a record stored as CRITICAL -confirms it, from CLEARED and from the HEALED latch alike, and no report leaves a record CRITICAL in -PREFAILED or PREPASSED. A row an earlier release stored that way keeps its status through the owner -migration until its next FAILED report. +CRITICAL severity bypasses debounce and confirms immediately. On a shared record the severity of +the report decides. On a per-source record the check reads the stored severity after the report is +applied, so any FAILED report on a record stored as CRITICAL confirms it, from CLEARED and from the +HEALED latch alike, and no report leaves a per-source record CRITICAL in PREFAILED or PREPASSED. The SQLite store records when the current occurrence of a record confirmed (``confirmed_at_ns``). A FAILED report on a CLEARED record resets it to 0, and the confirmation of the new occurrence sets diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp index 7841f391a..1f98796f1 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp @@ -283,7 +283,8 @@ class FaultManagerCore { void run_auto_confirm(int64_t wall_now_ns); /// Queues a capture unless @p id is inside its recapture cooldown. void capture_on_confirm(const FaultId & id); - /// Warns once per (code, source, severity) when a FAILED report disagrees with the stored severity. + /// Warns once per (code, source, severity) when a FAILED report disagrees with the stored severity of a + /// per-source record. void warn_severity_mismatch(const fd::FaultReport & report, uint8_t stored_severity); struct DebouncePolicyWitness { diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp index bd38cd133..cd41de7aa 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp @@ -61,8 +61,9 @@ struct DebounceConfig { /// Default: 3 (3 more PASSED than FAILED events to heal). int32_t healing_threshold{3}; - /// Whether a FAILED report on a record whose stored severity is CRITICAL confirms it immediately, - /// bypassing debounce and the HEALED latch. The stored severity decides, not the report's. + /// Whether a CRITICAL record confirms immediately on a FAILED report, bypassing debounce and the + /// HEALED latch. A shared record (owner kSharedOwner) goes by the report's severity, any other + /// record by its stored severity. bool critical_immediate_confirm{true}; /// Time-based auto-confirmation. If > 0, PREFAILED faults older than this are auto-confirmed. diff --git a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp index 34a4be867..fb1911bb0 100644 --- a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp +++ b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp @@ -436,7 +436,8 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { outcome.accepted = true; - if (fault_before && failed && report.severity != fault_before->severity) { + // Only a per-source record confirms on its stored severity, so only there a lower report needs explaining. + if (fault_before && failed && id.owner != kSharedOwner && report.severity != fault_before->severity) { warn_severity_mismatch(report, fault_before->severity); } diff --git a/src/ros2_medkit_fault_manager/src/fault_storage.cpp b/src/ros2_medkit_fault_manager/src/fault_storage.cpp index e538aff1e..44464cc5b 100644 --- a/src/ros2_medkit_fault_manager/src/fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_storage.cpp @@ -119,6 +119,7 @@ bool InMemoryFaultStorage::report_fault_event(const FaultId & id, fd::ReportKind std::lock_guard lock(mutex_); const bool is_failed = (kind == fd::ReportKind::kFailed); + const bool shared = id.owner == kSharedOwner; auto it = faults_.find(id); if (it == faults_.end()) { @@ -181,8 +182,8 @@ bool InMemoryFaultStorage::report_fault_event(const FaultId & id, fd::ReportKind if (!description.empty()) { state.description = description; } - // Immediate confirmation reads the stored severity: a CRITICAL record confirms on any FAILED report. - if (config.critical_immediate_confirm && state.severity == fd::severity::kCritical) { + // A per-source record confirms on its stored severity, a shared record on the report's own. + if (config.critical_immediate_confirm && (shared ? severity : state.severity) == fd::severity::kCritical) { state.status = fd::fault_status::kConfirmed; } else { update_status(state, config); @@ -230,8 +231,8 @@ bool InMemoryFaultStorage::report_fault_event(const FaultId & id, fd::ReportKind state.description = description; } - // Immediate confirmation reads the stored severity, so it also overrides the HEALED latch. - if (config.critical_immediate_confirm && state.severity == fd::severity::kCritical) { + // Same rule as on reactivation; it also overrides the HEALED latch. + if (config.critical_immediate_confirm && (shared ? severity : state.severity) == fd::severity::kCritical) { state.debounce_counter = std::min(state.debounce_counter, -1); state.status = fd::fault_status::kConfirmed; return false; diff --git a/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp b/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp index c86bb48dd..9818dca06 100644 --- a/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp @@ -730,10 +730,11 @@ bool PgFaultStorage::report_fault_event_locked(const FaultId & id, fd::ReportKin // Decrement towards confirmation, clamped to the thresholds. debounce_counter = clamp_debounce_counter(debounce_counter - 1, config); - // A CRITICAL record bypasses debounce and the HEALED latch. The stored severity decides, not the - // report's. Otherwise the shared state machine decides (with hysteresis). + // CRITICAL bypasses debounce and the HEALED latch. A per-source record decides on its stored severity, + // a shared record on the report's. Otherwise the shared state machine decides (with hysteresis). std::string new_status; - if (config.critical_immediate_confirm && new_severity == fd::severity::kCritical) { + if (config.critical_immediate_confirm && + (id.owner == kSharedOwner ? severity : new_severity) == fd::severity::kCritical) { debounce_counter = std::min(debounce_counter, -1); new_status = fd::fault_status::kConfirmed; } else { diff --git a/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp b/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp index bea6abf3b..ae2831af1 100644 --- a/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp @@ -992,10 +992,11 @@ bool SqliteFaultStorage::report_fault_event_locked(const FaultId & id, fd::Repor // Decrement towards confirmation, clamped to the thresholds. debounce_counter = clamp_debounce_counter(debounce_counter - 1, config); - // A CRITICAL record bypasses debounce and the HEALED latch. The stored severity decides, not the - // report's. Otherwise the shared state machine decides (with hysteresis). + // CRITICAL bypasses debounce and the HEALED latch. A per-source record decides on its stored severity, + // a shared record on the report's. Otherwise the shared state machine decides (with hysteresis). std::string new_status; - if (config.critical_immediate_confirm && new_severity == fd::severity::kCritical) { + if (config.critical_immediate_confirm && + (id.owner == kSharedOwner ? severity : new_severity) == fd::severity::kCritical) { debounce_counter = std::min(debounce_counter, -1); new_status = fd::fault_status::kConfirmed; } else { diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py index d0c670ac1..791ad9db6 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py @@ -24,6 +24,7 @@ import os import tempfile +import time import unittest from launch import LaunchDescription @@ -33,7 +34,8 @@ import launch_testing.markers import rclpy from rclpy.node import Node -from ros2_medkit_msgs.msg import Fault +from rclpy.qos import QoSProfile, ReliabilityPolicy +from ros2_medkit_msgs.msg import Fault, FaultEvent from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault TIMEOUT_SEC = 30.0 @@ -57,6 +59,11 @@ ID.PER_LIST: per_source: true confirmation_threshold: -1 +ID.CRIT_SHARED: + confirmation_threshold: -3 +ID.CRIT_PER: + per_source: true + confirmation_threshold: -3 """ THRESHOLDS_FILE = tempfile.NamedTemporaryFile( @@ -98,8 +105,16 @@ def setUpClass(cls): cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') cls.get_client = cls.node.create_client(GetFault, '/fault_manager/get_fault') cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + cls.events = [] + cls.event_sub = cls.node.create_subscription( + FaultEvent, '/fault_manager/events', cls.events.append, + QoSProfile(depth=100, reliability=ReliabilityPolicy.RELIABLE)) for client in (cls.report_client, cls.list_client, cls.get_client, cls.clear_client): assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + deadline = time.monotonic() + TIMEOUT_SEC + while cls.event_sub.get_publisher_count() == 0 and time.monotonic() < deadline: + rclpy.spin_once(cls.node, timeout_sec=0.1) + assert cls.event_sub.get_publisher_count() > 0 @classmethod def tearDownClass(cls): @@ -112,11 +127,12 @@ def _call(self, client, request): self.assertTrue(future.done(), 'service call timed out') return future.result() - def _report(self, code, source, event_type=ReportFault.Request.EVENT_FAILED): + def _report(self, code, source, event_type=ReportFault.Request.EVENT_FAILED, + severity=Fault.SEVERITY_ERROR): request = ReportFault.Request() request.fault_code = code request.event_type = event_type - request.severity = Fault.SEVERITY_ERROR + request.severity = severity request.description = f'{code} from {source}' request.source_id = source return self._call(self.report_client, request).accepted @@ -223,6 +239,51 @@ def test_default_and_per_source_codes_side_by_side(self): self.assertEqual(len(self._records('ID.PLAIN_LIST')), 1) self.assertEqual(len(self._records('ID.PER_LIST')), 3) + def _confirmed_events(self, code): + return [e for e in self.events + if e.fault.fault_code == code and e.event_type == FaultEvent.EVENT_CONFIRMED] + + def _wait_for_confirmed_event(self, code): + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline and not self._confirmed_events(code): + rclpy.spin_once(self.node, timeout_sec=0.1) + self.assertTrue(self._confirmed_events(code), f'no fault_confirmed event for {code}') + + def _critical_clear_warn(self, code, first_source, second_source): + """CRITICAL, clear the record, then a WARN: the sequence that tells the two rules apart.""" + self.assertTrue(self._report(code, first_source, severity=Fault.SEVERITY_CRITICAL)) + self._wait_for_confirmed_event(code) + cleared = self._clear(code) + self.assertTrue(cleared.success, cleared.message) + self.assertTrue(self._report(code, second_source, severity=Fault.SEVERITY_WARN)) + # The events arrive in order, so once a marker's confirmation is here, any event the WARN + # report produced is too. + marker = f'{code}.MARKER' + self.assertTrue(self._report(marker, '/crit/marker', severity=Fault.SEVERITY_CRITICAL)) + self._wait_for_confirmed_event(marker) + + def test_a_shared_record_confirms_on_the_severity_of_the_report(self): + """CRITICAL from /a, clear, WARN from /b on a default code stays pending.""" + code = 'ID.CRIT_SHARED' + self._critical_clear_warn(code, '/crit/a', '/crit/b') + record = self._get(code).fault + self.assertEqual(record.status, Fault.STATUS_PREFAILED) + self.assertEqual(record.occurrence_count, 2) + request = ListFaults.Request() + listed = self._call(self.list_client, request).faults + self.assertEqual([f for f in listed if f.fault_code == code], []) + self.assertEqual(len(self._confirmed_events(code)), 1) + + def test_a_per_source_record_confirms_on_its_stored_severity(self): + """CRITICAL from /a, clear, WARN from /a on a per_source code confirms at once.""" + code = 'ID.CRIT_PER' + self._critical_clear_warn(code, '/crit/a', '/crit/a') + record = self._get(code).fault + self.assertEqual(record.status, Fault.STATUS_CONFIRMED) + self.assertEqual(record.severity, Fault.SEVERITY_CRITICAL) + self.assertEqual(record.occurrence_count, 2) + self.assertEqual(len(self._confirmed_events(code)), 2) + def test_the_shared_owner_is_not_a_source_id(self): """A report claiming the reserved owner is refused and creates nothing.""" for code in ('ID.RESERVED_SHARED', 'ID.PER'): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py b/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py index 619368133..fa08205c4 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py @@ -88,8 +88,17 @@ def seed_legacy_database(): f'["{SOURCE}"]', 500_000_000, LEGACY_CONFIRMED_AT_NS)) +# Codes whose records are per source, so the stored severity decides immediate confirmation. The +# migrated rows and every other code are shared records, where the report's severity decides. +PER_SOURCE_CODES = ('SEV_CRIT', 'HEAL_CRIT') + tuple(f'SEV_MARK_{i}' for i in range(1, 9)) +THRESHOLDS_PATH = os.path.join(DB_DIR, 'fault_thresholds.yaml') + + def generate_test_description(): seed_legacy_database() + with open(THRESHOLDS_PATH, 'w') as f: + for code in PER_SOURCE_CODES: + f.write(f'{code}:\n per_source: true\n') fault_manager_node = launch_ros.actions.Node( package='ros2_medkit_fault_manager', executable='fault_manager_node', @@ -101,6 +110,7 @@ def generate_test_description(): 'confirmation_threshold': -3, 'healing_enabled': True, 'healing_threshold': 1, + 'fault_thresholds.config_file': THRESHOLDS_PATH, 'snapshots.enabled': False, 'snapshots.rosbag.enabled': False, }], @@ -277,11 +287,10 @@ def test_severity_contract_across_restarts(self, fault_manager_node, proc_output self.assertEqual(fault.status, Fault.STATUS_PREFAILED) self.assertEqual(fault.occurrence_count, 2) self.assertEqual(self._confirmed_at('LEGACY_CONF_AT'), 0) - with self.subTest(case='migrated CRITICAL, WARN'): + with self.subTest(case='migrated shared CRITICAL, WARN: the report decides'): fault = self._fault('LEGACY_CRIT') self.assertEqual(fault.severity, Fault.SEVERITY_CRITICAL) - self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) - self.assertGreater(self._confirmed_at('LEGACY_CRIT'), LEGACY_CONFIRMED_AT_NS) + self.assertEqual(fault.status, Fault.STATUS_PREFAILED) # Process 1: CRITICAL, cleared; CRITICAL, healed; ERROR confirmed, cleared. self._report('SEV_CRIT', Fault.SEVERITY_CRITICAL) @@ -321,10 +330,10 @@ def test_severity_contract_across_restarts(self, fault_manager_node, proc_output self.assertEqual(fault.occurrence_count, 1) with self.subTest(case='no pending CRITICAL'): - self.assertEqual( - self._list(statuses=[Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED], - severity=Fault.SEVERITY_CRITICAL), - {}) + pending = self._list(statuses=[Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED], + severity=Fault.SEVERITY_CRITICAL) + # LEGACY_CRIT is a shared record, which a WARN report leaves pending. + self.assertEqual(sorted(pending), ['LEGACY_CRIT']) for _ in range(9): self._report('SEV_CRIT', Fault.SEVERITY_WARN) diff --git a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp index e8b4096d2..d075a8a7e 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp @@ -1695,10 +1695,13 @@ class SeverityConsistencyNodeTest : public FaultEventPublishingTest, public ::te std::uniform_int_distribution dist; dir_ = std::filesystem::temp_directory_path() / ("test_severity_node_" + std::to_string(dist(gen))); std::filesystem::create_directories(dir_); - // The codes two sources report each get a record per source, which is what the owner-scoped - // assertions below are about. - std::ofstream(dir_ / "fault_thresholds.yaml") - << "MISMATCH:\n per_source: true\nENTITY_MIXED:\n per_source: true\n"; + // Every code these tests report is per_source: the stored severity of a record decides immediate + // confirmation there, and the owner-scoped assertions below are about one record per source. + std::ofstream thresholds(dir_ / "fault_thresholds.yaml"); + for (const char * code : {"EVT_SEV", "AUDIT_SEV", "AUDIT_REACT", "MISMATCH", "MISMATCH_A", "MISMATCH_B", + "ENTITY_WARN", "ENTITY_CRIT", "ENTITY_MIXED"}) { + thresholds << code << ":\n per_source: true\n"; + } } ~SeverityConsistencyNodeTest() override { @@ -1831,7 +1834,7 @@ TEST_P(SeverityConsistencyNodeTest, AuditRowsCarryTheStoredSeverity) { // The near-miss rows of the same reports keep what was reported. std::vector near_miss_severities; - for (const auto & record : fault_manager_->get_storage().get_near_misses({"AUDIT_REACT", "*"})) { + for (const auto & record : fault_manager_->get_storage().get_near_misses({"AUDIT_REACT", "/a"})) { near_miss_severities.push_back(record.severity); } EXPECT_EQ(near_miss_severities, (std::vector{error, error, warn, warn})); diff --git a/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp b/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp index 27efbae7e..24db91440 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp @@ -413,6 +413,21 @@ class SqliteSeverityTest : public ::testing::Test { open_legacy_database({LegacyRow{code, severity, status, confirmed_at_ns}}); } + /// open_legacy_database, then hand the migrated row to the test's reporter, so it is a per-source record + /// and the stored severity decides immediate confirmation. + void open_legacy_per_source_database(const std::string & code, int severity, const std::string & status, + int64_t confirmed_at_ns) { + open_legacy_database(code, severity, status, confirmed_at_ns); + storage_.reset(); + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path().c_str(), &raw), SQLITE_OK); + const std::string sql = std::string("UPDATE faults SET owner = '") + kOwner + "'"; + EXPECT_EQ(sqlite3_exec(raw, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK) << sqlite3_errmsg(raw); + sqlite3_close(raw); + record_owner_ = kOwner; + reopen(); + } + void failed(const std::string & code, uint8_t severity, int64_t second, const DebounceConfig & config) { storage_->report_fault_event(FaultId{code, record_owner_}, fd::ReportKind::kFailed, severity, "report", kOwner, at(second), config); @@ -551,7 +566,7 @@ TEST_F(SqliteSeverityTest, MigratedClearedRecordHasNoConfirmationTimeUntilItConf TEST_F(SqliteSeverityTest, MigratedClearedCriticalRecordConfirmsOnAWarn) { const auto config = confirm_at(-3); - open_legacy_database("MIGRATED_CRIT", Fault::SEVERITY_CRITICAL, Fault::STATUS_CLEARED, at(0)); + open_legacy_per_source_database("MIGRATED_CRIT", Fault::SEVERITY_CRITICAL, Fault::STATUS_CLEARED, at(0)); failed("MIGRATED_CRIT", Fault::SEVERITY_WARN, 2, config); const auto f = fault("MIGRATED_CRIT"); @@ -563,7 +578,7 @@ TEST_F(SqliteSeverityTest, MigratedClearedCriticalRecordConfirmsOnAWarn) { TEST_F(SqliteSeverityTest, PendingCriticalRowFromAnEarlierReleaseStaysUntilAReport) { // The row an earlier release left after CRITICAL, clear, WARN at threshold -3: CRITICAL in // PREFAILED, occurrence 2, still holding the first occurrence's confirmation time. - open_legacy_database("LEGACY", Fault::SEVERITY_CRITICAL, Fault::STATUS_PREFAILED, at(0)); + open_legacy_per_source_database("LEGACY", Fault::SEVERITY_CRITICAL, Fault::STATUS_PREFAILED, at(0)); auto f = fault("LEGACY"); EXPECT_EQ(f.status, Fault::STATUS_PREFAILED) << "the migration does not rewrite the status"; EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); @@ -657,7 +672,7 @@ TEST_F(SqliteSeverityTest, PendingCriticalRowConfirmsOnTheNextFailedOfAnySeverit {Fault::SEVERITY_INFO, Fault::SEVERITY_WARN, Fault::SEVERITY_ERROR, Fault::SEVERITY_CRITICAL}) { const std::string code = "LEGACY_FAILED_" + std::to_string(severity); SCOPED_TRACE(code); - open_legacy_database(code, Fault::SEVERITY_CRITICAL, Fault::STATUS_PREFAILED, at(0)); + open_legacy_per_source_database(code, Fault::SEVERITY_CRITICAL, Fault::STATUS_PREFAILED, at(0)); failed(code, severity, 2, config); const auto f = fault(code); @@ -670,7 +685,7 @@ TEST_F(SqliteSeverityTest, PendingCriticalRowConfirmsOnTheNextFailedOfAnySeverit TEST_F(SqliteSeverityTest, PendingCriticalRowCanMoveToPrepassedOnPassed) { const auto config = confirm_at(-3); - open_legacy_database("LEGACY_PASSED", Fault::SEVERITY_CRITICAL, Fault::STATUS_PREFAILED, at(0)); + open_legacy_per_source_database("LEGACY_PASSED", Fault::SEVERITY_CRITICAL, Fault::STATUS_PREFAILED, at(0)); passed("LEGACY_PASSED", 2, config); EXPECT_EQ(fault("LEGACY_PASSED").status, Fault::STATUS_PREFAILED) << "counter 0 keeps the status"; From 0b1bc957c0a6d35713cb9dd1813d2c5b5cde6563 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 17:22:16 +0100 Subject: [PATCH 05/52] fault_manager: test the shared-record CRITICAL rule on PostgreSQL --- .../test/test_postgres_storage.cpp | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp b/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp index 34b1dc6a6..59cfe7222 100644 --- a/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp +++ b/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp @@ -3236,6 +3236,31 @@ TEST_F(PgFaultStorageTest, NearMissSeriesIsPerRecord) { EXPECT_EQ(backup.front().severity, Fault::SEVERITY_ERROR); } +TEST_F(PgFaultStorageTest, ASharedRecordConfirmsCriticalOnTheReportsSeverity) { + rclcpp::Clock clock; + DebounceConfig config; + config.confirmation_threshold = -3; + ASSERT_TRUE(config.critical_immediate_confirm); + const FaultId shared{"CRIT_SHARED", ros2_medkit_fault_manager::kSharedOwner}; + const FaultId per_source{"CRIT_PER", "/a"}; + + storage_->report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "", "/a", + clock.now().nanoseconds(), config); + ASSERT_EQ(storage_->get_fault(shared)->status, Fault::STATUS_CONFIRMED); + ASSERT_TRUE(storage_->clear_fault(shared)); + EXPECT_TRUE(storage_->report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "", "/b", + clock.now().nanoseconds(), config)); + EXPECT_EQ(storage_->get_fault(shared)->severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(storage_->get_fault(shared)->status, Fault::STATUS_PREFAILED) << "the report's severity decides"; + + storage_->report_fault_event(per_source, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "", "/a", + clock.now().nanoseconds(), config); + ASSERT_TRUE(storage_->clear_fault(per_source)); + storage_->report_fault_event(per_source, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "", "/a", + clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(per_source)->status, Fault::STATUS_CONFIRMED) << "the stored severity decides"; +} + TEST_F(PgFaultStorageTest, ACriticalRecordConfirmsOnAnyFailedReportAndAnotherOwnersIsUntouched) { rclcpp::Clock clock; DebounceConfig config; From c02458dd15dd802332c3e4671dc3b35b27bbefb9 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 18:00:20 +0100 Subject: [PATCH 06/52] integration_tests: declare the severity consistency codes per_source --- .../features/test_fault_severity_consistency.test.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py b/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py index f7ede9451..a1a2bf025 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py @@ -16,8 +16,8 @@ """A fault record shows one severity on every gateway surface. Real fault manager (SQLite, confirmation threshold -3), real gateway, reports -through the ReportFault service. Each code has one owner, so the gateway's -code-addressed routes reach its one record. A CRITICAL fault is acknowledged +through the ReportFault service. Both codes are per_source, so a record belongs +to one owner and the gateway's code-addressed routes reach it. A CRITICAL fault is acknowledged and then reported again as WARN. The record keeps CRITICAL, so the WARN report confirms it at once, and GET /faults, the fault detail, the /faults/stream frame and the payload of an OnChange trigger all carry the same severity and @@ -50,9 +50,13 @@ CODE = 'SEV_CONSISTENCY' PRIME_CODE = 'SEV_CONSISTENCY_PRIME' DB_DIR = tempfile.mkdtemp(prefix='medkit_severity_e2e_') +THRESHOLDS_PATH = os.path.join(DB_DIR, 'fault_thresholds.yaml') def generate_test_description(): + with open(THRESHOLDS_PATH, 'w') as f: + for code in (CODE, PRIME_CODE): + f.write(f'{code}:\n per_source: true\n') return create_test_launch( demo_nodes=['temp_sensor'], fault_manager=True, @@ -60,6 +64,7 @@ def generate_test_description(): 'storage_type': 'sqlite', 'database_path': os.path.join(DB_DIR, 'faults.db'), 'confirmation_threshold': -3, + 'fault_thresholds.config_file': THRESHOLDS_PATH, 'snapshots.rosbag.enabled': False, }, gateway_params={ From 8894eff30dba205a83bf0566ff8313e97f7ca3dc Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 17:31:20 +0100 Subject: [PATCH 07/52] fault_manager: correlate records across owners Correlation rules match by fault code whoever reports it. A root cause of one node mutes, and with auto_clear_with_root auto-clears, the matching symptom records of any other node inside the window. An auto-cluster rule forms one pending cluster that collects the matching records of every node. The relations hold records: muted records, the root-to-symptom map, cluster members and the cluster representative are FaultIds, so a clear acts on the exact records the root cause muted and muting hides a record, not a code. ClusterInfo gains fault_owners, one entry per fault_codes entry in the same order, and representative_owner, so a consumer can address each member with the single-record services. --- docs/api/messages.rst | 8 +- docs/tutorials/fault-correlation.rst | 8 + src/ros2_medkit_fault_manager/CMakeLists.txt | 3 + src/ros2_medkit_fault_manager/README.md | 14 +- .../core/fault_manager_core.hpp | 2 + .../correlation/correlation_engine.hpp | 80 +++-- .../fault_storage.hpp | 4 + .../src/core/fault_manager_core.cpp | 10 +- .../src/correlation/correlation_engine.cpp | 180 ++++++----- .../src/fault_manager_node.cpp | 2 + .../test_fault_correlation_owners.test.py | 295 ++++++++++++++++++ .../test/test_correlation_engine.cpp | 44 ++- .../test/test_fault_manager.cpp | 28 +- src/ros2_medkit_msgs/msg/ClusterInfo.msg | 8 + src/ros2_medkit_msgs/msg/MutedFaultInfo.msg | 4 +- 15 files changed, 526 insertions(+), 164 deletions(-) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py diff --git a/docs/api/messages.rst b/docs/api/messages.rst index e6c2d6f02..2017cca57 100644 --- a/docs/api/messages.rst +++ b/docs/api/messages.rst @@ -154,7 +154,7 @@ MutedFaultInfo.msg ~~~~~~~~~~~~~~~~~~ Information about a correlated (muted) symptom record. One entry per muted RECORD: -a root cause mutes the symptoms of its own record owner only, so two per-source records +a root cause mutes the matching symptom records of every owner, so two per-source records muted on one ``fault_code`` produce two entries carrying that code, told apart by ``source_id``. @@ -183,6 +183,12 @@ Auto-detected fault cluster information. uint32 count # Number of faults builtin_interfaces/Time first_at # First fault timestamp builtin_interfaces/Time last_at # Last fault timestamp + string[] fault_owners # Owner of each member, parallel to fault_codes + string representative_owner # Owner of the representative record + +A cluster holds the matching records of every owner, so a member is the record +``(fault_codes[i], fault_owners[i])``. Pass the owner as ``source_id`` to the +single-record services. The owner of a shared record is ``*``. Services -------- diff --git a/docs/tutorials/fault-correlation.rst b/docs/tutorials/fault-correlation.rst index 07357780c..e5c429f3e 100644 --- a/docs/tutorials/fault-correlation.rst +++ b/docs/tutorials/fault-correlation.rst @@ -120,6 +120,14 @@ How It Works fm --> src: [SENSOR_003]\nclusters: [{id: "sensor_storm_1",\n fault_codes: [001,002,003,004],\n representative: "SENSOR_003"}] @enduml +**Rules match by fault code across reporting nodes.** A root cause reported by one +node mutes, and with ``auto_clear_with_root`` auto-clears, the matching symptoms +reported by any other node inside the window. An auto-cluster collects the matching +faults of every node. The relations hold fault records, not codes: for a code +configured with ``per_source: true`` each reporting node has its own record, and the +cascade clears exactly the records the root cause muted. For any other code there is +one shared record, and that record is what is muted or cleared. + Quick Start ----------- diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index 0af0a80fa..9d59daedd 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -300,6 +300,9 @@ endif() medkit_add_launch_test(test_fault_identity test/integration/test_fault_identity.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_correlation_owners + test/integration/test_fault_correlation_owners.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_identity_restart test/integration/test_fault_identity_restart.test.py TIMEOUT 180 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/README.md b/src/ros2_medkit_fault_manager/README.md index db523967d..4debedb33 100644 --- a/src/ros2_medkit_fault_manager/README.md +++ b/src/ros2_medkit_fault_manager/README.md @@ -496,6 +496,10 @@ subsequent matching symptom faults within a time window are correlated and optio **Auto-Cluster**: Automatically groups related faults that match a pattern within a time window. Useful for detecting "storms" of related faults (e.g., communication errors). +Rules match by fault code across reporting nodes: a root cause reported by one node mutes and +auto-clears the matching symptoms of any other node, and a cluster collects the matching faults of +every node. The relations hold records, so a clear acts on exactly the records the root cause muted. + ### Configuration Enable correlation by providing a YAML configuration file: @@ -577,10 +581,12 @@ Response includes: - `muted_count`: Number of muted symptom faults - `cluster_count`: Number of active fault clusters - `muted_faults[]`: Details of muted records (when `include_muted=true`), one entry per muted - record. Muting is per record, so a symptom muted under one owner's root cause never hides - another owner's record of the same code, and each entry names its owner in `source_id` (`*` for a - shared record). -- `clusters[]`: Details of active clusters (when `include_clusters=true`) + record. Muting is per record: it hides that record, not every record of the code, and each entry + names its owner in `source_id` (`*` for a shared record). +- `clusters[]`: Details of active clusters (when `include_clusters=true`). A member is a record: + `fault_owners[i]` is the owner of `fault_codes[i]`, and `(representative_code, + representative_owner)` names the representative. Pass the owner as `source_id` to GetFault and + ClearFault. ### REST API (via Gateway) diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp index 1f98796f1..0c5aa4fd3 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp @@ -143,8 +143,10 @@ struct ClusterSummary { std::string rule_name; std::string label; std::string representative_code; + std::string representative_owner; std::string representative_severity; std::vector fault_codes; + std::vector fault_owners; ///< Parallel to fault_codes uint32_t count{0}; int64_t first_at_ns{0}; int64_t last_at_ns{0}; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp index aaa7c533f..5bdc00df0 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp @@ -50,17 +50,16 @@ struct ProcessFaultResult { /// For auto-cluster: cluster ID if this fault triggered or joined a cluster std::string cluster_id; - /// For auto-cluster: fault codes that should be retroactively muted - /// When cluster reaches min_count threshold, previous non-representative faults + /// For auto-cluster: records that should be retroactively muted + /// When cluster reaches min_count threshold, previous non-representative records /// that were added before threshold was reached need to be muted retroactively - std::vector retroactive_mute_codes; + std::vector retroactive_mute_ids; }; /// Result of clearing a fault struct ProcessClearResult { - /// The symptom RECORDS that should be auto-cleared. Records, not codes: a root - /// cause of one owner must never auto-clear another owner's record of the same - /// symptom code. + /// The symptom RECORDS that should be auto-cleared. Records, not codes: a clear + /// acts on the exact records the root cause muted, whoever owns them. std::vector auto_cleared_symptoms; }; @@ -77,15 +76,19 @@ struct MutedFaultData { uint32_t delay_ms{0}; }; -/// Information about an active cluster (for ListFaults response) +/// Information about an active cluster (for ListFaults response). +/// A member is a record: fault_owners[i] is the owner of fault_codes[i], and +/// (representative_code, representative_owner) is the representative record. struct ClusterData { std::string cluster_id; std::string rule_id; std::string rule_name; std::string label; std::string representative_code; + std::string representative_owner; std::string representative_severity; std::vector fault_codes; + std::vector fault_owners; std::chrono::system_clock::time_point first_at; std::chrono::system_clock::time_point last_at; }; @@ -106,11 +109,9 @@ class CorrelationEngine { /// Process an incoming fault record /// - /// Rules match on the fault CODE - a rule says which codes are root causes and which - /// are symptoms, and says nothing about who reports them. Every RELATION the engine - /// then forms (root to symptom, muting, cluster membership, auto-clear cascade) is - /// between records of the SAME owner, so a root cause of owner X never mutes or - /// auto-clears a symptom of owner Y. + /// Rules match on the fault CODE, whoever owns the record: a root cause of one owner + /// mutes and auto-clears matching symptom records of any owner. Every RELATION the + /// engine forms (root to symptom, muting, cluster membership) holds exact records. /// @param id The fault record /// @param severity The fault severity (for representative selection) /// @param timestamp When the fault occurred @@ -120,7 +121,7 @@ class CorrelationEngine { /// Process a fault record being cleared /// @param id The record being cleared - /// @return Result with the symptom records to auto-clear, all of the same owner + /// @return Result with the symptom records to auto-clear ProcessClearResult process_clear(const FaultId & id); /// Get all currently muted faults @@ -132,7 +133,7 @@ class CorrelationEngine { /// Whether a fault record is currently muted as a symptom. /// @param id Record to test - /// @return True while the record is suppressed by a root cause of the same owner + /// @return True while the record is suppressed by a root cause bool is_muted(const FaultId & id) const; /// Get all active clusters @@ -151,12 +152,12 @@ class CorrelationEngine { /// @return Rule ID if matched, empty optional otherwise std::optional try_as_root_cause(const std::string & fault_code); - /// Check if the record is a symptom of a pending root cause OF THE SAME OWNER + /// Check if the record is a symptom of a pending root cause /// @return ProcessFaultResult with correlation info if matched std::optional try_as_symptom(const FaultId & id, std::chrono::steady_clock::time_point timestamp); - /// Check if the record's code matches an auto-cluster rule. A cluster holds records - /// of one owner, so the pending-cluster key carries the owner alongside the rule id. + /// Check if the record's code matches an auto-cluster rule. A cluster holds the records + /// of any owner that match the rule inside its window. /// @return ProcessFaultResult with cluster info if matched std::optional try_auto_cluster(const FaultId & id, const std::string & severity, std::chrono::steady_clock::time_point timestamp); @@ -164,14 +165,28 @@ class CorrelationEngine { /// Generate unique cluster ID std::string generate_cluster_id(const std::string & rule_id); + /// A cluster as the engine keeps it: members and the representative are records. + struct Cluster { + std::string cluster_id; + std::string rule_id; + std::string rule_name; + std::string label; + FaultId representative; + std::string representative_severity; + std::vector members; + std::chrono::system_clock::time_point first_at; + std::chrono::system_clock::time_point last_at; + }; + /// Record a re-reported member's severity and update the representative if it changes - void update_member_severity(const CorrelationRule & rule, ClusterData & cluster, - std::map & fault_severities, const std::string & fault_code, + void update_member_severity(const CorrelationRule & rule, Cluster & cluster, + std::map & severities, const FaultId & id, const std::string & severity) const; - /// Reassign cluster representative from remaining fault_codes and their severities - void reassign_representative(ClusterData & cluster, - const std::map & fault_severities) const; + /// Reassign cluster representative from the remaining members and their severities + void reassign_representative(Cluster & cluster, const std::map & severities) const; + + static ClusterData to_cluster_data(const Cluster & cluster); CorrelationConfig config_; std::unique_ptr matcher_; @@ -185,34 +200,31 @@ class CorrelationEngine { }; std::vector pending_root_causes_; - /// Mapping from a root cause RECORD to its symptom RECORDS, all of the same owner + /// Mapping from a root cause RECORD to its symptom RECORDS std::map> root_to_symptoms_; /// Muted records (record -> data) std::map muted_faults_; - /// Active clusters (cluster_id -> data) - std::map active_clusters_; + /// Active clusters (cluster_id -> cluster) + std::map active_clusters_; - /// Per-fault severities for active clusters (cluster_id -> (fault_code -> severity)). A cluster - /// holds the records of one owner, so a code appears once per cluster. - std::map> active_cluster_severities_; + /// Per-record severities for active clusters (cluster_id -> (record -> severity)) + std::map> active_cluster_severities_; /// Mapping from a record to its cluster ID (for records in clusters) std::map fault_to_cluster_; /// Pending cluster with steady_clock timestamp for window tracking struct PendingCluster { - ClusterData data; - std::string owner; ///< Every member record of this cluster has this owner + Cluster data; std::chrono::steady_clock::time_point steady_first_at; - std::map fault_severities; ///< fault_code -> severity + std::map fault_severities; ///< record -> severity }; - /// Pending clusters being formed ((rule_id, owner) -> cluster data). Keyed by owner - /// too, so one rule forms one cluster per owner and membership never crosses owners. + /// Pending clusters being formed (rule_id -> cluster data). /// Once min_count is reached, moved to active_clusters_ - std::map, PendingCluster> pending_clusters_; + std::map pending_clusters_; /// Counter for cluster ID generation uint64_t cluster_counter_{0}; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp index cd41de7aa..4a45b33a0 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp @@ -117,6 +117,10 @@ struct FaultId { return fault_code == other.fault_code && owner == other.owner; } + bool operator!=(const FaultId & other) const { + return !(*this == other); + } + /// Ordered by code first, so the records of one code are contiguous in a map and /// the ids of one code are a range scan rather than a full sweep. bool operator<(const FaultId & other) const { diff --git a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp index fb1911bb0..55322bfc7 100644 --- a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp +++ b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp @@ -473,9 +473,9 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { logf(log_, LogLevel::kDebug, logger, "Fault %s added to cluster %s", report.fault_code.c_str(), correlation_result.cluster_id.c_str()); // Log retroactively muted faults when cluster becomes active - if (!correlation_result.retroactive_mute_codes.empty()) { + if (!correlation_result.retroactive_mute_ids.empty()) { logf(log_, LogLevel::kDebug, logger, "Cluster %s activated: retroactively muting %zu faults", - correlation_result.cluster_id.c_str(), correlation_result.retroactive_mute_codes.size()); + correlation_result.cluster_id.c_str(), correlation_result.retroactive_mute_ids.size()); } } } @@ -595,8 +595,8 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, const std: return outcome; } - // Then the correlation engine (to get the auto-clear list). Symptoms come back as - // records of the same owner, so a cascade never crosses an owner boundary. + // Then the correlation engine (to get the auto-clear list). Symptoms come back as the + // exact records the root cause muted. // `skip_correlation_auto_clear` lets the caller opt out of cascade-clearing // correlated symptom faults. Per-entity DELETE routes set it to true // so they cannot reach across entity boundaries via the correlation graph. @@ -728,8 +728,10 @@ ListOutcome FaultManagerCore::list_locked(const ListQuery & query) { info.rule_name = cluster.rule_name; info.label = cluster.label; info.representative_code = cluster.representative_code; + info.representative_owner = cluster.representative_owner; info.representative_severity = cluster.representative_severity; info.fault_codes = cluster.fault_codes; + info.fault_owners = cluster.fault_owners; info.count = static_cast(cluster.fault_codes.size()); info.first_at_ns = std::chrono::duration_cast(cluster.first_at.time_since_epoch()).count(); diff --git a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp index f407f219a..a286c92aa 100644 --- a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp +++ b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp @@ -28,8 +28,7 @@ ProcessFaultResult CorrelationEngine::process_fault(const FaultId & id, const st std::chrono::steady_clock::time_point timestamp) { std::lock_guard lock(mutex_); - // Rules match on the CODE, and every relation formed below is between records of the - // same owner. + // Rules match on the CODE, whoever owns the record. Relations formed below hold exact records. const std::string & fault_code = id.fault_code; ProcessFaultResult result; @@ -45,7 +44,7 @@ ProcessFaultResult CorrelationEngine::process_fault(const FaultId & id, const st }), pending_root_causes_.end()); - // Check if this record is a symptom of an existing root cause of the same owner + // Check if this record is a symptom of an existing root cause auto symptom_result = try_as_symptom(id, timestamp); if (symptom_result) { return *symptom_result; @@ -93,8 +92,8 @@ ProcessClearResult CorrelationEngine::process_clear(const FaultId & id) { const std::string & fault_code = id.fault_code; ProcessClearResult result; - // Check if this record is a root cause with symptoms. The lookup is by record, so - // clearing owner X's root cause never reaches owner Y's symptoms. + // Check if this record is a root cause with symptoms. The lookup is by record, so the + // cascade reaches exactly the records this root cause muted. auto it = root_to_symptoms_.find(id); if (it != root_to_symptoms_.end()) { // Find the rule to check auto_clear_with_root @@ -152,36 +151,36 @@ ProcessClearResult CorrelationEngine::process_clear(const FaultId & id) { continue; } - auto & codes = pending_cluster.fault_codes; - codes.erase(std::remove(codes.begin(), codes.end(), fault_code), codes.end()); - pending_it->second.fault_severities.erase(fault_code); + auto & members = pending_cluster.members; + members.erase(std::remove(members.begin(), members.end(), id), members.end()); + pending_it->second.fault_severities.erase(id); - if (codes.empty()) { + if (members.empty()) { pending_it = pending_clusters_.erase(pending_it); continue; } - // Reassign representative if the cleared fault was the representative - if (pending_cluster.representative_code == fault_code) { + // Reassign representative if the cleared record was the representative + if (pending_cluster.representative == id) { reassign_representative(pending_cluster, pending_it->second.fault_severities); } ++pending_it; } - // Remove fault from active cluster + // Remove record from active cluster auto active_it = active_clusters_.find(cluster_id); if (active_it != active_clusters_.end()) { auto & active_cluster = active_it->second; - auto & codes = active_cluster.fault_codes; - codes.erase(std::remove(codes.begin(), codes.end(), fault_code), codes.end()); + auto & members = active_cluster.members; + members.erase(std::remove(members.begin(), members.end(), id), members.end()); auto & active_sevs = active_cluster_severities_[cluster_id]; - active_sevs.erase(fault_code); + active_sevs.erase(id); - if (codes.empty()) { + if (members.empty()) { active_clusters_.erase(active_it); active_cluster_severities_.erase(cluster_id); - } else if (active_cluster.representative_code == fault_code) { + } else if (active_cluster.representative == id) { reassign_representative(active_cluster, active_sevs); } } @@ -228,8 +227,8 @@ std::vector CorrelationEngine::get_clusters() const { std::vector result; result.reserve(active_clusters_.size()); - for (const auto & [id, data] : active_clusters_) { - result.push_back(data); + for (const auto & [cluster_id, cluster] : active_clusters_) { + result.push_back(to_cluster_data(cluster)); } return result; @@ -256,14 +255,14 @@ void CorrelationEngine::cleanup_expired() { pending_root_causes_.end()); // Clean up expired pending clusters - std::vector> expired_pending; - for (const auto & [key, pending] : pending_clusters_) { + std::vector expired_pending; + for (const auto & [rule_id, pending] : pending_clusters_) { // Find rule to get window_ms for (const auto & rule : config_.rules) { - if (rule.id == key.first) { + if (rule.id == rule_id) { auto elapsed = std::chrono::duration_cast(now - pending.steady_first_at).count(); if (elapsed > static_cast(rule.window_ms)) { - expired_pending.push_back(key); + expired_pending.push_back(rule_id); } break; } @@ -274,8 +273,8 @@ void CorrelationEngine::cleanup_expired() { auto it = pending_clusters_.find(key); if (it != pending_clusters_.end()) { if (active_clusters_.find(it->second.data.cluster_id) == active_clusters_.end()) { - for (const auto & fault_code : it->second.data.fault_codes) { - fault_to_cluster_.erase(FaultId{fault_code, it->second.owner}); + for (const auto & member : it->second.data.members) { + fault_to_cluster_.erase(member); } } pending_clusters_.erase(it); @@ -301,11 +300,6 @@ std::optional CorrelationEngine::try_as_symptom(const FaultI std::chrono::steady_clock::time_point timestamp) { const std::string & fault_code = id.fault_code; for (const auto & prc : pending_root_causes_) { - // A root cause only explains its OWN reporter's faults. Without this the first - // owner to report a root-cause code would mute every other owner's symptom. - if (prc.fault_id.owner != id.owner) { - continue; - } // Find the rule for (const auto & rule : config_.rules) { if (rule.id != prc.rule_id || rule.mode != CorrelationMode::HIERARCHICAL) { @@ -352,6 +346,7 @@ std::optional CorrelationEngine::try_as_symptom(const FaultI if (rule.mute_symptoms) { MutedFaultData muted; muted.fault_code = fault_code; + muted.owner = id.owner; muted.root_cause_code = prc.fault_id.fault_code; muted.rule_id = rule.id; muted.delay_ms = result.delay_ms; @@ -388,11 +383,9 @@ std::optional CorrelationEngine::try_auto_cluster(const Faul auto now_system = std::chrono::system_clock::now(); - // Check if we have a pending cluster for this rule AND this owner. One rule forms - // one cluster per owner, so a burst on owner A never counts owner B's faults - // towards min_count and never mutes them as non-representative members. - const auto pending_key = std::make_pair(rule.id, id.owner); - auto pending_it = pending_clusters_.find(pending_key); + // One pending cluster per rule: records of any owner that match the rule inside the + // window count towards min_count. + auto pending_it = pending_clusters_.find(rule.id); if (pending_it != pending_clusters_.end()) { // Check if within time window using steady_clock timestamp auto elapsed = @@ -401,8 +394,8 @@ std::optional CorrelationEngine::try_auto_cluster(const Faul if (elapsed > static_cast(rule.window_ms)) { // Window expired, start new cluster if (active_clusters_.find(pending_it->second.data.cluster_id) == active_clusters_.end()) { - for (const auto & old_code : pending_it->second.data.fault_codes) { - fault_to_cluster_.erase(FaultId{old_code, id.owner}); + for (const auto & old_member : pending_it->second.data.members) { + fault_to_cluster_.erase(old_member); } } pending_clusters_.erase(pending_it); @@ -417,16 +410,15 @@ std::optional CorrelationEngine::try_auto_cluster(const Faul if (active_it != active_clusters_.end() && active_it->second.rule_id == rule.id) { auto & active_cluster = active_it->second; active_cluster.last_at = now_system; - update_member_severity(rule, active_cluster, active_cluster_severities_[active_cluster.cluster_id], fault_code, + update_member_severity(rule, active_cluster, active_cluster_severities_[active_cluster.cluster_id], id, severity); if (pending_it != pending_clusters_.end() && pending_it->second.data.cluster_id == active_cluster.cluster_id) { pending_it->second.data.last_at = now_system; - update_member_severity(rule, pending_it->second.data, pending_it->second.fault_severities, fault_code, - severity); + update_member_severity(rule, pending_it->second.data, pending_it->second.fault_severities, id, severity); } ProcessFaultResult result; result.cluster_id = active_cluster.cluster_id; - if (rule.show_as_single && fault_code != active_cluster.representative_code) { + if (rule.show_as_single && id != active_cluster.representative) { result.should_mute = true; } return result; @@ -436,20 +428,19 @@ std::optional CorrelationEngine::try_auto_cluster(const Faul if (pending_it == pending_clusters_.end()) { // Start new pending cluster PendingCluster pending; - pending.owner = id.owner; pending.steady_first_at = timestamp; pending.data.cluster_id = generate_cluster_id(rule.id); pending.data.rule_id = rule.id; pending.data.rule_name = rule.name; pending.data.label = rule.name; // Use rule name as label - pending.data.representative_code = fault_code; + pending.data.representative = id; pending.data.representative_severity = severity; - pending.data.fault_codes.push_back(fault_code); - pending.fault_severities[fault_code] = severity; + pending.data.members.push_back(id); + pending.fault_severities[id] = severity; pending.data.first_at = now_system; pending.data.last_at = now_system; - pending_clusters_[pending_key] = pending; + pending_clusters_[rule.id] = pending; fault_to_cluster_[id] = pending.data.cluster_id; // Not enough faults yet for a cluster @@ -464,20 +455,19 @@ std::optional CorrelationEngine::try_auto_cluster(const Faul auto & cluster = pending.data; // Check for duplicate - if (std::find(cluster.fault_codes.begin(), cluster.fault_codes.end(), fault_code) != cluster.fault_codes.end()) { + if (std::find(cluster.members.begin(), cluster.members.end(), id) != cluster.members.end()) { // Already in cluster - ensure consistent muting for duplicates - update_member_severity(rule, cluster, pending.fault_severities, fault_code, severity); + update_member_severity(rule, cluster, pending.fault_severities, id, severity); ProcessFaultResult result; result.cluster_id = cluster.cluster_id; - if (rule.show_as_single && fault_code != cluster.representative_code && - cluster.fault_codes.size() >= rule.min_count) { + if (rule.show_as_single && id != cluster.representative && cluster.members.size() >= rule.min_count) { result.should_mute = true; } return result; } - cluster.fault_codes.push_back(fault_code); - pending.fault_severities[fault_code] = severity; + cluster.members.push_back(id); + pending.fault_severities[id] = severity; cluster.last_at = now_system; fault_to_cluster_[id] = cluster.cluster_id; @@ -498,7 +488,7 @@ std::optional CorrelationEngine::try_auto_cluster(const Faul } if (update_representative) { - cluster.representative_code = fault_code; + cluster.representative = id; cluster.representative_severity = severity; } @@ -507,21 +497,21 @@ std::optional CorrelationEngine::try_auto_cluster(const Faul // Sync active cluster if already active or if threshold reached for the first time bool already_active = (active_clusters_.find(cluster.cluster_id) != active_clusters_.end()); - if (already_active || cluster.fault_codes.size() >= rule.min_count) { + if (already_active || cluster.members.size() >= rule.min_count) { active_clusters_[cluster.cluster_id] = cluster; active_cluster_severities_[cluster.cluster_id] = pending.fault_severities; // Retroactively mute all non-representative faults added before threshold if (!already_active && rule.show_as_single) { - for (const auto & code : cluster.fault_codes) { - if (code != cluster.representative_code && code != fault_code) { - result.retroactive_mute_codes.push_back(code); + for (const auto & member : cluster.members) { + if (member != cluster.representative && member != id) { + result.retroactive_mute_ids.push_back(member); } } } // Mute non-representative faults - if (rule.show_as_single && fault_code != cluster.representative_code) { + if (rule.show_as_single && id != cluster.representative) { result.should_mute = true; } } @@ -539,29 +529,54 @@ std::string CorrelationEngine::generate_cluster_id(const std::string & rule_id) return oss.str(); } -void CorrelationEngine::update_member_severity(const CorrelationRule & rule, ClusterData & cluster, - std::map & fault_severities, - const std::string & fault_code, const std::string & severity) const { - fault_severities[fault_code] = severity; +ClusterData CorrelationEngine::to_cluster_data(const Cluster & cluster) { + ClusterData data; + data.cluster_id = cluster.cluster_id; + data.rule_id = cluster.rule_id; + data.rule_name = cluster.rule_name; + data.label = cluster.label; + data.representative_code = cluster.representative.fault_code; + data.representative_owner = cluster.representative.owner; + data.representative_severity = cluster.representative_severity; + data.fault_codes.reserve(cluster.members.size()); + data.fault_owners.reserve(cluster.members.size()); + for (const auto & member : cluster.members) { + data.fault_codes.push_back(member.fault_code); + data.fault_owners.push_back(member.owner); + } + data.first_at = cluster.first_at; + data.last_at = cluster.last_at; + return data; +} + +void CorrelationEngine::update_member_severity(const CorrelationRule & rule, Cluster & cluster, + std::map & severities, const FaultId & id, + const std::string & severity) const { + severities[id] = severity; if (rule.representative != Representative::HIGHEST_SEVERITY) { return; } - if (cluster.representative_code == fault_code) { + if (cluster.representative == id) { // A downgraded representative may no longer be the highest - reassign_representative(cluster, fault_severities); + reassign_representative(cluster, severities); } else if (severity_rank(severity) > severity_rank(cluster.representative_severity)) { - cluster.representative_code = fault_code; + cluster.representative = id; cluster.representative_severity = severity; } } -void CorrelationEngine::reassign_representative(ClusterData & cluster, - const std::map & fault_severities) const { - const auto & codes = cluster.fault_codes; - if (codes.empty()) { +void CorrelationEngine::reassign_representative(Cluster & cluster, + const std::map & severities) const { + const auto & members = cluster.members; + if (members.empty()) { return; } + const auto severity_of = [&severities](const FaultId & member) { + auto sev_it = severities.find(member); + return sev_it != severities.end() ? sev_it->second : std::string(); + }; + for (const auto & rule : config_.rules) { if (rule.id != cluster.rule_id) { continue; @@ -569,33 +584,28 @@ void CorrelationEngine::reassign_representative(ClusterData & cluster, switch (rule.representative) { case Representative::FIRST: { - const std::string & first_code = codes.front(); - cluster.representative_code = first_code; - auto sev_it = fault_severities.find(first_code); - cluster.representative_severity = (sev_it != fault_severities.end()) ? sev_it->second : ""; + cluster.representative = members.front(); + cluster.representative_severity = severity_of(members.front()); break; } case Representative::HIGHEST_SEVERITY: { - std::string best_code = codes.front(); + FaultId best = members.front(); int best_rank = -1; - for (const auto & code : codes) { - auto sev_it = fault_severities.find(code); - int rank = (sev_it != fault_severities.end()) ? severity_rank(sev_it->second) : 0; + for (const auto & member : members) { + auto sev_it = severities.find(member); + int rank = (sev_it != severities.end()) ? severity_rank(sev_it->second) : 0; if (rank > best_rank) { best_rank = rank; - best_code = code; + best = member; } } - cluster.representative_code = best_code; - auto best_sev_it = fault_severities.find(best_code); - cluster.representative_severity = (best_sev_it != fault_severities.end()) ? best_sev_it->second : ""; + cluster.representative = best; + cluster.representative_severity = severity_of(best); break; } case Representative::MOST_RECENT: { - const std::string & most_recent_code = codes.back(); - cluster.representative_code = most_recent_code; - auto sev_it = fault_severities.find(most_recent_code); - cluster.representative_severity = (sev_it != fault_severities.end()) ? sev_it->second : ""; + cluster.representative = members.back(); + cluster.representative_severity = severity_of(members.back()); break; } } diff --git a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp index dee60d96f..83a7159a5 100644 --- a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp @@ -384,8 +384,10 @@ void FaultManagerNode::handle_list_faults( info.rule_name = cluster.rule_name; info.label = cluster.label; info.representative_code = cluster.representative_code; + info.representative_owner = cluster.representative_owner; info.representative_severity = cluster.representative_severity; info.fault_codes = cluster.fault_codes; + info.fault_owners = cluster.fault_owners; info.count = cluster.count; info.first_at = ros2::ns_to_time_msg(cluster.first_at_ns); info.last_at = ros2::ns_to_time_msg(cluster.last_at_ns); diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py new file mode 100644 index 000000000..1d217bc54 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Correlation across owners, through a real fault manager. + +Rules match by fault code, whoever reports it. A root cause of one node mutes and +auto-clears the matching symptom records of every node; a cluster holds the matching +records of every node. The relations hold records, so they work the same for per_source +codes (one record per node) and for shared codes (one record, owner '*'). +""" + +import os +import tempfile +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import launch_testing.markers +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault + +TIMEOUT_SEC = 30.0 +ALL_STATUSES = [ + Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, + Fault.STATUS_HEALED, Fault.STATUS_CLEARED, +] + +THRESHOLDS = """\ +XO_PS_ESTOP: + per_source: true +XO_PS_MOTOR_A: + per_source: true +XO_PS_MOTOR_B: + per_source: true +XO_PS_STORM_1: + per_source: true +XO_PS_STORM_2: + per_source: true +XO_PS_STORM_3: + per_source: true +""" + +CORRELATION = """\ +correlation: + enabled: true + default_window_ms: 5000 + patterns: + ps_motor: + codes: ["XO_PS_MOTOR_*"] + sh_motor: + codes: ["XO_SH_MOTOR_*"] + ps_storm: + codes: ["XO_PS_STORM_*"] + sh_storm: + codes: ["XO_SH_STORM_*"] + rules: + - id: ps_estop + name: "Per-source e-stop" + mode: hierarchical + root_cause: + codes: ["XO_PS_ESTOP"] + symptoms: + - pattern: ps_motor + window_ms: 5000 + mute_symptoms: true + auto_clear_with_root: true + - id: sh_estop + name: "Shared e-stop" + mode: hierarchical + root_cause: + codes: ["XO_SH_ESTOP"] + symptoms: + - pattern: sh_motor + window_ms: 5000 + mute_symptoms: true + auto_clear_with_root: true + - id: ps_storm + name: "Per-source storm" + mode: auto_cluster + match: + - pattern: ps_storm + min_count: 3 + window_ms: 5000 + show_as_single: true + representative: first + - id: sh_storm + name: "Shared storm" + mode: auto_cluster + match: + - pattern: sh_storm + min_count: 3 + window_ms: 5000 + show_as_single: true + representative: first +""" + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file('correlation_owners_thresholds_', THRESHOLDS) +CORRELATION_FILE = _temp_file('correlation_owners_rules_', CORRELATION) + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'healing_enabled': False, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'correlation.config_file': CORRELATION_FILE, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestCorrelationAcrossOwners(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_correlation_owners_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + cls.get_client = cls.node.create_client(GetFault, '/fault_manager/get_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + for client in (cls.report_client, cls.list_client, cls.get_client, cls.clear_client): + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source, severity=Fault.SEVERITY_ERROR): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = severity + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _list(self, statuses=(), include_muted=False, include_clusters=False): + request = ListFaults.Request() + request.statuses = list(statuses) + request.include_muted = include_muted + request.include_clusters = include_clusters + return self._call(self.list_client, request) + + def _clear(self, code, source=''): + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + return self._call(self.clear_client, request) + + def _get(self, code, source=''): + request = GetFault.Request() + request.fault_code = code + request.source_id = source + return self._call(self.get_client, request) + + def _status_of(self, code, owner): + records = [f for f in self._list(ALL_STATUSES, include_muted=True).faults + if f.fault_code == code] + response = self._get(code, owner) + self.assertTrue(response.success, f'{code}/{owner}: {response.error_message} {records}') + return response.fault.status + + def _cascade(self, root, root_node, symptoms, symptom_owner_of_record): + """Report a root and symptoms from other nodes, expect them muted, then cleared with it.""" + self._report(root, root_node) + for code, node in symptoms: + self._report(code, node) + + muted = self._list(include_muted=True) + muted_records = sorted((m.fault_code, m.source_id) for m in muted.muted_faults) + expected = sorted((code, symptom_owner_of_record(code, node)) for code, node in symptoms) + self.assertEqual(muted_records, expected) + for m in muted.muted_faults: + self.assertEqual(m.root_cause_code, root) + + visible = {f.fault_code for f in self._list().faults} + self.assertIn(root, visible) + for code, _ in symptoms: + self.assertNotIn(code, visible, 'a muted symptom of another node stays hidden') + + cleared = self._clear(root, root_node) + self.assertTrue(cleared.success, cleared.message) + self.assertEqual(sorted(cleared.auto_cleared_codes), sorted(code for code, _ in symptoms)) + for code, node in symptoms: + self.assertEqual(self._status_of(code, symptom_owner_of_record(code, node)), + Fault.STATUS_CLEARED) + remaining = [m.fault_code for m in self._list(include_muted=True).muted_faults] + for code, _ in symptoms: + self.assertNotIn(code, remaining) + + def _storm(self, rule_id, reports, owner_of_record): + """Report records of different nodes inside one window and expect one cluster.""" + for code, node in reports: + self._report(code, node) + + clusters = [c for c in self._list(include_clusters=True).clusters if c.rule_id == rule_id] + self.assertEqual(len(clusters), 1) + cluster = clusters[0] + members = sorted(zip(cluster.fault_codes, cluster.fault_owners)) + expected = sorted((code, owner_of_record(node)) for code, node in reports) + self.assertEqual(members, expected) + self.assertEqual(len(cluster.fault_owners), len(cluster.fault_codes)) + self.assertEqual(cluster.count, len(reports)) + + # The representative resolves to exactly one record through the single-record services. + response = self._get(cluster.representative_code, cluster.representative_owner) + self.assertTrue(response.success, response.error_message) + self.assertEqual(response.fault.fault_code, cluster.representative_code) + self.assertIn((cluster.representative_code, cluster.representative_owner), members) + return cluster + + def test_per_source_root_mutes_and_clears_symptoms_of_other_nodes(self): + self._cascade( + 'XO_PS_ESTOP', '/estop_node', + [('XO_PS_MOTOR_A', '/motor_node'), ('XO_PS_MOTOR_B', '/drive_node')], + lambda code, node: node) + + def test_shared_root_mutes_and_clears_symptoms_of_other_nodes(self): + self._cascade( + 'XO_SH_ESTOP', '/estop_node', + [('XO_SH_MOTOR_A', '/motor_node'), ('XO_SH_MOTOR_B', '/drive_node')], + lambda code, node: '*') + + def test_per_source_storm_across_nodes_forms_one_cluster(self): + # One code from two nodes is two member records of the cluster. + cluster = self._storm( + 'ps_storm', + [('XO_PS_STORM_1', '/lidar'), ('XO_PS_STORM_1', '/camera'), + ('XO_PS_STORM_2', '/motor_node')], + lambda node: node) + self.assertEqual(cluster.representative_code, 'XO_PS_STORM_1') + self.assertEqual(cluster.representative_owner, '/lidar') + + def test_shared_storm_across_nodes_forms_one_cluster(self): + cluster = self._storm( + 'sh_storm', + [('XO_SH_STORM_1', '/lidar'), ('XO_SH_STORM_2', '/camera'), + ('XO_SH_STORM_3', '/motor_node')], + lambda node: '*') + self.assertEqual(cluster.representative_owner, '*') + + +@launch_testing.post_shutdown_test() +class TestCorrelationOwnersShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(CORRELATION_FILE) diff --git a/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp b/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp index b23fdb858..917581905 100644 --- a/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp +++ b/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp @@ -29,10 +29,9 @@ using ros2_medkit_fault_manager::FaultId; namespace { -// The engine relates fault RECORDS, and every relation it forms is between records -// of the same owner. Most of this suite exercises the rule vocabulary, which is -// keyed by code, so those tests report everything from one owner. The tests that -// name a second owner are the ones checking that a relation does not cross owners. +// The engine relates fault RECORDS. Rules match by code whoever owns the record, so most +// of this suite reports everything from one owner; the tests that name a second owner +// check that a relation crosses owners. constexpr const char * kOwner = "/reporter_a"; constexpr const char * kOtherOwner = "/reporter_b"; @@ -233,10 +232,8 @@ TEST_F(CorrelationEngineTest, ClearRootCauseClearsSymptoms) { EXPECT_EQ(0u, engine.get_muted_count()); } -// A root cause explains its own reporter's faults, not everyone's. Without the owner -// on the relation, the first reporter of a root-cause code would mute and auto-clear -// every other reporter's identical symptom. -TEST_F(CorrelationEngineTest, RootCauseDoesNotMuteAnotherOwnersSymptom) { +// Rules match by code across owners; the relation then holds the exact records. +TEST_F(CorrelationEngineTest, RootCauseMutesMatchingSymptomsOfAnyOwner) { auto config = create_hierarchical_config(); CorrelationEngine engine(config); @@ -246,27 +243,28 @@ TEST_F(CorrelationEngineTest, RootCauseDoesNotMuteAnotherOwnersSymptom) { const auto other_owner = engine.process_fault(other("MOTOR_COMM_FL"), "ERROR"); EXPECT_TRUE(same_owner.should_mute); - EXPECT_FALSE(other_owner.should_mute) << "a root cause of one owner must not mute another owner's record"; + EXPECT_TRUE(other_owner.should_mute); EXPECT_TRUE(engine.is_muted(rec("MOTOR_COMM_FL"))); - EXPECT_FALSE(engine.is_muted(other("MOTOR_COMM_FL"))); - EXPECT_EQ(1u, engine.get_muted_count()); + EXPECT_TRUE(engine.is_muted(other("MOTOR_COMM_FL"))); + EXPECT_EQ(2u, engine.get_muted_count()); } -// The auto-clear cascade is the same relation seen from the clear side. -TEST_F(CorrelationEngineTest, ClearRootCauseLeavesAnotherOwnersSymptomAlone) { +// The cascade acts on exactly the records the root cause muted. +TEST_F(CorrelationEngineTest, ClearRootCauseCascadesToTheRecordsItMuted) { auto config = create_hierarchical_config(); CorrelationEngine engine(config); engine.process_fault(rec("ESTOP_001"), "CRITICAL"); - engine.process_fault(other("ESTOP_001"), "CRITICAL"); - engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); engine.process_fault(other("MOTOR_COMM_FL"), "ERROR"); + engine.process_fault(rec("MOTOR_COMM_FR"), "ERROR"); const auto clear_result = engine.process_clear(rec("ESTOP_001")); - ASSERT_EQ(1u, clear_result.auto_cleared_symptoms.size()); - EXPECT_EQ(rec("MOTOR_COMM_FL"), clear_result.auto_cleared_symptoms.front()); - EXPECT_TRUE(engine.is_muted(other("MOTOR_COMM_FL"))) << "the other owner's symptom stays muted by its own root cause"; + const auto & symptoms = clear_result.auto_cleared_symptoms; + ASSERT_EQ(2u, symptoms.size()); + EXPECT_NE(std::find(symptoms.begin(), symptoms.end(), other("MOTOR_COMM_FL")), symptoms.end()); + EXPECT_NE(std::find(symptoms.begin(), symptoms.end(), rec("MOTOR_COMM_FR")), symptoms.end()); + EXPECT_EQ(0u, engine.get_muted_count()); } TEST_F(CorrelationEngineTest, UnrelatedFaultNotCorrelated) { @@ -611,18 +609,18 @@ TEST_F(CorrelationEngineTest, AutoClusterRetroactiveMuting) { auto result1 = engine.process_fault(rec("SENSOR_001"), "ERROR", t0); EXPECT_FALSE(result1.should_mute); // First fault is representative EXPECT_FALSE(result1.cluster_id.empty()); - EXPECT_TRUE(result1.retroactive_mute_codes.empty()); // Cluster not active yet + EXPECT_TRUE(result1.retroactive_mute_ids.empty()); // Cluster not active yet // Fault #2 - not muted because cluster not active auto result2 = engine.process_fault(rec("SENSOR_002"), "ERROR", t0 + std::chrono::milliseconds(10)); EXPECT_FALSE(result2.should_mute); // Cluster still not active - EXPECT_TRUE(result2.retroactive_mute_codes.empty()); + EXPECT_TRUE(result2.retroactive_mute_ids.empty()); // Fault #3 - triggers cluster activation (min_count=3) auto result3 = engine.process_fault(rec("SENSOR_003"), "ERROR", t0 + std::chrono::milliseconds(20)); - EXPECT_TRUE(result3.should_mute); // #3 is muted (not representative) - EXPECT_EQ(1u, result3.retroactive_mute_codes.size()); // #2 should be retroactively muted - EXPECT_EQ("SENSOR_002", result3.retroactive_mute_codes[0]); + EXPECT_TRUE(result3.should_mute); // #3 is muted (not representative) + EXPECT_EQ(1u, result3.retroactive_mute_ids.size()); // #2 should be retroactively muted + EXPECT_EQ(rec("SENSOR_002"), result3.retroactive_mute_ids[0]); // Cluster should be active with 3 faults EXPECT_EQ(1u, engine.get_cluster_count()); diff --git a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp index d075a8a7e..b6dfd59fc 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp @@ -2275,30 +2275,36 @@ class MutedFaultListingTest : public FaultEventPublishingTest { rclcpp::Client::SharedPtr list_faults_client_; }; -// Muting belongs to a RECORD, so the default listing hides the muted record alone. -// Owner A's symptom is suppressed by A's own root cause. Owner B reported the same -// code with no root cause of its own, so B's record was never muted and a listing -// that drops it hides a live CRITICAL fault from the default view. +// Muting belongs to a RECORD, so the default listing hides the muted record alone. A root cause +// of owner A mutes the symptom record of owner B too. Owner C reported the same code before any +// root cause was active, so C's record was never muted and a listing that drops it would hide a +// live CRITICAL fault from the default view. TEST_F(MutedFaultListingTest, DefaultListingHidesTheMutedRecordNotTheCode) { + ASSERT_TRUE(call_report_fault("SYMPTOM_SPEED", Fault::SEVERITY_CRITICAL, "/owner_c")); ASSERT_TRUE(call_report_fault("ROOT_CAUSE", Fault::SEVERITY_CRITICAL, "/owner_a")); ASSERT_TRUE(call_report_fault("SYMPTOM_SPEED", Fault::SEVERITY_CRITICAL, "/owner_a")); ASSERT_TRUE(call_report_fault("SYMPTOM_SPEED", Fault::SEVERITY_CRITICAL, "/owner_b")); auto visible = call_list_faults(false); ASSERT_TRUE(visible.has_value()); - EXPECT_EQ(visible->muted_count, 1u) << "only A's symptom record is muted"; - EXPECT_EQ(owners_of(*visible, "SYMPTOM_SPEED"), std::vector{"/owner_b"}); + EXPECT_EQ(visible->muted_count, 2u) << "the symptom records of A and B are muted"; + EXPECT_EQ(owners_of(*visible, "SYMPTOM_SPEED"), std::vector{"/owner_c"}); // The root cause itself is never muted, so the listing is not simply empty. EXPECT_EQ(owners_of(*visible, "ROOT_CAUSE"), std::vector{"/owner_a"}); auto everything = call_list_faults(true); ASSERT_TRUE(everything.has_value()); - EXPECT_EQ(owners_of(*everything, "SYMPTOM_SPEED"), (std::vector{"/owner_a", "/owner_b"})); + EXPECT_EQ(owners_of(*everything, "SYMPTOM_SPEED"), (std::vector{"/owner_a", "/owner_b", "/owner_c"})); // One entry per muted RECORD, naming both halves of its identity. - ASSERT_EQ(everything->muted_faults.size(), 1u); - EXPECT_EQ(everything->muted_faults.front().fault_code, "SYMPTOM_SPEED"); - EXPECT_EQ(everything->muted_faults.front().root_cause_code, "ROOT_CAUSE"); - EXPECT_EQ(everything->muted_faults.front().source_id, "/owner_a"); + ASSERT_EQ(everything->muted_faults.size(), 2u); + std::vector muted_owners; + for (const auto & muted : everything->muted_faults) { + EXPECT_EQ(muted.fault_code, "SYMPTOM_SPEED"); + EXPECT_EQ(muted.root_cause_code, "ROOT_CAUSE"); + muted_owners.push_back(muted.source_id); + } + std::sort(muted_owners.begin(), muted_owners.end()); + EXPECT_EQ(muted_owners, (std::vector{"/owner_a", "/owner_b"})); } // Two owners muted on one code are two entries of that code. Without the owner on the diff --git a/src/ros2_medkit_msgs/msg/ClusterInfo.msg b/src/ros2_medkit_msgs/msg/ClusterInfo.msg index d95fb9c2d..ce92abf6c 100644 --- a/src/ros2_medkit_msgs/msg/ClusterInfo.msg +++ b/src/ros2_medkit_msgs/msg/ClusterInfo.msg @@ -47,3 +47,11 @@ builtin_interfaces/Time first_at # Timestamp of the last fault in the cluster builtin_interfaces/Time last_at + +# Owner of each member record, one entry per fault_codes entry in the same order. A cluster holds +# the matching records of any owner, so (fault_codes[i], fault_owners[i]) names a record: pass the +# owner as source_id to the single-record services. +string[] fault_owners + +# Owner of the representative record: (representative_code, representative_owner) names it. +string representative_owner diff --git a/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg b/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg index d109ba634..7a664c601 100644 --- a/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg +++ b/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg @@ -18,8 +18,8 @@ # as events) but still stored. This message provides information about # muted records when queried with include_muted=true. # -# One entry per muted RECORD. A root cause mutes the symptoms of its own record owner -# only, so two per-source records muted on one fault_code produce two entries carrying +# One entry per muted RECORD. A root cause mutes the matching symptom records of every +# owner, so two per-source records muted on one fault_code produce two entries carrying # that code, told apart by source_id. # The fault code of the muted symptom From ffa3bd4a474e2b499f21c282aec01fec1eff9d70 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 17:37:02 +0100 Subject: [PATCH 08/52] fault_manager: audit rows keep the mechanism and name the record owner The source of an audit row is again the report's source for a row a report drove and the mechanism for an automatic one: auto_confirm_timer, auto_heal, clear_service and startup_reclassify. A symptom cleared by the correlation cascade is written as correlation_auto_clear, so it is told apart from an operator clear. AuditEvent gains a trailing owner: the owner of a per-source record, empty for a shared record. canonicalize hashes it only when it is not empty, so every row of a shared record and every existing chain hash as before. An audit log created without the column gets it on open. --- docs/config/fault-manager.rst | 10 + src/ros2_medkit_fault_manager/CMakeLists.txt | 3 + src/ros2_medkit_fault_manager/README.md | 2 +- .../core/fault_manager_core.hpp | 10 +- .../fault_audit_log.hpp | 12 +- .../fault_manager_node.hpp | 9 +- .../src/core/fault_manager_core.cpp | 19 +- .../src/fault_audit_log.cpp | 27 +- .../src/fault_manager_node.cpp | 14 +- .../integration/test_fault_audit_rows.test.py | 338 ++++++++++++++++++ .../test/test_fault_manager.cpp | 16 +- 11 files changed, 423 insertions(+), 37 deletions(-) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_fault_audit_rows.test.py diff --git a/docs/config/fault-manager.rst b/docs/config/fault-manager.rst index e0b6e7254..3118816b1 100644 --- a/docs/config/fault-manager.rst +++ b/docs/config/fault-manager.rst @@ -859,6 +859,16 @@ Every row records the stored severity of the fault record at the transition, the value ``GET /faults`` and the fault events show. The severity a single report carried is in the near-miss series and in the severity mismatch warning. +The ``source`` of a row says what drove the transition. A report drives +``occurred`` and ``confirmed`` rows and its ``source_id`` is the source. An +automatic transition carries its mechanism: ``auto_confirm_timer``, +``auto_heal``, ``startup_reclassify``, ``clear_service`` (an operator clear) and +``correlation_auto_clear`` (a symptom cleared with its root cause). The +``owner`` of a row is the owner of a per-source record and is empty for the +shared record of a code, so a shared record's rows are unchanged and an existing +chain still verifies. A log created before the ``owner`` column existed gets the +column on open. + .. code-block:: yaml fault_manager: diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index 9d59daedd..b4ac86756 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -303,6 +303,9 @@ endif() medkit_add_launch_test(test_fault_correlation_owners test/integration/test_fault_correlation_owners.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_audit_rows + test/integration/test_fault_audit_rows.test.py TIMEOUT 180 LABELS "integration") + medkit_add_launch_test(test_fault_identity_restart test/integration/test_fault_identity_restart.test.py TIMEOUT 180 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/README.md b/src/ros2_medkit_fault_manager/README.md index 4debedb33..b13c42bc0 100644 --- a/src/ros2_medkit_fault_manager/README.md +++ b/src/ros2_medkit_fault_manager/README.md @@ -323,7 +323,7 @@ storage API or the database file. ## Advanced: Tamper-Evident Audit Log -An optional append-only, hash-chained audit log records every fault state transition (`occurred`, `confirmed`, `healed`, `cleared`) so the fault history is independently verifiable. Auto-recovery (a record reaching the healing threshold via PASSED events) is recorded as a distinct `healed` row, so the record's END is in the timeline and is not confused with a manual `cleared`. The `source` of a row a report drove (`occurred`, `confirmed`) is the reporting source. The automatic and clear transitions name the owner of the record that moved, which for a code shared by all its sources is `*`. The manager has no acknowledge action separate from clearing, so `~/clear_fault` is recorded as `cleared` (clear == ack), and there is no `ack` kind. The log also records its own lifecycle with `logging_activated` / `logging_deactivated` markers at start and stop. Every row records the record's stored severity at the transition (see [Severity](#severity)). It is **off by default** because it adds a write and storage cost per transition. +An optional append-only, hash-chained audit log records every fault state transition (`occurred`, `confirmed`, `healed`, `cleared`) so the fault history is independently verifiable. Auto-recovery (a record reaching the healing threshold via PASSED events) is recorded as a distinct `healed` row, so the record's END is in the timeline and is not confused with a manual `cleared`. The `source` of a row says what drove it: the reporting source for a row a report drove (`occurred`, `confirmed`), and the mechanism for an automatic one - `auto_confirm_timer` (time-based confirmation), `auto_heal` (healing threshold reached), `startup_reclassify` (a stale HEALED record turned CLEARED at start), `clear_service` (an operator clear) and `correlation_auto_clear` (a symptom cleared with its root cause). The `owner` of a row is the owner of a per-source record, and empty for a shared record, so a row of a shared record is the row it was before the field existed and every existing chain still verifies. The manager has no acknowledge action separate from clearing, so `~/clear_fault` is recorded as `cleared` (clear == ack), and there is no `ack` kind. The log also records its own lifecycle with `logging_activated` / `logging_deactivated` markers at start and stop. Every row records the record's stored severity at the transition (see [Severity](#severity)). It is **off by default** because it adds a write and storage cost per transition. Each transition appends one immutable row holding `record_hash = sha256(prev_hash + canonical(event))` (OpenSSL EVP SHA-256), the `prev_hash` it links to, and a monotonic `seq`. The hash is computed once at insert and never recomputed. A persisted chain head lets the chain resume across restarts. The log is stored in its own SQLite database (separate from the fault store) and is treated as append-only: the manager only ever inserts rows, and `BEFORE UPDATE` / `BEFORE DELETE` triggers reject out-of-band edits (the guarded rotation prune excepted). diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp index 0c5aa4fd3..bee980b5d 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp @@ -227,8 +227,10 @@ class FaultManagerCore { /// Appends one audit event. No-op when the log is off, or when only confirmations are logged and /// this is not one. Throws when audit_log.fail_closed is set and the append fails. - void record_transition(const char * transition, const fd::FaultRecord & fault, const std::string & source_id, - int64_t occurred_at_ns); + /// @param id The record that moved: its owner is written to the row unless the record is shared. + /// @param source_id The report's source, or the kAuditSource* mechanism for an automatic transition. + void record_transition(const char * transition, const fd::FaultRecord & fault, const FaultId & id, + const std::string & source_id, int64_t occurred_at_ns); /// Runs the periodic jobs that are due at @p wall_now_ns (ns since the epoch): time-based /// auto-confirmation every kAutoConfirmPeriodSec and correlation cleanup every @@ -300,8 +302,8 @@ class FaultManagerCore { /// policies. A per-source record has one source and never warns. void warn_on_conflicting_debounce_policy(const FaultId & id, const std::string & source_id, const DebounceConfig & resolved); - void audit_transition(const char * transition, const fd::FaultRecord & fault, const std::string & source_id, - int64_t occurred_at_ns); + void audit_transition(const char * transition, const fd::FaultRecord & fault, const FaultId & id, + const std::string & source_id, int64_t occurred_at_ns); void publish_event(const char * event_type, const fd::FaultRecord & fault, const std::vector & auto_cleared_codes = {}); diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_audit_log.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_audit_log.hpp index 967090690..b5a7bf92c 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_audit_log.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_audit_log.hpp @@ -49,11 +49,21 @@ struct AuditEvent { std::string transition; ///< one of the kTransition* constants below uint8_t severity{0}; ///< severity at the time of the transition std::string status; ///< resulting fault status (e.g. CONFIRMED) - std::string source_id; ///< reporting source that drove the transition + std::string source_id; ///< the report's source, or a kAuditSource* mechanism for an automatic transition std::string description; ///< human-readable description int64_t occurred_at_ns{0}; ///< wall-clock timestamp of the transition + /// Owner of a per-source record. Empty for a shared record. Hashed only when non-empty, so every + /// row of a shared record hashes as it did before the field existed. + std::string owner; }; +/// `source_id` of the rows no report drives. Stored verbatim, so they are part of the hash. +constexpr const char * kAuditSourceAutoConfirm = "auto_confirm_timer"; +constexpr const char * kAuditSourceAutoHeal = "auto_heal"; +constexpr const char * kAuditSourceClearService = "clear_service"; +constexpr const char * kAuditSourceCorrelationAutoClear = "correlation_auto_clear"; +constexpr const char * kAuditSourceStartupReclassify = "startup_reclassify"; + /// Canonical transition kinds. Stored verbatim, so they are part of the hash. constexpr const char * kTransitionOccurred = "occurred"; constexpr const char * kTransitionConfirmed = "confirmed"; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_manager_node.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_manager_node.hpp index 8f62e21d1..167c9e2d2 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_manager_node.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_manager_node.hpp @@ -101,7 +101,7 @@ class FaultManagerNode : public rclcpp::Node { /// service handlers use. Throws when audit_log.fail_closed is set and the append /// fails; otherwise observe audit_dropped_writes()/audit_healthy(). void audit_transition_for_test(const char * transition, const fd::FaultRecord & fault) { - audit_transition(transition, fault, "test", 0); + audit_transition(transition, fault, FaultId{fault.fault_code, kSharedOwner}, "test", 0); } /// Get the storage type being used @@ -184,10 +184,11 @@ class FaultManagerNode : public rclcpp::Node { /// Append a fault state-transition to the audit log through the core. /// @param transition One of the kTransition* constants. /// @param fault The fault state at the time of the transition. - /// @param source_id The reporting source that drove the transition. + /// @param id The record that moved. + /// @param source_id The report's source, or the kAuditSource* mechanism for an automatic transition. /// @param occurred_at_ns Wall-clock timestamp of the transition. - void audit_transition(const char * transition, const fd::FaultRecord & fault, const std::string & source_id, - int64_t occurred_at_ns); + void audit_transition(const char * transition, const fd::FaultRecord & fault, const FaultId & id, + const std::string & source_id, int64_t occurred_at_ns); core::FaultManagerConfig config_; ///< Validated startup configuration std::string storage_type_; diff --git a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp index 55322bfc7..a61806daa 100644 --- a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp +++ b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp @@ -240,10 +240,10 @@ std::vector FaultManagerCore::list_for_entity(const std::string return result; } -void FaultManagerCore::record_transition(const char * transition, const fd::FaultRecord & fault, +void FaultManagerCore::record_transition(const char * transition, const fd::FaultRecord & fault, const FaultId & id, const std::string & source_id, int64_t occurred_at_ns) { std::lock_guard lock(mutex_); - audit_transition(transition, fault, source_id, occurred_at_ns); + audit_transition(transition, fault, id, source_id, occurred_at_ns); } void FaultManagerCore::tick(int64_t wall_now_ns) { @@ -299,7 +299,7 @@ void FaultManagerCore::run_auto_confirm(int64_t wall_now_ns) { if (!fault) { continue; } - audit_transition(kTransitionConfirmed, *fault, id.owner, wall_now_ns); + audit_transition(kTransitionConfirmed, *fault, id, kAuditSourceAutoConfirm, wall_now_ns); // Same rules as the report path: a symptom muted by a root cause is not announced, and the // capture is not gated by muting. if (!correlation_ || !correlation_->is_muted(id)) { @@ -522,13 +522,13 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { // Recorded regardless of correlation muting: muting affects display, not the // fact that the state transition occurred. if (is_new) { - audit_transition(kTransitionOccurred, *fault_after, report.source_id, event_time_ns); + audit_transition(kTransitionOccurred, *fault_after, id, report.source_id, event_time_ns); } if (just_confirmed) { - audit_transition(kTransitionConfirmed, *fault_after, report.source_id, event_time_ns); + audit_transition(kTransitionConfirmed, *fault_after, id, report.source_id, event_time_ns); } if (just_healed) { - audit_transition(kTransitionHealed, *fault_after, id.owner, event_time_ns); + audit_transition(kTransitionHealed, *fault_after, id, kAuditSourceAutoHeal, event_time_ns); } if (just_confirmed) { @@ -641,7 +641,7 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, const std: return outcome; } if (symptom) { - audit_transition(kTransitionCleared, *symptom, symptom_id.owner, clock_.now_ns()); + audit_transition(kTransitionCleared, *symptom, symptom_id, kAuditSourceCorrelationAutoClear, clock_.now_ns()); } } logf(log_, LogLevel::kDebug, logger, "Auto-cleared symptom: %s (root cause: %s)", symptom_id.fault_code.c_str(), @@ -674,7 +674,7 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, const std: } if (fault) { publish_event(fd::fault_event_type::kCleared, *fault, auto_cleared_codes); - audit_transition(kTransitionCleared, *fault, id.owner, clock_.now_ns()); + audit_transition(kTransitionCleared, *fault, id, kAuditSourceClearService, clock_.now_ns()); } return outcome; } @@ -923,7 +923,7 @@ void FaultManagerCore::warn_on_conflicting_debounce_policy(const FaultId & id, c resolved.confirmation_threshold, resolved.healing_enabled ? "true" : "false", resolved.healing_threshold); } -void FaultManagerCore::audit_transition(const char * transition, const fd::FaultRecord & fault, +void FaultManagerCore::audit_transition(const char * transition, const fd::FaultRecord & fault, const FaultId & id, const std::string & source_id, int64_t occurred_at_ns) { if (!audit_) { return; @@ -938,6 +938,7 @@ void FaultManagerCore::audit_transition(const char * transition, const fd::Fault event.severity = fault.severity; event.status = fault.status; event.source_id = source_id; + event.owner = id.owner == kSharedOwner ? std::string() : id.owner; event.description = fault.description; event.occurred_at_ns = occurred_at_ns; diff --git a/src/ros2_medkit_fault_manager/src/fault_audit_log.cpp b/src/ros2_medkit_fault_manager/src/fault_audit_log.cpp index fa7d45e27..3332c5d4a 100644 --- a/src/ros2_medkit_fault_manager/src/fault_audit_log.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_audit_log.cpp @@ -215,6 +215,10 @@ std::string FaultAuditLog::canonicalize(int64_t seq, const AuditEvent & event) { out += json_encode(event.source_id); out += ",\"description\":"; out += json_encode(event.description); + if (!event.owner.empty()) { + out += ",\"owner\":"; + out += json_encode(event.owner); + } out += "}"; return out; } @@ -233,12 +237,24 @@ void FaultAuditLog::initialize_schema() { source_id TEXT NOT NULL, description TEXT NOT NULL, prev_hash TEXT NOT NULL, - record_hash TEXT NOT NULL + record_hash TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '' ); CREATE INDEX IF NOT EXISTS idx_audit_log_fault_code ON audit_log(fault_code); )", "create audit_log table"); + // A log created before the owner column existed gets it empty, which is what its rows hash as. + bool has_owner_column = false; + { + Stmt columns(db_.get(), "SELECT 1 FROM pragma_table_info('audit_log') WHERE name = 'owner'"); + has_owner_column = columns.step() == SQLITE_ROW; + } + if (!has_owner_column) { + exec_or_throw(db_.get(), "ALTER TABLE audit_log ADD COLUMN owner TEXT NOT NULL DEFAULT ''", + "add audit_log owner column"); + } + // Single-row persisted chain head, so the chain resumes across restarts. exec_or_throw(db_.get(), R"( @@ -384,7 +400,7 @@ int64_t FaultAuditLog::append(const AuditEvent & event) { try { Stmt insert(db_.get(), "INSERT INTO audit_log (seq, occurred_at_ns, fault_code, transition, severity, status, " - "source_id, description, prev_hash, record_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + "source_id, description, prev_hash, record_hash, owner) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); insert.bind_int64(1, new_seq); insert.bind_int64(2, event.occurred_at_ns); insert.bind_text(3, event.fault_code); @@ -395,6 +411,7 @@ int64_t FaultAuditLog::append(const AuditEvent & event) { insert.bind_text(8, event.description); insert.bind_text(9, prev_hash); insert.bind_text(10, record_hash); + insert.bind_text(11, event.owner); if (insert.step() != SQLITE_DONE) { throw std::runtime_error(std::string("audit: failed to insert record: ") + sqlite3_errmsg(db_.get())); } @@ -509,7 +526,7 @@ std::vector FaultAuditLog::read(int64_t limit, int64_t after_seq) c std::string sql = "SELECT seq, occurred_at_ns, fault_code, transition, severity, status, source_id, description, " - "prev_hash, record_hash FROM audit_log WHERE seq > ? ORDER BY seq ASC"; + "prev_hash, record_hash, owner FROM audit_log WHERE seq > ? ORDER BY seq ASC"; if (limit > 0) { sql += " LIMIT ?"; } @@ -533,6 +550,7 @@ std::vector FaultAuditLog::read(int64_t limit, int64_t after_seq) c rec.event.description = stmt.column_text(7); rec.prev_hash = stmt.column_text(8); rec.record_hash = stmt.column_text(9); + rec.event.owner = stmt.column_text(10); result.push_back(std::move(rec)); } return result; @@ -565,7 +583,7 @@ AuditVerifyResult FaultAuditLog::verify() const { // Walk every retained row oldest-first, recomputing each link. Stmt stmt(db_.get(), "SELECT seq, occurred_at_ns, fault_code, transition, severity, status, source_id, description, " - "prev_hash, record_hash FROM audit_log ORDER BY seq ASC"); + "prev_hash, record_hash, owner FROM audit_log ORDER BY seq ASC"); bool first = true; int64_t expected_seq = 0; @@ -583,6 +601,7 @@ AuditVerifyResult FaultAuditLog::verify() const { rec.event.description = stmt.column_text(7); rec.prev_hash = stmt.column_text(8); rec.record_hash = stmt.column_text(9); + rec.event.owner = stmt.column_text(10); if (first) { first = false; diff --git a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp index 83a7159a5..77436bc3c 100644 --- a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp @@ -136,11 +136,11 @@ FaultManagerNode::FaultManagerNode(const rclcpp::NodeOptions & options) : Node(" // Runs after the override files are loaded, so a code with its own healing_enabled: true keeps // its HEALED status. Each flipped fault is audited, or the audit log's verify() would not see it. - core::reclassify_stale_healed(*storage_, core_->fault_code_resolver(), core_->global_debounce(), - core_->audit_log() != nullptr, log_sink_, config_.logger_name, - [this](const FaultId & id, const fd::FaultRecord & fault, int64_t reclassified_at_ns) { - audit_transition(kTransitionCleared, fault, id.owner, reclassified_at_ns); - }); + core::reclassify_stale_healed( + *storage_, core_->fault_code_resolver(), core_->global_debounce(), core_->audit_log() != nullptr, log_sink_, + config_.logger_name, [this](const FaultId & id, const fd::FaultRecord & fault, int64_t reclassified_at_ns) { + audit_transition(kTransitionCleared, fault, id, kAuditSourceStartupReclassify, reclassified_at_ns); + }); core::warn_on_identity_mode_mismatch(*storage_, core_->fault_code_resolver(), log_sink_, config_.logger_name); // Create service servers @@ -442,9 +442,9 @@ void FaultManagerNode::publish_fault_event(const std::string & event_type, const fault.fault_code.c_str()); } -void FaultManagerNode::audit_transition(const char * transition, const fd::FaultRecord & fault, +void FaultManagerNode::audit_transition(const char * transition, const fd::FaultRecord & fault, const FaultId & id, const std::string & source_id, int64_t occurred_at_ns) { - core_->record_transition(transition, fault, source_id, occurred_at_ns); + core_->record_transition(transition, fault, id, source_id, occurred_at_ns); } core::FaultManagerConfig FaultManagerNode::declare_config() { diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_audit_rows.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_audit_rows.test.py new file mode 100644 index 000000000..f47f25301 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_audit_rows.test.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +What the audit rows of a real fault manager say, read from the audit store it wrote. + +A row's source is the report's source for a row a report drove, and the mechanism for an +automatic one (auto_confirm_timer, auto_heal, clear_service, correlation_auto_clear, +startup_reclassify). A row's owner is the owner of a per-source record and empty for a shared +record. The hash chain is recomputed here with the serialization the format had before the +owner existed, so every row of a shared record must still match it byte for byte. +""" + +import hashlib +import json +import os +import signal +import sqlite3 +import subprocess +import tempfile +import time +import unittest + +from ament_index_python.packages import get_package_prefix +import launch +from launch import LaunchDescription +import launch_testing.actions +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetFault, ReportFault + +GENESIS = '0' * 64 +HEAL = (' confirmation_threshold: -1\n' + ' healing_enabled: true\n' + ' healing_threshold: 1\n') +TIMER = ' confirmation_threshold: -5\n' +PER_SOURCE = ' per_source: true\n' + +RUN1_CODES = ( + 'AU.SH.TIMER:\n' + TIMER + + 'AU.PS.TIMER:\n' + PER_SOURCE + TIMER + + 'AU.SH.HEAL:\n' + HEAL + + 'AU.PS.HEAL:\n' + PER_SOURCE + HEAL + + 'AU.PS.OP:\n' + PER_SOURCE + + 'AU.PS.ROOT:\n' + PER_SOURCE + + 'AU.PS.SYM:\n' + PER_SOURCE +) +# Healing is off for the HEAL codes, so a restart turns their HEALED records into CLEARED. +RUN2_CODES = 'AU.PS.HEAL:\n' + PER_SOURCE + +CORRELATION = """\ +correlation: + enabled: true + patterns: + sh_sym: + codes: ["AU.SH.SYM"] + ps_sym: + codes: ["AU.PS.SYM"] + rules: + - id: sh_cascade + name: "Shared cascade" + mode: hierarchical + root_cause: + codes: ["AU.SH.ROOT"] + symptoms: + - pattern: sh_sym + window_ms: 5000 + mute_symptoms: true + auto_clear_with_root: true + - id: ps_cascade + name: "Per-source cascade" + mode: hierarchical + root_cause: + codes: ["AU.PS.ROOT"] + symptoms: + - pattern: ps_sym + window_ms: 5000 + mute_symptoms: true + auto_clear_with_root: true +""" + + +def generate_test_description(): + """Start only a placeholder process, the test starts the fault manager itself.""" + keep_alive = launch.actions.ExecuteProcess(cmd=['sleep', '600']) + return LaunchDescription([keep_alive, launch_testing.actions.ReadyToTest()]) + + +def canonical_before_owner(seq, row): + """Serialize a row exactly as the audit format did before the owner column existed.""" + def enc(value): + return json.dumps(value, ensure_ascii=False) + return ('{"seq":%d,"ts":%d,"code":%s,"transition":%s,"severity":%d,"status":%s,' + '"source":%s,"description":%s}' % ( + seq, row['occurred_at_ns'], enc(row['fault_code']), enc(row['transition']), + row['severity'], enc(row['status']), enc(row['source_id']), + enc(row['description']))) + + +def canonical_with_owner(seq, row): + canonical = canonical_before_owner(seq, row) + if row['owner']: + canonical = canonical[:-1] + ',"owner":%s}' % json.dumps(row['owner'], ensure_ascii=False) + return canonical + + +class TestFaultAuditRows(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + cls.db_path = os.path.join(cls.tmp.name, 'faults.db') + cls.audit_path = os.path.join(cls.tmp.name, 'audit.db') + cls.run1_codes = cls._write('run1_codes.yaml', RUN1_CODES) + cls.run2_codes = cls._write('run2_codes.yaml', RUN2_CODES) + cls.correlation = cls._write('correlation.yaml', CORRELATION) + cls.exe = os.path.join( + get_package_prefix('ros2_medkit_fault_manager'), + 'lib', 'ros2_medkit_fault_manager', 'fault_manager_node') + rclpy.init() + cls.node = Node('test_fault_audit_rows_client') + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + cls.tmp.cleanup() + + @classmethod + def _write(cls, name, text): + path = os.path.join(cls.tmp.name, name) + with open(path, 'w') as f: + f.write(text) + return path + + def _start(self, codes_file): + params = self._write('params.yaml', ( + 'fault_manager:\n' + ' ros__parameters:\n' + ' storage_type: sqlite\n' + f' database_path: {self.db_path}\n' + ' confirmation_threshold: -1\n' + ' healing_enabled: false\n' + ' auto_confirm_after_sec: 1.0\n' + ' snapshots.enabled: false\n' + ' snapshots.rosbag.enabled: false\n' + ' audit_log.enabled: true\n' + f' audit_log.database_path: {self.audit_path}\n' + f' fault_thresholds.config_file: {codes_file}\n' + f' correlation.config_file: {self.correlation}\n' + )) + proc = subprocess.Popen( + [self.exe, '--ros-args', '-r', '__node:=fault_manager', '--params-file', params]) + self.addCleanup(self._kill, proc) + clients = { + 'report': self.node.create_client(ReportFault, '/fault_manager/report_fault'), + 'get': self.node.create_client(GetFault, '/fault_manager/get_fault'), + 'clear': self.node.create_client(ClearFault, '/fault_manager/clear_fault'), + } + for client in clients.values(): + self.assertTrue(client.wait_for_service(timeout_sec=15.0)) + # The graph can still list the previous node's services, so wait for an answer. + probe = GetFault.Request() + probe.fault_code = 'NOT.A.FAULT' + deadline = time.monotonic() + 15.0 + while time.monotonic() < deadline: + future = clients['get'].call_async(probe) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=1.0) + if future.result() is not None: + break + else: + self.fail('fault_manager did not answer after start') + return proc, clients + + @staticmethod + def _kill(proc): + if proc.poll() is None: + proc.kill() + proc.wait() + + def _stop(self, proc, clients): + for client in clients.values(): + self.node.destroy_client(client) + proc.send_signal(signal.SIGINT) + self.assertEqual(proc.wait(timeout=30), 0) + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=5.0) + self.assertIsNotNone(future.result(), 'service call timed out') + return future.result() + + def _report(self, clients, code, source, event_type=ReportFault.Request.EVENT_FAILED): + request = ReportFault.Request() + request.fault_code = code + request.event_type = event_type + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(clients['report'], request).accepted) + + def _clear(self, clients, code, source=''): + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + response = self._call(clients['clear'], request) + self.assertTrue(response.success, response.message) + + def _status(self, clients, code, source=''): + request = GetFault.Request() + request.fault_code = code + request.source_id = source + response = self._call(clients['get'], request) + self.assertTrue(response.success, response.error_message) + return response.fault.status + + def _wait_status(self, clients, code, expected): + deadline = time.monotonic() + 20.0 + while time.monotonic() < deadline: + if self._status(clients, code) == expected: + return + self.fail(f'{code} never reached {expected}') + + def _heal(self, clients, code, source): + self._report(clients, code, source) + for _ in range(4): + if self._status(clients, code) == Fault.STATUS_HEALED: + return + self._report(clients, code, source, ReportFault.Request.EVENT_PASSED) + self.assertEqual(self._status(clients, code), Fault.STATUS_HEALED) + + def _rows(self): + connection = sqlite3.connect(f'file:{self.audit_path}?mode=ro', uri=True) + connection.row_factory = sqlite3.Row + try: + return [dict(r) for r in connection.execute('SELECT * FROM audit_log ORDER BY seq')] + finally: + connection.close() + + def _of(self, rows, code): + return [(r['transition'], r['source_id'], r['owner']) for r in rows + if r['fault_code'] == code] + + def test_rows_say_what_drove_them_and_chain_matches_the_old_format(self): + proc, clients = self._start(self.run1_codes) + try: + # Timer confirmation: one FAILED leaves the record PREFAILED, the timer confirms it. + self._report(clients, 'AU.SH.TIMER', '/a') + self._report(clients, 'AU.PS.TIMER', '/a') + self._wait_status(clients, 'AU.SH.TIMER', Fault.STATUS_CONFIRMED) + self._wait_status(clients, 'AU.PS.TIMER', Fault.STATUS_CONFIRMED) + + # Healing. + self._heal(clients, 'AU.SH.HEAL', '/a') + self._heal(clients, 'AU.PS.HEAL', '/a') + + # Operator clear. + self._report(clients, 'AU.SH.OP', '/a') + self._report(clients, 'AU.PS.OP', '/a') + self._clear(clients, 'AU.SH.OP') + self._clear(clients, 'AU.PS.OP') + + # Correlation cascade: the symptom is cleared with its root, from another node. + self._report(clients, 'AU.SH.ROOT', '/root_node') + self._report(clients, 'AU.SH.SYM', '/sym_node') + self._clear(clients, 'AU.SH.ROOT') + self._report(clients, 'AU.PS.ROOT', '/root_node') + self._report(clients, 'AU.PS.SYM', '/sym_node') + self._clear(clients, 'AU.PS.ROOT') + finally: + self._stop(proc, clients) + + # Restart with healing off for both HEAL codes: their HEALED records become CLEARED. + proc, clients = self._start(self.run2_codes) + self._stop(proc, clients) + + rows = self._rows() + + # Shared records: no owner on any row. + self.assertEqual(self._of(rows, 'AU.SH.TIMER'), [ + ('occurred', '/a', ''), ('confirmed', 'auto_confirm_timer', '')]) + self.assertEqual(self._of(rows, 'AU.SH.HEAL'), [ + ('occurred', '/a', ''), ('confirmed', '/a', ''), ('healed', 'auto_heal', ''), + ('cleared', 'startup_reclassify', '')]) + self.assertEqual(self._of(rows, 'AU.SH.OP'), [ + ('occurred', '/a', ''), ('confirmed', '/a', ''), ('cleared', 'clear_service', '')]) + self.assertEqual(self._of(rows, 'AU.SH.ROOT'), [ + ('occurred', '/root_node', ''), ('confirmed', '/root_node', ''), + ('cleared', 'clear_service', '')]) + self.assertEqual(self._of(rows, 'AU.SH.SYM'), [ + ('occurred', '/sym_node', ''), ('confirmed', '/sym_node', ''), + ('cleared', 'correlation_auto_clear', '')]) + + # Per-source records: the same mechanisms, and the owner of the record. + self.assertEqual(self._of(rows, 'AU.PS.TIMER'), [ + ('occurred', '/a', '/a'), ('confirmed', 'auto_confirm_timer', '/a')]) + self.assertEqual(self._of(rows, 'AU.PS.HEAL'), [ + ('occurred', '/a', '/a'), ('confirmed', '/a', '/a'), ('healed', 'auto_heal', '/a'), + ('cleared', 'startup_reclassify', '/a')]) + self.assertEqual(self._of(rows, 'AU.PS.OP'), [ + ('occurred', '/a', '/a'), ('confirmed', '/a', '/a'), + ('cleared', 'clear_service', '/a')]) + self.assertEqual(self._of(rows, 'AU.PS.ROOT'), [ + ('occurred', '/root_node', '/root_node'), ('confirmed', '/root_node', '/root_node'), + ('cleared', 'clear_service', '/root_node')]) + self.assertEqual(self._of(rows, 'AU.PS.SYM'), [ + ('occurred', '/sym_node', '/sym_node'), ('confirmed', '/sym_node', '/sym_node'), + ('cleared', 'correlation_auto_clear', '/sym_node')]) + + # The chain verifies, and every row of a shared record hashes as it did before the + # owner existed. + previous = GENESIS + shared_rows = 0 + for row in rows: + self.assertEqual(row['prev_hash'], previous) + expected = hashlib.sha256( + (previous + canonical_with_owner(row['seq'], row)).encode()).hexdigest() + self.assertEqual(row['record_hash'], expected, row) + if not row['owner']: + shared_rows += 1 + old = hashlib.sha256( + (previous + canonical_before_owner(row['seq'], row)).encode()).hexdigest() + self.assertEqual(row['record_hash'], old, row) + previous = row['record_hash'] + self.assertGreater(shared_rows, 10) diff --git a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp index b6dfd59fc..476d59a59 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp @@ -3338,11 +3338,13 @@ TEST_F(FaultAuditIntegrationTest, TransitionsAppendVerifiableChain) { EXPECT_EQ(records[2].event.transition, ros2_medkit_fault_manager::kTransitionConfirmed); EXPECT_EQ(records[3].event.transition, ros2_medkit_fault_manager::kTransitionCleared); EXPECT_EQ(records[1].event.fault_code, "AUDIT_FAULT"); - // The rows a report drove name the reporting source. The clear row names the owner of the record, which - // for a code shared by every source is the shared owner. + // The rows a report drove name the reporting source, the clear row its mechanism. A shared record has no owner. EXPECT_EQ(records[1].event.source_id, "/plc/pump"); EXPECT_EQ(records[2].event.source_id, "/plc/pump"); - EXPECT_EQ(records[3].event.source_id, ros2_medkit_fault_manager::kSharedOwner); + EXPECT_EQ(records[3].event.source_id, ros2_medkit_fault_manager::kAuditSourceClearService); + for (size_t i = 1; i < records.size(); ++i) { + EXPECT_TRUE(records[i].event.owner.empty()) << "row " << i; + } auto result = audit.verify(); EXPECT_TRUE(result.ok) << result.error; @@ -3351,8 +3353,7 @@ TEST_F(FaultAuditIntegrationTest, TransitionsAppendVerifiableChain) { // Completeness: an auto-healed record must record its END. One FAILED confirms the // record (occurred + confirmed), two PASSED drive the debounce counter to the -// healing threshold, which must append a distinct "healed" row. That row names the -// record's owner, which for a code shared by every source is the shared owner. +// healing threshold, which must append a distinct "healed" row. That row names its mechanism. // The full occurred -> confirmed -> healed chain must verify. TEST_F(FaultAuditIntegrationTest, AutoHealAppendsHealedRow) { auto send_report = [&](uint8_t event_type) { @@ -3392,7 +3393,7 @@ TEST_F(FaultAuditIntegrationTest, AutoHealAppendsHealedRow) { EXPECT_EQ(transitions[0], ros2_medkit_fault_manager::kTransitionOccurred); EXPECT_EQ(transitions[1], ros2_medkit_fault_manager::kTransitionConfirmed); EXPECT_EQ(transitions[2], ros2_medkit_fault_manager::kTransitionHealed); - EXPECT_EQ(heal_source, ros2_medkit_fault_manager::kSharedOwner); + EXPECT_EQ(heal_source, ros2_medkit_fault_manager::kAuditSourceAutoHeal); auto result = audit.verify(); EXPECT_TRUE(result.ok) << result.error; @@ -3598,7 +3599,8 @@ TEST(FaultAuditStartupReclassifyTest, StartupReclassifyAppendsClearedRow) { for (const auto & rec : audit->read()) { if (rec.event.fault_code == "STALE_HEALED") { EXPECT_EQ(rec.event.transition, ros2_medkit_fault_manager::kTransitionCleared); - EXPECT_EQ(rec.event.source_id, "/robot/src") << "the audit row names the record's owner"; + EXPECT_EQ(rec.event.source_id, ros2_medkit_fault_manager::kAuditSourceStartupReclassify); + EXPECT_EQ(rec.event.owner, "/robot/src") << "a per-source record's row names its owner"; EXPECT_EQ(rec.event.status, Fault::STATUS_CLEARED); ++cleared_rows; } From 316c76dea56ddac58410467590b5458122d8326c Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 18:00:25 +0100 Subject: [PATCH 09/52] fault_manager: capture a code's topics once for its per-source records Per-source records of one code that confirm inside the recapture cooldown share one capture of the code's topics. The first worker to start samples; every record that joined while it ran, and every record that confirms after it finished and before the cooldown ends, gets the same rows and the same freeze frame under its own owner. The queue holds no job for a joining record unless a recording still needs one, so a storm of sources does not fill the capture queue with identical snapshot work. A record that confirms again after it was acknowledged gets a capture of its own. A shared record, a disabled cooldown, and a code whose topics come from the reporting node (entity default) are captured per record as before. --- docs/config/fault-manager.rst | 2 + docs/tutorials/snapshots.rst | 2 + src/ros2_medkit_fault_manager/CMakeLists.txt | 3 + src/ros2_medkit_fault_manager/README.md | 2 +- .../snapshot_capture.hpp | 51 +++++ .../src/fault_manager_node.cpp | 12 ++ .../src/snapshot_capture.cpp | 146 +++++++++++++ .../test_fault_snapshot_grouping.test.py | 194 ++++++++++++++++++ 8 files changed, 411 insertions(+), 1 deletion(-) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py diff --git a/docs/config/fault-manager.rst b/docs/config/fault-manager.rst index 3118816b1..bbe5df075 100644 --- a/docs/config/fault-manager.rst +++ b/docs/config/fault-manager.rst @@ -579,6 +579,8 @@ Basic Snapshot Settings - ``60.0`` - Minimum seconds between snapshot captures for the same fault record. Prevents snapshot storms when a fault is reported repeatedly. Set to 0 to disable. + Per-source records of one code that confirm inside the cooldown share one capture of + the code's topics: it is sampled once and every record lists it under its own owner. * - ``snapshots.max_per_fault`` - ``10`` - Maximum number of snapshot rows stored per fault record. One confirmation diff --git a/docs/tutorials/snapshots.rst b/docs/tutorials/snapshots.rst index 91d272535..2284b044d 100644 --- a/docs/tutorials/snapshots.rst +++ b/docs/tutorials/snapshots.rst @@ -124,6 +124,8 @@ Configure snapshot capture via fault manager parameters: - ``60.0`` - Minimum seconds between snapshot captures for the same fault record. Prevents snapshot storms when a fault is reported repeatedly. Set to 0 to disable. + Per-source records of one code that confirm inside the cooldown share one capture of + the code's topics: it is sampled once and every record lists it under its own owner. * - ``snapshots.max_per_fault`` - ``10`` - Maximum number of snapshot rows stored per fault record. One confirmation diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index b4ac86756..41a19d885 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -306,6 +306,9 @@ endif() medkit_add_launch_test(test_fault_audit_rows test/integration/test_fault_audit_rows.test.py TIMEOUT 180 LABELS "integration") + medkit_add_launch_test(test_fault_snapshot_grouping + test/integration/test_fault_snapshot_grouping.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_identity_restart test/integration/test_fault_identity_restart.test.py TIMEOUT 180 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/README.md b/src/ros2_medkit_fault_manager/README.md index b13c42bc0..66b42bea8 100644 --- a/src/ros2_medkit_fault_manager/README.md +++ b/src/ros2_medkit_fault_manager/README.md @@ -160,7 +160,7 @@ That single-writer property also shapes what each fault of a burst gets. Nothing | `snapshots.max_message_size` | int | `65536` | Maximum message size in bytes (larger messages skipped) | | `snapshots.default_topics` | string[] | `[]` | Topics to capture for all faults | | `snapshots.config_file` | string | `""` | Path to YAML config for `fault_specific` and `patterns` | -| `snapshots.recapture_cooldown_sec` | double | `60.0` | Min seconds between captures for the same fault record. | +| `snapshots.recapture_cooldown_sec` | double | `60.0` | Min seconds between captures for the same fault record. Per-source records of one code confirming inside it share one capture of the code's topics. | | `snapshots.max_per_fault` | int | `10` | Max snapshots retained per fault. | | `snapshots.capture_pool_size` | int | `2` | Max concurrent capture threads under a fault storm (>= 1). Parallelizes snapshot capture only; rosbag stays single-writer. | | `snapshots.capture_queue_depth` | int | `16` | Max pending captures before the full-queue policy applies (>= 1). | diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp index ce18563ec..3fb1e85b5 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp @@ -15,10 +15,12 @@ #pragma once #include +#include #include #include #include #include +#include #include #include @@ -132,6 +134,10 @@ struct SnapshotConfig { /// default_topics) always wins when present. bool entity_default{true}; + /// Recapture cooldown in seconds (0 = off). Per-source records of one code that confirm inside it + /// share one capture of the code's topics. + double recapture_cooldown_sec{0.0}; + /// Rosbag recording configuration (optional) RosbagConfig rosbag; }; @@ -182,6 +188,23 @@ class SnapshotCapture { /// @param id The fault record that was confirmed void capture(const FaultId & id); + /// What the caller does with the capture of a record. + enum class Admission { + kEnqueue, ///< Queue capture(): the record needs a worker. + kJoined ///< The record shares a capture already running or done: queue nothing for snapshots. + }; + + /// Decide, before capture() is queued, whether the record needs a capture of its own. + /// + /// Per-source records of one code confirming inside the recapture cooldown share ONE capture of the + /// code's topics, and every one of them gets the rows and the freeze frame written under its own + /// owner. Only a code whose topics are resolved by code qualifies: the entity-default topics depend + /// on the owner. A shared record, a disabled cooldown or an unconfigured code is always kEnqueue. + Admission admit(const FaultId & id); + + /// Undo admit() for a record whose capture could not be queued. + void withdraw(const FaultId & id); + /// Get current configuration const SnapshotConfig & config() const { return config_; @@ -208,6 +231,30 @@ class SnapshotCapture { /// throws; any failure degrades to empty. std::vector resolve_entity_topics(const FaultId & id) const; + /// Write the rows and the freeze frame of one capture under the record @p id. + void persist_capture(const FaultId & id, std::vector rows, const nlohmann::json & freeze_frame, + size_t captured_count, bool entity_scoped, int64_t capture_id); + + /// The capture of one code that its per-source records share inside the cooldown. + struct CaptureGroup { + enum class State { kQueued, kRunning, kDone }; + State state{State::kQueued}; + std::set owners; ///< Records the capture is written for + std::chrono::steady_clock::time_point started; + /// What the capture sampled, kept while the cooldown runs so a record confirming after it + /// finished gets the same rows. + std::vector rows; + nlohmann::json freeze_frame = nlohmann::json::object(); + size_t captured_count{0}; + int64_t capture_id{0}; + }; + + /// Drop groups older than the cooldown. Caller holds groups_mutex_. + void prune_groups_locked(std::chrono::steady_clock::time_point now); + + /// Whether @p id takes part in grouping: a per-source record of a code with by-code topics. + bool groups_by_code(const FaultId & id) const; + /// Capture a single topic on-demand (creates temporary subscription) /// On success also records the captured value into @p freeze_frame under the topic key. /// @return true if capture was successful @@ -256,6 +303,10 @@ class SnapshotCapture { std::atomic capture_seq_seeded_{false}; std::mutex capture_seq_seed_mutex_; + /// Captures shared by the per-source records of a code (code -> group), guarded by groups_mutex_. + std::mutex groups_mutex_; + std::map groups_; + /// Compiled regex patterns (cached for performance) std::vector>> compiled_patterns_; diff --git a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp index 77436bc3c..1189f84e2 100644 --- a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp @@ -307,6 +307,11 @@ core::CaptureResult FaultManagerNode::enqueue_capture(const FaultId & id) { if (!capture_pool_) { return {}; } + // A per-source record whose code is already captured inside the cooldown shares that capture. Its + // snapshots need no worker; only a recording still does. + if (snapshot_capture_ && snapshot_capture_->admit(id) == SnapshotCapture::Admission::kJoined && !rosbag_capture_) { + return {core::CaptureEnqueue::kAccepted, {}}; + } const EnqueueOutcome outcome = capture_pool_->enqueue(id); // RCLCPP_WARN_THROTTLE needs a non-const Clock lvalue (Humble/Lyrical compat); mirror // rosbag_capture.cpp's local-copy pattern. The counter is cast to unsigned long long for %llu @@ -324,6 +329,9 @@ core::CaptureResult FaultManagerNode::enqueue_capture(const FaultId & id) { static_cast(capture_pool_->dropped_captures())); return {core::CaptureEnqueue::kEvictedOldest, outcome.evicted_id.value_or(FaultId{})}; case EnqueueResult::kDroppedNewest: + if (snapshot_capture_) { + snapshot_capture_->withdraw(id); + } RCLCPP_WARN_THROTTLE(get_logger(), throttle_clock, 2000, "Capture queue full (reject_newest): dropped capture for '%s' " "(pool=%d, queue=%d, total_dropped=%llu)", @@ -331,6 +339,9 @@ core::CaptureResult FaultManagerNode::enqueue_capture(const FaultId & id) { static_cast(capture_pool_->dropped_captures())); return {core::CaptureEnqueue::kDroppedNewest, {}}; case EnqueueResult::kRejectedShuttingDown: + if (snapshot_capture_) { + snapshot_capture_->withdraw(id); + } RCLCPP_DEBUG(get_logger(), "Capture pool shutting down; skipped capture for '%s'", id.fault_code.c_str()); return {core::CaptureEnqueue::kShuttingDown, {}}; } @@ -541,6 +552,7 @@ SnapshotConfig FaultManagerNode::create_snapshot_config() { config.default_topics = config_.snapshots_default_topics; config.entity_default = config_.snapshots_entity_default; + config.recapture_cooldown_sec = config_.snapshots_recapture_cooldown_sec; if (!config_.snapshots_config_file.empty()) { load_snapshot_config_from_yaml(config_.snapshots_config_file, config); diff --git a/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp b/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp index dfe6a66b4..41e90786e 100644 --- a/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp +++ b/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp @@ -13,7 +13,9 @@ // limitations under the License. #include "ros2_medkit_fault_manager/snapshot_capture.hpp" +#include #include +#include #include #include @@ -146,6 +148,77 @@ bool SnapshotCapture::seed_capture_seq() { return true; } +bool SnapshotCapture::groups_by_code(const FaultId & id) const { + if (id.owner == kSharedOwner || config_.recapture_cooldown_sec <= 0.0) { + return false; + } + bool explicit_match = false; + return !resolve_topics(id.fault_code, explicit_match).empty(); +} + +void SnapshotCapture::prune_groups_locked(std::chrono::steady_clock::time_point now) { + const auto cooldown = std::chrono::duration(config_.recapture_cooldown_sec); + for (auto it = groups_.begin(); it != groups_.end();) { + if (now - it->second.started >= cooldown) { + it = groups_.erase(it); + } else { + ++it; + } + } +} + +SnapshotCapture::Admission SnapshotCapture::admit(const FaultId & id) { + if (!config_.enabled || !groups_by_code(id)) { + return Admission::kEnqueue; + } + const auto now = std::chrono::steady_clock::now(); + std::unique_lock lock(groups_mutex_); + prune_groups_locked(now); + auto it = groups_.find(id.fault_code); + if (it == groups_.end()) { + CaptureGroup group; + group.owners.insert(id.owner); + group.started = now; + groups_.emplace(id.fault_code, std::move(group)); + return Admission::kEnqueue; + } + CaptureGroup & group = it->second; + const bool is_new = group.owners.insert(id.owner).second; + switch (group.state) { + case CaptureGroup::State::kQueued: + // No worker has started yet. The record queues a job of its own, so that a capture evicted from + // the queue still runs for it. + return Admission::kEnqueue; + case CaptureGroup::State::kRunning: + return Admission::kJoined; + case CaptureGroup::State::kDone: + if (!is_new) { + // The record confirms again (it was acknowledged since): it needs a capture of its own. + groups_.erase(it); + CaptureGroup fresh; + fresh.owners.insert(id.owner); + fresh.started = now; + groups_.emplace(id.fault_code, std::move(fresh)); + return Admission::kEnqueue; + } + persist_capture(id, group.rows, group.freeze_frame, group.captured_count, false, group.capture_id); + return Admission::kJoined; + } + return Admission::kEnqueue; +} + +void SnapshotCapture::withdraw(const FaultId & id) { + std::lock_guard lock(groups_mutex_); + auto it = groups_.find(id.fault_code); + if (it == groups_.end() || it->second.state != CaptureGroup::State::kQueued) { + return; + } + it->second.owners.erase(id.owner); + if (it->second.owners.empty()) { + groups_.erase(it); + } +} + void SnapshotCapture::capture(const FaultId & id) { const std::string & fault_code = id.fault_code; if (!config_.enabled) { @@ -180,6 +253,34 @@ void SnapshotCapture::capture(const FaultId & id) { return; } + // Per-source records of one code share one capture: the first worker to start samples, and the + // record of any other worker is written by it, or from what it sampled. + const bool grouped = groups_by_code(id); + if (grouped) { + std::lock_guard lock(groups_mutex_); + prune_groups_locked(std::chrono::steady_clock::now()); + auto it = groups_.find(fault_code); + if (it == groups_.end()) { + CaptureGroup group; + group.owners.insert(id.owner); + group.state = CaptureGroup::State::kRunning; + group.started = std::chrono::steady_clock::now(); + groups_.emplace(fault_code, std::move(group)); + } else { + CaptureGroup & group = it->second; + const bool is_new = group.owners.insert(id.owner).second; + if (group.state != CaptureGroup::State::kQueued) { + if (is_new && group.state == CaptureGroup::State::kDone) { + persist_capture(id, group.rows, group.freeze_frame, group.captured_count, false, group.capture_id); + } + RCLCPP_DEBUG(node_->get_logger(), "Fault '%s' (source %s) shares the capture of its code", fault_code.c_str(), + id.owner.c_str()); + return; + } + group.state = CaptureGroup::State::kRunning; + } + } + RCLCPP_INFO(node_->get_logger(), "Capturing snapshots for fault '%s' (%zu topics%s)", fault_code.c_str(), topics.size(), entity_scoped ? ", entity-default" : ""); @@ -209,10 +310,55 @@ void SnapshotCapture::capture(const FaultId & id) { RCLCPP_INFO(node_->get_logger(), "Captured %zu/%zu snapshots for fault '%s'", captured_count, topics.size(), fault_code.c_str()); + if (!grouped) { + persist_capture(id, std::move(rows), freeze_frame, captured_count, entity_scoped, capture_id); + return; + } + + // Write for every record that joined while this ran, then for those that join while it writes. + std::set written; + for (;;) { + std::set todo; + { + std::lock_guard lock(groups_mutex_); + auto it = groups_.find(fault_code); + if (it == groups_.end()) { + // The group aged out while this ran: only the record of this worker is left to write. + if (written.count(id.owner) == 0) { + todo.insert(id.owner); + } + } else { + std::set_difference(it->second.owners.begin(), it->second.owners.end(), written.begin(), written.end(), + std::inserter(todo, todo.end())); + } + if (todo.empty()) { + if (it != groups_.end()) { + it->second.state = CaptureGroup::State::kDone; + it->second.rows = rows; + it->second.freeze_frame = freeze_frame; + it->second.captured_count = captured_count; + it->second.capture_id = capture_id; + } + return; + } + } + for (const auto & owner : todo) { + persist_capture(FaultId{fault_code, owner}, rows, freeze_frame, captured_count, false, capture_id); + written.insert(owner); + } + } +} + +void SnapshotCapture::persist_capture(const FaultId & id, std::vector rows, + const nlohmann::json & freeze_frame, size_t captured_count, bool entity_scoped, + int64_t capture_id) { + const std::string & fault_code = id.fault_code; + // One write for the whole capture. Storing topic by topic let the per-fault cap // reject the tail of a capture, leaving a freeze frame with holes and nothing // saying which values were dropped. for (auto & row : rows) { + row.owner = id.owner; row.capture_id = capture_id; } diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py new file mode 100644 index 000000000..ae6e2fe4c --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Snapshot capture of per-source records, through a real fault manager. + +Per-source records of one code that confirm inside the recapture cooldown share ONE capture of +the code's topics, and every record lists those snapshots under its own owner. The rows of one +capture all carry the same captured_at, so two captures of one code cannot be mistaken for one. +A record that confirms again after it was acknowledged gets a capture of its own. +""" + +import json +import os +import tempfile +import threading +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetSnapshots, ReportFault +from std_msgs.msg import Int32 + +TIMEOUT_SEC = 30.0 +SOURCES = ['/grp/a', '/grp/b', '/grp/c'] + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file( + 'snapshot_grouping_codes_', + 'XG.GROUP:\n per_source: true\nXG.AGAIN:\n per_source: true\n') +SNAPSHOTS_FILE = _temp_file( + 'snapshot_grouping_topics_', + 'fault_specific:\n' + ' XG.GROUP:\n - /grouping/counter\n' + ' XG.AGAIN:\n - /grouping/counter\n' + ' XG.SHARED:\n - /grouping/counter\n') + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'snapshots.enabled': True, + 'snapshots.config_file': SNAPSHOTS_FILE, + 'snapshots.timeout_sec': 5.0, + 'snapshots.capture_pool_size': 3, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestSnapshotGrouping(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_snapshot_grouping_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + cls.snapshots_client = cls.node.create_client( + GetSnapshots, '/fault_manager/get_snapshots') + for client in (cls.report_client, cls.clear_client, cls.snapshots_client): + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + cls.publisher = cls.node.create_publisher(Int32, '/grouping/counter', 10) + cls.stop = threading.Event() + cls.publisher_thread = threading.Thread(target=cls._publish) + cls.publisher_thread.start() + + @classmethod + def _publish(cls): + value = 0 + while not cls.stop.wait(0.2): + value += 1 + cls.publisher.publish(Int32(data=value)) + + @classmethod + def tearDownClass(cls): + cls.stop.set() + cls.publisher_thread.join() + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_CRITICAL + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _snapshot(self, code, source): + request = GetSnapshots.Request() + request.fault_code = code + request.source_id = source + return self._call(self.snapshots_client, request) + + def _wait_snapshot(self, code, source, newer_than=0.0): + """Poll GetSnapshots until the record lists a capture newer than @p newer_than.""" + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + response = self._snapshot(code, source) + if response.success and response.data: + data = json.loads(response.data) + if data.get('topics') and data['captured_at'] > newer_than: + return data + rclpy.spin_once(self.node, timeout_sec=0.1) + self.fail(f'{code}/{source} never listed a snapshot') + + def test_per_source_records_share_one_capture(self): + for source in SOURCES: + self._report('XG.GROUP', source) + + captures = [self._wait_snapshot('XG.GROUP', source) for source in SOURCES] + for data in captures: + self.assertIn('/grouping/counter', data['topics']) + # One capture: every owner lists the very same sampled values and the same moment. + self.assertEqual({d['captured_at'] for d in captures}, {captures[0]['captured_at']}) + self.assertEqual( + {json.dumps(d['topics'], sort_keys=True) for d in captures}, + {json.dumps(captures[0]['topics'], sort_keys=True)}) + + def test_a_record_that_confirms_again_gets_a_capture_of_its_own(self): + code = 'XG.AGAIN' + source = '/grp/again' + self._report(code, source) + first = self._wait_snapshot(code, source) + + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + self.assertTrue(self._call(self.clear_client, request).success) + + self._report(code, source) + second = self._wait_snapshot(code, source, newer_than=first['captured_at']) + self.assertGreater(second['captured_at'], first['captured_at']) + + def test_a_shared_record_is_captured_once(self): + for source in SOURCES: + self._report('XG.SHARED', source) + data = self._wait_snapshot('XG.SHARED', '') + self.assertIn('/grouping/counter', data['topics']) + + +@launch_testing.post_shutdown_test() +class TestSnapshotGroupingShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(SNAPSHOTS_FILE) From bd3fa384ab9d1016217e9b294c539a74e2cb20df Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 18:00:37 +0100 Subject: [PATCH 10/52] fault_manager: name the owner of every record a recording covers GetRosbag gains a trailing owners array, parallel to fault_codes, with one entry per fault record the recording is attached to: the reporting source for a per-source record, `*` for a shared one. fault_codes is now one entry per record too, so several per-source records of one code list the code once per owner and the two arrays stay parallel. A shared code is still one entry. --- docs/api/messages.rst | 4 ++ .../src/fault_manager_node.cpp | 24 ++++----- .../test_rosbag_integration.test.py | 52 +++++++++++++++++++ .../test/test_fault_manager.cpp | 18 ++++--- src/ros2_medkit_msgs/srv/GetRosbag.srv | 10 +++- 5 files changed, 87 insertions(+), 21 deletions(-) diff --git a/docs/api/messages.rst b/docs/api/messages.rst index 2017cca57..187211a2d 100644 --- a/docs/api/messages.rst +++ b/docs/api/messages.rst @@ -259,6 +259,10 @@ An ambiguous call clears nothing. ``GetFault``, ``GetSnapshots`` and ``GetRosbag the same field and resolve the same way. On ``GetRosbag`` it scopes the ``fault_code`` lookup only, and the ``recording_id`` path ignores it. +``GetRosbag`` answers with ``fault_codes`` and ``owners``, parallel arrays with one entry per +fault record the recording is attached to. A burst of several per-source records of one code +lists the code once per owner. A shared record is one entry carrying the owner ``*``. + When ``skip_correlation_auto_clear`` is ``false`` (default), clearing a root-cause fault also clears every symptom that the correlation engine attributes to it via ``auto_clear_with_root`` rules; the cleared diff --git a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp index 1189f84e2..281f534bc 100644 --- a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp @@ -780,17 +780,15 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptr rosbag_info; std::vector attached_codes; + std::vector attached_owners; std::string subject; - // fault_codes answers "which faults does this recording cover", so it is a set of - // codes. One row per RECORD means a burst of several owners of one code produces - // several rows pointing at the same bag, and pushing the code once per row would - // repeat it. First-seen order is kept, so the codes still read in the order the - // recording collected them. - const auto attach_code = [&attached_codes](const std::string & code) { - if (std::find(attached_codes.begin(), attached_codes.end(), code) == attached_codes.end()) { - attached_codes.push_back(code); - } + // fault_codes and owners are parallel: one entry per RECORD the recording is attached to, in the order + // the recording collected them. A burst of several owners of one code therefore lists the code once + // per owner, and a shared record is one entry carrying the shared owner. + const auto attach_row = [&attached_codes, &attached_owners](const RosbagFileInfo & row) { + attached_codes.push_back(row.fault_code); + attached_owners.push_back(row.owner); }; if (!request->recording_id.empty()) { @@ -816,7 +814,7 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptrrecording_id; rosbag_info = rows.front(); for (const auto & row : rows) { - attach_code(row.fault_code); + attach_row(row); } } else if (request->fault_code.empty()) { response->success = false; @@ -860,7 +858,7 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptrget_rosbag_files_by_recording(rosbag_info->recording_id)) { - attach_code(row.fault_code); + attach_row(row); } } catch (const FaultStorage::IgnorableConnectionException & e) { RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); @@ -869,7 +867,8 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptrfault_code); + attached_codes.push_back(target->fault_code); + attached_owners.push_back(target->owner); } } @@ -892,6 +891,7 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptrfile_path = rosbag_info->file_path; response->recording_id = rosbag_info->recording_id; response->fault_codes = attached_codes; + response->owners = attached_owners; response->format = rosbag_info->format; response->duration_sec = rosbag_info->duration_sec; response->size_bytes = core::rosbag_served_bytes(rosbag_info->file_path, rosbag_info->size_bytes); diff --git a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py index c1e41db99..52b9cab43 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py @@ -59,6 +59,12 @@ def get_coverage_env(): # Path to temp publisher script (set in generate_test_description, cleaned up in shutdown test) PUBLISHER_SCRIPT_PATH = None +# One code gets a record per reporting source; every other code is shared. +PER_SOURCE_CODES_FILE = tempfile.NamedTemporaryFile( + 'w', suffix='.yaml', prefix='rosbag_per_source_', delete=False) +PER_SOURCE_CODES_FILE.write('BURST_PER_SOURCE:\n per_source: true\n') +PER_SOURCE_CODES_FILE.close() + def generate_test_description(): """Generate launch description with fault_manager node with rosbag enabled.""" @@ -157,6 +163,7 @@ def main(): parameters=[{ 'storage_type': 'memory', 'confirmation_threshold': -1, # Single report confirms immediately + 'fault_thresholds.config_file': PER_SOURCE_CODES_FILE.name, # Rosbag configuration 'snapshots.rosbag.enabled': True, 'snapshots.rosbag.duration_sec': 2.0, # 2 second buffer @@ -586,6 +593,50 @@ def test_07_burst_faults_share_one_recording(self): 'faults of one burst must share a single recording') print(f'Shared recording: {responses["BURST_SHARED_A"].file_path}') + # Default codes have one shared record each: one entry per code, owned by '*'. + for response in responses.values(): + self.assertEqual(sorted(response.fault_codes), fault_codes) + self.assertEqual(list(response.owners), ['*'] * len(fault_codes)) + self.assertEqual(len(response.owners), len(response.fault_codes)) + + def test_08_per_source_burst_lists_every_owner_of_the_recording(self): + """Two per-source records of one code on one recording are two parallel entries.""" + code = 'BURST_PER_SOURCE' + owners = ['/burst_node_a', '/burst_node_b'] + + self.assertTrue(self._wait_for_buffered_data(), + 'ring buffer did not refill before reporting faults') + + for owner in owners: + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = f'Per-source burst from {owner}' + request.source_id = owner + self.assertTrue(self._call_service(self.report_fault_client, request).accepted) + + responses = {} + for owner in owners: + deadline = time.time() + 12.0 + while True: + request = GetRosbag.Request() + request.fault_code = code + request.source_id = owner + response = self._call_service(self.get_rosbag_client, request) + if response.success or time.time() > deadline: + break + rclpy.spin_once(self.node, timeout_sec=0.5) + self.assertTrue(response.success, + f'GetRosbag failed for {owner}: {response.error_message}') + responses[owner] = response + + self.assertEqual(responses[owners[0]].file_path, responses[owners[1]].file_path, + 'records of one burst must share a single recording') + for response in responses.values(): + self.assertEqual(list(response.fault_codes), [code, code]) + self.assertEqual(sorted(response.owners), owners) + @launch_testing.post_shutdown_test() class TestRosbagCaptureShutdown(unittest.TestCase): @@ -597,6 +648,7 @@ def test_exit_code(self, proc_info): proc_info, process='fault_manager_node' ) + os.unlink(PER_SOURCE_CODES_FILE.name) def test_cleanup_temp_directory(self): """Clean up temporary rosbag storage directory and publisher script.""" diff --git a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp index 476d59a59..94200ca06 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp @@ -2463,10 +2463,10 @@ class GetRosbagAttachedCodesTest : public FaultEventPublishingTest { rclcpp::Client::SharedPtr get_rosbag_client_; }; -// fault_codes answers which faults a recording covers, so it is a set of codes. Two -// owners of one code are two links to the one bag, and repeating the code would make -// the caller authorize the same download twice over. -TEST_F(GetRosbagAttachedCodesTest, ACodeSharedByTwoOwnersIsReportedOnce) { +// fault_codes and owners are parallel, one entry per record the recording is attached to. Two +// owners of one code are two links to the one bag, so the code is listed once per owner and the +// caller can tell whose record each entry is. +TEST_F(GetRosbagAttachedCodesTest, ACodeHeldByTwoOwnersIsListedOncePerOwner) { ASSERT_TRUE(call_report_fault("BURST_CODE", Fault::SEVERITY_CRITICAL, "/owner_a")); ASSERT_TRUE(call_report_fault("BURST_CODE", Fault::SEVERITY_CRITICAL, "/owner_b")); link_recording("BURST_CODE", "/owner_a"); @@ -2475,11 +2475,14 @@ TEST_F(GetRosbagAttachedCodesTest, ACodeSharedByTwoOwnersIsReportedOnce) { auto response = call_get_rosbag("BURST_CODE", "/owner_a"); ASSERT_TRUE(response.has_value()); ASSERT_TRUE(response->success) << response->error_message; - EXPECT_EQ(response->fault_codes, std::vector{"BURST_CODE"}); + EXPECT_EQ(response->fault_codes, (std::vector{"BURST_CODE", "BURST_CODE"})); + EXPECT_EQ(response->owners.size(), response->fault_codes.size()); + std::vector owners(response->owners.begin(), response->owners.end()); + std::sort(owners.begin(), owners.end()); + EXPECT_EQ(owners, (std::vector{"/owner_a", "/owner_b"})); } -// The control on the same harness: deduplicating must not collapse the set itself. -// A burst of two different codes on one bag still reports both, in first-seen order. +// A burst of two different codes on one bag reports both, in first-seen order, with their owners. TEST_F(GetRosbagAttachedCodesTest, EveryDistinctCodeOfTheBurstIsReported) { ASSERT_TRUE(call_report_fault("BURST_FIRST", Fault::SEVERITY_CRITICAL, "/owner_a")); ASSERT_TRUE(call_report_fault("BURST_SECOND", Fault::SEVERITY_CRITICAL, "/owner_a")); @@ -2490,6 +2493,7 @@ TEST_F(GetRosbagAttachedCodesTest, EveryDistinctCodeOfTheBurstIsReported) { ASSERT_TRUE(response.has_value()); ASSERT_TRUE(response->success) << response->error_message; EXPECT_EQ(response->fault_codes, (std::vector{"BURST_FIRST", "BURST_SECOND"})); + EXPECT_EQ(response->owners, (std::vector{"/owner_a", "/owner_a"})); } // @verifies REQ_INTEROP_012 diff --git a/src/ros2_medkit_msgs/srv/GetRosbag.srv b/src/ros2_medkit_msgs/srv/GetRosbag.srv index 3b02caceb..481595076 100644 --- a/src/ros2_medkit_msgs/srv/GetRosbag.srv +++ b/src/ros2_medkit_msgs/srv/GetRosbag.srv @@ -52,8 +52,9 @@ string file_path # as the bulk-data id. string recording_id -# Every fault this recording covers - a burst of correlated faults shares one bag. -# Callers authorize a download against this set, not against a single code. +# Every fault record this recording covers - a burst of correlated faults shares one bag. +# One entry per record, so a code appears once per owner that holds it. Callers authorize a +# download against this set, not against a single code. owners is parallel to this array. string[] fault_codes # Storage format: "sqlite3" or "mcap" @@ -70,3 +71,8 @@ uint64 size_bytes # Error message if success is false. string error_message + +# Owner of each record in fault_codes, one entry per fault_codes entry in the same order: +# the reporting source for a per_source code, "*" for the record that every source of any +# other code shares. +string[] owners From 258de7b42f71af9c6bf017cb39655954d51b79c9 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 18:00:57 +0100 Subject: [PATCH 11/52] fault_manager: index snapshots by record The per-record snapshot trims and reads filter on (fault_code, owner), so snapshots get an index on (fault_code, owner, capture_id) on SQLite and PostgreSQL. The near-miss index on (fault_code, owner, id) was already per record. An existing store gets both on open. --- .../src/postgres_fault_storage.cpp | 3 +- .../src/sqlite_fault_storage.cpp | 8 ++- .../test/test_postgres_storage.cpp | 24 ++++++++ .../test/test_sqlite_storage.cpp | 56 +++++++++++++++++++ 4 files changed, 87 insertions(+), 4 deletions(-) diff --git a/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp b/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp index 9818dca06..324e8abd9 100644 --- a/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp @@ -425,7 +425,8 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { capture_id BIGINT NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_snapshots_fault_code ON snapshots(fault_code); - CREATE INDEX IF NOT EXISTS idx_snapshots_fault_topic ON snapshots(fault_code, topic);)"); + CREATE INDEX IF NOT EXISTS idx_snapshots_fault_topic ON snapshots(fault_code, topic); + CREATE INDEX IF NOT EXISTS idx_snapshots_record ON snapshots(fault_code, owner, capture_id);)"); // Create freeze_frames table: one compact JSON dict of captured topic values per fault // RECORD. Unlike snapshots, freeze frames are keyed by the record identity and are NOT diff --git a/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp b/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp index ae2831af1..b29c5e161 100644 --- a/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp @@ -478,14 +478,16 @@ void SqliteFaultStorage::initialize_schema() { // before it. migrate_faults_add_owner(); - // After the migration, which adds near_misses.owner to a database that predates it. + // After the migration, which adds owner to near_misses and snapshots on a database that predates it. + // The per-record trims and reads of both tables go through these. if (sqlite3_exec(db_, "DROP INDEX IF EXISTS idx_near_misses_fault_code;" - "CREATE INDEX IF NOT EXISTS idx_near_misses_record ON near_misses(fault_code, owner, id);", + "CREATE INDEX IF NOT EXISTS idx_near_misses_record ON near_misses(fault_code, owner, id);" + "CREATE INDEX IF NOT EXISTS idx_snapshots_record ON snapshots(fault_code, owner, capture_id);", nullptr, nullptr, &err_msg) != SQLITE_OK) { std::string error = err_msg ? err_msg : "Unknown error"; sqlite3_free(err_msg); - throw std::runtime_error("Failed to create near_misses index: " + error); + throw std::runtime_error("Failed to create record indexes: " + error); } // The record identity, expressed as an index rather than a PRIMARY KEY so the next diff --git a/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp b/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp index 59cfe7222..5a85c38a2 100644 --- a/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp +++ b/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp @@ -151,6 +151,30 @@ class PgFaultStorageTest : public ::testing::Test { std::unique_ptr storage_; }; +// The per-record trims and reads of snapshots and near misses filter on (fault_code, owner): the +// indexes behind them exist, and a store that lacks them gets them when it is opened. +TEST_F(PgFaultStorageTest, RecordIndexesExist) { + const auto index_definition = [this](const std::string & name) { + auto conn = raw_connection(); + pqxx::nontransaction tx(*conn); + const auto rows = tx.exec( + "SELECT indexdef FROM pg_indexes WHERE schemaname = current_schema() AND " + "indexname = $1", + pqxx::params{name}); + return rows.empty() ? std::string() : rows[0][0].as(); + }; + + EXPECT_NE(index_definition("idx_snapshots_record").find("(fault_code, owner, capture_id)"), std::string::npos); + EXPECT_NE(index_definition("idx_near_misses_record").find("(fault_code, owner, id)"), std::string::npos); + + storage_.reset(); + exec_raw("DROP INDEX idx_snapshots_record; DROP INDEX idx_near_misses_record"); + EXPECT_TRUE(index_definition("idx_snapshots_record").empty()) << "the fixture must start without them"; + storage_ = std::make_unique(conn_info_, test_log_sink()); + EXPECT_NE(index_definition("idx_snapshots_record").find("(fault_code, owner, capture_id)"), std::string::npos); + EXPECT_NE(index_definition("idx_near_misses_record").find("(fault_code, owner, id)"), std::string::npos); +} + TEST_F(PgFaultStorageTest, ReportNewFaultEvent) { rclcpp::Clock clock; auto timestamp = clock.now(); diff --git a/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp b/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp index 4a7bdf261..813d34e9d 100644 --- a/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp +++ b/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp @@ -74,6 +74,62 @@ class SqliteFaultStorageTest : public ::testing::Test { std::unique_ptr storage_; }; +namespace { + +/// The detail column of every row EXPLAIN QUERY PLAN returns for @p sql, joined by newlines. +std::string query_plan(const std::string & db_path, const std::string & sql) { + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path.c_str(), &raw), SQLITE_OK); + std::string plan; + sqlite3_stmt * stmt = nullptr; + EXPECT_EQ(sqlite3_prepare_v2(raw, ("EXPLAIN QUERY PLAN " + sql).c_str(), -1, &stmt, nullptr), SQLITE_OK); + while (stmt != nullptr && sqlite3_step(stmt) == SQLITE_ROW) { + plan += reinterpret_cast(sqlite3_column_text(stmt, 3)); + plan += "\n"; + } + sqlite3_finalize(stmt); + sqlite3_close(raw); + return plan; +} + +} // namespace + +// The per-record trims and reads of snapshots and near misses filter on (fault_code, owner): they must +// be served by an index on exactly those columns, and a store that lacks them gets them on open. +TEST_F(SqliteFaultStorageTest, RecordIndexesServeTheTrims) { + const std::string path = temp_db_path_.string(); + const std::string snapshots_trim = "SELECT MAX(capture_id) FROM snapshots WHERE fault_code = 'C' AND owner = 'O'"; + const std::string snapshots_count = "SELECT COUNT(*) FROM snapshots WHERE fault_code = 'C' AND owner = 'O'"; + const std::string near_miss_trim = + "SELECT id FROM near_misses WHERE fault_code = 'C' AND owner = 'O' ORDER BY id DESC LIMIT 5"; + + const auto expect_served = [&path](const std::string & sql, const std::string & index) { + const std::string plan = query_plan(path, sql); + EXPECT_NE(plan.find(index), std::string::npos) << sql << "\n" << plan; + EXPECT_EQ(plan.find("SCAN"), std::string::npos) << sql << "\n" << plan; + }; + expect_served(snapshots_trim, "idx_snapshots_record"); + expect_served(snapshots_count, "idx_snapshots_record"); + expect_served(near_miss_trim, "idx_near_misses_record"); + + // A store written without them gets them when it is opened. + storage_.reset(); + { + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(path.c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, "DROP INDEX idx_snapshots_record; DROP INDEX idx_near_misses_record;", nullptr, nullptr, + nullptr), + SQLITE_OK); + sqlite3_close(raw); + } + EXPECT_EQ(query_plan(path, snapshots_trim).find("idx_snapshots_record"), std::string::npos) + << "the fixture must start without them"; + EXPECT_EQ(query_plan(path, near_miss_trim).find("idx_near_misses_record"), std::string::npos); + storage_ = std::make_unique(path, test_log_sink()); + expect_served(snapshots_trim, "idx_snapshots_record"); + expect_served(near_miss_trim, "idx_near_misses_record"); +} + TEST_F(SqliteFaultStorageTest, ReportNewFaultEvent) { rclcpp::Clock clock; auto timestamp = clock.now(); From 8b62b7301eaa29b37b96db209a6909ba95b00e56 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 18:00:59 +0100 Subject: [PATCH 12/52] docs: state that the owner migration is one way Opening a migrated SQLite or PostgreSQL store with an earlier release is unsupported: the earlier release's startup cleanup deletes rosbag_files rows that two owners of one code share. A copy taken before the upgrade is the only way back; there is no guard in the code. --- docs/config/fault-manager.rst | 4 ++-- src/ros2_medkit_fault_manager/README.md | 15 ++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/config/fault-manager.rst b/docs/config/fault-manager.rst index bbe5df075..40d42cd68 100644 --- a/docs/config/fault-manager.rst +++ b/docs/config/fault-manager.rst @@ -68,8 +68,8 @@ A build without the option stops at startup when ``storage_type`` is ``postgres` ``near_misses`` and ``rosbag_files``, the primary keys on the fault code are replaced by unique indexes that include it, and every existing row becomes the shared record of its code (owner ``*``), keeping its ``reporting_sources``. The same rules as for SQLite apply, see the package - README. The migration runs in one transaction. Keep a copy of the database first: an earlier - release cannot use the migrated tables. + README. The migration runs in one transaction. It is one way: opening a migrated database with + an earlier release is unsupported, and a copy taken before the upgrade is the only way back. - The audit log stays a local SQLite file, see ``audit_log.database_path``. - The node never logs ``database_url``. It logs host, port, database and user, or the service name. Error text from the server is logged and returned with the password from ``database_url`` diff --git a/src/ros2_medkit_fault_manager/README.md b/src/ros2_medkit_fault_manager/README.md index 66b42bea8..0a3b45a27 100644 --- a/src/ros2_medkit_fault_manager/README.md +++ b/src/ros2_medkit_fault_manager/README.md @@ -59,15 +59,20 @@ to that record, the near misses of every source included. `confirmed_at_ns` is c except on a PREFAILED or PREPASSED row, where it becomes 0 (see Storage Backends below). All of it runs in one transaction. -The rebuild is one way. A fault manager from an earlier release opened on a migrated store aborts on -the first report of a fault code the store does not hold yet (`NOT NULL constraint failed: -faults.owner`). If a rollback may be needed, keep a copy of `faults.db` from before the upgrade and -restore that copy instead of pointing the earlier release at the migrated file. +The rebuild is one way, and opening a migrated store with an earlier release is unsupported. That +release runs its own startup cleanup on the file before it serves a request: it deletes the +`rosbag_files` rows that two owners of one code share, so recording links are lost without a +message, and it aborts on the first report of a fault code the store does not hold yet (`NOT NULL +constraint failed: faults.owner`). There is no guard in the code. A copy of `faults.db` taken before +the upgrade is the only way back: restore it instead of pointing the earlier release at the migrated +file. A PostgreSQL database written by an earlier release is migrated in place on the first connection, by the same rules: the primary keys on the fault code become unique indexes over `(fault_code, owner)` and every row becomes the shared record of its code. It runs in one transaction, so a failure leaves -the tables as they were. +the tables as they were. This migration is one way too: an earlier release pointed at the migrated +database is unsupported, and a copy of the database taken before the upgrade (`pg_dump`) is the only +way back. The table names `faults_new` and `freeze_frames_new` are reserved for the rebuild. Anything found under those names is debris from a rebuild that did not finish and is dropped on open, with a From a3426c2fba3819e938870d0e23f65b774d107a78 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 18:29:29 +0100 Subject: [PATCH 13/52] opcua: name the reporting entity in every clear it sends Every ClearFault the OPC UA plugin sends carries the entity id the fault is reported under as source_id: the device alarm clear, the event alarm clear, the link-state clear and the SOVD scoped clear. The comms-lost probe names it too. For a code kept per source, a clear without it is refused as ambiguous as soon as a second source reported the code, and the plugin never reads the answer, so the fault stayed standing. --- .../ros2_medkit_opcua/opcua_plugin.hpp | 9 +- .../ros2_medkit_opcua/src/opcua_plugin.cpp | 17 ++- .../integration/test_opcua_comms_lost.test.py | 114 +++++++++++++++++- .../test/test_opcua_plugin.cpp | 9 +- 4 files changed, 136 insertions(+), 13 deletions(-) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp index 419b12020..52f4a892b 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp @@ -561,10 +561,13 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, } // Build the ClearFault request for one fault code. + // ``source_id`` is the entity id the fault is reported under: it picks the + // record when more than one source reported the code. // ``skip_correlation_auto_clear`` goes on the wire verbatim (see ClearOrigin - // for who sets it and why). Static so the wire field is assertable without a + // for who sets it and why). Static so the wire fields are assertable without a // fault manager. static ros2_medkit_msgs::srv::ClearFault::Request make_clear_fault_request(const std::string & fault_code, + const std::string & source_id, bool skip_correlation_auto_clear); // One entry in the bounded buffer of fault dispatches held while the @@ -644,9 +647,11 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // Report/clear fault via ROS 2 service (private helpers, not the FaultProvider overrides) void send_report_fault(const std::string & entity_id, const std::string & fault_code, const std::string & severity_str, const std::string & message); + // ``source_id`` is the entity id the fault is reported under. // ``origin`` says why the clear is being sent, which decides both the wire // flag and how the pending buffer ranks it. See ClearOrigin. - void send_clear_fault(const std::string & fault_code, ClearOrigin origin = ClearOrigin::DeviceAlarm); + void send_clear_fault(const std::string & fault_code, const std::string & source_id, + ClearOrigin origin = ClearOrigin::DeviceAlarm); // Record that a link-state decision about PLC_COMMS_LOST is due. // diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp index 226819639..26b2bfe21 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp @@ -1338,7 +1338,7 @@ void OpcuaPlugin::on_alarm_change(const std::string & entity_id, if (clear_origin_for_signal(signal.fault_code) == ClearOrigin::LinkState) { owe_comms_lost_decision(); } else { - send_clear_fault(signal.fault_code, ClearOrigin::DeviceAlarm); + send_clear_fault(signal.fault_code, entity_id, ClearOrigin::DeviceAlarm); } } } @@ -1522,7 +1522,7 @@ void OpcuaPlugin::on_event_alarm(const AlarmEventDelivery & delivery) { // this IS a resolution at the source and the correlation engine may act on // it. DeviceAlarm is also the default, spelled out here because this is // the one call site where the cascade is deliberately kept. - send_clear_fault(delivery.fault_code, ClearOrigin::DeviceAlarm); + send_clear_fault(delivery.fault_code, delivery.entity_id, ClearOrigin::DeviceAlarm); break; case AlarmAction::NoOp: break; @@ -1563,21 +1563,23 @@ void OpcuaPlugin::send_report_fault(const std::string & entity_id, const std::st } ros2_medkit_msgs::srv::ClearFault::Request OpcuaPlugin::make_clear_fault_request(const std::string & fault_code, + const std::string & source_id, bool skip_correlation_auto_clear) { ros2_medkit_msgs::srv::ClearFault::Request request; request.fault_code = fault_code; + request.source_id = source_id; request.skip_correlation_auto_clear = skip_correlation_auto_clear; return request; } -void OpcuaPlugin::send_clear_fault(const std::string & fault_code, ClearOrigin origin) { +void OpcuaPlugin::send_clear_fault(const std::string & fault_code, const std::string & source_id, ClearOrigin origin) { if (!fault_clients_->clear) { log_warn("ClearFault service client not available"); return; } auto request = std::make_shared( - make_clear_fault_request(fault_code, clear_skips_correlation(origin))); + make_clear_fault_request(fault_code, source_id, clear_skips_correlation(origin))); send_or_buffer({PendingFaultDispatch::Kind::Clear, fault_code, clear_is_link_state(origin), [this, request]() { fault_clients_->clear->async_send_request(request); @@ -1645,7 +1647,7 @@ void OpcuaPlugin::apply_comms_lost_answer(bool found, const std::vector weak_state = comms_lost_probe_state_; auto request = std::make_shared(); request->fault_code = kCommsLostFaultCode; + // The record this bridge reports it under, so a second source that raised the code does not make the + // probe ambiguous. + request->source_id = node_map_.component_id(); // rclcpp selects the response-callback overload of async_send_request only for // a callable whose parameter list is exactly (SharedFuture); a const reference // parameter matches no overload. The future is therefore taken by value. @@ -3304,7 +3309,7 @@ tl::expected OpcuaPlugin::clear_f // apps in other entities, and a plugin-owned entity must not be the hole in // that rule: the request goes through this provider instead, so the same flag // has to be set here. - send_clear_fault(fault_code, ClearOrigin::ScopedOperator); + send_clear_fault(fault_code, entity_id, ClearOrigin::ScopedOperator); return dto::FaultClearResult{ nlohmann::json{{"status", "cleared"}, {"fault_code", fault_code}, {"entity_id", entity_id}}}; } diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py index 6b8c82920..a5672a565 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py @@ -25,6 +25,9 @@ * foreign stand-in: a fault another endpoint's gateway raised stays. * late nameplate: a fault raised under the nameplate id stays while the device serves no nameplate, and is cleared when a later read names the device. + * per-source record: PLC_COMMS_LOST configured with one record per source and + a second source that reported it. The restored link clears this gateway's own + record and leaves the other source's alone. * binding directory: a binding file whose directory cannot be looked up is reported, and the plugin stays connected. Once at start, and once on the poll thread when a rescan adopts a PLC that came up later. @@ -228,11 +231,17 @@ def start_server(self, name, *extra, port=None): self.server_logs.append(log) return proc, port - def start_fault_manager(self, name): + def start_fault_manager(self, name, per_source_codes=()): """Start fault_manager_node with memory storage.""" log = self.workdir / f'{name}_fault_manager.log' + args = ['-p', 'storage_type:=memory'] + if per_source_codes: + thresholds = self.workdir / f'{name}_fault_thresholds.yaml' + thresholds.write_text(''.join(f'{code}:\n per_source: true\n' + for code in per_source_codes)) + args += ['-p', f'fault_thresholds.config_file:={thresholds}'] proc = self._start(['ros2', 'run', 'ros2_medkit_fault_manager', 'fault_manager_node', - '--ros-args', '-p', 'storage_type:=memory'], log, env=self.env) + '--ros-args', *args], log, env=self.env) # Logged at the end of the constructor, after the services exist. if not wait_log(log, 'FaultManager node started (', 30) or proc.poll() is not None: raise AssertionError(f'fault_manager_node did not start (rc={proc.poll()}):\n' @@ -378,6 +387,106 @@ def scenario_late_nameplate(run): 'after the device named itself') +# Talks to the fault manager as a second source: reports a fault, lists the records. +FAULT_MANAGER_CLIENT = """ +import json +import sys + +import rclpy +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ListFaults, ReportFault + +rclpy.init() +node = rclpy.create_node('comms_lost_test_client') + + +def call(client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(node, future, timeout_sec=10.0) + return future.result() + + +if sys.argv[1] == 'report': + client = node.create_client(ReportFault, '/fault_manager/report_fault') + client.wait_for_service(timeout_sec=10.0) + request = ReportFault.Request() + request.fault_code = sys.argv[2] + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = 'second source' + request.source_id = sys.argv[3] + print(json.dumps({'accepted': call(client, request).accepted})) +else: + client = node.create_client(ListFaults, '/fault_manager/list_faults') + client.wait_for_service(timeout_sec=10.0) + request = ListFaults.Request() + request.statuses = ['PREFAILED', 'PREPASSED', 'CONFIRMED', 'HEALED', 'CLEARED'] + print(json.dumps([{'code': f.fault_code, 'status': f.status, + 'sources': list(f.reporting_sources)} + for f in call(client, request).faults])) +node.destroy_node() +rclpy.shutdown() +""" + + +def fault_manager_client(run, *args): + """Run the fault manager client script on the scenario's ROS domain and return its JSON.""" + result = subprocess.run([sys.executable, '-c', FAULT_MANAGER_CLIENT, *args], env=run.env, + capture_output=True, text=True, timeout=60 * SCALE) + if result.returncode != 0: + raise AssertionError(f'fault manager client {args} failed:\n{result.stderr}') + return json.loads(result.stdout) + + +def comms_lost_records(run): + """Return {first reporting source: status} of every PLC_COMMS_LOST record.""" + return {f['sources'][0]: f['status'] for f in fault_manager_client(run, 'list') + if f['code'] == COMMS_LOST and f['sources']} + + +def wait_records(run, accept, deadline): + """Poll a scaled deadline until accept(records) holds. Return the last records.""" + end = time.monotonic() + deadline * SCALE + records = {} + while time.monotonic() < end: + records = comms_lost_records(run) + if accept(records): + return records + time.sleep(0.3) + return records + + +def scenario_per_source_record(run): + other = '/other_plc' + run.start_fault_manager('persrc', per_source_codes=[COMMS_LOST]) + named, port = run.start_server('persrc_named') + endpoint = f'opc.tcp://127.0.0.1:{port}' + gateway = run.start_gateway('persrc', endpoint=endpoint) + if not wait_log(gateway.log, 'Connected to OPC-UA server', 30): + raise AssertionError('the gateway never connected:\n' + tail(gateway)) + + # A second source holds a record of the same code before the outage. + if not fault_manager_client(run, 'report', COMMS_LOST, other)['accepted']: + raise AssertionError('the second source was refused') + + run.stop(named) + records = wait_records( + run, lambda r: len(r) == 2 and all(s == 'CONFIRMED' for s in r.values()), 60) + if len(records) != 2 or any(s != 'CONFIRMED' for s in records.values()): + raise AssertionError(f'the outage was never reported as its own record: {records}\n' + + tail(gateway)) + own = next(source for source in records if source != other) + + # The same device comes back on its port: the link is restored. + run.start_server('persrc_back', port=port) + records = wait_records(run, lambda r: r.get(own) == 'CLEARED', 60) + if records.get(own) != 'CLEARED': + raise AssertionError('the restored link did not clear this gateway record: ' + f'{records}\n' + tail(gateway)) + if records.get(other) != 'CONFIRMED': + raise AssertionError(f"the other source's record was cleared too: {records}") + + def start_discovering_gateway(run, name): """Start a config-less gateway whose binding file sits under a symlink loop.""" loop_a = run.workdir / 'loop_a' @@ -440,6 +549,7 @@ def scenario_binding_directory_on_reconnect(run): ('pinned stand-in', scenario_pinned_stand_in), ('foreign stand-in', scenario_foreign_stand_in), ('late nameplate', scenario_late_nameplate), + ('per-source record', scenario_per_source_record), ('binding directory at start', scenario_binding_directory_at_start), ('binding directory on reconnect', scenario_binding_directory_on_reconnect), ] diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp index 550ecd9e9..90949fa5d 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp @@ -1903,13 +1903,15 @@ TEST(CommsLostProbeDue, AllFourConditionsHold) { EXPECT_FALSE(OpcuaPlugin::comms_lost_probe_due(true, true, false, false)) << "the store cannot be asked"; } -TEST(MakeClearFaultRequest, CarriesTheSkipFlagAndCodeVerbatim) { - const auto skipping = OpcuaPlugin::make_clear_fault_request(kCommsLostFaultCode, true); +TEST(MakeClearFaultRequest, CarriesTheSkipFlagCodeAndSourceVerbatim) { + const auto skipping = OpcuaPlugin::make_clear_fault_request(kCommsLostFaultCode, "plc_component", true); EXPECT_EQ(skipping.fault_code, kCommsLostFaultCode); + EXPECT_EQ(skipping.source_id, "plc_component"); EXPECT_TRUE(skipping.skip_correlation_auto_clear); - const auto cascading = OpcuaPlugin::make_clear_fault_request("PLC_TANK_HIGH", false); + const auto cascading = OpcuaPlugin::make_clear_fault_request("PLC_TANK_HIGH", "tank", false); EXPECT_EQ(cascading.fault_code, "PLC_TANK_HIGH"); + EXPECT_EQ(cascading.source_id, "tank"); EXPECT_FALSE(cascading.skip_correlation_auto_clear); } @@ -2780,6 +2782,7 @@ component_id: scoped_runtime std::lock_guard lock(received_mutex); ASSERT_FALSE(received.empty()) << "the scoped DELETE never reached the fault manager"; EXPECT_EQ(received.front().fault_code, "PLC_TANK_HIGH"); + EXPECT_EQ(received.front().source_id, "tank") << "the clear names the entity the route addressed"; EXPECT_TRUE(received.front().skip_correlation_auto_clear) << "a per-entity DELETE served by the plugin cascade-cleared correlated symptoms"; } From f04a20c9fcb135d07b7c5249d7423b07550696b9 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 19:33:50 +0100 Subject: [PATCH 14/52] fault_manager: keep the shared snapshot capture of a code whole A code has at most one queued job for its group: a record joining a queued capture queues nothing, so a storm of sources no longer fills the capture queue and evicts other faults' captures. When that one job is evicted or refused, the group's capture is lost for its members and the group goes, so a later confirmation starts a new one. Only finished groups age out with the cooldown. A queued or running group lives until its capture completes, so a capture longer than the cooldown still writes every record that joined it, and a group whose capture fails is dropped. A record that confirms again while the capture of its previous occurrence still runs leaves that group and is captured alone, after the new confirmation, instead of being given the old sample. --- src/ros2_medkit_fault_manager/CMakeLists.txt | 3 + .../snapshot_capture.hpp | 17 +- .../src/fault_manager_node.cpp | 3 + .../src/snapshot_capture.cpp | 150 ++++++----- .../test_fault_snapshot_queue.test.py | 239 ++++++++++++++++++ 5 files changed, 342 insertions(+), 70 deletions(-) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index 41a19d885..be4e635d4 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -309,6 +309,9 @@ endif() medkit_add_launch_test(test_fault_snapshot_grouping test/integration/test_fault_snapshot_grouping.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_snapshot_queue + test/integration/test_fault_snapshot_queue.test.py TIMEOUT 180 LABELS "integration") + medkit_add_launch_test(test_fault_identity_restart test/integration/test_fault_identity_restart.test.py TIMEOUT 180 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp index 3fb1e85b5..1b02e7f0d 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp @@ -198,11 +198,16 @@ class SnapshotCapture { /// /// Per-source records of one code confirming inside the recapture cooldown share ONE capture of the /// code's topics, and every one of them gets the rows and the freeze frame written under its own - /// owner. Only a code whose topics are resolved by code qualifies: the entity-default topics depend - /// on the owner. A shared record, a disabled cooldown or an unconfigured code is always kEnqueue. + /// owner. A code has at most one queued job for its group: a record joining a queued or running + /// capture queues nothing. A record that confirms again while its old capture still runs gets a + /// capture of its own, taken after that confirmation. Only a code whose topics are resolved by code + /// qualifies: the entity-default topics depend on the owner. A shared record, a disabled cooldown + /// or an unconfigured code is always kEnqueue. Admission admit(const FaultId & id); - /// Undo admit() for a record whose capture could not be queued. + /// The queued capture job of @p id did not run (the queue refused or evicted it). When it was the + /// job of a group that has not started, the group's capture is lost for every member and the group + /// goes, so that a later confirmation starts a new one. void withdraw(const FaultId & id); /// Get current configuration @@ -239,7 +244,10 @@ class SnapshotCapture { struct CaptureGroup { enum class State { kQueued, kRunning, kDone }; State state{State::kQueued}; + std::string primary; ///< Owner whose job is queued for the group std::set owners; ///< Records the capture is written for + /// Records that confirmed again while the group ran: each takes a capture of its own. + std::set solo; std::chrono::steady_clock::time_point started; /// What the capture sampled, kept while the cooldown runs so a record confirming after it /// finished gets the same rows. @@ -249,7 +257,8 @@ class SnapshotCapture { int64_t capture_id{0}; }; - /// Drop groups older than the cooldown. Caller holds groups_mutex_. + /// Drop finished groups older than the cooldown. A queued or running group stays until its capture + /// completes. Caller holds groups_mutex_. void prune_groups_locked(std::chrono::steady_clock::time_point now); /// Whether @p id takes part in grouping: a per-source record of a code with by-code topics. diff --git a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp index 281f534bc..ed44f5b02 100644 --- a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp @@ -321,6 +321,9 @@ core::CaptureResult FaultManagerNode::enqueue_capture(const FaultId & id) { case EnqueueResult::kAccepted: return {core::CaptureEnqueue::kAccepted, {}}; case EnqueueResult::kEvictedOldest: + if (snapshot_capture_ && outcome.evicted_id) { + snapshot_capture_->withdraw(*outcome.evicted_id); + } RCLCPP_WARN_THROTTLE(get_logger(), throttle_clock, 2000, "Capture queue full (drop_oldest): evicted pending '%s' for '%s' " "(pool=%d, queue=%d, total_dropped=%llu)", diff --git a/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp b/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp index 41e90786e..b6fc94d41 100644 --- a/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp +++ b/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp @@ -159,7 +159,7 @@ bool SnapshotCapture::groups_by_code(const FaultId & id) const { void SnapshotCapture::prune_groups_locked(std::chrono::steady_clock::time_point now) { const auto cooldown = std::chrono::duration(config_.recapture_cooldown_sec); for (auto it = groups_.begin(); it != groups_.end();) { - if (now - it->second.started >= cooldown) { + if (it->second.state == CaptureGroup::State::kDone && now - it->second.started >= cooldown) { it = groups_.erase(it); } else { ++it; @@ -177,6 +177,7 @@ SnapshotCapture::Admission SnapshotCapture::admit(const FaultId & id) { auto it = groups_.find(id.fault_code); if (it == groups_.end()) { CaptureGroup group; + group.primary = id.owner; group.owners.insert(id.owner); group.started = now; groups_.emplace(id.fault_code, std::move(group)); @@ -186,16 +187,23 @@ SnapshotCapture::Admission SnapshotCapture::admit(const FaultId & id) { const bool is_new = group.owners.insert(id.owner).second; switch (group.state) { case CaptureGroup::State::kQueued: - // No worker has started yet. The record queues a job of its own, so that a capture evicted from - // the queue still runs for it. - return Admission::kEnqueue; + // The group's one job is queued and will sample after this confirmation. + return Admission::kJoined; case CaptureGroup::State::kRunning: + if (!is_new) { + // The record confirms again while the capture of its previous occurrence still runs: that + // sample is not this occurrence's, so the record leaves the group and is captured alone. + group.owners.erase(id.owner); + group.solo.insert(id.owner); + return Admission::kEnqueue; + } return Admission::kJoined; case CaptureGroup::State::kDone: if (!is_new) { // The record confirms again (it was acknowledged since): it needs a capture of its own. groups_.erase(it); CaptureGroup fresh; + fresh.primary = id.owner; fresh.owners.insert(id.owner); fresh.started = now; groups_.emplace(id.fault_code, std::move(fresh)); @@ -210,13 +218,10 @@ SnapshotCapture::Admission SnapshotCapture::admit(const FaultId & id) { void SnapshotCapture::withdraw(const FaultId & id) { std::lock_guard lock(groups_mutex_); auto it = groups_.find(id.fault_code); - if (it == groups_.end() || it->second.state != CaptureGroup::State::kQueued) { + if (it == groups_.end() || it->second.state != CaptureGroup::State::kQueued || it->second.primary != id.owner) { return; } - it->second.owners.erase(id.owner); - if (it->second.owners.empty()) { - groups_.erase(it); - } + groups_.erase(it); } void SnapshotCapture::capture(const FaultId & id) { @@ -250,18 +255,22 @@ void SnapshotCapture::capture(const FaultId & id) { if (!seed_capture_seq()) { RCLCPP_WARN(node_->get_logger(), "Skipping snapshot capture for fault '%s': fault storage unavailable", fault_code.c_str()); + withdraw(id); return; } // Per-source records of one code share one capture: the first worker to start samples, and the // record of any other worker is written by it, or from what it sampled. - const bool grouped = groups_by_code(id); + bool grouped = groups_by_code(id); if (grouped) { std::lock_guard lock(groups_mutex_); prune_groups_locked(std::chrono::steady_clock::now()); auto it = groups_.find(fault_code); - if (it == groups_.end()) { + if (it != groups_.end() && it->second.solo.erase(id.owner) > 0) { + grouped = false; // confirmed again while the group ran: sampled and written for this record alone + } else if (it == groups_.end()) { CaptureGroup group; + group.primary = id.owner; group.owners.insert(id.owner); group.state = CaptureGroup::State::kRunning; group.started = std::chrono::steady_clock::now(); @@ -281,71 +290,80 @@ void SnapshotCapture::capture(const FaultId & id) { } } - RCLCPP_INFO(node_->get_logger(), "Capturing snapshots for fault '%s' (%zu topics%s)", fault_code.c_str(), - topics.size(), entity_scoped ? ", entity-default" : ""); - - // Accumulate a compact dict of captured topic values alongside the per-topic snapshots. - nlohmann::json freeze_frame = nlohmann::json::object(); - // One id for the whole capture, minted here so every row of this confirmation - // shares it. Monotonic rather than a clock read: the rows are written seconds - // apart under load and a wall clock is not guaranteed to move forward. - const int64_t capture_id = ++capture_seq_; - std::vector rows; - size_t captured_count = 0; - for (const auto & topic : topics) { - bool success = false; - // Entity-default topics are not known at construction, so the background - // cache never holds them - always sample those on demand. - if (config_.background_capture && !entity_scoped) { - success = capture_topic_from_cache(id, topic, freeze_frame, rows); - } else { - success = capture_topic_on_demand(id, topic, freeze_frame, rows); - } + // A group whose capture fails is dropped, or its members would wait for a capture that never completes. + try { + RCLCPP_INFO(node_->get_logger(), "Capturing snapshots for fault '%s' (%zu topics%s)", fault_code.c_str(), + topics.size(), entity_scoped ? ", entity-default" : ""); + + // Accumulate a compact dict of captured topic values alongside the per-topic snapshots. + nlohmann::json freeze_frame = nlohmann::json::object(); + // One id for the whole capture, minted here so every row of this confirmation + // shares it. Monotonic rather than a clock read: the rows are written seconds + // apart under load and a wall clock is not guaranteed to move forward. + const int64_t capture_id = ++capture_seq_; + std::vector rows; + size_t captured_count = 0; + for (const auto & topic : topics) { + bool success = false; + // Entity-default topics are not known at construction, so the background + // cache never holds them - always sample those on demand. + if (config_.background_capture && !entity_scoped) { + success = capture_topic_from_cache(id, topic, freeze_frame, rows); + } else { + success = capture_topic_on_demand(id, topic, freeze_frame, rows); + } - if (success) { - ++captured_count; + if (success) { + ++captured_count; + } } - } - RCLCPP_INFO(node_->get_logger(), "Captured %zu/%zu snapshots for fault '%s'", captured_count, topics.size(), - fault_code.c_str()); + RCLCPP_INFO(node_->get_logger(), "Captured %zu/%zu snapshots for fault '%s'", captured_count, topics.size(), + fault_code.c_str()); - if (!grouped) { - persist_capture(id, std::move(rows), freeze_frame, captured_count, entity_scoped, capture_id); - return; - } + if (!grouped) { + persist_capture(id, std::move(rows), freeze_frame, captured_count, entity_scoped, capture_id); + return; + } - // Write for every record that joined while this ran, then for those that join while it writes. - std::set written; - for (;;) { - std::set todo; - { - std::lock_guard lock(groups_mutex_); - auto it = groups_.find(fault_code); - if (it == groups_.end()) { - // The group aged out while this ran: only the record of this worker is left to write. - if (written.count(id.owner) == 0) { - todo.insert(id.owner); + // Write for every record that joined while this ran, then for those that join while it writes. + std::set written; + for (;;) { + std::set todo; + { + std::lock_guard lock(groups_mutex_); + auto it = groups_.find(fault_code); + if (it == groups_.end()) { + // The group aged out while this ran: only the record of this worker is left to write. + if (written.count(id.owner) == 0) { + todo.insert(id.owner); + } + } else { + std::set_difference(it->second.owners.begin(), it->second.owners.end(), written.begin(), written.end(), + std::inserter(todo, todo.end())); } - } else { - std::set_difference(it->second.owners.begin(), it->second.owners.end(), written.begin(), written.end(), - std::inserter(todo, todo.end())); - } - if (todo.empty()) { - if (it != groups_.end()) { - it->second.state = CaptureGroup::State::kDone; - it->second.rows = rows; - it->second.freeze_frame = freeze_frame; - it->second.captured_count = captured_count; - it->second.capture_id = capture_id; + if (todo.empty()) { + if (it != groups_.end()) { + it->second.state = CaptureGroup::State::kDone; + it->second.rows = rows; + it->second.freeze_frame = freeze_frame; + it->second.captured_count = captured_count; + it->second.capture_id = capture_id; + } + return; } - return; + } + for (const auto & owner : todo) { + persist_capture(FaultId{fault_code, owner}, rows, freeze_frame, captured_count, false, capture_id); + written.insert(owner); } } - for (const auto & owner : todo) { - persist_capture(FaultId{fault_code, owner}, rows, freeze_frame, captured_count, false, capture_id); - written.insert(owner); + } catch (...) { + if (grouped) { + std::lock_guard lock(groups_mutex_); + groups_.erase(fault_code); } + throw; } } diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py new file mode 100644 index 000000000..47257339d --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Shared snapshot capture of per-source records under queue pressure, through a real fault manager. + +One worker, a queue of two and drop_oldest, so every extra queued job evicts another fault's +capture. A topic nobody publishes on keeps a capture running for its whole timeout. + + * a record joining the queued capture of its code queues no job of its own; + * a capture that outlives the recapture cooldown keeps every record that joined it; + * a record that confirms again while its old capture runs gets a sample taken afterwards. +""" + +import json +import os +import tempfile +import threading +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetSnapshots, ReportFault +from std_msgs.msg import Int32 + +TIMEOUT_SEC = 40.0 +FAST = '/queue/fast' +SLOW = '/queue/slow' + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file( + 'snapshot_queue_codes_', + ''.join(f'{code}:\n per_source: true\n' + for code in ('XQ.QUEUE', 'XQ.PRUNE', 'XQ.PRUNE.TRIGGER', 'XQ.AGAIN'))) +SNAPSHOTS_FILE = _temp_file( + 'snapshot_queue_topics_', + 'fault_specific:\n' + f' XQ.BUSY:\n - {SLOW}\n' + f' XQ.OTHER:\n - {FAST}\n' + f' XQ.QUEUE:\n - {FAST}\n' + f' XQ.PRUNE:\n - {FAST}\n - {SLOW}\n' + f' XQ.PRUNE.TRIGGER:\n - {FAST}\n' + f' XQ.AGAIN:\n - {FAST}\n - {SLOW}\n') + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'snapshots.enabled': True, + 'snapshots.config_file': SNAPSHOTS_FILE, + 'snapshots.timeout_sec': 2.0, + 'snapshots.capture_pool_size': 1, + 'snapshots.capture_queue_depth': 2, + 'snapshots.capture_queue_full_policy': 'drop_oldest', + 'snapshots.recapture_cooldown_sec': 0.3, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestSnapshotQueue(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_snapshot_queue_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + cls.snapshots_client = cls.node.create_client( + GetSnapshots, '/fault_manager/get_snapshots') + for client in (cls.report_client, cls.clear_client, cls.snapshots_client): + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + cls.fast = cls.node.create_publisher(Int32, FAST, 10) + # Nothing is ever published here, so a capture of this topic runs for its whole timeout. + cls.slow = cls.node.create_publisher(Int32, SLOW, 10) + cls.stop = threading.Event() + cls.publisher_thread = threading.Thread(target=cls._publish) + cls.publisher_thread.start() + + @classmethod + def _publish(cls): + value = 0 + while not cls.stop.wait(0.1): + value += 1 + cls.fast.publish(Int32(data=value)) + + @classmethod + def tearDownClass(cls): + cls.stop.set() + cls.publisher_thread.join() + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_CRITICAL + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _clear(self, code, source): + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + self.assertTrue(self._call(self.clear_client, request).success) + + def _snapshot(self, code, source): + """Return the record's snapshot data, or None while it lists none.""" + request = GetSnapshots.Request() + request.fault_code = code + request.source_id = source + response = self._call(self.snapshots_client, request) + if response.success and response.data: + data = json.loads(response.data) + if data.get('topics'): + return data + return None + + def _wait_snapshot(self, code, source, between=None): + """Poll until the record lists a snapshot; run @p between after every miss.""" + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + data = self._snapshot(code, source) + if data: + return data + if between: + between() + rclpy.spin_once(self.node, timeout_sec=0.1) + self.fail(f'{code}/{source} never listed a snapshot') + + def _wait_capture_started(self, proc_output, fault_manager_node, code): + proc_output.assertWaitFor( + f"Capturing snapshots for fault '{code}'", + process=fault_manager_node, timeout=TIMEOUT_SEC) + + def test_a_joining_record_does_not_evict_another_faults_capture( + self, proc_output, fault_manager_node): + # The worker is busy with a capture that waits on a silent topic. + self._report('XQ.BUSY', '/q/busy') + self._wait_capture_started(proc_output, fault_manager_node, 'XQ.BUSY') + # The oldest queued job belongs to another fault. + self._report('XQ.OTHER', '/q/other') + # Two owners of one code: one queued job for both. A second job would fill the queue and + # evict the oldest one, which is the other fault's. + self._report('XQ.QUEUE', '/q/a') + self._report('XQ.QUEUE', '/q/b') + + self._wait_snapshot('XQ.OTHER', '/q/other') + first = self._wait_snapshot('XQ.QUEUE', '/q/a') + second = self._wait_snapshot('XQ.QUEUE', '/q/b') + self.assertEqual(first['captured_at'], second['captured_at']) + + def test_a_capture_that_outlives_the_cooldown_keeps_its_records( + self, proc_output, fault_manager_node): + self._report('XQ.PRUNE', '/q/a') + self._wait_capture_started(proc_output, fault_manager_node, 'XQ.PRUNE') + self._report('XQ.PRUNE', '/q/b') + + # Every admission of another grouped code is a chance to drop an aged-out group. The + # capture above runs for the whole topic timeout, much longer than the cooldown. + triggers = iter(range(10000)) + first = self._wait_snapshot( + 'XQ.PRUNE', '/q/a', + between=lambda: self._report('XQ.PRUNE.TRIGGER', f'/q/trigger{next(triggers)}')) + second = self._wait_snapshot('XQ.PRUNE', '/q/b') + self.assertEqual(first['captured_at'], second['captured_at']) + + def test_a_record_confirming_again_is_not_given_the_old_sample( + self, proc_output, fault_manager_node): + self._report('XQ.AGAIN', '/q/a') + self._wait_capture_started(proc_output, fault_manager_node, 'XQ.AGAIN') + self._report('XQ.AGAIN', '/q/b') + + # The record is acknowledged and confirms again while the first capture still waits on + # the silent topic. That capture sampled the previous occurrence. + self._clear('XQ.AGAIN', '/q/a') + confirmed_again = time.time() + self._report('XQ.AGAIN', '/q/a') + + old = self._wait_snapshot('XQ.AGAIN', '/q/b') + again = self._wait_snapshot('XQ.AGAIN', '/q/a') + self.assertNotEqual(again['captured_at'], old['captured_at'], + 'the record was given the sample of its previous occurrence') + self.assertGreater(again['captured_at'], confirmed_again) + + +@launch_testing.post_shutdown_test() +class TestSnapshotQueueShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(SNAPSHOTS_FILE) From 7af545461700d1dd05fe4bfe6c788340193174a8 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 19:36:04 +0100 Subject: [PATCH 15/52] opcua: probe and clear PLC_COMMS_LOST under the id it was reported under The plugin remembers the entity id under which it reported the standing comms-lost fault, and the probe and the clear name that id until the clear is sent; after it the current component id is used again. A per-source record raised under the placeholder derived from the endpoint is therefore cleared after the component is renamed, instead of being left standing. Shared codes are unaffected. --- .../ros2_medkit_opcua/opcua_plugin.hpp | 10 +++++++++ .../ros2_medkit_opcua/src/opcua_plugin.cpp | 18 ++++++++++++++-- .../integration/test_opcua_comms_lost.test.py | 21 +++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp index 52f4a892b..d5fd6cc93 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp @@ -993,6 +993,16 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // on the wire, so one further decision follows any that had a raise behind // it. Set on the poll thread, taken on the poll thread. std::atomic comms_lost_raise_since_decision_{false}; + + // The entity id this process last reported PLC_COMMS_LOST under, empty once that fault was cleared. + // The component can be renamed while the fault stands, and a per-source record belongs to the id it + // was reported under, so the probe and the clear use this id and not the current one. + mutable std::mutex comms_lost_entity_mutex_; + std::string comms_lost_entity_id_; + + // The id the probe and the clear of PLC_COMMS_LOST name: the one it was reported under, else the + // current component id. + std::string comms_lost_entity_id() const; // How long a probe may stay outstanding (kDefaultFaultServiceTimeout unless // ``fault_service_timeout_ms`` says otherwise). std::chrono::milliseconds fault_service_timeout_{kDefaultFaultServiceTimeout}; diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp index 26b2bfe21..9cedf4cc9 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp @@ -1324,6 +1324,8 @@ void OpcuaPlugin::on_alarm_change(const std::string & entity_id, // before it has processed this report, so the decision that follows earns // one more (see comms_lost_raise_since_decision_). comms_lost_raise_since_decision_.store(true); + std::lock_guard lock(comms_lost_entity_mutex_); + comms_lost_entity_id_ = entity_id; } send_report_fault(entity_id, signal.fault_code, signal.severity, signal.message); } else { @@ -1647,7 +1649,19 @@ void OpcuaPlugin::apply_comms_lost_answer(bool found, const std::vector lock(comms_lost_entity_mutex_); + comms_lost_entity_id_.clear(); +} + +std::string OpcuaPlugin::comms_lost_entity_id() const { + { + std::lock_guard lock(comms_lost_entity_mutex_); + if (!comms_lost_entity_id_.empty()) { + return comms_lost_entity_id_; + } + } + return node_map_.component_id(); } void OpcuaPlugin::drive_comms_lost_decision() { @@ -1741,7 +1755,7 @@ void OpcuaPlugin::drive_comms_lost_decision() { request->fault_code = kCommsLostFaultCode; // The record this bridge reports it under, so a second source that raised the code does not make the // probe ambiguous. - request->source_id = node_map_.component_id(); + request->source_id = comms_lost_entity_id(); // rclcpp selects the response-callback overload of async_send_request only for // a callable whose parameter list is exactly (SharedFuture); a const reference // parameter matches no overload. The future is therefore taken by value. diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py index a5672a565..3342fcd33 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py @@ -28,6 +28,9 @@ * per-source record: PLC_COMMS_LOST configured with one record per source and a second source that reported it. The restored link clears this gateway's own record and leaves the other source's alone. + * per-source record under a stand-in: the gateway raises the fault under the stand-in + of its endpoint, the PLC comes up and names the component, and the same gateway + clears the record it raised under the stand-in. * binding directory: a binding file whose directory cannot be looked up is reported, and the plugin stays connected. Once at start, and once on the poll thread when a rescan adopts a PLC that came up later. @@ -487,6 +490,23 @@ def scenario_per_source_record(run): raise AssertionError(f"the other source's record was cleared too: {records}") +def scenario_per_source_placeholder_rename(run): + run.start_fault_manager('rename', per_source_codes=[COMMS_LOST]) + port = free_port() + gateway = run.start_gateway('rename', endpoint=f'opc.tcp://127.0.0.1:{port}') + records = wait_records(run, lambda r: len(r) == 1 and 'CONFIRMED' in r.values(), 60) + if len(records) != 1 or 'CONFIRMED' not in records.values(): + raise AssertionError(f'the outage was never reported: {records}\n' + tail(gateway)) + stand_in = next(iter(records)) + + # The PLC comes up on the endpoint the gateway was pinned to and names the component. + run.start_server('rename', port=port) + records = wait_records(run, lambda r: r.get(stand_in) == 'CLEARED', 60) + if records.get(stand_in) != 'CLEARED': + raise AssertionError(f'the record raised under the stand-in {stand_in} was stranded ' + f'after the component was renamed: {records}\n' + tail(gateway)) + + def start_discovering_gateway(run, name): """Start a config-less gateway whose binding file sits under a symlink loop.""" loop_a = run.workdir / 'loop_a' @@ -550,6 +570,7 @@ def scenario_binding_directory_on_reconnect(run): ('foreign stand-in', scenario_foreign_stand_in), ('late nameplate', scenario_late_nameplate), ('per-source record', scenario_per_source_record), + ('per-source record under a stand-in', scenario_per_source_placeholder_rename), ('binding directory at start', scenario_binding_directory_at_start), ('binding directory on reconnect', scenario_binding_directory_on_reconnect), ] From f19267c7b86f005d5272b52a36e814dba645838c Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 20:37:21 +0100 Subject: [PATCH 16/52] docs: state what the correlation cascade clears The cascade clears the records the root cause has muted since its last report, because a new report of the root cause starts that list again. --- docs/tutorials/fault-correlation.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/fault-correlation.rst b/docs/tutorials/fault-correlation.rst index e5c429f3e..5e8636378 100644 --- a/docs/tutorials/fault-correlation.rst +++ b/docs/tutorials/fault-correlation.rst @@ -125,8 +125,9 @@ node mutes, and with ``auto_clear_with_root`` auto-clears, the matching symptoms reported by any other node inside the window. An auto-cluster collects the matching faults of every node. The relations hold fault records, not codes: for a code configured with ``per_source: true`` each reporting node has its own record, and the -cascade clears exactly the records the root cause muted. For any other code there is -one shared record, and that record is what is muted or cleared. +cascade clears the records the root cause has muted since its last report. A new report +of the root cause starts that list again. For any other code there is one shared record, +and that record is what is muted or cleared. Quick Start ----------- From c468d756c7e41d689bd9198b8a4e58fc747f9c3d Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 20:37:24 +0100 Subject: [PATCH 17/52] docs: every client must be rebuilt against the new messages Fields added inside the elements of the ListFaults arrays change the wire layout, so a client built against earlier messages misreads a response that carries a muted or clustered entry without any error. The compatibility notes say so for every service and topic of the fault manager. --- docs/api/messages.rst | 14 ++++++++++---- src/ros2_medkit_msgs/README.md | 11 ++++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/api/messages.rst b/docs/api/messages.rst index 187211a2d..3e800c0fc 100644 --- a/docs/api/messages.rst +++ b/docs/api/messages.rst @@ -279,10 +279,16 @@ cluster-wide clearing still works. ``skip_correlation_auto_clear`` was added in ``ros2_medkit_msgs`` post-0.4.0, and ``source_id`` after it (the same field was added to ``GetFault``, - ``GetSnapshots`` and ``GetRosbag``). Adding a request field changes the - service type hash, so out-of-tree callers that invoke those services directly - via ``ros2 service call`` or a generated client must rebuild against the new - ``ros2_medkit_msgs`` release to keep talking to ``fault_manager``. + ``GetSnapshots`` and ``GetRosbag``). Adding a field changes the type hash, and + a field added inside an array element changes the wire layout, so every client + of the fault manager's services and topics must be rebuilt against this + ``ros2_medkit_msgs``. This includes out-of-tree callers that use ``ros2 + service call`` or a generated client. The case that breaks without an error is + ``ListFaults``: ``MutedFaultInfo.source_id`` and ``ClusterInfo.fault_owners`` and + ``representative_owner`` sit inside the elements of the ``muted_faults`` and + ``clusters`` arrays, CDR has no per-element length, and a client built against + an earlier ``ros2_medkit_msgs`` misreads any response that carries a muted or + clustered entry. Trailing fields do not keep old clients working. ListFaults.srv ~~~~~~~~~~~~~~ diff --git a/src/ros2_medkit_msgs/README.md b/src/ros2_medkit_msgs/README.md index 1492dc8eb..04c940470 100644 --- a/src/ros2_medkit_msgs/README.md +++ b/src/ros2_medkit_msgs/README.md @@ -151,9 +151,14 @@ Clear/acknowledge one fault record. Cleared records are retained and queryable w > **Note:** `skip_correlation_auto_clear` was added in `ros2_medkit_msgs` post-0.4.0, and `source_id` > after it. `source_id` was added the same way to `GetFault`, `GetSnapshots` and `GetRosbag`. Adding a -> request field changes the service type hash, so out-of-tree callers that invoke those services -> directly (via `ros2 service call` or a generated client) must rebuild against the new -> `ros2_medkit_msgs` release to keep talking to `fault_manager`. +> field changes the type hash, and a field added inside an array element changes the wire layout, so +> every client of the fault manager's services and topics must be rebuilt against this +> `ros2_medkit_msgs`. This includes out-of-tree callers that use `ros2 service call` or a generated +> client. The case that breaks without an error is `ListFaults`: `MutedFaultInfo.source_id` and +> `ClusterInfo.fault_owners` / `representative_owner` sit inside the elements of the `muted_faults` +> and `clusters` arrays, CDR has no per-element length, and a client built against an earlier +> `ros2_medkit_msgs` misreads any response that carries a muted or clustered entry. Trailing fields +> do not keep old clients working. ## Usage From a7a54e345c2663756c67bed983a8a1499bc8e883 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 21:10:50 +0100 Subject: [PATCH 18/52] fault_manager: pin the shared record of a default code again The tests that origin/main had for a record several sources report into are restored: UpdateExistingFaultPublishesUpdatedEvent and GetFaultReturnsExtendedDataRecords report the second time from another source, and the in-memory cases for confirmation, debounce depth, healing and recurrence use the shared owner. The confirmed-latch tests use the shared owner too: with a per-source owner the final report took the stored-severity rule and never reached the latch. A launch test on the in-memory and the SQLite store checks that two sources reporting a default code give one confirmed event and one updated event that names both sources with the occurrence count still 1. --- src/ros2_medkit_fault_manager/CMakeLists.txt | 3 + .../test_fault_shared_events.test.py | 132 ++++++++++++++++++ .../test/test_fault_manager.cpp | 119 ++++++++-------- .../test/test_postgres_storage.cpp | 21 +-- .../test/test_sqlite_storage.cpp | 18 +-- 5 files changed, 215 insertions(+), 78 deletions(-) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index be4e635d4..f57316de0 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -312,6 +312,9 @@ endif() medkit_add_launch_test(test_fault_snapshot_queue test/integration/test_fault_snapshot_queue.test.py TIMEOUT 180 LABELS "integration") + medkit_add_launch_test(test_fault_shared_events + test/integration/test_fault_shared_events.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_identity_restart test/integration/test_fault_identity_restart.test.py TIMEOUT 180 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py new file mode 100644 index 000000000..98873bc06 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The shared record of a default code, through a real fault manager on both stores. + +Two sources reporting one default code feed one record: the first report confirms it, the +second updates it, and the update names both sources with the occurrence count still 1. +""" + +import os +import shutil +import tempfile +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import launch_testing.markers +import rclpy +from rclpy.node import Node +from rclpy.qos import QoSProfile, ReliabilityPolicy +from ros2_medkit_msgs.msg import Fault, FaultEvent +from ros2_medkit_msgs.srv import ReportFault + +TIMEOUT_SEC = 30.0 +_temp_dirs = [] + + +@launch_testing.parametrize('storage_type', ['memory', 'sqlite']) +@launch_testing.markers.keep_alive +def generate_test_description(storage_type): + temp_dir = tempfile.mkdtemp(prefix='shared_events_') + _temp_dirs.append(temp_dir) + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': storage_type, + 'database_path': os.path.join(temp_dir, 'faults.db'), + 'confirmation_threshold': -1, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node, 'storage_type': storage_type}, + ) + + +class TestSharedRecordEvents(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_shared_events_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + assert cls.report_client.wait_for_service(timeout_sec=TIMEOUT_SEC) + cls.events = [] + cls.subscription = cls.node.create_subscription( + FaultEvent, '/fault_manager/events', cls.events.append, + QoSProfile(depth=100, reliability=ReliabilityPolicy.RELIABLE)) + deadline = time.monotonic() + TIMEOUT_SEC + while cls.subscription.get_publisher_count() == 0 and time.monotonic() < deadline: + rclpy.spin_once(cls.node, timeout_sec=0.1) + assert cls.subscription.get_publisher_count() > 0 + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + future = self.report_client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.result().accepted) + + def _wait_events(self, count): + deadline = time.monotonic() + TIMEOUT_SEC + while len(self.events) < count and time.monotonic() < deadline: + rclpy.spin_once(self.node, timeout_sec=0.1) + self.assertGreaterEqual(len(self.events), count) + + def test_a_second_source_updates_the_one_shared_record(self, storage_type): + self._report('SE.DEFAULT', '/a') + self._report('SE.DEFAULT', '/b') + self._wait_events(2) + + events = [(e.event_type, e.fault) for e in self.events + if e.fault.fault_code == 'SE.DEFAULT'] + self.assertEqual([t for t, _ in events], + [FaultEvent.EVENT_CONFIRMED, FaultEvent.EVENT_UPDATED], storage_type) + updated = events[1][1] + self.assertEqual(list(updated.reporting_sources), ['/a', '/b']) + self.assertEqual(updated.occurrence_count, 1) + + +@launch_testing.post_shutdown_test() +class TestSharedRecordEventsShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info, fault_manager_node): + launch_testing.asserts.assertExitCodes(proc_info, process=fault_manager_node) + + def test_temp_dirs_removed(self): + for path in _temp_dirs: + shutil.rmtree(path, ignore_errors=True) + self.assertFalse(os.path.exists(path)) diff --git a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp index 94200ca06..865fbf4e5 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp @@ -365,14 +365,12 @@ TEST_F(FaultStorageTest, FaultStaysPrefailedAboveThreshold) { config.confirmation_threshold = -3; storage_.set_debounce_config(config); - // Both reports from one source: the counter belongs to that source's record, so - // debounce depth is what one reporter has said, not what several have said between them. - storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", - "/node1", clock.now().nanoseconds(), config); - storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", - "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node2", clock.now().nanoseconds(), config); - auto fault = storage_.get_fault({"FAULT_1", "/node1"}); + auto fault = storage_.get_fault({"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Still debouncing towards confirmation, same occurrence EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); @@ -386,12 +384,14 @@ TEST_F(FaultStorageTest, FaultConfirmsAtThreshold) { config.confirmation_threshold = -3; storage_.set_debounce_config(config); - for (int i = 0; i < 3; ++i) { - storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", - "/node1", clock.now().nanoseconds(), config); - } + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node2", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node3", clock.now().nanoseconds(), config); - auto fault = storage_.get_fault({"FAULT_1", "/node1"}); + auto fault = storage_.get_fault({"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Debounce build-up is still one occurrence EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -415,21 +415,21 @@ TEST_F(FaultStorageTest, ConfirmedFaultStaysConfirmed) { TEST_F(FaultStorageTest, MultiSourceConfirmsFault) { rclcpp::Clock clock; - // Report the same fault code from 3 different sources - for (const auto * source : {"/sensor1", "/sensor2", "/sensor3"}) { - storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", source}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "Test", source, clock.now().nanoseconds(), default_config()); - } - - // Three records, each confirmed on its own report and naming only its own source. - EXPECT_EQ(storage_.size(), 3u); - for (const auto * source : {"/sensor1", "/sensor2", "/sensor3"}) { - auto fault = storage_.get_fault({"MOTOR_OVERHEAT", source}); - ASSERT_TRUE(fault.has_value()) << source; - EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); - ASSERT_EQ(fault->reporting_sources.size(), 1u); - EXPECT_EQ(fault->reporting_sources.front(), source); - } + // Report same fault from 3 different sources + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", ros2_medkit_fault_manager::kSharedOwner}, + fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/sensor1", + clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", ros2_medkit_fault_manager::kSharedOwner}, + fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/sensor2", + clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", ros2_medkit_fault_manager::kSharedOwner}, + fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/sensor3", + clock.now().nanoseconds(), default_config()); + + auto fault = storage_.get_fault({"MOTOR_OVERHEAT", ros2_medkit_fault_manager::kSharedOwner}); + ASSERT_TRUE(fault.has_value()); + EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(fault->reporting_sources.size(), 3u); } TEST_F(FaultStorageTest, SameSourceMultipleReportsConfirms) { @@ -615,19 +615,19 @@ TEST_F(FaultStorageTest, PassedEventIncrementsCounter) { config.confirmation_threshold = -3; config.healing_threshold = 3; // explicit upper clamp (don't depend on the default) - // Report 2 FAILED events from one source (its counter = -2, PREFAILED) - for (int i = 0; i < 2; ++i) { - storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", - "/node1", clock.now().nanoseconds(), config); - } + // Report 2 FAILED events (counter = -2, PREFAILED) + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node2", clock.now().nanoseconds(), config); // Report 3 PASSED events (counter = -2 + 3 = +1) for (int i = 0; i < 3; ++i) { - storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kPassed, 0, + "", "/node1", clock.now().nanoseconds(), config); } - auto fault = storage_.get_fault({"FAULT_1", "/node1"}); + auto fault = storage_.get_fault({"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREPASSED); // Counter > 0 } @@ -843,26 +843,26 @@ TEST_F(FaultStorageTest, HealedFaultCanRecurWithFailedEvents) { storage_.set_debounce_config(config); // Report 1 FAILED event (counter = -1) - storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", - "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); // Report 4 PASSED events to heal (counter = -1 + 4 = +3) for (int i = 0; i < 4; ++i) { - storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kPassed, 0, + "", "/node1", clock.now().nanoseconds(), config); } - auto fault = storage_.get_fault({"FAULT_1", "/node1"}); + auto fault = storage_.get_fault({"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); // Report 3 FAILED events - fault should recur and confirm (counter = +3 - 3 = 0, then -3) for (int i = 0; i < 6; ++i) { - storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "Recurrence", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Recurrence", "/node2", clock.now().nanoseconds(), config); } - fault = storage_.get_fault({"FAULT_1", "/node1"}); + fault = storage_.get_fault({"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -910,17 +910,17 @@ TEST_F(FaultStorageTest, ConfirmedLatchSurvivesFailedAtPositiveCounter) { config.healing_threshold = 3; config.critical_immediate_confirm = true; - storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", - clock.now().nanoseconds(), config); - ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); + storage_.report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); + ASSERT_EQ(storage_.get_fault({"F", ros2_medkit_fault_manager::kSharedOwner})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { - storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), - config); + storage_.report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kPassed, 0, "", + "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); - storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", - clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_.get_fault({"F", ros2_medkit_fault_manager::kSharedOwner})->status, Fault::STATUS_CONFIRMED); + storage_.report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); + EXPECT_EQ(storage_.get_fault({"F", ros2_medkit_fault_manager::kSharedOwner})->status, Fault::STATUS_CONFIRMED); } // HEALED with a negative counter stays HEALED on PASSED. @@ -1488,19 +1488,18 @@ TEST_F(FaultEventPublishingTest, UpdateExistingFaultPublishesUpdatedEvent) { // Clear received events received_events_.clear(); - // The SAME owner reports again - its own record is updated, so EVENT_UPDATED. - ASSERT_TRUE(call_report_fault("TEST_FAULT_2", Fault::SEVERITY_ERROR, "/test_node1")); + // Report same fault again from another source - should trigger EVENT_UPDATED on the shared record + ASSERT_TRUE(call_report_fault("TEST_FAULT_2", Fault::SEVERITY_ERROR, "/test_node2")); ASSERT_TRUE(spin_until([this]() { return !received_events_.empty(); })); - // Verify EVENT_UPDATED was published (severity changed, still one occurrence) + // Verify EVENT_UPDATED was published (severity/sources changed; still one occurrence) ASSERT_EQ(received_events_.size(), 1u); EXPECT_EQ(received_events_[0].event_type, FaultEvent::EVENT_UPDATED); EXPECT_EQ(received_events_[0].fault.fault_code, "TEST_FAULT_2"); EXPECT_EQ(received_events_[0].fault.occurrence_count, 1u); - ASSERT_EQ(received_events_[0].fault.reporting_sources.size(), 1u); - EXPECT_EQ(received_events_[0].fault.reporting_sources.front(), "/test_node1"); + EXPECT_EQ(received_events_[0].fault.reporting_sources, (std::vector{"/test_node1", "/test_node2"})); } /// A node where the codes the tests below report are configured per_source, so each source gets its own record. @@ -2073,17 +2072,17 @@ TEST_F(FaultEventPublishingTest, GetFaultReturnsEnvironmentData) { } TEST_F(FaultEventPublishingTest, GetFaultReturnsExtendedDataRecords) { - // Report twice from one owner so first and last occurrence are both stamped + // Report fault twice to have first and last occurrence timestamps differ ASSERT_TRUE(call_report_fault("EDR_TEST", Fault::SEVERITY_ERROR, "/node1")); ASSERT_TRUE(spin_until([this]() { return !received_events_.empty(); })); - ASSERT_TRUE(call_report_fault("EDR_TEST", Fault::SEVERITY_ERROR, "/node1")); + ASSERT_TRUE(call_report_fault("EDR_TEST", Fault::SEVERITY_ERROR, "/node2")); ASSERT_TRUE(spin_until([this]() { return received_events_.size() >= 2; })); - // One record carries the code, so the unscoped call resolves to it. + // One shared record carries the code, so the unscoped call resolves to it. auto response = call_get_fault("EDR_TEST"); ASSERT_TRUE(response.has_value()); EXPECT_TRUE(response->success); diff --git a/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp b/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp index 5a85c38a2..1df9f61d1 100644 --- a/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp +++ b/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp @@ -1557,19 +1557,22 @@ TEST_F(PgFaultStorageTest, ConfirmedLatchSurvivesFailedAtPositiveCounter) { config.critical_immediate_confirm = true; // CRITICAL confirms immediately at counter -1. - storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", - clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); + ASSERT_EQ(storage_->get_fault(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner})->status, + Fault::STATUS_CONFIRMED); // PASSED events raise the counter into positive territory; latch keeps it CONFIRMED. for (int i = 0; i < 3; ++i) { - storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kPassed, 0, "", + "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner})->status, + Fault::STATUS_CONFIRMED); // One normal FAILED: counter still positive, must NOT become PREPASSED. - storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", - clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner})->status, + Fault::STATUS_CONFIRMED); } // A HEALED fault with a negative counter stays HEALED on PASSED (opposite direction of the bug above). diff --git a/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp b/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp index 813d34e9d..b668f9eff 100644 --- a/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp +++ b/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp @@ -805,19 +805,19 @@ TEST_F(SqliteFaultStorageTest, ConfirmedLatchSurvivesFailedAtPositiveCounter) { config.critical_immediate_confirm = true; // CRITICAL confirms immediately at counter -1. - storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", - clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); + ASSERT_EQ(storage_->get_fault({"F", ros2_medkit_fault_manager::kSharedOwner})->status, Fault::STATUS_CONFIRMED); // PASSED events raise the counter into positive territory; latch keeps it CONFIRMED. for (int i = 0; i < 3; ++i) { - storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kPassed, 0, "", + "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault({"F", ros2_medkit_fault_manager::kSharedOwner})->status, Fault::STATUS_CONFIRMED); // One normal FAILED: counter still positive, must NOT become PREPASSED. - storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", - clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault({"F", ros2_medkit_fault_manager::kSharedOwner})->status, Fault::STATUS_CONFIRMED); } // A HEALED fault with a negative counter stays HEALED on PASSED (opposite direction of the bug above). From 1be2ed5b3d2cbccbb4b05885e1c06dfb2b56546e Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 21:10:58 +0100 Subject: [PATCH 19/52] fault_manager: test the promises of the per-source model through the process - An unscoped GetSnapshots or GetRosbag on a per-source code with two owners is refused as ambiguous and returns no evidence. - A recapture cooldown of 0 turns snapshot grouping off: two owners confirmed back to back are two captures. - With max_per_fault 1 and snapshots retained on clear, an owner that confirms twice keeps only its newest capture, both topics of it, and another owner's capture is intact. - A cleared cluster member leaves the cluster, the representative moves off it, and the member re-reported later is in the cluster exactly once. - A thresholds file with a non-bool per_source is rejected as a whole, so the code stays one shared record listing both sources. --- src/ros2_medkit_fault_manager/CMakeLists.txt | 3 + .../test_fault_correlation_owners.test.py | 45 ++++ .../test_fault_identity_restart.test.py | 17 ++ .../test_fault_snapshot_cap.test.py | 213 ++++++++++++++++++ .../test_fault_snapshot_grouping.test.py | 6 + .../test_rosbag_integration.test.py | 8 + 6 files changed, 292 insertions(+) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index f57316de0..07b79cc66 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -315,6 +315,9 @@ endif() medkit_add_launch_test(test_fault_shared_events test/integration/test_fault_shared_events.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_snapshot_cap + test/integration/test_fault_snapshot_cap.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_identity_restart test/integration/test_fault_identity_restart.test.py TIMEOUT 180 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py index 1d217bc54..3b1020074 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py @@ -55,6 +55,10 @@ per_source: true XO_PS_STORM_3: per_source: true +XO_PS_CHURN_1: + per_source: true +XO_PS_CHURN_2: + per_source: true """ CORRELATION = """\ @@ -70,6 +74,8 @@ codes: ["XO_PS_STORM_*"] sh_storm: codes: ["XO_SH_STORM_*"] + ps_churn: + codes: ["XO_PS_CHURN_*"] rules: - id: ps_estop name: "Per-source e-stop" @@ -100,6 +106,15 @@ window_ms: 5000 show_as_single: true representative: first + - id: ps_churn + name: "Per-source churn" + mode: auto_cluster + match: + - pattern: ps_churn + min_count: 3 + window_ms: 30000 + show_as_single: true + representative: first - id: sh_storm name: "Shared storm" mode: auto_cluster @@ -277,6 +292,36 @@ def test_per_source_storm_across_nodes_forms_one_cluster(self): self.assertEqual(cluster.representative_code, 'XO_PS_STORM_1') self.assertEqual(cluster.representative_owner, '/lidar') + def test_a_cleared_member_leaves_the_cluster_and_comes_back_once(self): + for code, node in [('XO_PS_CHURN_1', '/lidar'), ('XO_PS_CHURN_1', '/camera'), + ('XO_PS_CHURN_2', '/motor_node')]: + self._report(code, node) + + def cluster(): + clusters = [c for c in self._list(include_clusters=True).clusters + if c.rule_id == 'ps_churn'] + self.assertEqual(len(clusters), 1) + return clusters[0] + + before = cluster() + self.assertEqual((before.representative_code, before.representative_owner), + ('XO_PS_CHURN_1', '/lidar')) + + self._clear('XO_PS_CHURN_1', '/lidar') + after = cluster() + self.assertEqual(sorted(zip(after.fault_codes, after.fault_owners)), + [('XO_PS_CHURN_1', '/camera'), ('XO_PS_CHURN_2', '/motor_node')]) + self.assertNotEqual((after.representative_code, after.representative_owner), + ('XO_PS_CHURN_1', '/lidar'), 'a cleared record is representative') + self.assertIn((after.representative_code, after.representative_owner), + zip(after.fault_codes, after.fault_owners)) + + self._report('XO_PS_CHURN_1', '/lidar') + again = cluster() + members = list(zip(again.fault_codes, again.fault_owners)) + self.assertEqual(members.count(('XO_PS_CHURN_1', '/lidar')), 1) + self.assertEqual(len(members), 3) + def test_shared_storm_across_nodes_forms_one_cluster(self): cluster = self._storm( 'sh_storm', diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py index a42277456..5c434634d 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py @@ -295,6 +295,23 @@ def test_changing_the_mode_keeps_stored_records_and_routes_new_reports(self): ('N.PER', '*'), ('N.PER', '/a'), ('N.PER', '/b'), ]) + def test_a_malformed_per_source_value_rejects_the_whole_file(self): + thresholds = 'A:\n per_source: true\nB:\n per_source: "yes"\n' + process = self._start(self._path('malformed.db'), thresholds) + try: + self.assertIsNotNone( + process.wait_for_line('Failed to parse fault thresholds config'), + 'the malformed per_source value was not reported') + # The file is dropped as a whole, so A is not per_source either: one shared record. + self._report('A', '/a') + self._report('A', '/b') + records = self._records('A') + self.assertEqual(len(records), 1) + self.assertEqual(list(records[0].reporting_sources), ['/a', '/b']) + self.assertTrue(self._get('A', '*').success) + finally: + self._stop(process) + def test_a_store_written_with_one_record_per_code_opens_as_shared_records(self): db_path = self._path('legacy.db') bag_path = self._path('fault_MIG.CODE_1') diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py new file mode 100644 index 000000000..e0097a032 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Snapshots of per-source records with the recapture cooldown off, through a real fault manager. + + * a cooldown of 0 turns grouping off: two owners confirmed back to back are two captures; + * the per-record cap keeps the newest capture of each record, both topics of it, and one + owner's captures never trim another owner's. + +The cap is read from the store the fault manager writes, because GetSnapshots only ever +returns the newest capture of a record. +""" + +import json +import os +import shutil +import sqlite3 +import tempfile +import threading +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetSnapshots, ReportFault +from std_msgs.msg import Int32 + +TIMEOUT_SEC = 40.0 +FIRST = '/cap/first' +SECOND = '/cap/second' +TEMP_DIR = tempfile.mkdtemp(prefix='snapshot_cap_') +DB_PATH = os.path.join(TEMP_DIR, 'faults.db') + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file( + 'snapshot_cap_codes_', ''.join(f'{c}:\n per_source: true\n' for c in ('XC.TWO', 'XC.CAP'))) +SNAPSHOTS_FILE = _temp_file( + 'snapshot_cap_topics_', + 'fault_specific:\n' + f' XC.TWO:\n - {FIRST}\n' + f' XC.CAP:\n - {FIRST}\n - {SECOND}\n') + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'sqlite', + 'database_path': DB_PATH, + 'confirmation_threshold': -1, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'snapshots.enabled': True, + 'snapshots.config_file': SNAPSHOTS_FILE, + 'snapshots.timeout_sec': 3.0, + 'snapshots.capture_pool_size': 3, + 'snapshots.recapture_cooldown_sec': 0.0, + 'snapshots.max_per_fault': 1, + 'snapshots.retain_on_clear': True, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestSnapshotCap(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_snapshot_cap_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + cls.snapshots_client = cls.node.create_client( + GetSnapshots, '/fault_manager/get_snapshots') + for client in (cls.report_client, cls.clear_client, cls.snapshots_client): + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + cls.publishers = [cls.node.create_publisher(Int32, topic, 10) for topic in (FIRST, SECOND)] + cls.stop = threading.Event() + cls.publisher_thread = threading.Thread(target=cls._publish) + cls.publisher_thread.start() + + @classmethod + def _publish(cls): + value = 0 + while not cls.stop.wait(0.1): + value += 1 + for publisher in cls.publishers: + publisher.publish(Int32(data=value)) + + @classmethod + def tearDownClass(cls): + cls.stop.set() + cls.publisher_thread.join() + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_CRITICAL + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _clear(self, code, source): + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + self.assertTrue(self._call(self.clear_client, request).success) + + def _wait_snapshot(self, code, source, newer_than=0.0): + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + request = GetSnapshots.Request() + request.fault_code = code + request.source_id = source + response = self._call(self.snapshots_client, request) + if response.success and response.data: + data = json.loads(response.data) + if data.get('topics') and data['captured_at'] > newer_than: + return data + rclpy.spin_once(self.node, timeout_sec=0.1) + self.fail(f'{code}/{source} never listed a snapshot newer than {newer_than}') + + def _stored(self, code): + """Return {owner: {capture_id: [topics]}} as the store holds it.""" + with sqlite3.connect(f'file:{DB_PATH}?mode=ro', uri=True) as db: + rows = db.execute( + 'SELECT owner, capture_id, topic FROM snapshots WHERE fault_code = ?', + (code,)).fetchall() + stored = {} + for owner, capture_id, topic in rows: + stored.setdefault(owner, {}).setdefault(capture_id, []).append(topic) + return stored + + def test_a_cooldown_of_zero_turns_grouping_off(self): + self._report('XC.TWO', '/cap/a') + self._report('XC.TWO', '/cap/b') + first = self._wait_snapshot('XC.TWO', '/cap/a') + second = self._wait_snapshot('XC.TWO', '/cap/b') + self.assertNotEqual(first['captured_at'], second['captured_at']) + stored = self._stored('XC.TWO') + self.assertEqual(sorted(stored), ['/cap/a', '/cap/b']) + capture_ids = {cid for captures in stored.values() for cid in captures} + self.assertEqual(len(capture_ids), 2, 'two owners must be two captures') + + def test_the_cap_keeps_the_newest_capture_of_each_record(self): + self._report('XC.CAP', '/cap/a') + first = self._wait_snapshot('XC.CAP', '/cap/a') + self._clear('XC.CAP', '/cap/a') + self._report('XC.CAP', '/cap/a') + second = self._wait_snapshot('XC.CAP', '/cap/a', newer_than=first['captured_at']) + self._report('XC.CAP', '/cap/b') + own = self._wait_snapshot('XC.CAP', '/cap/b') + + self.assertEqual(sorted(second['topics']), [FIRST, SECOND]) + self.assertEqual(sorted(own['topics']), [FIRST, SECOND]) + stored = self._stored('XC.CAP') + self.assertEqual(len(stored['/cap/a']), 1, 'the cap keeps one capture per record') + self.assertEqual(sorted(next(iter(stored['/cap/a'].values()))), [FIRST, SECOND]) + self.assertEqual(len(stored['/cap/b']), 1) + self.assertEqual(sorted(next(iter(stored['/cap/b'].values()))), [FIRST, SECOND]) + self.assertNotEqual(stored['/cap/a'].keys(), stored['/cap/b'].keys()) + + +@launch_testing.post_shutdown_test() +class TestSnapshotCapShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(SNAPSHOTS_FILE) + shutil.rmtree(TEMP_DIR, ignore_errors=True) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py index ae6e2fe4c..147c1632f 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py @@ -163,6 +163,12 @@ def test_per_source_records_share_one_capture(self): {json.dumps(d['topics'], sort_keys=True) for d in captures}, {json.dumps(captures[0]['topics'], sort_keys=True)}) + # Three records carry the code, so an unscoped call is refused and returns no evidence. + unscoped = self._snapshot('XG.GROUP', '') + self.assertFalse(unscoped.success) + self.assertTrue(unscoped.error_message.startswith('ambiguous:'), unscoped.error_message) + self.assertEqual(unscoped.data, '') + def test_a_record_that_confirms_again_gets_a_capture_of_its_own(self): code = 'XG.AGAIN' source = '/grp/again' diff --git a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py index 52b9cab43..5ec519eb3 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py @@ -633,6 +633,14 @@ def test_08_per_source_burst_lists_every_owner_of_the_recording(self): self.assertEqual(responses[owners[0]].file_path, responses[owners[1]].file_path, 'records of one burst must share a single recording') + + # Two records carry the code, so an unscoped call is refused and returns no recording. + request = GetRosbag.Request() + request.fault_code = code + unscoped = self._call_service(self.get_rosbag_client, request) + self.assertFalse(unscoped.success) + self.assertTrue(unscoped.error_message.startswith('ambiguous:'), unscoped.error_message) + self.assertEqual(unscoped.file_path, '') for response in responses.values(): self.assertEqual(list(response.fault_codes), [code, code]) self.assertEqual(sorted(response.owners), owners) From 3eb882f8403939a86cd57244b3297f5216ab2037 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 21:13:13 +0100 Subject: [PATCH 20/52] opcua: find a per-source comms-lost record raised under the stand-in after a restart After a restart the plugin remembers no reporting id, so the comms-lost probe under the current component id found nothing for a per-source code and the record raised under the stand-in derived from the endpoint stayed standing. When the probe under the remembered or current id finds no record, the plugin now probes under each other id this process reports under, and clears the record it finds under the id that found it. A shared code is found by the first probe, as before. --- .../ros2_medkit_opcua/opcua_plugin.hpp | 9 +++++- .../ros2_medkit_opcua/src/opcua_plugin.cpp | 32 ++++++++++++++++--- .../integration/test_opcua_comms_lost.test.py | 23 +++++++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp index d5fd6cc93..bd02bf9e0 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp @@ -675,7 +675,9 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // Apply the gate to an answer the store gave and send the clear it permits. // Poll thread only. - void apply_comms_lost_answer(bool found, const std::vector & reporting_sources); + // ``clear_source`` is the id the probe that found the record was sent under. + void apply_comms_lost_answer(bool found, const std::vector & reporting_sources, + const std::string & clear_source); // The ids a fault reported by THIS process can carry: the component id being // served, every placeholder this process assigned, and the NodeMap default. @@ -993,6 +995,11 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // on the wire, so one further decision follows any that had a raise behind // it. Set on the poll thread, taken on the poll thread. std::atomic comms_lost_raise_since_decision_{false}; + // Ids this process reports under that the current decision has not probed yet, and the id the + // outstanding probe was sent under. A per-source record belongs to the id it was raised under, which + // after a restart is not the one the plugin remembers. Poll thread only. + std::vector comms_lost_untried_ids_; + std::string comms_lost_probe_id_; // The entity id this process last reported PLC_COMMS_LOST under, empty once that fault was cleared. // The component can be renamed while the fault stands, and a per-source record belongs to the id it diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp index 9cedf4cc9..5c6f64302 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp @@ -1626,7 +1626,8 @@ void OpcuaPlugin::owe_comms_lost_decision() { comms_lost_decision_owed_.store(true); } -void OpcuaPlugin::apply_comms_lost_answer(bool found, const std::vector & reporting_sources) { +void OpcuaPlugin::apply_comms_lost_answer(bool found, const std::vector & reporting_sources, + const std::string & clear_source) { if (!found) { // No row, so nothing to clear and nothing standing to report. return; @@ -1649,7 +1650,7 @@ void OpcuaPlugin::apply_comms_lost_answer(bool found, const std::vector lock(comms_lost_entity_mutex_); comms_lost_entity_id_.clear(); } @@ -1695,11 +1696,19 @@ void OpcuaPlugin::drive_comms_lost_decision() { // the store from before an outage that has since been reported. It is // dropped and the decision is taken again on this session. comms_lost_decision_owed_.store(true); + comms_lost_untried_ids_.clear(); RCLCPP_DEBUG(opcua_plugin_logger(), "%s answer belongs to an earlier session; asking again on this one", kCommsLostFaultCode); return; } - apply_comms_lost_answer(found, sources); + if (!found && !comms_lost_untried_ids_.empty()) { + // No record under that id. A per-source record belongs to the id it was raised under, so ask + // under the next id this process reports under before concluding there is nothing to clear. + comms_lost_decision_owed_.store(true); + return; + } + comms_lost_untried_ids_.clear(); + apply_comms_lost_answer(found, sources, comms_lost_probe_id_); // The store may have answered from before a ReportFault this process had // already sent, so a decision with a raise behind it earns one more. if (comms_lost_raise_since_decision_.exchange(false)) { @@ -1726,6 +1735,7 @@ void OpcuaPlugin::drive_comms_lost_decision() { } comms_lost_probe_in_flight_ = false; comms_lost_decision_owed_.store(true); + comms_lost_untried_ids_.clear(); log_warn(std::string("fault store did not answer within ") + std::to_string(fault_service_timeout_.count()) + " ms; the " + kCommsLostFaultCode + " decision stays owed"); } @@ -1743,6 +1753,19 @@ void OpcuaPlugin::drive_comms_lost_decision() { return; } + // The id this probe asks under: the next untried one, else the one the fault is reported under + // now, with every other id this process reports under queued behind it. + if (!comms_lost_untried_ids_.empty()) { + comms_lost_probe_id_ = comms_lost_untried_ids_.back(); + comms_lost_untried_ids_.pop_back(); + } else { + comms_lost_probe_id_ = comms_lost_entity_id(); + std::unordered_set others = reporting_ids_of_this_process(); + others.erase(comms_lost_probe_id_); + comms_lost_untried_ids_.assign(others.begin(), others.end()); + std::sort(comms_lost_untried_ids_.begin(), comms_lost_untried_ids_.end()); + } + uint64_t generation = 0; { std::lock_guard lock(comms_lost_probe_state_->mutex); @@ -1755,7 +1778,7 @@ void OpcuaPlugin::drive_comms_lost_decision() { request->fault_code = kCommsLostFaultCode; // The record this bridge reports it under, so a second source that raised the code does not make the // probe ambiguous. - request->source_id = comms_lost_entity_id(); + request->source_id = comms_lost_probe_id_; // rclcpp selects the response-callback overload of async_send_request only for // a callable whose parameter list is exactly (SharedFuture); a const reference // parameter matches no overload. The future is therefore taken by value. @@ -1796,6 +1819,7 @@ void OpcuaPlugin::drive_comms_lost_decision() { // the poll thread has no catch of its own and a lost owe is a fault left // standing for good. comms_lost_decision_owed_.store(true); + comms_lost_untried_ids_.clear(); RCLCPP_WARN(opcua_plugin_logger(), "fault store request failed to go out (%s); the %s decision stays owed", e.what(), kCommsLostFaultCode); } diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py index 3342fcd33..d6d21ba36 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py @@ -31,6 +31,8 @@ * per-source record under a stand-in: the gateway raises the fault under the stand-in of its endpoint, the PLC comes up and names the component, and the same gateway clears the record it raised under the stand-in. + * per-source pinned stand-in: the same, with the gateway restarted in between, so it remembers + nothing and the record is found under the stand-in derived from its endpoint. * binding directory: a binding file whose directory cannot be looked up is reported, and the plugin stays connected. Once at start, and once on the poll thread when a rescan adopts a PLC that came up later. @@ -507,6 +509,26 @@ def scenario_per_source_placeholder_rename(run): f'after the component was renamed: {records}\n' + tail(gateway)) +def scenario_per_source_pinned_stand_in(run): + run.start_fault_manager('persrc_pinned', per_source_codes=[COMMS_LOST]) + port = free_port() + endpoint = f'opc.tcp://127.0.0.1:{port}' + gateway = run.start_gateway('persrc_pinned_before', endpoint=endpoint) + records = wait_records(run, lambda r: len(r) == 1 and 'CONFIRMED' in r.values(), 60) + if len(records) != 1 or 'CONFIRMED' not in records.values(): + raise AssertionError(f'the outage was never reported: {records}\n' + tail(gateway)) + stand_in = next(iter(records)) + run.stop(gateway.proc) + + # The restarted gateway remembers nothing, and the PLC names the component. + run.start_server('persrc_pinned', port=port) + after = run.start_gateway('persrc_pinned_after', endpoint=endpoint) + records = wait_records(run, lambda r: r.get(stand_in) == 'CLEARED', 60) + if records.get(stand_in) != 'CLEARED': + raise AssertionError(f'the record raised under the stand-in {stand_in} stayed standing ' + f'after a restart: {records}\n' + tail(after)) + + def start_discovering_gateway(run, name): """Start a config-less gateway whose binding file sits under a symlink loop.""" loop_a = run.workdir / 'loop_a' @@ -571,6 +593,7 @@ def scenario_binding_directory_on_reconnect(run): ('late nameplate', scenario_late_nameplate), ('per-source record', scenario_per_source_record), ('per-source record under a stand-in', scenario_per_source_placeholder_rename), + ('per-source pinned stand-in', scenario_per_source_pinned_stand_in), ('binding directory at start', scenario_binding_directory_at_start), ('binding directory on reconnect', scenario_binding_directory_on_reconnect), ] From 12bb14b26fe9c91bfd9ba04f6efad93cfba68c30 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 21:27:43 +0100 Subject: [PATCH 21/52] test(opcua): a cleared alarm clears only its own per-source record Two sources report one per-source code. The alarm bit of the bridge falls, and the test requires the bridge's record to clear while the other source's record stays confirmed. Without the source in the ClearFault request the call is refused as ambiguous and the test fails. --- .../ros2_medkit_opcua/CMakeLists.txt | 10 + .../test_opcua_per_source_clear.test.py | 369 ++++++++++++++++++ 2 files changed, 379 insertions(+) create mode 100644 src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_per_source_clear.test.py diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/CMakeLists.txt b/src/ros2_medkit_plugins/ros2_medkit_opcua/CMakeLists.txt index 3adf51951..b5c3bb48d 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/CMakeLists.txt +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/CMakeLists.txt @@ -645,6 +645,7 @@ if(BUILD_TESTING) test/integration/test_opcua_subscription_values.test.py test/integration/test_opcua_raise_qualifier.test.py test/integration/test_opcua_comms_lost.test.py + test/integration/test_opcua_per_source_clear.test.py test/integration/test_opcua_startup_sweep_signal.test.py DESTINATION lib/${PROJECT_NAME}) @@ -718,6 +719,15 @@ if(BUILD_TESTING) RESOURCE_LOCK opcua_tcp_4840 TIMEOUT 1200) + # A device alarm of a per-source code clears the gateway's own record only. + medkit_add_wrapped_test(test_opcua_per_source_clear + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/test/integration/test_opcua_per_source_clear.test.py" + "${CMAKE_BINARY_DIR}/test_alarm_server") + set_tests_properties(test_opcua_per_source_clear PROPERTIES + LABELS "integration" + TIMEOUT 300) + # A signal during the startup discovery sweep ends the sweep and the gateway. # Two cases, each bounded by its own 60 s start and 21 s exit deadlines. # The silent listener binds TCP 4840. diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_per_source_clear.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_per_source_clear.test.py new file mode 100644 index 000000000..9a13a12ac --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_per_source_clear.test.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A device alarm of a per-source code clears the gateway's own record, through a real gateway. + +The node map gives StatusWord bit 0 a fault code that the fault manager keeps one record per +source for, and a second source holds a record of the same code. Raising the bit raises the +gateway's record and clearing it clears that record alone: the plugin names the entity it +reports under in its ClearFault, so the clear is not refused as ambiguous. + +Usage: test_opcua_per_source_clear.test.py +""" + +import json +import os +from pathlib import Path +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request + +COMPONENT = 'per_source_runtime' +ENTITY = 'plc_app' +CODE = 'PS_STATUS_BIT0' +OTHER = '/other_source' +POLL_MS = 200 + +SCALE = 1.0 + + +def time_scale(): + """Return MEDKIT_TEST_TIME_SCALE, the factor the sanitizer jobs apply to wall-clock budgets.""" + try: + scale = float(os.environ.get('MEDKIT_TEST_TIME_SCALE', '1')) + except ValueError: + return 1.0 + return scale if scale >= 1.0 else 1.0 + + +SCALE = time_scale() + + +def free_port(): + """Grab an OS-assigned free TCP port on the loopback and release it.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('127.0.0.1', 0)) + return s.getsockname()[1] + + +def find_plugin(): + """Locate libros2_medkit_opcua_plugin.so under AMENT_PREFIX_PATH.""" + for prefix in os.environ.get('AMENT_PREFIX_PATH', '').split(os.pathsep): + if not prefix: + continue + for root, _dirs, files in os.walk(prefix): + if 'libros2_medkit_opcua_plugin.so' in files: + return os.path.join(root, 'libros2_medkit_opcua_plugin.so') + return None + + +def http_json(url, timeout=5): + """Return the parsed JSON of a GET, or None on any failure.""" + try: + with urllib.request.urlopen(url, timeout=timeout) as resp: + return json.loads(resp.read().decode()) + except (urllib.error.URLError, OSError, ValueError): + return None + + +def wait_log(path, needle, deadline=30, period=0.5): + """Poll a log file until appears; return True/False.""" + for _ in range(int(deadline / period) + 1): + try: + if needle in Path(path).read_text(errors='replace'): + return True + except OSError: + pass + time.sleep(period) + return False + + +def terminate(proc): + """SIGTERM then SIGKILL a child process group started with start_new_session.""" + if proc is None: + return + pgid = None + if proc.returncode is None: + try: + pgid = os.getpgid(proc.pid) + except ProcessLookupError: + pgid = None + + def signal_group(sig): + if pgid is None: + return + try: + os.killpg(pgid, sig) + except ProcessLookupError: + pass + + signal_group(signal.SIGTERM) + proc.terminate() + try: + proc.wait(timeout=8) + except subprocess.TimeoutExpired: + signal_group(signal.SIGKILL) + proc.kill() + proc.wait() + signal_group(signal.SIGKILL) + log = getattr(proc, '_log', None) + if log is not None: + log.close() + + +def start_server(server_bin, port, log_path): + """Start the fixture server and wait for its READY line.""" + log = open(log_path, 'w') + proc = subprocess.Popen( + [str(server_bin), '--port', str(port)], + stdin=subprocess.PIPE, stdout=log, stderr=subprocess.STDOUT, + text=True, start_new_session=True, + ) + proc._log = log + if not wait_log(log_path, 'READY ', deadline=25 * SCALE): + terminate(proc) + return None + return proc + + +def set_register(server, log_path, name, value): + """Write one Int32 register through the fixture and wait for its OK line.""" + needle = f'OK {name}={value}' + before = Path(log_path).read_text(errors='replace').count(needle) + server.stdin.write(f'set {name} {value}\n') + server.stdin.flush() + for _ in range(int(40 * SCALE)): + if Path(log_path).read_text(errors='replace').count(needle) > before: + return True + time.sleep(0.05) + return False + + +def start_gateway(params_file, log_path, env): + """Launch a gateway_node bound to the given params file.""" + log = open(log_path, 'w') + proc = subprocess.Popen( + ['ros2', 'run', 'ros2_medkit_gateway', 'gateway_node', + '--ros-args', '--params-file', str(params_file)], + stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True, + ) + proc._log = log + return proc + + +def start_fault_manager(log_path, env): + """Start fault_manager_node with memory storage and wait until its services exist.""" + log = open(log_path, 'w') + proc = subprocess.Popen( + ['ros2', 'run', 'ros2_medkit_fault_manager', 'fault_manager_node', + '--ros-args', '-p', 'storage_type:=memory'], + stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True, + ) + proc._log = log + # Logged at the end of the constructor, after the services exist. + started = wait_log(log_path, 'FaultManager node started (', 30 * SCALE) + if not started or proc.poll() is not None: + terminate(proc) + return None + return proc + + +FAULT_MANAGER_CLIENT = """ +import json +import sys + +import rclpy +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ListFaults, ReportFault + +rclpy.init() +node = rclpy.create_node('per_source_clear_test_client') + + +def call(client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(node, future, timeout_sec=10.0) + return future.result() + + +if sys.argv[1] == 'report': + client = node.create_client(ReportFault, '/fault_manager/report_fault') + client.wait_for_service(timeout_sec=10.0) + request = ReportFault.Request() + request.fault_code = sys.argv[2] + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = 'second source' + request.source_id = sys.argv[3] + print(json.dumps({'accepted': call(client, request).accepted})) +else: + client = node.create_client(ListFaults, '/fault_manager/list_faults') + client.wait_for_service(timeout_sec=10.0) + request = ListFaults.Request() + request.statuses = ['PREFAILED', 'PREPASSED', 'CONFIRMED', 'HEALED', 'CLEARED'] + print(json.dumps({f.reporting_sources[0]: f.status for f in call(client, request).faults + if f.fault_code == sys.argv[2] and f.reporting_sources})) +node.destroy_node() +rclpy.shutdown() +""" + + +def fault_manager_client(env, *args): + result = subprocess.run([sys.executable, '-c', FAULT_MANAGER_CLIENT, *args], env=env, + capture_output=True, text=True, timeout=60 * SCALE) + if result.returncode != 0: + raise AssertionError(f'fault manager client {args} failed:\n{result.stderr}') + return json.loads(result.stdout) + + +def wait_records(env, accept, deadline): + """Poll a scaled deadline until accept(records) holds. Return the last records.""" + end = time.monotonic() + deadline * SCALE + records = {} + while time.monotonic() < end: + records = fault_manager_client(env, 'list', CODE) + if accept(records): + return records + time.sleep(0.3) + return records + + +def main(): + global SCALE + if len(sys.argv) < 2: + print('usage: test_opcua_per_source_clear.test.py ', file=sys.stderr) + return 2 + SCALE = time_scale() + server_bin = Path(sys.argv[1]).resolve() + ros_domain_id = os.environ.get('ROS_DOMAIN_ID') + if not ros_domain_id: + print('ROS_DOMAIN_ID is not set: run this test through CTest', file=sys.stderr) + return 1 + plugin = find_plugin() + if plugin is None or not server_bin.is_file(): + print('plugin or fixture missing', file=sys.stderr) + return 1 + + workdir = Path(tempfile.mkdtemp(prefix='opcua_per_source_clear_')) + env = dict(os.environ, ROS_DOMAIN_ID=ros_domain_id) + procs = [] + try: + node_map = workdir / 'nodes.yaml' + node_map.write_text( + 'area_id: plc_systems\n' + f'component_id: {COMPONENT}\n' + 'nodes:\n' + ' - node_id: "ns=2;s=StatusWord"\n' + f' entity_id: {ENTITY}\n' + ' data_name: status_word\n' + ' data_type: int\n' + ' status_bits:\n' + ' - bit: 0\n' + f' fault_code: {CODE}\n') + manifest = workdir / 'manifest.yaml' + manifest.write_text('manifest_version: "1.0"\n') + thresholds = workdir / 'thresholds.yaml' + thresholds.write_text(f'{CODE}:\n per_source: true\n') + + server_port = free_port() + server_log = workdir / 'alarm_server.log' + server = start_server(server_bin, server_port, server_log) + if server is None: + print('fixture server did not become READY', file=sys.stderr) + return 1 + procs.append(server) + + fm_log = workdir / 'fault_manager.log' + fm_proc_log = open(fm_log, 'w') + fm = subprocess.Popen( + ['ros2', 'run', 'ros2_medkit_fault_manager', 'fault_manager_node', '--ros-args', + '-p', 'storage_type:=memory', '-p', f'fault_thresholds.config_file:={thresholds}'], + stdout=fm_proc_log, stderr=subprocess.STDOUT, env=env, start_new_session=True) + fm._log = fm_proc_log + procs.append(fm) + if not wait_log(fm_log, 'FaultManager node started (', 30 * SCALE): + print(fm_log.read_text(errors='replace'), file=sys.stderr) + return 1 + + port = free_port() + params = workdir / 'gateway.yaml' + params.write_text('\n'.join([ + 'ros2_medkit_gateway:', + ' ros__parameters:', + ' server:', + ' host: "127.0.0.1"', + f' port: {port}', + ' plugins: ["opcua"]', + f' plugins.opcua.path: "{plugin}"', + f' plugins.opcua.endpoint_url: "opc.tcp://127.0.0.1:{server_port}"', + f' plugins.opcua.node_map_path: "{node_map}"', + f' plugins.opcua.poll_interval_ms: {POLL_MS}', + ' discovery.mode: "hybrid"', + f' discovery.manifest_path: "{manifest}"', + ' discovery.manifest_strict_validation: false', + ]) + '\n') + gateway_log = workdir / 'gateway.log' + gateway = start_gateway(params, gateway_log, env) + procs.append(gateway) + status_url = f'http://127.0.0.1:{port}/api/v1/components/{COMPONENT}/x-plc-status' + end = time.monotonic() + 90 * SCALE + while time.monotonic() < end: + status = http_json(status_url) + if isinstance(status, dict) and status.get('connected') is True: + break + time.sleep(0.5) + else: + tail = gateway_log.read_text(errors='replace')[-3000:] + print('the gateway never connected:\n' + tail, file=sys.stderr) + return 1 + + # A second source holds a record of the same code. + if not fault_manager_client(env, 'report', CODE, OTHER)['accepted']: + print('the second source was refused', file=sys.stderr) + return 1 + + set_register(server, server_log, 'StatusWord', 1) + records = wait_records( + env, lambda r: r.get(ENTITY) == 'CONFIRMED' and r.get(OTHER) == 'CONFIRMED', 30) + if records.get(ENTITY) != 'CONFIRMED': + print(f'the alarm never raised the gateway record: {records}', file=sys.stderr) + return 1 + + set_register(server, server_log, 'StatusWord', 0) + records = wait_records(env, lambda r: r.get(ENTITY) == 'CLEARED', 30) + if records.get(ENTITY) != 'CLEARED': + print(f"the cleared alarm did not clear the gateway's record: {records}\n" + + fm_log.read_text(errors='replace')[-3000:], file=sys.stderr) + return 1 + if records.get(OTHER) != 'CONFIRMED': + print(f"the other source's record changed: {records}", file=sys.stderr) + return 1 + print('PASS: a device alarm clear reaches only the gateway record of a per-source code') + return 0 + finally: + for proc in reversed(procs): + terminate(proc) + shutil.rmtree(workdir, ignore_errors=True) + + +if __name__ == '__main__': + sys.exit(main()) From 7b0bcd5767b923a647ac5889eb268c8fe0deaaf4 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 21:32:17 +0100 Subject: [PATCH 22/52] fault_manager: a cleared root cause ends its relations whatever the cascade flag says Clearing a root cause with skip_correlation_auto_clear left its symptoms muted and kept the root muting later reports, so the symptoms vanished from the default listing under a root that no longer existed. The correlation state of the root is now always released on clear; the flag only decides whether the symptoms are cleared too. This changes the behaviour of default (shared) codes against origin/main on purpose: the same defect existed there. --- docs/api/messages.rst | 3 +- docs/tutorials/fault-correlation.rst | 4 +- src/ros2_medkit_fault_manager/CMakeLists.txt | 3 + .../src/core/fault_manager_core.cpp | 11 +- .../test_fault_correlation_clear.test.py | 197 ++++++++++++++++++ .../test_correlation_root_clear.test.py | 149 +++++++++++++ 6 files changed, 361 insertions(+), 6 deletions(-) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py create mode 100644 src/ros2_medkit_integration_tests/test/features/test_correlation_root_clear.test.py diff --git a/docs/api/messages.rst b/docs/api/messages.rst index 3e800c0fc..34da2fb8e 100644 --- a/docs/api/messages.rst +++ b/docs/api/messages.rst @@ -268,7 +268,8 @@ root-cause fault also clears every symptom that the correlation engine attributes to it via ``auto_clear_with_root`` rules; the cleared symptom codes are returned in ``auto_cleared_codes``. When ``true``, only the requested ``fault_code`` is cleared and ``auto_cleared_codes`` -is empty. The gateway's per-entity ``DELETE +is empty. Either way the cleared root cause ends its relations: the symptoms it +muted are unmuted, and it no longer mutes later reports. The gateway's per-entity ``DELETE /{entity-path}/faults/{fault_code}`` route sets this to ``true`` so that an operator with access to one entity cannot cascade-clear correlated symptoms reported by apps in other entities. The global diff --git a/docs/tutorials/fault-correlation.rst b/docs/tutorials/fault-correlation.rst index 5e8636378..5b5cfda85 100644 --- a/docs/tutorials/fault-correlation.rst +++ b/docs/tutorials/fault-correlation.rst @@ -126,7 +126,9 @@ reported by any other node inside the window. An auto-cluster collects the match faults of every node. The relations hold fault records, not codes: for a code configured with ``per_source: true`` each reporting node has its own record, and the cascade clears the records the root cause has muted since its last report. A new report -of the root cause starts that list again. For any other code there is one shared record, +of the root cause starts that list again. Clearing the root cause ends its relations: with +``skip_correlation_auto_clear`` its symptoms are not cleared, they are unmuted and show in +the default listing again. For any other code there is one shared record, and that record is what is muted or cleared. Quick Start diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index 07b79cc66..3ee1d149c 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -318,6 +318,9 @@ endif() medkit_add_launch_test(test_fault_snapshot_cap test/integration/test_fault_snapshot_cap.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_correlation_clear + test/integration/test_fault_correlation_clear.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_identity_restart test/integration/test_fault_identity_restart.test.py TIMEOUT 180 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp index a61806daa..b03e8c077 100644 --- a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp +++ b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp @@ -595,15 +595,18 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, const std: return outcome; } - // Then the correlation engine (to get the auto-clear list). Symptoms come back as the - // exact records the root cause muted. + // Then the correlation engine. A cleared record ends its relations whatever the flag says: its muted + // symptoms are unmuted and it stops muting later reports. Symptoms come back as the exact records the + // root cause muted. // `skip_correlation_auto_clear` lets the caller opt out of cascade-clearing // correlated symptom faults. Per-entity DELETE routes set it to true // so they cannot reach across entity boundaries via the correlation graph. std::vector auto_cleared; - if (correlation_ && !skip_correlation_auto_clear) { + if (correlation_) { auto clear_result = correlation_->process_clear(id); - auto_cleared = clear_result.auto_cleared_symptoms; + if (!skip_correlation_auto_clear) { + auto_cleared = clear_result.auto_cleared_symptoms; + } } outcome.success = cleared; diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py new file mode 100644 index 000000000..ff7052692 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The relations of a root cause across its clear, through a real fault manager. + +A root cleared with skip_correlation_auto_clear leaves its symptoms standing and unmuted, +and stops muting later reports. + +The scenario runs on a default (shared) code and on a per_source code whose root and symptom +come from different nodes. +""" + +import os +import tempfile +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault + +TIMEOUT_SEC = 30.0 +ALL_STATUSES = [ + Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, + Fault.STATUS_HEALED, Fault.STATUS_CLEARED, +] +KINDS = ('P7',) +MODES = ('SH', 'PS') + +THRESHOLDS = ''.join( + f'CL_{kind}_PS_{role}:\n per_source: true\n' for kind in KINDS for role in ('ROOT', 'SYM')) +CORRELATION = 'correlation:\n enabled: true\n rules:\n' + ''.join( + f""" - id: {kind.lower()}_{mode.lower()} + name: "{kind} {mode}" + mode: hierarchical + root_cause: + codes: ["CL_{kind}_{mode}_ROOT"] + symptoms: + - codes: ["CL_{kind}_{mode}_SYM"] + window_ms: 30000 + mute_symptoms: true + auto_clear_with_root: true +""" for kind in KINDS for mode in MODES) + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file('correlation_clear_thresholds_', THRESHOLDS) +CORRELATION_FILE = _temp_file('correlation_clear_rules_', CORRELATION) + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'healing_enabled': False, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'correlation.config_file': CORRELATION_FILE, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestCorrelationClear(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_correlation_clear_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + cls.get_client = cls.node.create_client(GetFault, '/fault_manager/get_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + for client in (cls.report_client, cls.list_client, cls.get_client, cls.clear_client): + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _clear(self, code, source, skip): + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + request.skip_correlation_auto_clear = skip + response = self._call(self.clear_client, request) + self.assertTrue(response.success, response.message) + return response + + def _status(self, code, source): + request = GetFault.Request() + request.fault_code = code + request.source_id = source + response = self._call(self.get_client, request) + self.assertTrue(response.success, response.error_message) + return response.fault.status + + def _visible(self): + """Return the codes the default listing shows.""" + return {f.fault_code for f in self._call(self.list_client, ListFaults.Request()).faults} + + def _muted(self): + request = ListFaults.Request() + request.include_muted = True + return {m.fault_code for m in self._call(self.list_client, request).muted_faults} + + def _scenario_sources(self, mode): + """Return (root owner, symptom owner, root source, symptom source).""" + # A per-source record is addressed by its owner, a shared one by '*'. + if mode == 'PS': + return '/root_node', '/sym_node', '/root_node', '/sym_node' + return '*', '*', '/root_node', '/sym_node' + + def _skip_clear_ends_the_relations(self, mode): + root, symptom = f'CL_P7_{mode}_ROOT', f'CL_P7_{mode}_SYM' + root_owner, sym_owner, root_source, sym_source = self._scenario_sources(mode) + self._report(root, root_source) + self._report(symptom, sym_source) + self.assertIn(symptom, self._muted()) + self.assertNotIn(symptom, self._visible()) + + response = self._clear(root, root_owner, skip=True) + self.assertEqual(list(response.auto_cleared_codes), [], 'skip must not cascade') + self.assertEqual(self._status(root, root_owner), Fault.STATUS_CLEARED) + self.assertEqual(self._status(symptom, sym_owner), Fault.STATUS_CONFIRMED) + self.assertNotIn(symptom, self._muted()) + self.assertIn(symptom, self._visible(), 'the symptom stayed muted after its root cleared') + + # The cleared root no longer mutes what is reported after it. + self._report(symptom, sym_source) + self.assertNotIn(symptom, self._muted()) + self.assertIn(symptom, self._visible()) + + def test_p7_default_code_skip_clear_ends_the_relations(self): + self._skip_clear_ends_the_relations('SH') + + def test_p7_per_source_code_skip_clear_ends_the_relations(self): + self._skip_clear_ends_the_relations('PS') + + +@launch_testing.post_shutdown_test() +class TestCorrelationClearShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(CORRELATION_FILE) diff --git a/src/ros2_medkit_integration_tests/test/features/test_correlation_root_clear.test.py b/src/ros2_medkit_integration_tests/test/features/test_correlation_root_clear.test.py new file mode 100644 index 000000000..a0d29529d --- /dev/null +++ b/src/ros2_medkit_integration_tests/test/features/test_correlation_root_clear.test.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Integration test: clearing a root cause through its app route unmutes its symptoms. + +The gateway's per-entity DELETE sends skip_correlation_auto_clear, so it clears the root cause +alone. The symptom the root cause muted must then show on its own app again, and stay a +CONFIRMED fault. +""" + +import atexit +import os +import tempfile +import time +import unittest + +import launch_testing +import rclpy +from rclpy.node import Node +import requests +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ListFaults, ReportFault + +from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES +from ros2_medkit_test_utils.gateway_test_case import GatewayTestCase +from ros2_medkit_test_utils.launch_helpers import create_test_launch + +ROOT_CODE = 'LIDAR_RANGE_INVALID' +SYMPTOM_CODE = 'ROOT_CLEAR_SYMPTOM' +SYMPTOM_SOURCE = '/powertrain/engine/temp_sensor' + +CORRELATION = """\ +correlation: + enabled: true + rules: + - id: lidar_cascade + name: "Lidar cascade" + mode: hierarchical + root_cause: + codes: ["LIDAR_RANGE_INVALID"] + symptoms: + - codes: ["ROOT_CLEAR_SYMPTOM"] + window_ms: 60000 + mute_symptoms: true + auto_clear_with_root: true +""" + + +def generate_test_description(): + """Launch gateway, fault manager with a cascade rule, a faulty lidar and a temp sensor.""" + rules = tempfile.NamedTemporaryFile( + 'w', suffix='.yaml', prefix='root_clear_rules_', delete=False) + rules.write(CORRELATION) + rules.close() + atexit.register(os.unlink, rules.name) + return create_test_launch( + demo_nodes=['lidar_sensor', 'temp_sensor'], + gateway_params={}, + fault_manager=True, + fault_manager_params={ + 'confirmation_threshold': -1, + 'correlation.config_file': rules.name, + }, + lidar_faulty=True, + ) + + +class TestCorrelationRootClear(GatewayTestCase): + """DELETE of a root cause on its app route ends its relations.""" + + MIN_EXPECTED_APPS = 2 + REQUIRED_APPS = {'lidar_sensor', 'temp_sensor'} + + @classmethod + def setUpClass(cls): + super().setUpClass() + rclpy.init() + cls._node = Node('root_clear_reporter') + cls._report = cls._node.create_client(ReportFault, '/fault_manager/report_fault') + cls._list = cls._node.create_client(ListFaults, '/fault_manager/list_faults') + assert cls._report.wait_for_service(timeout_sec=30.0) + assert cls._list.wait_for_service(timeout_sec=30.0) + + @classmethod + def tearDownClass(cls): + cls._node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self._node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _muted_codes(self): + request = ListFaults.Request() + request.include_muted = True + return {m.fault_code for m in self._call(self._list, request).muted_faults} + + def test_deleting_the_root_cause_unmutes_its_symptom(self): + self.wait_for_fault('/apps/lidar_sensor', ROOT_CODE) + + request = ReportFault.Request() + request.fault_code = SYMPTOM_CODE + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = 'symptom of the lidar' + request.source_id = SYMPTOM_SOURCE + self.assertTrue(self._call(self._report, request).accepted) + + deadline = time.monotonic() + 30.0 + while SYMPTOM_CODE not in self._muted_codes(): + self.assertLess(time.monotonic(), deadline, 'the symptom was never muted') + rclpy.spin_once(self._node, timeout_sec=0.2) + resp = requests.get(f'{self.BASE_URL}/apps/temp_sensor/faults', timeout=10) + self.assertNotIn(SYMPTOM_CODE, [f['fault_code'] for f in resp.json().get('items', [])]) + + resp = requests.delete( + f'{self.BASE_URL}/apps/lidar_sensor/faults/{ROOT_CODE}', timeout=10) + self.assertEqual(resp.status_code, 204, resp.text) + + fault = self.wait_for_fault('/apps/temp_sensor', SYMPTOM_CODE, max_wait=30.0) + self.assertEqual(fault['status'], 'CONFIRMED') + self.assertNotIn(SYMPTOM_CODE, self._muted_codes()) + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + + def test_exit_codes(self, proc_info): + """Check all processes exited cleanly.""" + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'{info.process_name} exited with code {info.returncode}', + ) From 0d351c385e0848a9c771abed74b2f3ad8d7cb804 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 21:32:51 +0100 Subject: [PATCH 23/52] fault_manager: a re-reported active root cause keeps the symptoms it muted A report of a root cause that is still active reset its symptom list, so its later clear cascaded only to the symptoms muted after the last report. The report now renews the window only. A symptom is muted when it is reported within window_ms of the latest report of the root cause, and it stays muted until the root cause is cleared. This changes the behaviour of default (shared) codes against origin/main on purpose: the same defect existed there. --- docs/tutorials/fault-correlation.rst | 12 ++++---- .../src/correlation/correlation_engine.cpp | 4 +-- .../test_fault_correlation_clear.test.py | 30 +++++++++++++++---- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/docs/tutorials/fault-correlation.rst b/docs/tutorials/fault-correlation.rst index 5b5cfda85..48cdef4d1 100644 --- a/docs/tutorials/fault-correlation.rst +++ b/docs/tutorials/fault-correlation.rst @@ -125,11 +125,13 @@ node mutes, and with ``auto_clear_with_root`` auto-clears, the matching symptoms reported by any other node inside the window. An auto-cluster collects the matching faults of every node. The relations hold fault records, not codes: for a code configured with ``per_source: true`` each reporting node has its own record, and the -cascade clears the records the root cause has muted since its last report. A new report -of the root cause starts that list again. Clearing the root cause ends its relations: with -``skip_correlation_auto_clear`` its symptoms are not cleared, they are unmuted and show in -the default listing again. For any other code there is one shared record, -and that record is what is muted or cleared. +cascade clears exactly the records the root cause muted. A symptom is muted when it is +reported within ``window_ms`` of the latest report of the root cause, and it stays muted +until the root cause is cleared. A new report of a root cause that is still active renews +that window and keeps the symptoms it already muted. Clearing the root cause ends its +relations: with ``skip_correlation_auto_clear`` its symptoms are not cleared, they are +unmuted and show in the default listing again. For any other code there is one shared +record, and that record is what is muted or cleared. Quick Start ----------- diff --git a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp index a286c92aa..9f9d9abea 100644 --- a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp +++ b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp @@ -67,8 +67,8 @@ ProcessFaultResult CorrelationEngine::process_fault(const FaultId & id, const st prc.window_ms = rule.window_ms; pending_root_causes_.push_back(prc); - // Initialize symptom list - root_to_symptoms_[id] = {}; + // A root that is already active keeps the symptoms it muted: a re-report only renews its window. + root_to_symptoms_.try_emplace(id); break; } } diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py index ff7052692..a8c8ef950 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py @@ -14,12 +14,14 @@ # limitations under the License. """ -The relations of a root cause across its clear, through a real fault manager. +The relations of a root cause across its re-reports and its clear, through a real fault manager. -A root cleared with skip_correlation_auto_clear leaves its symptoms standing and unmuted, -and stops muting later reports. + * a root cleared with skip_correlation_auto_clear leaves its symptoms standing and unmuted, + and stops muting later reports; + * a root reported again while active keeps the symptoms it already muted, so its clear + cascades to all of them. -The scenario runs on a default (shared) code and on a per_source code whose root and symptom +Each scenario runs on a default (shared) code and on a per_source code whose root and symptom come from different nodes. """ @@ -41,7 +43,7 @@ Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, Fault.STATUS_HEALED, Fault.STATUS_CLEARED, ] -KINDS = ('P7',) +KINDS = ('P7', 'P8') MODES = ('SH', 'PS') THRESHOLDS = ''.join( @@ -181,12 +183,30 @@ def _skip_clear_ends_the_relations(self, mode): self.assertNotIn(symptom, self._muted()) self.assertIn(symptom, self._visible()) + def _re_reported_root_keeps_its_symptoms(self, mode): + root, symptom = f'CL_P8_{mode}_ROOT', f'CL_P8_{mode}_SYM' + root_owner, sym_owner, root_source, sym_source = self._scenario_sources(mode) + self._report(root, root_source) + self._report(symptom, sym_source) + self.assertIn(symptom, self._muted()) + self._report(root, root_source) + + response = self._clear(root, root_owner, skip=False) + self.assertIn(symptom, list(response.auto_cleared_codes)) + self.assertEqual(self._status(symptom, sym_owner), Fault.STATUS_CLEARED) + def test_p7_default_code_skip_clear_ends_the_relations(self): self._skip_clear_ends_the_relations('SH') def test_p7_per_source_code_skip_clear_ends_the_relations(self): self._skip_clear_ends_the_relations('PS') + def test_p8_default_code_re_reported_root_keeps_its_symptoms(self): + self._re_reported_root_keeps_its_symptoms('SH') + + def test_p8_per_source_code_re_reported_root_keeps_its_symptoms(self): + self._re_reported_root_keeps_its_symptoms('PS') + @launch_testing.post_shutdown_test() class TestCorrelationClearShutdown(unittest.TestCase): From c22f2b505c5f74ef90fddbffca2ba17835dca993 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 21:46:14 +0100 Subject: [PATCH 24/52] test(fault_manager): a PostgreSQL database of the old layout is migrated by the node The test writes the old one-record-per-code layout with faults, a snapshot, a freeze frame and a rosbag link, starts the real node on it, and reads the records back through the services, before and after a restart. A small helper executable runs the SQL because the test has no PostgreSQL client. The test is built only with POSTGRES_SUPPORT and runs in the CI postgres job. --- src/ros2_medkit_fault_manager/CMakeLists.txt | 8 + .../test_fault_postgres_migration.test.py | 285 ++++++++++++++++++ .../test/pg_exec.cpp | 39 +++ 3 files changed, 332 insertions(+) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py create mode 100644 src/ros2_medkit_fault_manager/test/pg_exec.cpp diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index 3ee1d149c..75f205454 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -213,6 +213,14 @@ if(POSTGRES_SUPPORT) medkit_add_gtest(test_postgres_storage test/test_postgres_storage.cpp) target_link_libraries(test_postgres_storage fault_manager_lib) medkit_target_dependencies(test_postgres_storage rclcpp ros2_medkit_msgs) + + # SQL runner for the launch test that migrates a PostgreSQL database through the node. + add_executable(pg_exec test/pg_exec.cpp) + target_link_libraries(pg_exec fault_manager_lib) + install(TARGETS pg_exec DESTINATION lib/${PROJECT_NAME}) + + medkit_add_launch_test(test_fault_postgres_migration + test/integration/test_fault_postgres_migration.test.py TIMEOUT 180 LABELS "integration") endif() # Rosbag retention parity: every assertion runs against both storage backends. diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py new file mode 100644 index 000000000..bee79059f --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A PostgreSQL database written by the release before record identity, migrated through the node. + +The tables are created with that release's layout and rows, the fault manager is started on them +with storage_type: postgres, and the migrated records are read over its services: every code is one +shared record (owner '*') with its reporting sources, and its snapshot, freeze frame and recording +link attach to it. A report from a new source joins the record, and a restart changes nothing. +Needs a server: ROS2_MEDKIT_TEST_PG_CONN, as for the storage tests. +""" + +import os +import queue +import signal +import subprocess +import tempfile +import threading +import time +import unittest +import uuid + +from ament_index_python.packages import get_package_prefix +import launch +from launch import LaunchDescription +import launch_testing.actions +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import GetFault, GetRosbag, GetSnapshots, ListFaults, ReportFault + +TIMEOUT_SEC = 30.0 +ALL_STATUSES = [ + Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, + Fault.STATUS_HEALED, Fault.STATUS_CLEARED, +] +SERVICE = '/fault_manager/report_fault' + +# The tables as the release before record identity created them: one row per fault code. +MAIN_SCHEMA = """ +CREATE TABLE faults ( + fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, description TEXT NOT NULL, + first_occurred_ns BIGINT NOT NULL, last_occurred_ns BIGINT NOT NULL, + occurrence_count BIGINT NOT NULL, status TEXT NOT NULL, reporting_sources TEXT NOT NULL, + debounce_counter INTEGER NOT NULL DEFAULT 0, last_failed_ns BIGINT NOT NULL DEFAULT 0, + last_passed_ns BIGINT NOT NULL DEFAULT 0, confirmed_at_ns BIGINT NOT NULL DEFAULT 0); +CREATE TABLE snapshots ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + topic TEXT NOT NULL, message_type TEXT NOT NULL, data TEXT NOT NULL, + captured_at_ns BIGINT NOT NULL, capture_id BIGINT NOT NULL DEFAULT 0); +CREATE INDEX idx_snapshots_fault_code ON snapshots(fault_code); +CREATE INDEX idx_snapshots_fault_topic ON snapshots(fault_code, topic); +CREATE TABLE freeze_frames ( + fault_code TEXT PRIMARY KEY, data TEXT NOT NULL, captured_at_ns BIGINT NOT NULL); +CREATE TABLE near_misses ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + occurred_at_ns BIGINT NOT NULL, debounce_counter INTEGER NOT NULL, + confirmation_threshold INTEGER NOT NULL, severity INTEGER NOT NULL, + source_id TEXT NOT NULL, resulting_status TEXT NOT NULL DEFAULT ''); +CREATE INDEX idx_near_misses_fault_code ON near_misses(fault_code, id); +CREATE TABLE rosbag_files ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + recording_id TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL, format TEXT NOT NULL, + duration_sec DOUBLE PRECISION NOT NULL, size_bytes BIGINT NOT NULL, + created_at_ns BIGINT NOT NULL); +CREATE INDEX idx_rosbag_files_fault_code ON rosbag_files(fault_code); +CREATE INDEX idx_rosbag_files_created_at ON rosbag_files(created_at_ns); +CREATE INDEX idx_rosbag_files_fault_created ON rosbag_files(fault_code, created_at_ns, id); +CREATE INDEX idx_rosbag_files_recording ON rosbag_files(recording_id); +CREATE INDEX idx_rosbag_files_path ON rosbag_files(file_path); +CREATE UNIQUE INDEX idx_rosbag_files_fault_path ON rosbag_files(fault_code, file_path); +""" + + +def generate_test_description(): + """Start only a placeholder process, so launch keeps the test running.""" + keep_alive = launch.actions.ExecuteProcess(cmd=['sleep', '600']) + return LaunchDescription([keep_alive, launch_testing.actions.ReadyToTest()]) + + +class FaultManagerProcess: + """One fault_manager_node process with its output readable line by line.""" + + def __init__(self, exe, params_file): + self.proc = subprocess.Popen( + [exe, '--ros-args', '-r', '__node:=fault_manager', '--params-file', params_file], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, errors='replace') + self.lines = queue.Queue() + self.seen = [] + threading.Thread(target=self._pump, daemon=True).start() + + def _pump(self): + for line in self.proc.stdout: + self.lines.put(line) + self.lines.put(None) + + def wait_for_line(self, text): + """Return the first output line containing text, or None at the end or the timeout.""" + for line in self.seen: + if text in line: + return line + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + try: + line = self.lines.get(timeout=1.0) + except queue.Empty: + continue + if line is None: + return None + self.seen.append(line) + if text in line: + return line + return None + + def stop(self): + if self.proc.poll() is None: + self.proc.send_signal(signal.SIGINT) + return self.proc.wait(timeout=TIMEOUT_SEC) + + def kill(self): + if self.proc.poll() is None: + self.proc.kill() + self.proc.wait() + + +class TestPostgresMigration(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + prefix = get_package_prefix('ros2_medkit_fault_manager') + cls.exe = os.path.join(prefix, 'lib', 'ros2_medkit_fault_manager', 'fault_manager_node') + cls.pg_exec = os.path.join(prefix, 'lib', 'ros2_medkit_fault_manager', 'pg_exec') + cls.base = os.environ.get('ROS2_MEDKIT_TEST_PG_CONN') + assert cls.base, 'set ROS2_MEDKIT_TEST_PG_CONN to a PostgreSQL server' + cls.schema = 'medkit_migrate_' + uuid.uuid4().hex[:12] + cls.conn = cls.base + '?options=-csearch_path%3D' + cls.schema + cls._sql(cls.base, f'CREATE SCHEMA {cls.schema}') + rclpy.init() + cls.node = Node('test_postgres_migration_client') + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + cls._sql(cls.base, f'DROP SCHEMA IF EXISTS {cls.schema} CASCADE') + cls.tmp.cleanup() + + @classmethod + def _sql(cls, conn, statements): + result = subprocess.run( + [cls.pg_exec, conn], input=statements, text=True, + capture_output=True) + assert result.returncode == 0, result.stderr + + def _wait_until_service_is_gone(self): + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + names = [name for name, _ in self.node.get_service_names_and_types()] + if SERVICE not in names: + return + rclpy.spin_once(self.node, timeout_sec=0.1) + self.fail('the stopped fault manager is still listed in the graph') + + def _start(self): + self._wait_until_service_is_gone() + params = os.path.join(self.tmp.name, 'params.yaml') + with open(params, 'w') as f: + f.write('fault_manager:\n ros__parameters:\n' + ' storage_type: postgres\n' + f' database_url: "{self.conn}"\n' + ' confirmation_threshold: -1\n' + ' snapshots.enabled: false\n' + ' snapshots.rosbag.enabled: false\n') + process = FaultManagerProcess(self.exe, params) + self.addCleanup(process.kill) + self.clients = { + 'report': self.node.create_client(ReportFault, SERVICE), + 'get': self.node.create_client(GetFault, '/fault_manager/get_fault'), + 'list': self.node.create_client(ListFaults, '/fault_manager/list_faults'), + 'snapshots': self.node.create_client(GetSnapshots, '/fault_manager/get_snapshots'), + 'rosbag': self.node.create_client(GetRosbag, '/fault_manager/get_rosbag'), + } + for client in self.clients.values(): + self.assertTrue(client.wait_for_service(timeout_sec=TIMEOUT_SEC)) + self.assertIsNotNone( + process.wait_for_line('FaultManager node started'), 'the node never finished starting') + return process + + def _stop(self, process): + for client in self.clients.values(): + self.node.destroy_client(client) + self.assertEqual(process.stop(), 0) + + def _call(self, name, request): + future = self.clients[name].call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call('report', request).accepted) + + def _records(self, code): + request = ListFaults.Request() + request.statuses = ALL_STATUSES + return [f for f in self._call('list', request).faults if f.fault_code == code] + + def _check_migrated(self, sources): + records = self._records('MIG.CODE') + self.assertEqual([list(r.reporting_sources) for r in records], [sources]) + for source in ('', '*', '/s1'): + request = GetFault.Request() + request.fault_code = 'MIG.CODE' + request.source_id = source + record = self._call('get', request) + self.assertTrue(record.success, f'source {source!r}: {record.error_message}') + self.assertEqual(record.fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(list(record.fault.reporting_sources), sources) + self.assertEqual([list(r.reporting_sources) for r in self._records('MIG.OLD')], [['/s3']]) + + snapshots = GetSnapshots.Request() + snapshots.fault_code = 'MIG.CODE' + snapshots.source_id = '*' + response = self._call('snapshots', snapshots) + self.assertTrue(response.success, response.error_message) + self.assertIn('/t', response.data) + + rosbag = GetRosbag.Request() + rosbag.fault_code = 'MIG.CODE' + rosbag.source_id = '*' + bag = self._call('rosbag', rosbag) + self.assertTrue(bag.success, bag.error_message) + self.assertEqual(list(bag.fault_codes), ['MIG.CODE']) + self.assertEqual(list(bag.owners), ['*']) + + def test_a_database_written_with_one_record_per_code_is_migrated_by_the_node(self): + bag_path = os.path.join(self.tmp.name, 'fault_MIG.CODE_1') + os.makedirs(bag_path) + self._sql(self.conn, MAIN_SCHEMA + f""" +INSERT INTO faults VALUES ('MIG.CODE', 2, 'from the old release', 1, 1, 1, 'CONFIRMED', + '["/s1","/s2"]', -1, 1, 0, 0); +INSERT INTO faults VALUES ('MIG.OLD', 1, 'cleared long ago', 1, 1, 1, 'CLEARED', + '["/s3"]', 0, 1, 0, 0); +INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) + VALUES ('MIG.CODE', '/t', 'std_msgs/msg/Float64', '{{"data": 1.5}}', 9); +INSERT INTO freeze_frames VALUES ('MIG.CODE', '{{"/t": 1.5}}', 9); +INSERT INTO rosbag_files (fault_code, recording_id, file_path, format, duration_sec, + size_bytes, created_at_ns) + VALUES ('MIG.CODE', 'fault_MIG.CODE_1', '{bag_path}', 'mcap', 5.0, 100, 11); +""") + + process = self._start() + try: + self._check_migrated(['/s1', '/s2']) + # A new source joins the shared record. + self._report('MIG.CODE', '/s4') + self._check_migrated(['/s1', '/s2', '/s4']) + finally: + self._stop(process) + + process = self._start() + try: + self._check_migrated(['/s1', '/s2', '/s4']) + finally: + self._stop(process) diff --git a/src/ros2_medkit_fault_manager/test/pg_exec.cpp b/src/ros2_medkit_fault_manager/test/pg_exec.cpp new file mode 100644 index 000000000..29f6a1619 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/pg_exec.cpp @@ -0,0 +1,39 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Runs the SQL on stdin against the PostgreSQL server of the connection string in argv[1]. +// A launch test uses it to write and drop tables, since the test has no PostgreSQL client of its own. + +#include + +#include +#include +#include + +int main(int argc, char ** argv) { + if (argc != 2) { + std::cerr << "usage: pg_exec < statements.sql\n"; + return 2; + } + try { + const std::string sql((std::istreambuf_iterator(std::cin)), std::istreambuf_iterator()); + pqxx::connection connection(argv[1]); + pqxx::nontransaction tx(connection); + tx.exec(sql); + } catch (const std::exception & e) { + std::cerr << "pg_exec failed: " << e.what() << "\n"; + return 1; + } + return 0; +} From 88f686efc546cf8b97a5331bab805c136aae5aa4 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 22:06:08 +0100 Subject: [PATCH 25/52] test(opcua): wait for the next probe instead of sleeping after a late answer The negative proof that an answer to a timed-out probe decides nothing slept a fixed two seconds. It now waits until a further probe is parked, which shows the decision ran again after the late answer arrived. --- .../ros2_medkit_opcua/test/test_opcua_identity.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp index bb481e58a..c78cd4091 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp @@ -1799,7 +1799,12 @@ TEST_F(OpcuaIdentityE2ETest, AnAnswerToATimedOutProbeIsNotReadAsTheNextOnes) { // A answers late. ASSERT_TRUE(store.release_one_read()); - std::this_thread::sleep_for(std::chrono::seconds(2)); + // B times out in turn and a third probe parks: the decision ran again after A's answer arrived. + const auto next_probe_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (store.parked_read_count() < 2 && std::chrono::steady_clock::now() < next_probe_deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + ASSERT_GE(store.parked_read_count(), 2u) << "no further probe was sent after the late answer"; EXPECT_TRUE(store.cleared_codes().empty()) << "an answer to a probe already given up on decided the clear"; // B answers, and that is the answer the decision is waiting for. From 775fb72749ed8cdb5f61eb8d9eee4f577380b9b2 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 12 Nov 2026 22:25:24 +0100 Subject: [PATCH 26/52] test(fault_manager): the snapshot cap test no longer races topic discovery The first test raced the node's discovery of the published topics, and compared capture times that two captures can share when the topics publish slower than the captures run. A throw-away fault now waits until a capture holds both topics, and the separation of two captures is asserted by their capture ids alone. --- .../test_fault_snapshot_cap.test.py | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py index e0097a032..d8d8e64f9 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py @@ -120,6 +120,31 @@ def _publish(cls): for publisher in cls.publishers: publisher.publish(Int32(data=value)) + warm = False + + def setUp(self): + # The node samples only topics it has discovered. Raise a throw-away fault until a + # capture of it holds both topics, so the first test does not race the discovery. + if TestSnapshotCap.warm: + return + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + self._report('XC.CAP', '/cap/warmup') + request = GetSnapshots.Request() + request.fault_code = 'XC.CAP' + request.source_id = '/cap/warmup' + for _ in range(20): + response = self._call(self.snapshots_client, request) + if response.success and response.data and json.loads(response.data).get('topics'): + break + rclpy.spin_once(self.node, timeout_sec=0.1) + data = json.loads(response.data) if response.success and response.data else {} + if sorted(data.get('topics', {})) == [FIRST, SECOND]: + TestSnapshotCap.warm = True + return + self._clear('XC.CAP', '/cap/warmup') + self.fail('the fault manager never captured both topics') + @classmethod def tearDownClass(cls): cls.stop.set() @@ -176,9 +201,8 @@ def _stored(self, code): def test_a_cooldown_of_zero_turns_grouping_off(self): self._report('XC.TWO', '/cap/a') self._report('XC.TWO', '/cap/b') - first = self._wait_snapshot('XC.TWO', '/cap/a') - second = self._wait_snapshot('XC.TWO', '/cap/b') - self.assertNotEqual(first['captured_at'], second['captured_at']) + self._wait_snapshot('XC.TWO', '/cap/a') + self._wait_snapshot('XC.TWO', '/cap/b') stored = self._stored('XC.TWO') self.assertEqual(sorted(stored), ['/cap/a', '/cap/b']) capture_ids = {cid for captures in stored.values() for cid in captures} From 9754097b1823ca2bc9e4042a0fd9377c07a9883e Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 08:37:54 +0100 Subject: [PATCH 27/52] fault_manager: a root cause lists exactly the symptoms it mutes A symptom that was cleared or ended stayed in the list of the root cause that had muted it. When the root cause was reported again and the symptom came back outside the window, the root cause's clear cascaded to that later occurrence, which it never muted, and the list also grew for every owner of a per-source code. A record that ends now leaves the list of every root cause. This changes the behaviour of default (shared) codes against origin/main on purpose: the same defect existed there. --- .../correlation/correlation_engine.hpp | 3 ++ .../src/correlation/correlation_engine.cpp | 7 ++++ .../test_fault_correlation_clear.test.py | 41 ++++++++++++++++--- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp index 5bdc00df0..dd0e78fc8 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp @@ -156,6 +156,9 @@ class CorrelationEngine { /// @return ProcessFaultResult with correlation info if matched std::optional try_as_symptom(const FaultId & id, std::chrono::steady_clock::time_point timestamp); + /// Drops a record from the symptom list of every root cause. Caller holds the mutex. + void forget_symptom(const FaultId & id); + /// Check if the record's code matches an auto-cluster rule. A cluster holds the records /// of any owner that match the rule inside its window. /// @return ProcessFaultResult with cluster info if matched diff --git a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp index 9f9d9abea..9f591c2e4 100644 --- a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp +++ b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp @@ -190,10 +190,17 @@ ProcessClearResult CorrelationEngine::process_clear(const FaultId & id) { // Remove from muted records if it was a symptom muted_faults_.erase(id); + forget_symptom(id); return result; } +void CorrelationEngine::forget_symptom(const FaultId & id) { + for (auto & [root_id, symptoms] : root_to_symptoms_) { + symptoms.erase(std::remove(symptoms.begin(), symptoms.end(), id), symptoms.end()); + } +} + std::vector CorrelationEngine::get_muted_faults() const { std::lock_guard lock(mutex_); diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py index a8c8ef950..ec98cb5d4 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py @@ -19,7 +19,7 @@ * a root cleared with skip_correlation_auto_clear leaves its symptoms standing and unmuted, and stops muting later reports; * a root reported again while active keeps the symptoms it already muted, so its clear - cascades to all of them. + cascades only to the ones it still mutes. Each scenario runs on a default (shared) code and on a per_source code whose root and symptom come from different nodes. @@ -27,6 +27,7 @@ import os import tempfile +import time import unittest from launch import LaunchDescription @@ -43,8 +44,10 @@ Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, Fault.STATUS_HEALED, Fault.STATUS_CLEARED, ] -KINDS = ('P7', 'P8') +KINDS = ('P7', 'P8', 'P11') MODES = ('SH', 'PS') +# A short window lets a test move past it by reporting until the symptom is no longer muted. +WINDOW_MS = {'P11': 200} THRESHOLDS = ''.join( f'CL_{kind}_PS_{role}:\n per_source: true\n' for kind in KINDS for role in ('ROOT', 'SYM')) @@ -56,7 +59,7 @@ codes: ["CL_{kind}_{mode}_ROOT"] symptoms: - codes: ["CL_{kind}_{mode}_SYM"] - window_ms: 30000 + window_ms: {WINDOW_MS.get(kind, 30000)} mute_symptoms: true auto_clear_with_root: true """ for kind in KINDS for mode in MODES) @@ -121,10 +124,10 @@ def _call(self, client, request): self.assertTrue(future.done(), 'service call timed out') return future.result() - def _report(self, code, source): + def _report(self, code, source, event=ReportFault.Request.EVENT_FAILED): request = ReportFault.Request() request.fault_code = code - request.event_type = ReportFault.Request.EVENT_FAILED + request.event_type = event request.severity = Fault.SEVERITY_ERROR request.description = f'{code} from {source}' request.source_id = source @@ -195,6 +198,34 @@ def _re_reported_root_keeps_its_symptoms(self, mode): self.assertIn(symptom, list(response.auto_cleared_codes)) self.assertEqual(self._status(symptom, sym_owner), Fault.STATUS_CLEARED) + def _list_exact_symptoms(self, mode): + root, symptom = f'CL_P11_{mode}_ROOT', f'CL_P11_{mode}_SYM' + root_owner, sym_owner, root_source, sym_source = self._scenario_sources(mode) + self._report(root, root_source) + self._report(symptom, sym_source) + self.assertIn(symptom, self._muted()) + self._clear(symptom, sym_owner, skip=False) + self._report(root, root_source) + # Report the symptom until it falls outside the root's window and is no longer muted. + deadline = time.monotonic() + TIMEOUT_SEC + while True: + self._report(symptom, sym_source) + if symptom not in self._muted(): + break + self.assertLess(time.monotonic(), deadline, 'the symptom stayed muted past its window') + self._clear(symptom, sym_owner, skip=False) + + response = self._clear(root, root_owner, skip=False) + self.assertEqual(list(response.auto_cleared_codes), [], + 'the root cleared a symptom it did not mute') + self.assertEqual(self._status(symptom, sym_owner), Fault.STATUS_CONFIRMED) + + def test_p11_default_code_cascade_clears_only_the_muted_symptoms(self): + self._list_exact_symptoms('SH') + + def test_p11_per_source_code_cascade_clears_only_the_muted_symptoms(self): + self._list_exact_symptoms('PS') + def test_p7_default_code_skip_clear_ends_the_relations(self): self._skip_clear_ends_the_relations('SH') From dbc0f0ce9b9e642c8268173eb5ba6adf1c60944c Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 08:38:19 +0100 Subject: [PATCH 28/52] fault_manager: every end of a record releases the symptoms it muted Only the record a ClearFault named released its relations. A symptom cleared by that cascade, and a root cause that reached HEALED, kept the symptoms they had muted hidden under a root that no longer existed. A cascade-cleared record and a healed record now end their relations as a direct clear does: their symptoms are unmuted and keep their status. Only a ClearFault without the skip flag cascades, one level. This changes the behaviour of default (shared) codes against origin/main on purpose: the same defect existed there. --- docs/tutorials/fault-correlation.rst | 5 +- .../src/core/fault_manager_core.cpp | 6 ++ .../test_fault_correlation_clear.test.py | 65 +++++++++++++++++-- 3 files changed, 71 insertions(+), 5 deletions(-) diff --git a/docs/tutorials/fault-correlation.rst b/docs/tutorials/fault-correlation.rst index 48cdef4d1..a838c0db8 100644 --- a/docs/tutorials/fault-correlation.rst +++ b/docs/tutorials/fault-correlation.rst @@ -130,7 +130,10 @@ reported within ``window_ms`` of the latest report of the root cause, and it sta until the root cause is cleared. A new report of a root cause that is still active renews that window and keeps the symptoms it already muted. Clearing the root cause ends its relations: with ``skip_correlation_auto_clear`` its symptoms are not cleared, they are -unmuted and show in the default listing again. For any other code there is one shared +unmuted and show in the default listing again. Every other end of a record does the +same: a root cause that heals, or that a cascade clears, releases the symptoms it muted +and they keep their status. Only a ``ClearFault`` without the flag cascades, and it goes +one level. For any other code there is one shared record, and that record is what is muted or cleared. Quick Start diff --git a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp index b03e8c077..e343d4cf1 100644 --- a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp +++ b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp @@ -516,6 +516,10 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { // an acknowledged clear. if (just_healed) { publish_event(fd::fault_event_type::kCleared, *fault_after); + // A healed record has ended: its muted symptoms are unmuted and keep their status. + if (correlation_) { + correlation_->process_clear(id); + } } // Append tamper-evident audit records for the transitions that just happened. @@ -634,6 +638,8 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, const std: return outcome; } auto_cleared_codes.push_back(symptom_id.fault_code); + // The cascade goes one level: a cleared symptom that was itself a root releases its own symptoms. + correlation_->process_clear(symptom_id); if (audit_) { std::optional symptom; try { diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py index ec98cb5d4..ae4da4e18 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py @@ -19,7 +19,8 @@ * a root cleared with skip_correlation_auto_clear leaves its symptoms standing and unmuted, and stops muting later reports; * a root reported again while active keeps the symptoms it already muted, so its clear - cascades only to the ones it still mutes. + cascades to all of them, and only to the ones it still mutes; + * every way a record ends (a cascade clear, a heal) releases the symptoms it muted. Each scenario runs on a default (shared) code and on a per_source code whose root and symptom come from different nodes. @@ -44,7 +45,7 @@ Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, Fault.STATUS_HEALED, Fault.STATUS_CLEARED, ] -KINDS = ('P7', 'P8', 'P11') +KINDS = ('P7', 'P8', 'P11', 'P12') MODES = ('SH', 'PS') # A short window lets a test move past it by reporting until the symptom is no longer muted. WINDOW_MS = {'P11': 200} @@ -62,7 +63,27 @@ window_ms: {WINDOW_MS.get(kind, 30000)} mute_symptoms: true auto_clear_with_root: true -""" for kind in KINDS for mode in MODES) +""" for kind in KINDS for mode in MODES) + """ - id: p10_outer + name: "P10 outer" + mode: hierarchical + root_cause: + codes: ["CL_P10_OUTER"] + symptoms: + - codes: ["CL_P10_MID"] + window_ms: 30000 + mute_symptoms: true + auto_clear_with_root: true + - id: p10_inner + name: "P10 inner" + mode: hierarchical + root_cause: + codes: ["CL_P10_MID"] + symptoms: + - codes: ["CL_P10_LEAF"] + window_ms: 30000 + mute_symptoms: true + auto_clear_with_root: true +""" def _temp_file(prefix, content): @@ -85,7 +106,8 @@ def generate_test_description(): parameters=[{ 'storage_type': 'memory', 'confirmation_threshold': -1, - 'healing_enabled': False, + 'healing_enabled': True, + 'healing_threshold': 0, 'fault_thresholds.config_file': THRESHOLDS_FILE, 'correlation.config_file': CORRELATION_FILE, 'snapshots.enabled': False, @@ -226,6 +248,41 @@ def test_p11_default_code_cascade_clears_only_the_muted_symptoms(self): def test_p11_per_source_code_cascade_clears_only_the_muted_symptoms(self): self._list_exact_symptoms('PS') + def test_p10_a_cascade_cleared_root_releases_its_own_symptoms(self): + self._report('CL_P10_MID', '/node') + self._report('CL_P10_LEAF', '/node') + self._report('CL_P10_OUTER', '/node') + self._report('CL_P10_MID', '/node') + self.assertIn('CL_P10_MID', self._muted()) + self.assertIn('CL_P10_LEAF', self._muted()) + + response = self._clear('CL_P10_OUTER', '*', skip=False) + self.assertEqual( + list(response.auto_cleared_codes), ['CL_P10_MID'], 'the cascade is one level') + self.assertEqual(self._status('CL_P10_MID', '*'), Fault.STATUS_CLEARED) + self.assertEqual(self._status('CL_P10_LEAF', '*'), Fault.STATUS_CONFIRMED) + self.assertNotIn('CL_P10_LEAF', self._muted()) + self.assertIn('CL_P10_LEAF', self._visible()) + + def _healed_root_releases_its_symptoms(self, mode): + root, symptom = f'CL_P12_{mode}_ROOT', f'CL_P12_{mode}_SYM' + root_owner, sym_owner, root_source, sym_source = self._scenario_sources(mode) + self._report(root, root_source) + self._report(symptom, sym_source) + self.assertIn(symptom, self._muted()) + + self._report(root, root_source, event=ReportFault.Request.EVENT_PASSED) + self.assertEqual(self._status(root, root_owner), Fault.STATUS_HEALED) + self.assertEqual(self._status(symptom, sym_owner), Fault.STATUS_CONFIRMED) + self.assertNotIn(symptom, self._muted()) + self.assertIn(symptom, self._visible(), 'the symptom stayed muted after its root healed') + + def test_p10_default_code_healed_root_releases_its_symptoms(self): + self._healed_root_releases_its_symptoms('SH') + + def test_p10_per_source_code_healed_root_releases_its_symptoms(self): + self._healed_root_releases_its_symptoms('PS') + def test_p7_default_code_skip_clear_ends_the_relations(self): self._skip_clear_ends_the_relations('SH') From a1d5bf4af82a575fa9b967045d86a37890ecd334 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 08:39:46 +0100 Subject: [PATCH 29/52] opcua: report under the default component id only when the node map names none The set of ids a bridge reports under always held the NodeMap default id, although only a node map that names no component_id reports under it. A bridge with an explicit component_id therefore treated a record held under the default id as its own: it probed that id after its own probe missed, and cleared a per-source PLC_COMMS_LOST record that belongs to a default-id bridge. For a shared code the same set let it clear a default-id bridge's record, a defect that exists on origin/main as well. A decision that finds no record of the bridge now says so in the log. This changes the behaviour of default (shared) codes against origin/main on purpose. --- .../ros2_medkit_opcua/src/opcua_plugin.cpp | 6 ++--- .../integration/test_opcua_comms_lost.test.py | 27 ++++++++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp index 5c6f64302..d2b6281ef 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp @@ -1600,9 +1600,8 @@ bool OpcuaPlugin::link_state_clear_permitted(bool fault_found, const std::vector std::unordered_set OpcuaPlugin::reporting_ids_of_this_process() const { std::unordered_set ids(placeholder_ids_.begin(), placeholder_ids_.end()); - // A node map that names no component_id reports under the NodeMap default, so - // a fault carrying it is this bridge's. - ids.insert(NodeMap::kDefaultComponentId); + // A node map that names no component_id reports under the NodeMap default, which + // component_id() then returns. A map that names one does not report under the default. const std::string current = node_map_.component_id(); if (!current.empty()) { ids.insert(current); @@ -1630,6 +1629,7 @@ void OpcuaPlugin::apply_comms_lost_answer(bool found, const std::vector my_ids = reporting_ids_of_this_process(); diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py index d6d21ba36..f169f6ed0 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py @@ -33,6 +33,8 @@ clears the record it raised under the stand-in. * per-source pinned stand-in: the same, with the gateway restarted in between, so it remembers nothing and the record is found under the stand-in derived from its endpoint. + * explicit component id: a gateway whose node map names a component_id leaves the + per-source record a default-id bridge holds under 'opcua_device' alone. * binding directory: a binding file whose directory cannot be looked up is reported, and the plugin stays connected. Once at start, and once on the poll thread when a rescan adopts a PLC that came up later. @@ -253,7 +255,7 @@ def start_fault_manager(self, name, per_source_codes=()): + log.read_text(errors='replace')) return proc - def start_gateway(self, name, endpoint=None, poll_ms=200, extra_env=None): + def start_gateway(self, name, endpoint=None, poll_ms=200, extra_env=None, node_map=None): """Start gateway_node with the opcua plugin and wait for HTTP.""" port = free_port() params = self.workdir / f'{name}_gateway.yaml' @@ -270,6 +272,8 @@ def start_gateway(self, name, endpoint=None, poll_ms=200, extra_env=None): ] if endpoint is not None: lines.append(f' plugins.opcua.endpoint_url: "{endpoint}"') + if node_map is not None: + lines.append(f' plugins.opcua.node_map_path: "{node_map}"') params.write_text('\n'.join(lines) + '\n') env = dict(self.env) env.update(extra_env or {}) @@ -529,6 +533,26 @@ def scenario_per_source_pinned_stand_in(run): f'after a restart: {records}\n' + tail(after)) +def scenario_explicit_component_id_keeps_the_default_id_record(run): + run.start_fault_manager('explicit_id', per_source_codes=[COMMS_LOST]) + default_id = 'opcua_device' + # A bridge whose node map names no component_id holds an outage record under the default id. + if not fault_manager_client(run, 'report', COMMS_LOST, default_id)['accepted']: + raise AssertionError('the seeded record was refused') + node_map = run.workdir / 'explicit_id_nodes.yaml' + node_map.write_text('area_id: plc_area\ncomponent_id: plc_a\nnodes: []\n') + _, port = run.start_server('explicit_id') + gateway = run.start_gateway('explicit_id', endpoint=f'opc.tcp://127.0.0.1:{port}', + node_map=node_map) + # The gateway asked for its own record, found none, and went through every id it reports under. + if not wait_log(gateway.log, 'nothing to clear', 30): + raise AssertionError('the gateway never decided on PLC_COMMS_LOST:\n' + tail(gateway)) + records = comms_lost_records(run) + if records != {default_id: 'CONFIRMED'}: + raise AssertionError('a bridge with an explicit component_id touched the record of a ' + f'default-id bridge: {records}\n' + tail(gateway)) + + def start_discovering_gateway(run, name): """Start a config-less gateway whose binding file sits under a symlink loop.""" loop_a = run.workdir / 'loop_a' @@ -594,6 +618,7 @@ def scenario_binding_directory_on_reconnect(run): ('per-source record', scenario_per_source_record), ('per-source record under a stand-in', scenario_per_source_placeholder_rename), ('per-source pinned stand-in', scenario_per_source_pinned_stand_in), + ('explicit component id', scenario_explicit_component_id_keeps_the_default_id_record), ('binding directory at start', scenario_binding_directory_at_start), ('binding directory on reconnect', scenario_binding_directory_on_reconnect), ] From 9936cb4a32c991e666a2a5dbba4ca0a41d0895a5 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 09:02:18 +0100 Subject: [PATCH 30/52] test(fault_manager): the zero-message bag test records a topic only it knows The test recorded topics "all", so the bag stayed empty only while the whole ROS domain was silent. Another process on the domain filled the bag with its traffic and the empty-bag assertion failed. The test now records a topic nobody publishes on, with every assertion unchanged. --- src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp b/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp index 2d6917206..2abc2e8a0 100644 --- a/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp +++ b/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp @@ -2385,11 +2385,13 @@ TEST_F(RosbagCaptureIntegrationTest, ZeroMessagePostFaultOnlyBagFinalizesCleanly rosbag_config.duration_sec = 2.0; rosbag_config.duration_after_sec = 0.5; rosbag_config.format = format; + // A topic only this test knows, so traffic from the rest of the ROS domain cannot reach the bag. + rosbag_config.topics = "/zero_message_post_fault_probe"; auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); - // No publisher at all: the buffer is empty and the post-roll records nothing. + // No publisher on that topic: the buffer is empty and the post-roll records nothing. const std::string fault_code = "ZERO_MSG_" + format; capture.on_fault_confirmed(rec(fault_code)); From d9c6da040463de0e8397444283bf1c6f76eaa5ba Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 10:32:14 +0100 Subject: [PATCH 31/52] fault_manager: a show_as_single cluster mutes every member but its representative The cluster only logged the members it meant to hide and never wrote them into the muted set, so the default listing showed all of them. Members other than the representative are now muted while the cluster is active, on activation and on arrival. The set follows the representative: when it changes, the new one is shown and the old one is muted, and when the shown member is cleared the next representative is shown. A cluster ends when its last member is cleared. Muting is display only: GetFault and the storage see a muted member as they see an unmuted one, and a member that stops being muted keeps its status. This changes the behaviour of default (shared) codes against origin/main on purpose: the listing now matches what the documentation promised. --- docs/tutorials/fault-correlation.rst | 10 +- src/ros2_medkit_fault_manager/CMakeLists.txt | 3 + .../correlation/correlation_engine.hpp | 4 + .../src/correlation/correlation_engine.cpp | 34 +++ .../test_fault_cluster_muting.test.py | 248 ++++++++++++++++++ .../test_fault_correlation_owners.test.py | 8 +- .../test/test_correlation_engine.cpp | 5 +- 7 files changed, 306 insertions(+), 6 deletions(-) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_fault_cluster_muting.test.py diff --git a/docs/tutorials/fault-correlation.rst b/docs/tutorials/fault-correlation.rst index a838c0db8..74e308e47 100644 --- a/docs/tutorials/fault-correlation.rst +++ b/docs/tutorials/fault-correlation.rst @@ -317,7 +317,15 @@ Auto-cluster mode groups similar faults into clusters. 1. First ``SENSOR_*`` fault starts a potential cluster 2. Additional ``SENSOR_*`` faults within 2000ms join the cluster 3. When 3+ faults accumulate, cluster becomes active -4. Only the representative fault is shown; others are muted +4. Only the representative fault is shown; the other members are muted. They are hidden + from the default listing and show under ``include_muted``. Muting is display only: + the record of a muted member, as ``GetFault`` returns it, is the record it has unmuted. +5. When the representative changes, the current representative is shown and the previous + one is muted. A member that escalates under ``highest_severity``, or a newer member + under ``most_recent``, becomes the shown one. When the shown member is cleared, the + next representative the policy selects is shown. +6. A cluster ends when its last member is cleared. A member that stops being muted keeps + its status: nothing is cleared for the operator. Querying Correlation Data ------------------------- diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index 75f205454..a94ab10ab 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -329,6 +329,9 @@ endif() medkit_add_launch_test(test_fault_correlation_clear test/integration/test_fault_correlation_clear.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_cluster_muting + test/integration/test_fault_cluster_muting.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_identity_restart test/integration/test_fault_identity_restart.test.py TIMEOUT 180 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp index dd0e78fc8..77f12d3cb 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp @@ -189,6 +189,10 @@ class CorrelationEngine { /// Reassign cluster representative from the remaining members and their severities void reassign_representative(Cluster & cluster, const std::map & severities) const; + /// Mutes every member of a show_as_single cluster except its representative and unmutes the + /// representative. Records muted by another rule are left alone. Caller holds the mutex. + void sync_cluster_muting(const Cluster & cluster); + static ClusterData to_cluster_data(const Cluster & cluster); CorrelationConfig config_; diff --git a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp index 9f591c2e4..6009cda67 100644 --- a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp +++ b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp @@ -182,6 +182,7 @@ ProcessClearResult CorrelationEngine::process_clear(const FaultId & id) { active_cluster_severities_.erase(cluster_id); } else if (active_cluster.representative == id) { reassign_representative(active_cluster, active_sevs); + sync_cluster_muting(active_cluster); } } @@ -195,6 +196,37 @@ ProcessClearResult CorrelationEngine::process_clear(const FaultId & id) { return result; } +void CorrelationEngine::sync_cluster_muting(const Cluster & cluster) { + bool show_as_single = false; + for (const auto & rule : config_.rules) { + if (rule.id == cluster.rule_id) { + show_as_single = rule.show_as_single; + break; + } + } + if (!show_as_single) { + return; + } + for (const auto & member : cluster.members) { + const auto muted_it = muted_faults_.find(member); + if (member == cluster.representative) { + if (muted_it != muted_faults_.end() && muted_it->second.rule_id == cluster.rule_id) { + muted_faults_.erase(muted_it); + } + continue; + } + if (muted_it != muted_faults_.end() && muted_it->second.rule_id != cluster.rule_id) { + continue; + } + MutedFaultData muted; + muted.fault_code = member.fault_code; + muted.owner = member.owner; + muted.root_cause_code = cluster.representative.fault_code; + muted.rule_id = cluster.rule_id; + muted_faults_[member] = muted; + } +} + void CorrelationEngine::forget_symptom(const FaultId & id) { for (auto & [root_id, symptoms] : root_to_symptoms_) { symptoms.erase(std::remove(symptoms.begin(), symptoms.end(), id), symptoms.end()); @@ -419,6 +451,7 @@ std::optional CorrelationEngine::try_auto_cluster(const Faul active_cluster.last_at = now_system; update_member_severity(rule, active_cluster, active_cluster_severities_[active_cluster.cluster_id], id, severity); + sync_cluster_muting(active_cluster); if (pending_it != pending_clusters_.end() && pending_it->second.data.cluster_id == active_cluster.cluster_id) { pending_it->second.data.last_at = now_system; update_member_severity(rule, pending_it->second.data, pending_it->second.fault_severities, id, severity); @@ -507,6 +540,7 @@ std::optional CorrelationEngine::try_auto_cluster(const Faul if (already_active || cluster.members.size() >= rule.min_count) { active_clusters_[cluster.cluster_id] = cluster; active_cluster_severities_[cluster.cluster_id] = pending.fault_severities; + sync_cluster_muting(active_clusters_[cluster.cluster_id]); // Retroactively mute all non-representative faults added before threshold if (!already_active && rule.show_as_single) { diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_cluster_muting.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_cluster_muting.test.py new file mode 100644 index 000000000..42fa4c86f --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_cluster_muting.test.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A show_as_single cluster shows its current representative and mutes the other members. + +Muting is display only: the default listing hides a muted member, include_muted lists it, and +the record GetFault returns is the one an unmuted member has. The representative the policy +selects is the one shown. When a member leaves, the representative that follows is shown. + +Each scenario runs on shared codes (one code per member) and on a per_source code (one record +per reporting node). +""" + +import os +import tempfile +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault + +TIMEOUT_SEC = 30.0 +MODES = ('SH', 'PS') +# scenario -> representative policy +SCENARIOS = { + 'ONLY': 'first', + 'NEXT': 'first', + 'ESC': 'highest_severity', + 'REC': 'most_recent', + 'GET': 'first', +} + +THRESHOLDS = ''.join( + f'CM_PS_{scenario}_X:\n per_source: true\n' for scenario in SCENARIOS) +CORRELATION = 'correlation:\n enabled: true\n patterns:\n' + ''.join( + f' cm_{mode.lower()}_{scenario.lower()}:\n codes: ["CM_{mode}_{scenario}_*"]\n' + for mode in MODES for scenario in SCENARIOS) + ' rules:\n' + ''.join( + f""" - id: cm_{mode.lower()}_{scenario.lower()} + name: "CM {mode} {scenario}" + mode: auto_cluster + match: + - pattern: cm_{mode.lower()}_{scenario.lower()} + min_count: 3 + window_ms: 30000 + show_as_single: true + representative: {policy} +""" for mode in MODES for scenario, policy in SCENARIOS.items()) + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file('cluster_muting_thresholds_', THRESHOLDS) +CORRELATION_FILE = _temp_file('cluster_muting_rules_', CORRELATION) + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'healing_enabled': False, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'correlation.config_file': CORRELATION_FILE, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestClusterMuting(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_cluster_muting_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + cls.get_client = cls.node.create_client(GetFault, '/fault_manager/get_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + for client in (cls.report_client, cls.list_client, cls.get_client, cls.clear_client): + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + @staticmethod + def _members(mode, scenario): + """Return [(code, source, owner)] of the three members, in report order.""" + if mode == 'PS': + return [(f'CM_PS_{scenario}_X', f'/n{i}', f'/n{i}') for i in (1, 2, 3)] + return [(f'CM_SH_{scenario}_{i}', '/n', '*') for i in (1, 2, 3)] + + def _report(self, member, severity=Fault.SEVERITY_WARN): + code, source, _ = member + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = severity + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _clear(self, member): + code, _, owner = member + request = ClearFault.Request() + request.fault_code = code + request.source_id = owner + response = self._call(self.clear_client, request) + self.assertTrue(response.success, response.message) + + def _get(self, member): + code, _, owner = member + request = GetFault.Request() + request.fault_code = code + request.source_id = owner + response = self._call(self.get_client, request) + self.assertTrue(response.success, response.error_message) + return response.fault + + def _shown(self, members): + """Return the (code, owner) pairs of the default listing that belong to members.""" + owner_of = {(code, source): owner for code, source, owner in members} + faults = self._call(self.list_client, ListFaults.Request()).faults + return {(f.fault_code, owner_of[(f.fault_code, f.reporting_sources[0])]) + for f in faults if (f.fault_code, f.reporting_sources[0]) in owner_of} + + def _muted(self, members): + """Return the (code, owner) pairs of include_muted that belong to members.""" + request = ListFaults.Request() + request.include_muted = True + response = self._call(self.list_client, request) + ours = {self._key(member) for member in members} + return {(m.fault_code, m.source_id) for m in response.muted_faults} & ours + + def _key(self, member): + return (member[0], member[2]) + + def _each_mode(self, scenario, check): + for mode in MODES: + with self.subTest(mode=mode): + check(self._members(mode, scenario)) + + def test_the_representative_is_shown_and_the_others_are_muted(self): + def check(members): + for member in members: + self._report(member) + first, second, third = members + self.assertEqual(self._shown(members), {self._key(first)}) + self.assertEqual(self._muted(members), {self._key(second), self._key(third)}) + self._each_mode('ONLY', check) + + def test_the_next_representative_is_shown_when_the_shown_one_is_cleared(self): + def check(members): + first, second, third = members + for member in members: + self._report(member) + self._clear(first) + self.assertEqual(self._shown(members), {self._key(second)}) + self.assertEqual(self._muted(members), {self._key(third)}) + self._clear(second) + self.assertEqual(self._shown(members), {self._key(third)}) + self.assertEqual(self._muted(members), set()) + self._clear(third) + self.assertEqual(self._shown(members), set()) + self.assertEqual(self._muted(members), set()) + self._each_mode('NEXT', check) + + def test_a_member_that_escalates_becomes_the_shown_one(self): + def check(members): + first, second, third = members + for member in members: + self._report(member) + self.assertEqual(self._shown(members), {self._key(first)}) + self._report(third, Fault.SEVERITY_CRITICAL) + self.assertEqual(self._shown(members), {self._key(third)}) + self.assertEqual(self._muted(members), {self._key(first), self._key(second)}) + self._each_mode('ESC', check) + + def test_the_most_recent_member_is_shown(self): + def check(members): + first, second, third = members + for member in members: + self._report(member) + self.assertEqual(self._shown(members), {self._key(third)}) + self.assertEqual(self._muted(members), {self._key(first), self._key(second)}) + self._each_mode('REC', check) + + def test_muting_does_not_change_the_record(self): + def check(members): + for member in members: + self._report(member, Fault.SEVERITY_ERROR) + shown, muted = self._get(members[0]), self._get(members[1]) + self.assertIn(self._key(members[1]), self._muted(members)) + self.assertEqual(muted.status, shown.status) + self.assertEqual(muted.status, Fault.STATUS_CONFIRMED) + self.assertEqual(muted.severity, Fault.SEVERITY_ERROR) + self.assertEqual(muted.occurrence_count, shown.occurrence_count) + self._each_mode('GET', check) + + +@launch_testing.post_shutdown_test() +class TestClusterMutingShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(CORRELATION_FILE) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py index 3b1020074..bd99b3172 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py @@ -227,11 +227,13 @@ def _cascade(self, root, root_node, symptoms, symptom_owner_of_record): for code, node in symptoms: self._report(code, node) - muted = self._list(include_muted=True) - muted_records = sorted((m.fault_code, m.source_id) for m in muted.muted_faults) + # Cluster members muted by the storm rules are not this root's symptoms. + muted = [m for m in self._list(include_muted=True).muted_faults + if m.rule_id.endswith('_estop')] + muted_records = sorted((m.fault_code, m.source_id) for m in muted) expected = sorted((code, symptom_owner_of_record(code, node)) for code, node in symptoms) self.assertEqual(muted_records, expected) - for m in muted.muted_faults: + for m in muted: self.assertEqual(m.root_cause_code, root) visible = {f.fault_code for f in self._list().faults} diff --git a/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp b/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp index 917581905..68f2f5f12 100644 --- a/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp +++ b/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp @@ -538,8 +538,9 @@ TEST_F(CorrelationEngineTest, HierarchicalAndClusterCoexist) { engine.process_fault(rec("SENSOR_IMU"), "ERROR"); EXPECT_EQ(1u, engine.get_cluster_count()); - // Both should coexist - EXPECT_EQ(1u, engine.get_muted_count()); // MOTOR_COMM_FL + // Both should coexist: the symptom and the cluster member that is not the representative are muted + EXPECT_EQ(2u, engine.get_muted_count()); + EXPECT_TRUE(engine.is_muted(rec("MOTOR_COMM_FL"))); EXPECT_EQ(1u, engine.get_cluster_count()); // SENSOR cluster } From 2c2ebb80031fc059fea740e67cd675dd81ca4cac Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 13:46:43 +0100 Subject: [PATCH 32/52] opcua: a bridge with a node map reports under its component id only The ids a bridge reports under included the stand-ins derived from its endpoint for every bridge. A bridge with a node map never reports under a stand-in, and the derivation uses the host only, so two bridges on one host collided: the bridge with a node map probed the stand-in of a config-less bridge and cleared its PLC_COMMS_LOST record. The stand-ins are now part of the set only when no node map is configured. A bridge reconfigured from config-less to a node map no longer clears the record it raised under its old stand-in. This changes the behaviour of shared codes against origin/main on purpose: main has the same loop. --- .../ros2_medkit_opcua/src/opcua_plugin.cpp | 16 ++++--- .../integration/test_opcua_comms_lost.test.py | 44 +++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp index d2b6281ef..da86ac871 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp @@ -1606,13 +1606,15 @@ std::unordered_set OpcuaPlugin::reporting_ids_of_this_process() con if (!current.empty()) { ids.insert(current); } - // A process that never reached its PLC reported under the stand-in of the - // endpoint it started on, or of the one it found and never opened a session - // on. This process derives both from the same configuration. - for (const std::string & endpoint : {start_endpoint_url_, client_ ? client_->endpoint_url() : std::string{}}) { - const std::string stand_in = derive_component_identity(OpcuaClient::DeviceInfo{}, endpoint).id; - if (!stand_in.empty()) { - ids.insert(stand_in); + // A bridge without a node map that never reached its PLC reported under the stand-in of the + // endpoint it started on, or of the one it found and never opened a session on. A bridge with a + // node map reports under its component_id only. + if (node_map_path_.empty()) { + for (const std::string & endpoint : {start_endpoint_url_, client_ ? client_->endpoint_url() : std::string{}}) { + const std::string stand_in = derive_component_identity(OpcuaClient::DeviceInfo{}, endpoint).id; + if (!stand_in.empty()) { + ids.insert(stand_in); + } } } return ids; diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py index f169f6ed0..cf5151471 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py @@ -35,6 +35,8 @@ nothing and the record is found under the stand-in derived from its endpoint. * explicit component id: a gateway whose node map names a component_id leaves the per-source record a default-id bridge holds under 'opcua_device' alone. + * explicit component id, stand-in record: the same, for a record held under the + endpoint-derived stand-in of a config-less bridge on the same host, per-source and shared. * binding directory: a binding file whose directory cannot be looked up is reported, and the plugin stays connected. Once at start, and once on the poll thread when a rescan adopts a PLC that came up later. @@ -553,6 +555,44 @@ def scenario_explicit_component_id_keeps_the_default_id_record(run): f'default-id bridge: {records}\n' + tail(gateway)) +def scenario_explicit_component_id_keeps_a_stand_in_record(run): + stand_in = 'opcua-127_0_0_1' + run.start_fault_manager('stand_in_id', per_source_codes=[COMMS_LOST]) + # A config-less bridge on the same host holds an outage record under its stand-in. + if not fault_manager_client(run, 'report', COMMS_LOST, stand_in)['accepted']: + raise AssertionError('the seeded record was refused') + node_map = run.workdir / 'stand_in_id_nodes.yaml' + node_map.write_text('area_id: plc_area\ncomponent_id: plc_a\nnodes: []\n') + _, port = run.start_server('stand_in_id') + gateway = run.start_gateway('stand_in_id', endpoint=f'opc.tcp://127.0.0.1:{port}', + node_map=node_map) + if not wait_log(gateway.log, 'nothing to clear', 30): + raise AssertionError('the gateway never decided on PLC_COMMS_LOST:\n' + tail(gateway)) + records = comms_lost_records(run) + if records != {stand_in: 'CONFIRMED'}: + raise AssertionError('a bridge with a node map touched the record of a stand-in bridge: ' + f'{records}\n' + tail(gateway)) + + +def scenario_explicit_component_id_keeps_a_shared_stand_in_record(run): + stand_in = 'opcua-127_0_0_1' + run.start_fault_manager('stand_in_shared') + if not fault_manager_client(run, 'report', COMMS_LOST, stand_in)['accepted']: + raise AssertionError('the seeded record was refused') + node_map = run.workdir / 'stand_in_shared_nodes.yaml' + node_map.write_text('area_id: plc_area\ncomponent_id: plc_a\nnodes: []\n') + _, port = run.start_server('stand_in_shared') + gateway = run.start_gateway('stand_in_shared', endpoint=f'opc.tcp://127.0.0.1:{port}', + node_map=node_map) + if not wait_log(gateway.log, HELD_BY_OTHERS, 30): + raise AssertionError('the gateway never left the shared record standing:\n' + + tail(gateway)) + records = comms_lost_records(run) + if records != {stand_in: 'CONFIRMED'}: + raise AssertionError('a bridge with a node map cleared the shared record of a stand-in ' + f'bridge: {records}\n' + tail(gateway)) + + def start_discovering_gateway(run, name): """Start a config-less gateway whose binding file sits under a symlink loop.""" loop_a = run.workdir / 'loop_a' @@ -619,6 +659,10 @@ def scenario_binding_directory_on_reconnect(run): ('per-source record under a stand-in', scenario_per_source_placeholder_rename), ('per-source pinned stand-in', scenario_per_source_pinned_stand_in), ('explicit component id', scenario_explicit_component_id_keeps_the_default_id_record), + ('explicit component id, stand-in record', + scenario_explicit_component_id_keeps_a_stand_in_record), + ('explicit component id, shared stand-in record', + scenario_explicit_component_id_keeps_a_shared_stand_in_record), ('binding directory at start', scenario_binding_directory_at_start), ('binding directory on reconnect', scenario_binding_directory_on_reconnect), ] From e75fe6408f3ad393f41e23c8b55621bca48d08a9 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 13:53:54 +0100 Subject: [PATCH 33/52] fault_manager: a record stays muted while any live relation mutes it A record had one muted entry, and ending a root cause erased the entry of every symptom it listed, although another root cause that lists the same symptom under a muting rule, or an active show_as_single cluster, still muted it. The entry is now set from the relations that are still live whenever one relation ends: a direct clear with or without the skip flag, a cascade and a heal. A root of a rule with mute_symptoms false no longer erases a mute it never wrote. A record that no relation mutes is unmuted and keeps its status. This changes the behaviour of default (shared) codes against origin/main on purpose: the same defect existed there. --- src/ros2_medkit_fault_manager/CMakeLists.txt | 3 + .../correlation/correlation_engine.hpp | 8 + .../src/correlation/correlation_engine.cpp | 102 ++++--- .../test_fault_muting_relations.test.py | 259 ++++++++++++++++++ 4 files changed, 331 insertions(+), 41 deletions(-) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index a94ab10ab..6330d9d0a 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -332,6 +332,9 @@ endif() medkit_add_launch_test(test_fault_cluster_muting test/integration/test_fault_cluster_muting.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_muting_relations + test/integration/test_fault_muting_relations.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_identity_restart test/integration/test_fault_identity_restart.test.py TIMEOUT 180 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp index 77f12d3cb..e09250205 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp @@ -193,6 +193,14 @@ class CorrelationEngine { /// representative. Records muted by another rule are left alone. Caller holds the mutex. void sync_cluster_muting(const Cluster & cluster); + /// Sets the muted entry of a record from the relations that are still live: a root cause that + /// lists it under a muting rule, else its active show_as_single cluster. With none, the entry + /// goes. Caller holds the mutex. + void reconcile_muting(const FaultId & id); + + /// Whether a fault code is one of the symptoms of a hierarchical rule. + bool matches_symptom(const CorrelationRule & rule, const std::string & fault_code) const; + static ClusterData to_cluster_data(const Cluster & cluster); CorrelationConfig config_; diff --git a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp index 6009cda67..1e6a18ce3 100644 --- a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp +++ b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp @@ -122,13 +122,12 @@ ProcessClearResult CorrelationEngine::process_clear(const FaultId & id) { } } - // Clean up muted records - for (const auto & symptom_id : it->second) { - muted_faults_.erase(symptom_id); - } - - // Remove from root_to_symptoms + // The symptoms stay muted while another root cause or an active cluster still mutes them + const std::vector released = it->second; root_to_symptoms_.erase(it); + for (const auto & symptom_id : released) { + reconcile_muting(symptom_id); + } } // Remove from pending root causes @@ -197,34 +196,69 @@ ProcessClearResult CorrelationEngine::process_clear(const FaultId & id) { } void CorrelationEngine::sync_cluster_muting(const Cluster & cluster) { - bool show_as_single = false; for (const auto & rule : config_.rules) { - if (rule.id == cluster.rule_id) { - show_as_single = rule.show_as_single; - break; + if (rule.id == cluster.rule_id && rule.show_as_single) { + for (const auto & member : cluster.members) { + reconcile_muting(member); + } + return; } } - if (!show_as_single) { - return; +} + +bool CorrelationEngine::matches_symptom(const CorrelationRule & rule, const std::string & fault_code) const { + for (const auto & pattern_id : rule.symptom_pattern_ids) { + if (matcher_->matches(fault_code, pattern_id)) { + return true; + } } - for (const auto & member : cluster.members) { - const auto muted_it = muted_faults_.find(member); - if (member == cluster.representative) { - if (muted_it != muted_faults_.end() && muted_it->second.rule_id == cluster.rule_id) { - muted_faults_.erase(muted_it); - } + return !rule.inline_symptom_codes.empty() && matcher_->matches_any(fault_code, rule.inline_symptom_codes); +} + +void CorrelationEngine::reconcile_muting(const FaultId & id) { + for (const auto & [root_id, symptoms] : root_to_symptoms_) { + if (std::find(symptoms.begin(), symptoms.end(), id) == symptoms.end()) { continue; } - if (muted_it != muted_faults_.end() && muted_it->second.rule_id != cluster.rule_id) { - continue; + for (const auto & rule : config_.rules) { + if (rule.mode != CorrelationMode::HIERARCHICAL || !rule.mute_symptoms || + !matcher_->matches_any(root_id.fault_code, rule.root_cause_codes) || !matches_symptom(rule, id.fault_code)) { + continue; + } + MutedFaultData muted; + const auto existing = muted_faults_.find(id); + if (existing != muted_faults_.end() && existing->second.rule_id == rule.id && + existing->second.root_cause_code == root_id.fault_code) { + muted = existing->second; + } + muted.fault_code = id.fault_code; + muted.owner = id.owner; + muted.root_cause_code = root_id.fault_code; + muted.rule_id = rule.id; + muted_faults_[id] = muted; + return; } - MutedFaultData muted; - muted.fault_code = member.fault_code; - muted.owner = member.owner; - muted.root_cause_code = cluster.representative.fault_code; - muted.rule_id = cluster.rule_id; - muted_faults_[member] = muted; } + + const auto cluster_it = fault_to_cluster_.find(id); + if (cluster_it != fault_to_cluster_.end()) { + const auto active_it = active_clusters_.find(cluster_it->second); + if (active_it != active_clusters_.end() && active_it->second.representative != id) { + const auto & cluster = active_it->second; + for (const auto & rule : config_.rules) { + if (rule.id == cluster.rule_id && rule.show_as_single) { + MutedFaultData muted; + muted.fault_code = id.fault_code; + muted.owner = id.owner; + muted.root_cause_code = cluster.representative.fault_code; + muted.rule_id = cluster.rule_id; + muted_faults_[id] = muted; + return; + } + } + } + } + muted_faults_.erase(id); } void CorrelationEngine::forget_symptom(const FaultId & id) { @@ -345,21 +379,7 @@ std::optional CorrelationEngine::try_as_symptom(const FaultI continue; } - // Check if fault matches any symptom pattern - bool matches_symptom = false; - for (const auto & pattern_id : rule.symptom_pattern_ids) { - if (matcher_->matches(fault_code, pattern_id)) { - matches_symptom = true; - break; - } - } - - // Also check inline symptom codes (direct codes with wildcard support) - if (!matches_symptom && !rule.inline_symptom_codes.empty()) { - matches_symptom = matcher_->matches_any(fault_code, rule.inline_symptom_codes); - } - - if (!matches_symptom) { + if (!matches_symptom(rule, fault_code)) { continue; } diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py new file mode 100644 index 000000000..543ce6afa --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A record stays muted while any live relation still mutes it. + +Two relations can mute one record: a hierarchical root cause that lists it, and an active +show_as_single cluster. Ending one relation, by a clear with or without the skip flag or by a +heal, must not unmute a record the other relation still mutes. The same holds for two root +causes that list one symptom. Each scenario runs on shared codes (one code per record) and on +per_source codes (one record per reporting node). +""" + +import os +import tempfile +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, ListFaults, ReportFault + +TIMEOUT_SEC = 30.0 +MODES = ('SH', 'PS') +# variant -> mute_symptoms of the hierarchical rule +VARIANTS = {'CLR': 'true', 'HEAL': 'true', 'NOMUTE': 'false'} + + +def _code(variant, mode, role): + return f'CL_MX_{variant}_{mode}_{role}' + + +THRESHOLDS = ''.join( + f'{_code(v, "PS", role)}:\n per_source: true\n' + for v in VARIANTS for role in ('ROOT', 'T_X')) + ''.join( + f'CL_TR_PS_{role}:\n per_source: true\n' for role in ('R1', 'R2', 'S')) +CORRELATION = 'correlation:\n enabled: true\n patterns:\n' + ''.join( + f' mx_{v.lower()}_{m.lower()}:\n codes: ["{_code(v, m, "T")}_*"]\n' + for v in VARIANTS for m in MODES) + ' rules:\n' + ''.join( + f""" - id: mx_h_{v.lower()}_{m.lower()} + name: "MX hierarchical {v} {m}" + mode: hierarchical + root_cause: + codes: ["{_code(v, m, 'ROOT')}"] + symptoms: + - codes: ["{_code(v, m, 'T')}_*"] + window_ms: 30000 + mute_symptoms: {mute} + auto_clear_with_root: true + - id: mx_c_{v.lower()}_{m.lower()} + name: "MX cluster {v} {m}" + mode: auto_cluster + match: + - pattern: mx_{v.lower()}_{m.lower()} + min_count: 3 + window_ms: 30000 + show_as_single: true + representative: first +""" for v, mute in VARIANTS.items() for m in MODES) + ''.join( + f""" - id: tr_{m.lower()} + name: "TR {m}" + mode: hierarchical + root_cause: + codes: ["CL_TR_{m}_R1", "CL_TR_{m}_R2"] + symptoms: + - codes: ["CL_TR_{m}_S", "CL_TR_{m}_SENT_*"] + window_ms: 300 + mute_symptoms: true + auto_clear_with_root: true +""" for m in MODES) + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file('muting_relations_thresholds_', THRESHOLDS) +CORRELATION_FILE = _temp_file('muting_relations_rules_', CORRELATION) + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'healing_enabled': True, + 'healing_threshold': 0, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'correlation.config_file': CORRELATION_FILE, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestMutingRelations(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_muting_relations_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + for client in (cls.report_client, cls.list_client, cls.clear_client): + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source, event=ReportFault.Request.EVENT_FAILED): + request = ReportFault.Request() + request.fault_code = code + request.event_type = event + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _clear(self, code, owner, skip): + request = ClearFault.Request() + request.fault_code = code + request.source_id = owner + request.skip_correlation_auto_clear = skip + response = self._call(self.clear_client, request) + self.assertTrue(response.success, response.message) + + def _shown(self, codes): + """Return the (code, first source) pairs of the default listing among codes.""" + faults = self._call(self.list_client, ListFaults.Request()).faults + return {(f.fault_code, f.reporting_sources[0]) for f in faults if f.fault_code in codes} + + def _muted(self, codes): + """Return {(code, owner): root_cause_code} of include_muted among codes.""" + request = ListFaults.Request() + request.include_muted = True + muted = self._call(self.list_client, request).muted_faults + return {(m.fault_code, m.source_id): m.root_cause_code for m in muted + if m.fault_code in codes} + + @staticmethod + def _records(variant, mode): + """Return (root, root source, root owner, [(code, source, owner)] of three records).""" + root = _code(variant, mode, 'ROOT') + if mode == 'PS': + return root, '/r', '/r', [(_code(variant, mode, 'T_X'), f'/n{i}', f'/n{i}') + for i in (1, 2, 3)] + return root, '/r', '*', [(f'{_code(variant, mode, "T")}_{i}', '/n', '*') + for i in (1, 2, 3)] + + def _cluster_and_root(self, variant, mode, end_root): + root, root_source, root_owner, records = self._records(variant, mode) + codes = {code for code, _, _ in records} + for code, source, _ in records: + self._report(code, source) + first, second, third = records + self.assertEqual(self._shown(codes), {(first[0], first[1])}) + self._report(root, root_source) + self._report(second[0], second[1]) + end_root(root, root_source, root_owner) + self.assertEqual( + self._shown(codes), {(first[0], first[1])}, + 'a record its active cluster mutes was shown when the root cause ended') + self.assertEqual(set(self._muted(codes)), {(second[0], second[2]), (third[0], third[2])}) + + def _each_mode(self, check): + for mode in MODES: + with self.subTest(mode=mode): + check(mode) + + def test_a_cleared_root_leaves_the_cluster_mute(self): + def end_root(root, source, owner): + self._clear(root, owner, skip=True) + self._each_mode(lambda mode: self._cluster_and_root('CLR', mode, end_root)) + + def test_a_healed_root_leaves_the_cluster_mute(self): + def end_root(root, source, owner): + self._report(root, source, event=ReportFault.Request.EVENT_PASSED) + self._each_mode(lambda mode: self._cluster_and_root('HEAL', mode, end_root)) + + def test_a_root_that_does_not_mute_leaves_the_cluster_mute(self): + def end_root(root, source, owner): + self._clear(root, owner, skip=True) + self._each_mode(lambda mode: self._cluster_and_root('NOMUTE', mode, end_root)) + + def _two_roots(self, mode): + r1, r2 = f'CL_TR_{mode}_R1', f'CL_TR_{mode}_R2' + symptom = f'CL_TR_{mode}_S' + owner = lambda source: source if mode == 'PS' else '*' # noqa: E731 + self._report(r1, '/r1') + self._report(symptom, '/s') + self.assertIn((symptom, owner('/s')), self._muted({symptom})) + # Report fresh symptoms until one comes back unmuted: R1's window has passed. + deadline = time.monotonic() + TIMEOUT_SEC + index = 0 + while True: + index += 1 + sentinel = f'CL_TR_{mode}_SENT_{index}' + self._report(sentinel, '/s') + if not self._muted({sentinel}): + break + self.assertLess(time.monotonic(), deadline, 'the window of the first root never ended') + self._report(r2, '/r2') + self._report(symptom, '/s') + self.assertEqual(self._muted({symptom}), {(symptom, owner('/s')): r2}) + + self._clear(r1, owner('/r1'), skip=True) + self.assertEqual(self._shown({symptom}), set(), 'the second root still mutes the symptom') + self.assertEqual(self._muted({symptom}), {(symptom, owner('/s')): r2}) + + def test_a_cleared_root_leaves_the_mute_of_another_root(self): + self._each_mode(self._two_roots) + + +@launch_testing.post_shutdown_test() +class TestMutingRelationsShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(CORRELATION_FILE) From 9709e41c082a0ad6fd13f22314227fa75e1574c3 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 13:53:59 +0100 Subject: [PATCH 34/52] test(fault_manager): the zero-message bag test pins that traffic elsewhere stays out A publisher on the probe topic that never publishes and a publisher on an unrelated topic now exist before the capture. The test waits until the capture has subscribed to the probe topic, requires that it has not subscribed to the unrelated one, and publishes on the unrelated topic through the post-roll, so the empty bag is evidence of the topic selection and not of a quiet domain. --- .../test/test_rosbag_capture.cpp | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp b/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp index 2abc2e8a0..a6c1d6ffb 100644 --- a/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp +++ b/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp @@ -2388,15 +2388,38 @@ TEST_F(RosbagCaptureIntegrationTest, ZeroMessagePostFaultOnlyBagFinalizesCleanly // A topic only this test knows, so traffic from the rest of the ROS domain cannot reach the bag. rosbag_config.topics = "/zero_message_post_fault_probe"; auto snapshot_config = create_snapshot_config(); + // The probe topic never publishes. Traffic on another topic must stay out of the bag. Both + // publishers exist before the capture, which subscribes to the topics it finds on construction. + auto probe_pub = node_->create_publisher(rosbag_config.topics, 10); + const std::string unrelated_topic = "/zero_message_unrelated_traffic"; + auto unrelated_pub = node_->create_publisher(unrelated_topic, 10); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); - // No publisher on that topic: the buffer is empty and the post-roll records nothing. + const auto discovery_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (node_->count_subscribers(rosbag_config.topics) < 1 && + std::chrono::steady_clock::now() < discovery_deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + ASSERT_GE(node_->count_subscribers(rosbag_config.topics), 1u) << "the capture never subscribed to its topic"; + ASSERT_EQ(node_->count_subscribers(unrelated_topic), 0u) << "the capture subscribed to a topic it was not given"; + + std::atomic stop_traffic{false}; + std::thread traffic([&]() { + std_msgs::msg::String msg; + msg.data = "unrelated"; + while (!stop_traffic.load()) { + unrelated_pub->publish(msg); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + }); const std::string fault_code = "ZERO_MSG_" + format; capture.on_fault_confirmed(rec(fault_code)); - ASSERT_TRUE(wait_for_row(rec(fault_code), std::chrono::milliseconds(8000))) - << "zero-message post-fault-only bag did not finalise on " << format; + const bool finalised = wait_for_row(rec(fault_code), std::chrono::milliseconds(8000)); + stop_traffic.store(true); + traffic.join(); + ASSERT_TRUE(finalised) << "zero-message post-fault-only bag did not finalise on " << format; auto row = storage_->get_rosbag_file(rec(fault_code)); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->format, format); From f90eb62d4da4faefb72109a028e6cc31e5d4b02f Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 13:56:24 +0100 Subject: [PATCH 35/52] docs: state what a root cause lists, mutes and clears for both rule kinds The tutorial said the cascade clears exactly the records the root cause muted. That holds for a rule with mute_symptoms true. A rule with mute_symptoms false lists symptoms it does not mute, and its cascade clears the listed records. The text now says what holds for both, and that a record stays muted while any root cause or active cluster still mutes it. --- docs/tutorials/fault-correlation.rst | 34 +++++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/docs/tutorials/fault-correlation.rst b/docs/tutorials/fault-correlation.rst index 74e308e47..316334509 100644 --- a/docs/tutorials/fault-correlation.rst +++ b/docs/tutorials/fault-correlation.rst @@ -124,17 +124,29 @@ How It Works node mutes, and with ``auto_clear_with_root`` auto-clears, the matching symptoms reported by any other node inside the window. An auto-cluster collects the matching faults of every node. The relations hold fault records, not codes: for a code -configured with ``per_source: true`` each reporting node has its own record, and the -cascade clears exactly the records the root cause muted. A symptom is muted when it is -reported within ``window_ms`` of the latest report of the root cause, and it stays muted -until the root cause is cleared. A new report of a root cause that is still active renews -that window and keeps the symptoms it already muted. Clearing the root cause ends its -relations: with ``skip_correlation_auto_clear`` its symptoms are not cleared, they are -unmuted and show in the default listing again. Every other end of a record does the -same: a root cause that heals, or that a cascade clears, releases the symptoms it muted -and they keep their status. Only a ``ClearFault`` without the flag cascades, and it goes -one level. For any other code there is one shared -record, and that record is what is muted or cleared. +configured with ``per_source: true`` each reporting node has its own record. For any +other code there is one shared record, and that record is what is muted or cleared. + +A root cause lists the symptom records reported within ``window_ms`` of its latest +report. A new report of a root cause that is still active renews that window and keeps +its list. A record that is cleared or ends leaves the list of every root cause, so the +list holds only records of the current occurrence. + +- With ``mute_symptoms: true`` the listed records are muted: hidden from the default + listing and shown under ``include_muted``. They stay muted until the root cause ends. + The cascade of ``auto_clear_with_root`` clears exactly the records the root cause muted. +- With ``mute_symptoms: false`` the listed records are not muted. The cascade of + ``auto_clear_with_root`` still clears the records the root cause lists. + +Clearing the root cause ends its relations: with ``skip_correlation_auto_clear`` its +symptoms are not cleared, they keep their status. Every other end of a record does the +same: a root cause that heals, or that a cascade clears, releases the symptoms it +muted. Only a ``ClearFault`` without the flag cascades, and it goes one level. + +A record stays muted while any root cause that lists it under a muting rule, or its +active ``show_as_single`` cluster, still mutes it. When the relation that muted it ends, +a record that no other relation mutes is unmuted and shows in the default listing again. +The reason shown under ``include_muted`` names the relation that mutes it now. Quick Start ----------- From f0cc31a008b1f718d169ebd0d1a864a3d3a2fe36 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 14:23:59 +0100 Subject: [PATCH 36/52] fault_manager: a root mutes through its own rule and its own delay Reconciling the muted entry of a record looped over every hierarchical rule that matched both codes, so a later rule that the root is not bound to could mute a symptom whose real relation does not mute. A root is bound to the first rule its code matches, and only that rule decides whether it mutes. The delay of a symptom behind a root is now kept per relation, recorded when the record last attached to that root and erased with the relation, so a mute that moves to another root carries that root's delay. --- .../correlation/correlation_engine.hpp | 3 + .../src/correlation/correlation_engine.cpp | 22 +++-- .../test_fault_muting_relations.test.py | 98 ++++++++++++++++++- 3 files changed, 115 insertions(+), 8 deletions(-) diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp index e09250205..02db12002 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp @@ -221,6 +221,9 @@ class CorrelationEngine { /// Muted records (record -> data) std::map muted_faults_; + /// Delay of a symptom record behind a root cause record, from the last attach ((root, symptom) -> ms) + std::map, uint32_t> relation_delay_ms_; + /// Active clusters (cluster_id -> cluster) std::map active_clusters_; diff --git a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp index 1e6a18ce3..0e646f891 100644 --- a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp +++ b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp @@ -125,6 +125,9 @@ ProcessClearResult CorrelationEngine::process_clear(const FaultId & id) { // The symptoms stay muted while another root cause or an active cluster still mutes them const std::vector released = it->second; root_to_symptoms_.erase(it); + for (auto delay_it = relation_delay_ms_.begin(); delay_it != relation_delay_ms_.end();) { + delay_it = delay_it->first.first == id ? relation_delay_ms_.erase(delay_it) : std::next(delay_it); + } for (const auto & symptom_id : released) { reconcile_muting(symptom_id); } @@ -220,21 +223,22 @@ void CorrelationEngine::reconcile_muting(const FaultId & id) { if (std::find(symptoms.begin(), symptoms.end(), id) == symptoms.end()) { continue; } + // A root is bound to the first rule its code matches. That rule decides whether it mutes. + const auto root_rule_id = try_as_root_cause(root_id.fault_code); + if (!root_rule_id) { + continue; + } for (const auto & rule : config_.rules) { - if (rule.mode != CorrelationMode::HIERARCHICAL || !rule.mute_symptoms || - !matcher_->matches_any(root_id.fault_code, rule.root_cause_codes) || !matches_symptom(rule, id.fault_code)) { + if (rule.id != *root_rule_id || !rule.mute_symptoms || !matches_symptom(rule, id.fault_code)) { continue; } MutedFaultData muted; - const auto existing = muted_faults_.find(id); - if (existing != muted_faults_.end() && existing->second.rule_id == rule.id && - existing->second.root_cause_code == root_id.fault_code) { - muted = existing->second; - } muted.fault_code = id.fault_code; muted.owner = id.owner; muted.root_cause_code = root_id.fault_code; muted.rule_id = rule.id; + const auto delay_it = relation_delay_ms_.find({root_id, id}); + muted.delay_ms = delay_it != relation_delay_ms_.end() ? delay_it->second : 0; muted_faults_[id] = muted; return; } @@ -262,6 +266,9 @@ void CorrelationEngine::reconcile_muting(const FaultId & id) { } void CorrelationEngine::forget_symptom(const FaultId & id) { + for (auto delay_it = relation_delay_ms_.begin(); delay_it != relation_delay_ms_.end();) { + delay_it = delay_it->first.second == id ? relation_delay_ms_.erase(delay_it) : std::next(delay_it); + } for (auto & [root_id, symptoms] : root_to_symptoms_) { symptoms.erase(std::remove(symptoms.begin(), symptoms.end(), id), symptoms.end()); } @@ -401,6 +408,7 @@ std::optional CorrelationEngine::try_as_symptom(const FaultI if (std::find(symptoms.begin(), symptoms.end(), id) == symptoms.end()) { symptoms.push_back(id); } + relation_delay_ms_[{prc.fault_id, id}] = result.delay_ms; if (rule.mute_symptoms) { MutedFaultData muted; diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py index 543ce6afa..d1078230e 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py @@ -50,7 +50,8 @@ def _code(variant, mode, role): THRESHOLDS = ''.join( f'{_code(v, "PS", role)}:\n per_source: true\n' for v in VARIANTS for role in ('ROOT', 'T_X')) + ''.join( - f'CL_TR_PS_{role}:\n per_source: true\n' for role in ('R1', 'R2', 'S')) + f'CL_TR_PS_{role}:\n per_source: true\n' for role in ('R1', 'R2', 'S')) + ''.join( + f'CL_F5_PS_{role}:\n per_source: true\n' for role in ('X', 'Y', 'S_1')) CORRELATION = 'correlation:\n enabled: true\n patterns:\n' + ''.join( f' mx_{v.lower()}_{m.lower()}:\n codes: ["{_code(v, m, "T")}_*"]\n' for v in VARIANTS for m in MODES) + ' rules:\n' + ''.join( @@ -84,6 +85,37 @@ def _code(variant, mode, role): window_ms: 300 mute_symptoms: true auto_clear_with_root: true +""" for m in MODES) + ''.join( + f""" - id: f5_a_{m.lower()} + name: "F5 A {m}" + mode: hierarchical + root_cause: + codes: ["CL_F5_{m}_X"] + symptoms: + - codes: ["CL_F5_{m}_S_*"] + window_ms: 30000 + mute_symptoms: false + auto_clear_with_root: true + - id: f5_b_{m.lower()} + name: "F5 B {m}" + mode: hierarchical + root_cause: + codes: ["CL_F5_{m}_X"] + symptoms: + - codes: ["CL_F5_{m}_S_*"] + window_ms: 30000 + mute_symptoms: true + auto_clear_with_root: true + - id: f5_c_{m.lower()} + name: "F5 C {m}" + mode: hierarchical + root_cause: + codes: ["CL_F5_{m}_Y"] + symptoms: + - codes: ["CL_F5_{m}_S_*", "CL_F5_{m}_Z_*"] + window_ms: 300 + mute_symptoms: true + auto_clear_with_root: true """ for m in MODES) @@ -249,6 +281,70 @@ def _two_roots(self, mode): def test_a_cleared_root_leaves_the_mute_of_another_root(self): self._each_mode(self._two_roots) + def _muted_info(self, codes): + """Return {(code, owner): (root_cause_code, delay_ms)} of include_muted among codes.""" + request = ListFaults.Request() + request.include_muted = True + muted = self._call(self.list_client, request).muted_faults + return {(m.fault_code, m.source_id): (m.root_cause_code, m.delay_ms) for m in muted + if m.fault_code in codes} + + def _cross_window(self, prefix, source): + """Report fresh symptoms until one is no longer muted: the window of the root passed.""" + deadline = time.monotonic() + TIMEOUT_SEC + index = 0 + while True: + index += 1 + sentinel = f'{prefix}_{index}' + self._report(sentinel, source) + if not self._muted({sentinel}): + return + self.assertLess(time.monotonic(), deadline, 'the window of the root never ended') + + def _root_bound_to_its_own_rule(self, mode): + x, y = f'CL_F5_{mode}_X', f'CL_F5_{mode}_Y' + symptom = f'CL_F5_{mode}_S_1' + owner = '/s' if mode == 'PS' else '*' + self._report(y, '/y') + self._report(symptom, '/s') + self.assertEqual(self._muted({symptom}), {(symptom, owner): y}) + self._cross_window(f'CL_F5_{mode}_Z', '/s') + # X is bound to its first rule, which does not mute. The later rule for X never applies. + self._report(x, '/x') + self._report(symptom, '/s') + + self._clear(y, '/y' if mode == 'PS' else '*', skip=True) + self.assertEqual(self._muted({symptom}), {}, 'a rule the root is not bound to muted it') + self.assertEqual(self._shown({symptom}), {(symptom, '/s')}) + + def test_a_root_mutes_only_through_its_own_rule(self): + self._each_mode(self._root_bound_to_its_own_rule) + + def _delay_follows_the_relation(self, mode): + r1, r2 = f'CL_TR_{mode}_R1', f'CL_TR_{mode}_R2' + symptom = f'CL_TR_{mode}_S' + owner = '/s' if mode == 'PS' else '*' + self._report(r1, '/r1') + # Report again until the delay behind the first root is more than a millisecond. + deadline = time.monotonic() + TIMEOUT_SEC + while True: + self._report(symptom, '/s') + root, delay = self._muted_info({symptom})[(symptom, owner)] + if delay > 0: + break + self.assertLess(time.monotonic(), deadline, 'the delay behind the first root stayed 0') + self.assertEqual(root, r1) + self._cross_window(f'CL_TR_{mode}_SENT', '/s') + self._report(r2, '/r2') + self._report(symptom, '/s') + self.assertEqual(self._muted({symptom}), {(symptom, owner): r2}) + + self._clear(r2, '/r2' if mode == 'PS' else '*', skip=True) + self.assertEqual(self._muted_info({symptom}), {(symptom, owner): (r1, delay)}) + + def test_the_delay_moves_with_the_mute(self): + self._each_mode(self._delay_follows_the_relation) + @launch_testing.post_shutdown_test() class TestMutingRelationsShutdown(unittest.TestCase): From c58677136e4fd38859b581787258aca3b3172567 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 17:40:53 +0100 Subject: [PATCH 37/52] test(fault_manager): wait for the events publisher through the node Subscription.get_publisher_count does not exist in the Humble rclpy. Node.count_publishers does on every distro. --- .../test/integration/test_fault_identity.test.py | 5 +++-- .../test/integration/test_fault_shared_events.test.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py index 791ad9db6..1606f4e08 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py @@ -112,9 +112,10 @@ def setUpClass(cls): for client in (cls.report_client, cls.list_client, cls.get_client, cls.clear_client): assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) deadline = time.monotonic() + TIMEOUT_SEC - while cls.event_sub.get_publisher_count() == 0 and time.monotonic() < deadline: + while (cls.node.count_publishers('/fault_manager/events') == 0 + and time.monotonic() < deadline): rclpy.spin_once(cls.node, timeout_sec=0.1) - assert cls.event_sub.get_publisher_count() > 0 + assert cls.node.count_publishers('/fault_manager/events') > 0 @classmethod def tearDownClass(cls): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py index 98873bc06..6133ff42c 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py @@ -80,9 +80,10 @@ def setUpClass(cls): FaultEvent, '/fault_manager/events', cls.events.append, QoSProfile(depth=100, reliability=ReliabilityPolicy.RELIABLE)) deadline = time.monotonic() + TIMEOUT_SEC - while cls.subscription.get_publisher_count() == 0 and time.monotonic() < deadline: + while (cls.node.count_publishers('/fault_manager/events') == 0 + and time.monotonic() < deadline): rclpy.spin_once(cls.node, timeout_sec=0.1) - assert cls.subscription.get_publisher_count() > 0 + assert cls.node.count_publishers('/fault_manager/events') > 0 @classmethod def tearDownClass(cls): From 879e8ba7d404c1fb0e9aedf789c262e01038d5d6 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 17:54:09 +0100 Subject: [PATCH 38/52] scripts: port the ROS-free cores check to the record identity API The program overrode the capture hooks and called the storage and the core with a bare fault code. It now passes the record id to the hooks and the storage, and the source id to the core's clear. Every check it makes is kept. --- scripts/check_cores_without_ros/main.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/check_cores_without_ros/main.cpp b/scripts/check_cores_without_ros/main.cpp index 57c325552..a162ef9ff 100644 --- a/scripts/check_cores_without_ros/main.cpp +++ b/scripts/check_cores_without_ros/main.cpp @@ -109,13 +109,13 @@ class RecordingEventSink : public ros2_medkit_fault_manager::core::EventSink { class IgnoreCaptureHooks : public ros2_medkit_fault_manager::core::CaptureHooks { public: - ros2_medkit_fault_manager::core::CaptureResult capture_now(const std::string & code) override { - captured.push_back(code); + ros2_medkit_fault_manager::core::CaptureResult capture_now(const ros2_medkit_fault_manager::FaultId & id) override { + captured.push_back(id.fault_code); return {ros2_medkit_fault_manager::core::CaptureEnqueue::kAccepted, {}}; } - void on_prefailed(const std::string & /*code*/) override { + void on_prefailed(const ros2_medkit_fault_manager::FaultId & /*id*/) override { } - void on_cleared(const std::string & /*code*/) override { + void on_cleared(const ros2_medkit_fault_manager::FaultId & /*id*/) override { } std::vector captured; }; @@ -1524,8 +1524,8 @@ static int run() { fm::SqliteFaultStorage storage(db_path.string(), sink); fm::DebounceConfig debounce; debounce.confirmation_threshold = -1; - ok &= check(storage.report_fault_event("NO_ROS_FAULT", fd::ReportKind::kFailed, fd::severity::kError, "d", "/src", - 1000000000LL, debounce), + ok &= check(storage.report_fault_event(fm::FaultId{"NO_ROS_FAULT", fm::kSharedOwner}, fd::ReportKind::kFailed, + fd::severity::kError, "d", "/src", 1000000000LL, debounce), "a FAILED report creates the fault"); const auto faults = storage.list_faults(false, 0, {}); ok &= check(faults.size() == 1, "list_faults returns one record"); @@ -1596,7 +1596,7 @@ static int run() { "core list returns the fault CONFIRMED"); ok &= check(events.types.size() == 1 && events.types[0] == fd::fault_event_type::kConfirmed, "the sink saw exactly one fault_confirmed"); - const auto cleared = fault_core.clear("CORE_FAULT", false); + const auto cleared = fault_core.clear("CORE_FAULT", "", false); ok &= check(cleared.success, "core clear succeeds"); ok &= check(events.types.size() == 2 && events.types[1] == fd::fault_event_type::kCleared, "the sink saw fault_cleared after the clear"); From 4b04d714441fe58e8abafbae1b077b5fe3da3e0c Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 18:01:23 +0100 Subject: [PATCH 39/52] test(fault_manager): the snapshot tests wait for topic discovery before reporting The grouping and queue tests reported a fault at once after creating their publisher. A fault manager that had not discovered the topic yet skipped the snapshot and the first test failed. A snapshot pattern for probe codes on the published topic lets setUpClass report fresh probe codes until one lists a snapshot, so the tests start after discovery. --- .../test_fault_snapshot_grouping.test.py | 34 ++++++++++++++++++- .../test_fault_snapshot_queue.test.py | 34 ++++++++++++++++++- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py index 147c1632f..8afc169d4 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py @@ -58,7 +58,9 @@ def _temp_file(prefix, content): 'fault_specific:\n' ' XG.GROUP:\n - /grouping/counter\n' ' XG.AGAIN:\n - /grouping/counter\n' - ' XG.SHARED:\n - /grouping/counter\n') + ' XG.SHARED:\n - /grouping/counter\n' + 'patterns:\n' + ' "XG_PROBE_.*":\n - /grouping/counter\n') def generate_test_description(): @@ -102,6 +104,36 @@ def setUpClass(cls): cls.stop = threading.Event() cls.publisher_thread = threading.Thread(target=cls._publish) cls.publisher_thread.start() + cls._wait_for_discovery('XG_PROBE_') + + @classmethod + def _wait_for_discovery(cls, topic_probe_prefix): + """Report fresh probe codes until the fault manager has discovered the probed topic.""" + deadline = time.monotonic() + TIMEOUT_SEC + index = 0 + while time.monotonic() < deadline: + index += 1 + code = f'{topic_probe_prefix}{index}' + report = ReportFault.Request() + report.fault_code = code + report.event_type = ReportFault.Request.EVENT_FAILED + report.severity = Fault.SEVERITY_CRITICAL + report.description = 'discovery probe' + report.source_id = '/probe' + future = cls.report_client.call_async(report) + rclpy.spin_until_future_complete(cls.node, future, timeout_sec=10.0) + request = GetSnapshots.Request() + request.fault_code = code + attempt_end = time.monotonic() + 1.5 + while time.monotonic() < attempt_end: + future = cls.snapshots_client.call_async(request) + rclpy.spin_until_future_complete(cls.node, future, timeout_sec=10.0) + response = future.result() + if response is not None and response.success and response.data: + if json.loads(response.data).get('topics'): + return + rclpy.spin_once(cls.node, timeout_sec=0.1) + raise AssertionError('the fault manager never discovered the published topic') @classmethod def _publish(cls): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py index 47257339d..56a5bc832 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py @@ -65,7 +65,9 @@ def _temp_file(prefix, content): f' XQ.QUEUE:\n - {FAST}\n' f' XQ.PRUNE:\n - {FAST}\n - {SLOW}\n' f' XQ.PRUNE.TRIGGER:\n - {FAST}\n' - f' XQ.AGAIN:\n - {FAST}\n - {SLOW}\n') + f' XQ.AGAIN:\n - {FAST}\n - {SLOW}\n' + 'patterns:\n' + f' "XQ_PROBE_.*":\n - {FAST}\n') def generate_test_description(): @@ -114,6 +116,36 @@ def setUpClass(cls): cls.stop = threading.Event() cls.publisher_thread = threading.Thread(target=cls._publish) cls.publisher_thread.start() + cls._wait_for_discovery('XQ_PROBE_') + + @classmethod + def _wait_for_discovery(cls, topic_probe_prefix): + """Report fresh probe codes until the fault manager has discovered the probed topic.""" + deadline = time.monotonic() + TIMEOUT_SEC + index = 0 + while time.monotonic() < deadline: + index += 1 + code = f'{topic_probe_prefix}{index}' + report = ReportFault.Request() + report.fault_code = code + report.event_type = ReportFault.Request.EVENT_FAILED + report.severity = Fault.SEVERITY_CRITICAL + report.description = 'discovery probe' + report.source_id = '/probe' + future = cls.report_client.call_async(report) + rclpy.spin_until_future_complete(cls.node, future, timeout_sec=10.0) + request = GetSnapshots.Request() + request.fault_code = code + attempt_end = time.monotonic() + 1.5 + while time.monotonic() < attempt_end: + future = cls.snapshots_client.call_async(request) + rclpy.spin_until_future_complete(cls.node, future, timeout_sec=10.0) + response = future.result() + if response is not None and response.success and response.data: + if json.loads(response.data).get('topics'): + return + rclpy.spin_once(cls.node, timeout_sec=0.1) + raise AssertionError('the fault manager never discovered the published topic') @classmethod def _publish(cls): From ada9c910308f8c042b28bfdda7f1bbb473e6de3f Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Fri, 13 Nov 2026 18:11:17 +0100 Subject: [PATCH 40/52] test(fault_manager): the snapshot queue test fails if the silent topic was not discovered The silent topic is created before the probed one, so the fault manager has processed it once it has discovered the probed topic. After shutdown the test requires that the fault manager never logged the silent topic as not found, because every capture that skipped it at once would leave nothing busy and the tests would pass without exercising the queue. --- .../test/integration/test_fault_snapshot_queue.test.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py index 56a5bc832..badff0e2d 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py @@ -110,9 +110,10 @@ def setUpClass(cls): GetSnapshots, '/fault_manager/get_snapshots') for client in (cls.report_client, cls.clear_client, cls.snapshots_client): assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) - cls.fast = cls.node.create_publisher(Int32, FAST, 10) # Nothing is ever published here, so a capture of this topic runs for its whole timeout. + # Created first: the fault manager has processed it once it has discovered FAST. cls.slow = cls.node.create_publisher(Int32, SLOW, 10) + cls.fast = cls.node.create_publisher(Int32, FAST, 10) cls.stop = threading.Event() cls.publisher_thread = threading.Thread(target=cls._publish) cls.publisher_thread.start() @@ -265,6 +266,12 @@ def test_a_record_confirming_again_is_not_given_the_old_sample( @launch_testing.post_shutdown_test() class TestSnapshotQueueShutdown(unittest.TestCase): + def test_the_silent_topic_was_discovered(self, proc_output, fault_manager_node): + for text in proc_output[fault_manager_node]: + self.assertNotIn( + f"Topic '{SLOW}' not found", text.text.decode(errors='replace'), + 'a capture skipped the silent topic, so nothing was kept busy') + def test_exit_code(self, proc_info): launch_testing.asserts.assertExitCodes(proc_info) os.unlink(THRESHOLDS_FILE) From c8461c24ce76143260ab629c6b6eb2b3bc166d26 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Sat, 14 Nov 2026 15:06:39 +0100 Subject: [PATCH 41/52] msgs: every Fault names the record it describes Fault.msg gains `owner`, appended as its last field: the reporting source of a per-source record, "*" for a shared record. Passed back as source_id it addresses exactly that record, also when a stored shared record and a per-source record of one source carry the same reporting_sources after a code changed from shared to per_source. The fault manager fills it on every Fault it sends (ListFaults, GetFault, ListFaultsForEntity, the events topic) from the record identity each storage keeps, in SQLite, PostgreSQL and in-memory storage alike. --- docs/api/messages.rst | 8 ++++ .../fault_types.hpp | 2 + src/ros2_medkit_fault_manager/README.md | 4 ++ .../src/fault_storage.cpp | 1 + .../src/postgres_fault_storage.cpp | 1 + .../src/ros2/fault_msg_conversions.cpp | 1 + .../src/sqlite_fault_storage.cpp | 1 + .../integration/test_fault_identity.test.py | 41 +++++++++++++++++++ .../test_fault_identity_restart.test.py | 33 +++++++++++++++ .../test_fault_postgres_migration.test.py | 25 ++++++++++- src/ros2_medkit_msgs/msg/Fault.msg | 4 ++ 11 files changed, 120 insertions(+), 1 deletion(-) diff --git a/docs/api/messages.rst b/docs/api/messages.rst index 34da2fb8e..7e14a432c 100644 --- a/docs/api/messages.rst +++ b/docs/api/messages.rst @@ -23,6 +23,10 @@ the ``source_id`` of the ``ReportFault`` call. Every other code has one record s its sources, owned by the reserved id ``*``. Each record has its own status, debounce counter, ``occurrence_count``, severity and timestamps, and is cleared on its own. +``owner`` names the record a ``Fault`` describes. Passed back as ``source_id``, it addresses exactly +that record, also when a stored shared record and a per-source record of one source carry the same +``reporting_sources`` after the code changed from shared to ``per_source``. + .. code-block:: text # Global fault identifier (e.g., "MOTOR_OVERHEAT", "SENSOR_FAILURE_001") @@ -59,6 +63,10 @@ its sources, owned by the reserved id ``*``. Each record has its own status, deb # them for a shared one. string[] reporting_sources + # Owner of the record this Fault describes: the reporting source of a + # per-source record, "*" for a shared record. + string owner + **Severity Constants:** .. list-table:: diff --git a/src/ros2_medkit_fault_detection/include/ros2_medkit_fault_detection/fault_types.hpp b/src/ros2_medkit_fault_detection/include/ros2_medkit_fault_detection/fault_types.hpp index 66c378829..3fc0d6588 100644 --- a/src/ros2_medkit_fault_detection/include/ros2_medkit_fault_detection/fault_types.hpp +++ b/src/ros2_medkit_fault_detection/include/ros2_medkit_fault_detection/fault_types.hpp @@ -70,6 +70,8 @@ struct FaultRecord { uint32_t occurrence_count{0}; std::string status; std::vector reporting_sources; + /// Owner of the record: the reporting source of a per-source record, "*" for a shared one. + std::string owner; }; /// FaultEvent.msg field for field. diff --git a/src/ros2_medkit_fault_manager/README.md b/src/ros2_medkit_fault_manager/README.md index 0a3b45a27..e806cc8c3 100644 --- a/src/ros2_medkit_fault_manager/README.md +++ b/src/ros2_medkit_fault_manager/README.md @@ -24,6 +24,10 @@ knows only a reporting source still reaches the shared record of a default code. unscoped and applies only when exactly one record carries the `fault_code`, otherwise the call fails with a message beginning `ambiguous:` that lists the owners and changes nothing. +Every `Fault` the node sends (`~/list_faults`, `~/get_fault`, `~/list_faults_for_entity` and +`~/events`) carries the record's `owner`: the reporting source of a per-source record, `*` for a +shared one. It is the `source_id` that addresses exactly that record. + ### Identity per fault code ```yaml diff --git a/src/ros2_medkit_fault_manager/src/fault_storage.cpp b/src/ros2_medkit_fault_manager/src/fault_storage.cpp index 44464cc5b..9e4923204 100644 --- a/src/ros2_medkit_fault_manager/src/fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_storage.cpp @@ -94,6 +94,7 @@ fd::FaultRecord FaultState::to_record() const { msg.occurrence_count = occurrence_count; msg.status = status; msg.reporting_sources.assign(reporting_sources.begin(), reporting_sources.end()); + msg.owner = owner; return msg; } diff --git a/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp b/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp index 324e8abd9..3da734531 100644 --- a/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp @@ -645,6 +645,7 @@ fd::FaultRecord PgFaultStorage::read_fault_row(const pqxx::row & r) const { fault.status = r["status"].as(); fault.reporting_sources = parse_json_array(r["reporting_sources"].as()); fault.last_passed_ns = r["last_passed_ns"].as(); + fault.owner = r["owner"].as(); return fault; } diff --git a/src/ros2_medkit_fault_manager/src/ros2/fault_msg_conversions.cpp b/src/ros2_medkit_fault_manager/src/ros2/fault_msg_conversions.cpp index c354d7bf8..82d5873ac 100644 --- a/src/ros2_medkit_fault_manager/src/ros2/fault_msg_conversions.cpp +++ b/src/ros2_medkit_fault_manager/src/ros2/fault_msg_conversions.cpp @@ -43,6 +43,7 @@ ros2_medkit_msgs::msg::Fault to_msg(const fd::FaultRecord & record) { msg.occurrence_count = record.occurrence_count; msg.status = record.status; msg.reporting_sources = record.reporting_sources; + msg.owner = record.owner; return msg; } diff --git a/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp b/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp index b29c5e161..b343108cc 100644 --- a/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp @@ -208,6 +208,7 @@ fd::FaultRecord read_fault_row(SqliteStatement & stmt, core::LogSink & sink) { fault.status = stmt.column_text(6); fault.reporting_sources = parse_json_array(stmt.column_text(7), sink); fault.last_passed_ns = stmt.column_int64(8); + fault.owner = stmt.column_text(9); return fault; } diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py index 1606f4e08..c031131cc 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py @@ -64,6 +64,9 @@ ID.CRIT_PER: per_source: true confirmation_threshold: -3 +ID.PER_OWNER: + per_source: true + confirmation_threshold: -1 """ THRESHOLDS_FILE = tempfile.NamedTemporaryFile( @@ -285,6 +288,44 @@ def test_a_per_source_record_confirms_on_its_stored_severity(self): self.assertEqual(record.occurrence_count, 2) self.assertEqual(len(self._confirmed_events(code)), 2) + def _wait_for_confirmed_events(self, code, count): + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline and len(self._confirmed_events(code)) < count: + rclpy.spin_once(self.node, timeout_sec=0.1) + self.assertEqual( + len(self._confirmed_events(code)), count, f'fault_confirmed events of {code}') + + def test_every_fault_names_the_record_it_describes(self): + """ListFaults, GetFault and the confirmation event carry the owner of the record.""" + sources = ['/owner/a', '/owner/b'] + for source in sources: + self.assertTrue(self._report('ID.PER_OWNER', source)) + self.assertTrue(self._report('ID.SHARED_OWNER', source)) + + # A per_source code: one record per source, each owned by its source. + listed = sorted( + (r.owner, list(r.reporting_sources)) for r in self._records('ID.PER_OWNER')) + self.assertEqual(listed, [(s, [s]) for s in sources]) + for source in sources: + with self.subTest(per_source_get=source): + response = self._get('ID.PER_OWNER', source) + self.assertTrue(response.success, response.error_message) + self.assertEqual(response.fault.owner, source) + self._wait_for_confirmed_events('ID.PER_OWNER', 2) + self.assertEqual( + sorted(e.fault.owner for e in self._confirmed_events('ID.PER_OWNER')), sources) + + # A shared code: one record owned by '*', whichever source addresses it. + listed = [(r.owner, list(r.reporting_sources)) for r in self._records('ID.SHARED_OWNER')] + self.assertEqual(listed, [('*', sources)]) + for source in ['', '*'] + sources: + with self.subTest(shared_get=source): + response = self._get('ID.SHARED_OWNER', source) + self.assertTrue(response.success, response.error_message) + self.assertEqual(response.fault.owner, '*') + self._wait_for_confirmed_events('ID.SHARED_OWNER', 1) + self.assertEqual(self._confirmed_events('ID.SHARED_OWNER')[0].fault.owner, '*') + def test_the_shared_owner_is_not_a_source_id(self): """A report claiming the reserved owner is refused and creates nothing.""" for code in ('ID.RESERVED_SHARED', 'ID.PER'): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py index 5c434634d..df55e12eb 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py @@ -295,6 +295,39 @@ def test_changing_the_mode_keeps_stored_records_and_routes_new_reports(self): ('N.PER', '*'), ('N.PER', '/a'), ('N.PER', '/b'), ]) + def test_the_owner_tells_a_kept_shared_record_from_a_new_per_source_one(self): + """After the switch, the kept shared record of /s and the new one differ by owner.""" + db_path = self._path('owner_switch.db') + process = self._start(db_path, 'O.SWITCH:\n confirmation_threshold: -1\n') + try: + self._report('O.SWITCH', '/s') + request = ClearFault.Request() + request.fault_code = 'O.SWITCH' + self.assertTrue(self._call(self.clear, request).success) + self.assertEqual( + [(r.owner, list(r.reporting_sources)) for r in self._records('O.SWITCH')], + [('*', ['/s'])]) + finally: + self._stop(process) + + process = self._start( + db_path, 'O.SWITCH:\n per_source: true\n confirmation_threshold: -1\n') + try: + self._report('O.SWITCH', '/s') + expected = [('*', Fault.STATUS_CLEARED), ('/s', Fault.STATUS_CONFIRMED)] + listed = sorted( + (r.owner, r.status) for r in self._records('O.SWITCH') + if list(r.reporting_sources) == ['/s']) + self.assertEqual(listed, expected) + for owner, status in expected: + with self.subTest(owner=owner): + record = self._get('O.SWITCH', owner) + self.assertTrue(record.success, record.error_message) + self.assertEqual(record.fault.owner, owner) + self.assertEqual(record.fault.status, status) + finally: + self._stop(process) + def test_a_malformed_per_source_value_rejects_the_whole_file(self): thresholds = 'A:\n per_source: true\nB:\n per_source: "yes"\n' process = self._start(self._path('malformed.db'), thresholds) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py index bee79059f..32fc97e11 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py @@ -20,6 +20,7 @@ with storage_type: postgres, and the migrated records are read over its services: every code is one shared record (owner '*') with its reporting sources, and its snapshot, freeze frame and recording link attach to it. A report from a new source joins the record, and a restart changes nothing. +Every record names its owner: '*' for the shared ones, the source for a per_source code. Needs a server: ROS2_MEDKIT_TEST_PG_CONN, as for the storage tests. """ @@ -177,6 +178,9 @@ def _wait_until_service_is_gone(self): def _start(self): self._wait_until_service_is_gone() + thresholds = os.path.join(self.tmp.name, 'thresholds.yaml') + with open(thresholds, 'w') as f: + f.write('PG.PER:\n per_source: true\n') params = os.path.join(self.tmp.name, 'params.yaml') with open(params, 'w') as f: f.write('fault_manager:\n ros__parameters:\n' @@ -184,7 +188,8 @@ def _start(self): f' database_url: "{self.conn}"\n' ' confirmation_threshold: -1\n' ' snapshots.enabled: false\n' - ' snapshots.rosbag.enabled: false\n') + ' snapshots.rosbag.enabled: false\n' + f' fault_thresholds.config_file: {thresholds}\n') process = FaultManagerProcess(self.exe, params) self.addCleanup(process.kill) self.clients = { @@ -228,6 +233,7 @@ def _records(self, code): def _check_migrated(self, sources): records = self._records('MIG.CODE') self.assertEqual([list(r.reporting_sources) for r in records], [sources]) + self.assertEqual([r.owner for r in records], ['*']) for source in ('', '*', '/s1'): request = GetFault.Request() request.fault_code = 'MIG.CODE' @@ -236,6 +242,7 @@ def _check_migrated(self, sources): self.assertTrue(record.success, f'source {source!r}: {record.error_message}') self.assertEqual(record.fault.status, Fault.STATUS_CONFIRMED) self.assertEqual(list(record.fault.reporting_sources), sources) + self.assertEqual(record.fault.owner, '*') self.assertEqual([list(r.reporting_sources) for r in self._records('MIG.OLD')], [['/s3']]) snapshots = GetSnapshots.Request() @@ -253,6 +260,17 @@ def _check_migrated(self, sources): self.assertEqual(list(bag.fault_codes), ['MIG.CODE']) self.assertEqual(list(bag.owners), ['*']) + def _check_per_source_owners(self, sources): + listed = sorted((r.owner, list(r.reporting_sources)) for r in self._records('PG.PER')) + self.assertEqual(listed, [(s, [s]) for s in sources]) + for source in sources: + request = GetFault.Request() + request.fault_code = 'PG.PER' + request.source_id = source + record = self._call('get', request) + self.assertTrue(record.success, f'source {source!r}: {record.error_message}') + self.assertEqual(record.fault.owner, source) + def test_a_database_written_with_one_record_per_code_is_migrated_by_the_node(self): bag_path = os.path.join(self.tmp.name, 'fault_MIG.CODE_1') os.makedirs(bag_path) @@ -275,11 +293,16 @@ def test_a_database_written_with_one_record_per_code_is_migrated_by_the_node(sel # A new source joins the shared record. self._report('MIG.CODE', '/s4') self._check_migrated(['/s1', '/s2', '/s4']) + # A per_source code gets one record per source, each owned by its source. + for source in ('/p1', '/p2'): + self._report('PG.PER', source) + self._check_per_source_owners(['/p1', '/p2']) finally: self._stop(process) process = self._start() try: self._check_migrated(['/s1', '/s2', '/s4']) + self._check_per_source_owners(['/p1', '/p2']) finally: self._stop(process) diff --git a/src/ros2_medkit_msgs/msg/Fault.msg b/src/ros2_medkit_msgs/msg/Fault.msg index c0f2e4250..ae36bbba6 100644 --- a/src/ros2_medkit_msgs/msg/Fault.msg +++ b/src/ros2_medkit_msgs/msg/Fault.msg @@ -61,6 +61,10 @@ string status # the owner, or any source listed here. string[] reporting_sources +# Owner of the record this Fault describes: the reporting source of a per-source record, +# "*" for a shared record. The source_id that addresses exactly this record. +string owner + # Severity level constants # Informational message: No action required, used for logging or tracking. uint8 SEVERITY_INFO = 0 From 4535403f99d5683118a49d8657984ef60a0ca181 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Sat, 14 Nov 2026 15:21:33 +0100 Subject: [PATCH 42/52] test: no null dereference warnings in the Release build of the fault tests The fault detection tests dereferenced a find() result without a check, so a missing signal crashed the whole test binary, and GCC warned about it at -O2. Each looked-up signal is now asserted non-null before use. pg_exec reads stdin into a string stream through rdbuf(); GCC flagged its iterator range the same way. --- .../test/test_fault_detection.cpp | 70 ++++++++++++++----- .../test/pg_exec.cpp | 7 +- 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/ros2_medkit_fault_detection/test/test_fault_detection.cpp b/src/ros2_medkit_fault_detection/test/test_fault_detection.cpp index 6420ed377..5c4fa0313 100644 --- a/src/ros2_medkit_fault_detection/test/test_fault_detection.cpp +++ b/src/ros2_medkit_fault_detection/test/test_fault_detection.cpp @@ -127,10 +127,16 @@ TEST(StatusWordRule, NamedBitDecode) { // 0b1001 -> bit 0 and bit 3 set, bit 7 clear. auto out = fd::evaluate(fd::Value{static_cast(0b1001)}, r); ASSERT_EQ(out.size(), 3u); - EXPECT_TRUE(find(out, "E_STOP")->active); - EXPECT_TRUE(find(out, "PUMP_OVERLOAD")->active); - EXPECT_FALSE(find(out, "FILTER_DIRTY")->active); - EXPECT_EQ(find(out, "PUMP_OVERLOAD")->message, "Pump overload"); + const auto * e_stop = find(out, "E_STOP"); + const auto * pump = find(out, "PUMP_OVERLOAD"); + const auto * filter = find(out, "FILTER_DIRTY"); + ASSERT_NE(e_stop, nullptr); + ASSERT_NE(pump, nullptr); + ASSERT_NE(filter, nullptr); + EXPECT_TRUE(e_stop->active); + EXPECT_TRUE(pump->active); + EXPECT_FALSE(filter->active); + EXPECT_EQ(pump->message, "Pump overload"); } TEST(StatusWordRule, AllClearWhenZero) { @@ -174,13 +180,20 @@ TEST(StatusWordRule, WidthMasksSignExtendedHighBits) { const auto sign_extended = static_cast(0xFFFFFFFFFFFF8000ULL); auto out = fd::evaluate(fd::Value{sign_extended}, r); - EXPECT_TRUE(find(out, "REAL_FAULT")->active); - EXPECT_FALSE(find(out, "SPURIOUS")->active); + const auto * real = find(out, "REAL_FAULT"); + const auto * spurious = find(out, "SPURIOUS"); + ASSERT_NE(real, nullptr); + ASSERT_NE(spurious, nullptr); + EXPECT_TRUE(real->active); + EXPECT_FALSE(spurious->active); // Without a width the same value would spuriously fire bit 20. fd::StatusWordRule wide; wide.bits = {{20u, {"SPURIOUS", "ERROR", "spurious"}}}; - EXPECT_TRUE(find(fd::evaluate(fd::Value{sign_extended}, wide), "SPURIOUS")->active); + const auto wide_out = fd::evaluate(fd::Value{sign_extended}, wide); + const auto * wide_spurious = find(wide_out, "SPURIOUS"); + ASSERT_NE(wide_spurious, nullptr); + EXPECT_TRUE(wide_spurious->active); } // -- Fault-code enum --------------------------------------------------------- @@ -196,10 +209,16 @@ TEST(EnumMapRule, CodeMapsToFaultAndText) { auto out = fd::evaluate(fd::Value{static_cast(11)}, r); ASSERT_EQ(out.size(), 3u); - EXPECT_FALSE(find(out, "VFD_OVERVOLTAGE")->active); - EXPECT_TRUE(find(out, "VFD_OVERCURRENT")->active); - EXPECT_EQ(find(out, "VFD_OVERCURRENT")->message, "Output overcurrent"); - EXPECT_FALSE(find(out, "VFD_OVERTEMP")->active); + const auto * overvoltage = find(out, "VFD_OVERVOLTAGE"); + const auto * overcurrent = find(out, "VFD_OVERCURRENT"); + const auto * overtemp = find(out, "VFD_OVERTEMP"); + ASSERT_NE(overvoltage, nullptr); + ASSERT_NE(overcurrent, nullptr); + ASSERT_NE(overtemp, nullptr); + EXPECT_FALSE(overvoltage->active); + EXPECT_TRUE(overcurrent->active); + EXPECT_EQ(overcurrent->message, "Output overcurrent"); + EXPECT_FALSE(overtemp->active); } TEST(EnumMapRule, OkValueRaisesNothing) { @@ -221,18 +240,29 @@ TEST(EnumMapRule, UnmappedNonOkRaisesCatchAll) { auto out = fd::evaluate(fd::Value{static_cast(99)}, r); ASSERT_EQ(out.size(), 3u); - EXPECT_FALSE(find(out, "OVERVOLT")->active); - EXPECT_FALSE(find(out, "OVERCURR")->active); + const auto * overvolt = find(out, "OVERVOLT"); + const auto * overcurr = find(out, "OVERCURR"); const auto * unk = find(out, "VFD_UNMAPPED"); + ASSERT_NE(overvolt, nullptr); + ASSERT_NE(overcurr, nullptr); ASSERT_NE(unk, nullptr); + EXPECT_FALSE(overvolt->active); + EXPECT_FALSE(overcurr->active); EXPECT_TRUE(unk->active); EXPECT_EQ(unk->message, "unmapped fault code 99"); // ok and mapped values keep the catch-all inactive so the tracker clears it. - EXPECT_FALSE(find(fd::evaluate(fd::Value{static_cast(0)}, r), "VFD_UNMAPPED")->active); + const auto ok = fd::evaluate(fd::Value{static_cast(0)}, r); + const auto * ok_unk = find(ok, "VFD_UNMAPPED"); + ASSERT_NE(ok_unk, nullptr); + EXPECT_FALSE(ok_unk->active); auto mapped = fd::evaluate(fd::Value{static_cast(10)}, r); - EXPECT_TRUE(find(mapped, "OVERVOLT")->active); - EXPECT_FALSE(find(mapped, "VFD_UNMAPPED")->active); + const auto * mapped_overvolt = find(mapped, "OVERVOLT"); + const auto * mapped_unk = find(mapped, "VFD_UNMAPPED"); + ASSERT_NE(mapped_overvolt, nullptr); + ASSERT_NE(mapped_unk, nullptr); + EXPECT_TRUE(mapped_overvolt->active); + EXPECT_FALSE(mapped_unk->active); } TEST(EnumMapRule, NonFiniteValueHoldsState) { @@ -314,8 +344,12 @@ TEST(TransitionTracker, EnumCodeChangeClearsOldRaisesNew) { // Code switches 10 -> 11: clear OVERVOLT, raise OVERCURR in one step. auto changes = tracker.apply(fd::evaluate(fd::Value{static_cast(11)}, r)); ASSERT_EQ(changes.size(), 2u); - EXPECT_FALSE(find(changes, "OVERVOLT")->active); - EXPECT_TRUE(find(changes, "OVERCURR")->active); + const auto * overvolt = find(changes, "OVERVOLT"); + const auto * overcurr = find(changes, "OVERCURR"); + ASSERT_NE(overvolt, nullptr); + ASSERT_NE(overcurr, nullptr); + EXPECT_FALSE(overvolt->active); + EXPECT_TRUE(overcurr->active); } // -- Raise qualifier --------------------------------------------------------- diff --git a/src/ros2_medkit_fault_manager/test/pg_exec.cpp b/src/ros2_medkit_fault_manager/test/pg_exec.cpp index 29f6a1619..91aaf1fd5 100644 --- a/src/ros2_medkit_fault_manager/test/pg_exec.cpp +++ b/src/ros2_medkit_fault_manager/test/pg_exec.cpp @@ -18,7 +18,7 @@ #include #include -#include +#include #include int main(int argc, char ** argv) { @@ -27,10 +27,11 @@ int main(int argc, char ** argv) { return 2; } try { - const std::string sql((std::istreambuf_iterator(std::cin)), std::istreambuf_iterator()); + std::ostringstream sql; + sql << std::cin.rdbuf(); pqxx::connection connection(argv[1]); pqxx::nontransaction tx(connection); - tx.exec(sql); + tx.exec(sql.str()); } catch (const std::exception & e) { std::cerr << "pg_exec failed: " << e.what() << "\n"; return 1; From fcc6a5a89e56caba4285a5482d72bd24adf31869 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 18 Nov 2026 14:38:59 +0100 Subject: [PATCH 43/52] test(fault_manager): get an answer on each client before the restart test changes a record The fault manager drops an answer while its reply writer has not yet matched the client's reader, and rmw_fastrtps does not send it again: the server logs "failed to send response ... client will not receive response" and the client waits until its deadline. wait_for_service only says the client has matched the server, not the reverse. The first call after each start could therefore hang although the request was processed. After each start every client now sends a request that changes nothing (a report or clear with no fault code, a get, a list) until one is answered. --- .../test_fault_identity_restart.test.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py index df55e12eb..bc4c50633 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py @@ -47,6 +47,8 @@ from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault TIMEOUT_SEC = 30.0 +# How long one request that changes nothing waits for its answer before it is sent again. +ANSWER_WAIT_SEC = 1.0 ALL_STATUSES = [ Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, Fault.STATUS_HEALED, Fault.STATUS_CLEARED, @@ -199,6 +201,14 @@ def _start(self, db_path, thresholds): self.assertTrue(client.wait_for_service(timeout_sec=TIMEOUT_SEC)) self.assertIsNotNone( process.wait_for_line('FaultManager node started'), 'the node never finished starting') + # The fault manager drops an answer while its reply writer has not matched the + # client's reader, and never sends it again. Each client asks something that changes + # nothing until one answer arrives; after that its answers are delivered. + for client, request in ((self.report, ReportFault.Request()), + (self.get, GetFault.Request()), + (self.list, ListFaults.Request()), + (self.clear, ClearFault.Request())): + self._until_answered(client, request) return process def _stop(self, process): @@ -206,6 +216,16 @@ def _stop(self, process): self.node.destroy_client(client) self.assertEqual(process.stop(), 0) + def _until_answered(self, client, request): + """Send request until one copy is answered. Only for a request that changes nothing.""" + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=ANSWER_WAIT_SEC) + if future.done(): + return + self.fail(f'{client.srv_name} never answered') + def _call(self, client, request): future = client.call_async(request) rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) From 5bf028845096e3147e26837f2e1efba385fec50a Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 18 Nov 2026 15:40:30 +0100 Subject: [PATCH 44/52] opcua: make the first reconnect wait a setting plugins.opcua.reconnect_interval_ms sets the wait after the first failed connect (default 5000 ms, as before). Each further failure doubles it up to the backoff ceiling. A value outside [100, 60000] or not an integer is refused with a warning and the default is kept. The comms-lost scenarios that lose and regain the link set it to 200 ms. One of them waits for four failed connects within 4 s, which takes 15 s at the default. The binding-on-reconnect scenario keeps the default. --- .../ros2_medkit_opcua/README.md | 1 + .../ros2_medkit_opcua/opcua_poller.hpp | 1 + .../ros2_medkit_opcua/src/opcua_plugin.cpp | 10 ++++++ .../integration/test_opcua_comms_lost.test.py | 34 +++++++++++++------ 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md index 4c580db38..09c033313 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md @@ -688,6 +688,7 @@ ros2_medkit_gateway: | `auto_browse` | `false` | Boolean or object; recursively discover the SOVD tree from the live address space instead of (or in addition to) `node_map_path` - see above | | `auto_alarms` | `false` | Boolean or object; subscribe a server EventNotifier and auto-derive a fault per AlarmCondition with no per-alarm `event_alarms` mapping - the zero-config native A&C surface (see above) | | `poll_interval_ms` | `1000` | Polling interval in ms (clamped to [100, 60000]) | +| `reconnect_interval_ms` | `5000` | Wait after the first failed connect while the server is down. Each further failure doubles it, up to 60 s, or up to the discovery `interval_s` while discovery re-scans, but never below this value (out of range [100, 60000] ms or not an integer is refused with a warning and the default is kept) | | `prefer_subscriptions` | `false` | Use OPC-UA subscriptions instead of polling | | `subscription_interval_ms` | `500` | Publishing interval for OPC-UA subscriptions when `prefer_subscriptions: true`. Between two polls the client is also iterated at this interval (at least 20 ms apart, at most once per poll), so changes arrive at this cadence and not once per `poll_interval_ms` | | `condition_replay_strategy` | `auto` | Active-condition replay on reconnect: `method`, `read`, `auto`, `off` (see below) | diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp index 9c6c617a3..373755c63 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp @@ -104,6 +104,7 @@ struct PollerConfig { bool prefer_subscriptions{false}; // poll mode by default (subscriptions need event loop) double subscription_interval_ms{500.0}; std::chrono::milliseconds poll_interval{1000}; + /// First wait after a failed connect. Config: plugins.opcua.reconnect_interval_ms. std::chrono::milliseconds reconnect_interval{5000}; /// Ceiling for the exponential reconnect backoff (it doubles from /// ``reconnect_interval`` up to this). A plugin whose reconnect arm also diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp index da86ac871..96793a207 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp @@ -331,6 +331,16 @@ void OpcuaPlugin::configure(const nlohmann::json & config) { auto ms = std::clamp(config["poll_interval_ms"].get(), 100, 60000); poller_config_.poll_interval = std::chrono::milliseconds(ms); } + if (config.contains("reconnect_interval_ms")) { + const auto & raw = config["reconnect_interval_ms"]; + const auto ceiling = poller_config_.max_reconnect_interval.count(); + if (raw.is_number_integer() && raw.get() >= 100 && raw.get() <= ceiling) { + poller_config_.reconnect_interval = std::chrono::milliseconds(raw.get()); + } else { + log_warn("plugins.opcua.reconnect_interval_ms must be an integer in [100, " + std::to_string(ceiling) + + "] - keeping " + std::to_string(poller_config_.reconnect_interval.count()) + " ms"); + } + } if (config.contains("condition_replay_strategy")) { poller_config_.condition_replay_strategy = OpcuaPoller::parse_replay_strategy(config["condition_replay_strategy"].get()); diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py index cf5151471..90fc7b07a 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py @@ -63,6 +63,9 @@ HELD_BY_OTHERS = f'{COMMS_LOST} is held by sources this bridge did not report under [' # Discovery identifies OPC-UA only on this port. DISCOVERY_PORT = 4840 +# First reconnect wait of the scenarios that set one. The production default is 5000 ms. +RECONNECT_MS = 200 +CONNECT_FAILED = 'OPC-UA connect to ' def time_scale(): @@ -109,12 +112,12 @@ def http_json(url, timeout=2): return None -def wait_log(path, needle, deadline): - """Poll a log file until appears, for a scaled deadline.""" +def wait_log(path, needle, deadline, count=1): + """Poll a log file until appears times, for a scaled deadline.""" end = time.monotonic() + deadline * SCALE while time.monotonic() < end: try: - if needle in Path(path).read_text(errors='replace'): + if Path(path).read_text(errors='replace').count(needle) >= count: return True except OSError: pass @@ -257,7 +260,8 @@ def start_fault_manager(self, name, per_source_codes=()): + log.read_text(errors='replace')) return proc - def start_gateway(self, name, endpoint=None, poll_ms=200, extra_env=None, node_map=None): + def start_gateway(self, name, endpoint=None, poll_ms=200, extra_env=None, node_map=None, + reconnect_ms=None): """Start gateway_node with the opcua plugin and wait for HTTP.""" port = free_port() params = self.workdir / f'{name}_gateway.yaml' @@ -276,6 +280,8 @@ def start_gateway(self, name, endpoint=None, poll_ms=200, extra_env=None, node_m lines.append(f' plugins.opcua.endpoint_url: "{endpoint}"') if node_map is not None: lines.append(f' plugins.opcua.node_map_path: "{node_map}"') + if reconnect_ms is not None: + lines.append(f' plugins.opcua.reconnect_interval_ms: {reconnect_ms}') params.write_text('\n'.join(lines) + '\n') env = dict(self.env) env.update(extra_env or {}) @@ -472,7 +478,7 @@ def scenario_per_source_record(run): run.start_fault_manager('persrc', per_source_codes=[COMMS_LOST]) named, port = run.start_server('persrc_named') endpoint = f'opc.tcp://127.0.0.1:{port}' - gateway = run.start_gateway('persrc', endpoint=endpoint) + gateway = run.start_gateway('persrc', endpoint=endpoint, reconnect_ms=RECONNECT_MS) if not wait_log(gateway.log, 'Connected to OPC-UA server', 30): raise AssertionError('the gateway never connected:\n' + tail(gateway)) @@ -501,7 +507,12 @@ def scenario_per_source_record(run): def scenario_per_source_placeholder_rename(run): run.start_fault_manager('rename', per_source_codes=[COMMS_LOST]) port = free_port() - gateway = run.start_gateway('rename', endpoint=f'opc.tcp://127.0.0.1:{port}') + gateway = run.start_gateway('rename', endpoint=f'opc.tcp://127.0.0.1:{port}', + reconnect_ms=RECONNECT_MS) + # Four failed connects take 15 s at the default wait and well under 1 s at RECONNECT_MS. + if not wait_log(gateway.log, CONNECT_FAILED, 4, count=4): + raise AssertionError('the gateway did not retry at reconnect_interval_ms:\n' + + tail(gateway)) records = wait_records(run, lambda r: len(r) == 1 and 'CONFIRMED' in r.values(), 60) if len(records) != 1 or 'CONFIRMED' not in records.values(): raise AssertionError(f'the outage was never reported: {records}\n' + tail(gateway)) @@ -519,7 +530,8 @@ def scenario_per_source_pinned_stand_in(run): run.start_fault_manager('persrc_pinned', per_source_codes=[COMMS_LOST]) port = free_port() endpoint = f'opc.tcp://127.0.0.1:{port}' - gateway = run.start_gateway('persrc_pinned_before', endpoint=endpoint) + gateway = run.start_gateway('persrc_pinned_before', endpoint=endpoint, + reconnect_ms=RECONNECT_MS) records = wait_records(run, lambda r: len(r) == 1 and 'CONFIRMED' in r.values(), 60) if len(records) != 1 or 'CONFIRMED' not in records.values(): raise AssertionError(f'the outage was never reported: {records}\n' + tail(gateway)) @@ -528,7 +540,7 @@ def scenario_per_source_pinned_stand_in(run): # The restarted gateway remembers nothing, and the PLC names the component. run.start_server('persrc_pinned', port=port) - after = run.start_gateway('persrc_pinned_after', endpoint=endpoint) + after = run.start_gateway('persrc_pinned_after', endpoint=endpoint, reconnect_ms=RECONNECT_MS) records = wait_records(run, lambda r: r.get(stand_in) == 'CLEARED', 60) if records.get(stand_in) != 'CLEARED': raise AssertionError(f'the record raised under the stand-in {stand_in} stayed standing ' @@ -545,7 +557,7 @@ def scenario_explicit_component_id_keeps_the_default_id_record(run): node_map.write_text('area_id: plc_area\ncomponent_id: plc_a\nnodes: []\n') _, port = run.start_server('explicit_id') gateway = run.start_gateway('explicit_id', endpoint=f'opc.tcp://127.0.0.1:{port}', - node_map=node_map) + node_map=node_map, reconnect_ms=RECONNECT_MS) # The gateway asked for its own record, found none, and went through every id it reports under. if not wait_log(gateway.log, 'nothing to clear', 30): raise AssertionError('the gateway never decided on PLC_COMMS_LOST:\n' + tail(gateway)) @@ -565,7 +577,7 @@ def scenario_explicit_component_id_keeps_a_stand_in_record(run): node_map.write_text('area_id: plc_area\ncomponent_id: plc_a\nnodes: []\n') _, port = run.start_server('stand_in_id') gateway = run.start_gateway('stand_in_id', endpoint=f'opc.tcp://127.0.0.1:{port}', - node_map=node_map) + node_map=node_map, reconnect_ms=RECONNECT_MS) if not wait_log(gateway.log, 'nothing to clear', 30): raise AssertionError('the gateway never decided on PLC_COMMS_LOST:\n' + tail(gateway)) records = comms_lost_records(run) @@ -583,7 +595,7 @@ def scenario_explicit_component_id_keeps_a_shared_stand_in_record(run): node_map.write_text('area_id: plc_area\ncomponent_id: plc_a\nnodes: []\n') _, port = run.start_server('stand_in_shared') gateway = run.start_gateway('stand_in_shared', endpoint=f'opc.tcp://127.0.0.1:{port}', - node_map=node_map) + node_map=node_map, reconnect_ms=RECONNECT_MS) if not wait_log(gateway.log, HELD_BY_OTHERS, 30): raise AssertionError('the gateway never left the shared record standing:\n' + tail(gateway)) From 74ee0fcfbe62a85752a9cd77b9cb307224568636 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 18 Nov 2026 15:45:42 +0100 Subject: [PATCH 45/52] test(fault_manager): the refused clear test waits on a later event The check that a refused unscoped clear publishes nothing spun 200 ms. A report from a third owner now follows the refused clear, and the test waits for its event: events arrive in publish order, so an event of the clear would come first. --- .../test/test_fault_manager.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp index 865fbf4e5..510da55db 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp @@ -2139,9 +2139,16 @@ TEST_F(PerSourceNodeTest, UnscopedClearOfTwoRecordsIsRefusedAndClearsNothing) { EXPECT_NE(response->message.find("Set source_id to pick one."), std::string::npos) << "got: " << response->message; EXPECT_EQ(response->message.find(';'), std::string::npos) << "got: " << response->message; - // Nothing was cleared, and nothing was published. - spin_for(std::chrono::milliseconds(200)); - EXPECT_TRUE(received_events_.empty()); + // Nothing was cleared, and nothing was published: events arrive in publish order, so the + // refused clear published nothing if a later report's event comes first. + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_c")); + ASSERT_TRUE(spin_until([this]() { + return !received_events_.empty(); + })); + ASSERT_EQ(received_events_.size(), 1u); + EXPECT_EQ(received_events_[0].event_type, FaultEvent::EVENT_CONFIRMED); + ASSERT_EQ(received_events_[0].fault.reporting_sources.size(), 1u); + EXPECT_EQ(received_events_[0].fault.reporting_sources.front(), "/owner_c"); for (const char * owner : {"/owner_a", "/owner_b"}) { auto still = call_get_fault("SHARED_CODE", owner); ASSERT_TRUE(still.has_value()); From daaf79f2cb7ab434228eb18b29760a5dd543064d Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 18 Nov 2026 16:00:01 +0100 Subject: [PATCH 46/52] test(fault_manager): the snapshot queue test ends each slow capture itself A capture of the slow topic held the single worker for the 2 s capture timeout, four times a run. The topic now stays silent until the test has queued what it checks, then the test publishes on it and the capture ends. The capture timeout is longer than any test, so only the release ends a capture. The cooldown case releases once a trigger was admitted after the cooldown. --- .../test_fault_snapshot_queue.test.py | 40 ++++++++++++++----- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py index badff0e2d..e5f7fcd92 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py @@ -17,7 +17,8 @@ Shared snapshot capture of per-source records under queue pressure, through a real fault manager. One worker, a queue of two and drop_oldest, so every extra queued job evicts another fault's -capture. A topic nobody publishes on keeps a capture running for its whole timeout. +capture. A capture of the slow topic runs until the test releases it: the topic is silent until +then, and the capture timeout is longer than any test. * a record joining the queued capture of its code queues no job of its own; * a capture that outlives the recapture cooldown keeps every record that joined it; @@ -42,6 +43,7 @@ from std_msgs.msg import Int32 TIMEOUT_SEC = 40.0 +COOLDOWN_SEC = 0.3 FAST = '/queue/fast' SLOW = '/queue/slow' @@ -82,11 +84,11 @@ def generate_test_description(): 'fault_thresholds.config_file': THRESHOLDS_FILE, 'snapshots.enabled': True, 'snapshots.config_file': SNAPSHOTS_FILE, - 'snapshots.timeout_sec': 2.0, + 'snapshots.timeout_sec': TIMEOUT_SEC, 'snapshots.capture_pool_size': 1, 'snapshots.capture_queue_depth': 2, 'snapshots.capture_queue_full_policy': 'drop_oldest', - 'snapshots.recapture_cooldown_sec': 0.3, + 'snapshots.recapture_cooldown_sec': COOLDOWN_SEC, 'snapshots.rosbag.enabled': False, }], sigterm_timeout='30', @@ -110,11 +112,12 @@ def setUpClass(cls): GetSnapshots, '/fault_manager/get_snapshots') for client in (cls.report_client, cls.clear_client, cls.snapshots_client): assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) - # Nothing is ever published here, so a capture of this topic runs for its whole timeout. - # Created first: the fault manager has processed it once it has discovered FAST. + # Silent until a test sets release. Created first: the fault manager has processed it + # once it has discovered FAST. cls.slow = cls.node.create_publisher(Int32, SLOW, 10) cls.fast = cls.node.create_publisher(Int32, FAST, 10) cls.stop = threading.Event() + cls.release = threading.Event() cls.publisher_thread = threading.Thread(target=cls._publish) cls.publisher_thread.start() cls._wait_for_discovery('XQ_PROBE_') @@ -154,6 +157,8 @@ def _publish(cls): while not cls.stop.wait(0.1): value += 1 cls.fast.publish(Int32(data=value)) + if cls.release.is_set(): + cls.slow.publish(Int32(data=value)) @classmethod def tearDownClass(cls): @@ -162,6 +167,10 @@ def tearDownClass(cls): cls.node.destroy_node() rclpy.shutdown() + def setUp(self): + # A capture of SLOW subscribes when it starts, so it gets nothing published before. + self.release.clear() + def _call(self, client, request): future = client.call_async(request) rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) @@ -223,6 +232,7 @@ def test_a_joining_record_does_not_evict_another_faults_capture( # evict the oldest one, which is the other fault's. self._report('XQ.QUEUE', '/q/a') self._report('XQ.QUEUE', '/q/b') + self.release.set() self._wait_snapshot('XQ.OTHER', '/q/other') first = self._wait_snapshot('XQ.QUEUE', '/q/a') @@ -232,15 +242,22 @@ def test_a_joining_record_does_not_evict_another_faults_capture( def test_a_capture_that_outlives_the_cooldown_keeps_its_records( self, proc_output, fault_manager_node): self._report('XQ.PRUNE', '/q/a') + admitted = time.monotonic() self._wait_capture_started(proc_output, fault_manager_node, 'XQ.PRUNE') self._report('XQ.PRUNE', '/q/b') # Every admission of another grouped code is a chance to drop an aged-out group. The - # capture above runs for the whole topic timeout, much longer than the cooldown. + # capture runs until a trigger is admitted after the cooldown. triggers = iter(range(10000)) - first = self._wait_snapshot( - 'XQ.PRUNE', '/q/a', - between=lambda: self._report('XQ.PRUNE.TRIGGER', f'/q/trigger{next(triggers)}')) + + def admit_trigger(): + sent = time.monotonic() + self._report('XQ.PRUNE.TRIGGER', f'/q/trigger{next(triggers)}') + if sent - admitted > COOLDOWN_SEC: + self.release.set() + + first = self._wait_snapshot('XQ.PRUNE', '/q/a', between=admit_trigger) + self.assertTrue(self.release.is_set(), 'the capture ended before it was released') second = self._wait_snapshot('XQ.PRUNE', '/q/b') self.assertEqual(first['captured_at'], second['captured_at']) @@ -255,6 +272,7 @@ def test_a_record_confirming_again_is_not_given_the_old_sample( self._clear('XQ.AGAIN', '/q/a') confirmed_again = time.time() self._report('XQ.AGAIN', '/q/a') + self.release.set() old = self._wait_snapshot('XQ.AGAIN', '/q/b') again = self._wait_snapshot('XQ.AGAIN', '/q/a') @@ -266,11 +284,11 @@ def test_a_record_confirming_again_is_not_given_the_old_sample( @launch_testing.post_shutdown_test() class TestSnapshotQueueShutdown(unittest.TestCase): - def test_the_silent_topic_was_discovered(self, proc_output, fault_manager_node): + def test_the_slow_topic_was_discovered(self, proc_output, fault_manager_node): for text in proc_output[fault_manager_node]: self.assertNotIn( f"Topic '{SLOW}' not found", text.text.decode(errors='replace'), - 'a capture skipped the silent topic, so nothing was kept busy') + 'a capture skipped the slow topic, so nothing was kept busy') def test_exit_code(self, proc_info): launch_testing.asserts.assertExitCodes(proc_info) From f38df618efb8abd7fb31a42d0a8236e23627e057 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 18 Nov 2026 16:13:26 +0100 Subject: [PATCH 47/52] opcua: state the first reconnect wait when the poller starts The poller-started line now names the reconnect_interval_ms in effect. A comms-lost scenario starts a gateway per value: 100 and 60000 are used, 99, 60001 and 250.5 are refused with a warning and 5000 is used. At 100 the gateway also logs four failed connects within 4 s. --- .../ros2_medkit_opcua/src/opcua_plugin.cpp | 2 +- .../integration/test_opcua_comms_lost.test.py | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp index 96793a207..81fe57d2e 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp @@ -893,7 +893,7 @@ void OpcuaPlugin::set_context(PluginContext & context) { } poller_->start(poller_config_); log_info("OPC-UA poller started (mode: " + std::string(poller_->using_subscriptions() ? "subscription" : "poll") + - ")"); + ", first reconnect wait " + std::to_string(poller_config_.reconnect_interval.count()) + " ms)"); } std::vector OpcuaPlugin::get_routes() { diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py index 90fc7b07a..307b44e97 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py @@ -40,6 +40,9 @@ * binding directory: a binding file whose directory cannot be looked up is reported, and the plugin stays connected. Once at start, and once on the poll thread when a rescan adopts a PLC that came up later. + * reconnect interval setting: reconnect_interval_ms at both ends of its range is + used, and a value just outside each end or a non-integer is refused with a + warning, so the default 5000 ms is used. Usage: test_opcua_comms_lost.test.py """ @@ -663,6 +666,31 @@ def scenario_binding_directory_on_reconnect(run): expect_binding_failure_survived(gateway, binding, 60) +REFUSED_INTERVAL = ('plugins.opcua.reconnect_interval_ms must be an integer in [100, 60000] ' + '- keeping 5000 ms') + + +def scenario_reconnect_interval_setting(run): + endpoint = f'opc.tcp://127.0.0.1:{free_port()}' + for value, effective in ((100, 100), (60000, 60000), (99, 5000), (60001, 5000), + (250.5, 5000)): + name = 'interval_' + str(value).replace('.', '_') + gateway = run.start_gateway(name, endpoint=endpoint, reconnect_ms=value) + started = f'OPC-UA poller started (mode: poll, first reconnect wait {effective} ms)' + if not wait_log(gateway.log, started, 30): + raise AssertionError(f'reconnect_interval_ms={value}: the poller did not start with ' + f'a {effective} ms wait:\n' + tail(gateway)) + # The value is read before the poller starts, so its warning is already logged. + warned = REFUSED_INTERVAL in gateway.log.read_text(errors='replace') + if warned != (effective != value): + raise AssertionError(f'reconnect_interval_ms={value}: refused warning ' + f'{"logged" if warned else "missing"}:\n' + tail(gateway)) + # Four failed connects take 15 s at the default wait and 0.3 s at 100 ms. + if value == 100 and not wait_log(gateway.log, CONNECT_FAILED, 4, count=4): + raise AssertionError('the gateway did not retry at 100 ms:\n' + tail(gateway)) + run.stop(gateway.proc) + + SCENARIOS = [ ('pinned stand-in', scenario_pinned_stand_in), ('foreign stand-in', scenario_foreign_stand_in), @@ -677,6 +705,7 @@ def scenario_binding_directory_on_reconnect(run): scenario_explicit_component_id_keeps_a_shared_stand_in_record), ('binding directory at start', scenario_binding_directory_at_start), ('binding directory on reconnect', scenario_binding_directory_on_reconnect), + ('reconnect interval setting', scenario_reconnect_interval_setting), ] From 4082e5d14c0e1e55c78ecdc35c76fdca09e3aa9c Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 18 Nov 2026 16:28:13 +0100 Subject: [PATCH 48/52] test(opcua): the comms-lost test talks to the fault manager from one node Every report and listing started a Python process that initialised rclpy and discovered the fault manager again, 0.7 to 3 s a call. One node in the test process now serves every scenario. A call is sent again after 1 s (scaled) without an answer, since a reply can be lost while each scenario's new fault manager matches the client. The scenarios are unchanged. --- .../integration/test_opcua_comms_lost.test.py | 97 ++++++++++--------- 1 file changed, 50 insertions(+), 47 deletions(-) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py index 307b44e97..2e1a68878 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py @@ -61,6 +61,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'fixtures' / 'test_alarm_server')) import alarm_fixture # noqa: E402 +import rclpy # noqa: E402 +from ros2_medkit_msgs.msg import Fault # noqa: E402 +from ros2_medkit_msgs.srv import ListFaults, ReportFault # noqa: E402 COMMS_LOST = 'PLC_COMMS_LOST' HELD_BY_OTHERS = f'{COMMS_LOST} is held by sources this bridge did not report under [' @@ -197,11 +200,12 @@ def __init__(self, proc, port, log): class Run: """The processes of one scenario. stop_all() ends every one of them.""" - def __init__(self, workdir, server_bin, plugin, env): + def __init__(self, workdir, server_bin, plugin, env, client): self.workdir = workdir self.server_bin = server_bin self.plugin = plugin self.env = env + self.client = client self.procs = [] self.logs = {} # Logs of fixtures that printed READY. A failed start printed its own. @@ -407,55 +411,52 @@ def scenario_late_nameplate(run): 'after the device named itself') -# Talks to the fault manager as a second source: reports a fault, lists the records. -FAULT_MANAGER_CLIENT = """ -import json -import sys +# A reply is asked for again after this long. A reply can be lost while the fault manager has not +# matched this client yet, and every scenario starts a new fault manager. +ANSWER_WAIT_SEC = 1.0 * SCALE -import rclpy -from ros2_medkit_msgs.msg import Fault -from ros2_medkit_msgs.srv import ListFaults, ReportFault - -rclpy.init() -node = rclpy.create_node('comms_lost_test_client') - - -def call(client, request): - future = client.call_async(request) - rclpy.spin_until_future_complete(node, future, timeout_sec=10.0) - return future.result() - - -if sys.argv[1] == 'report': - client = node.create_client(ReportFault, '/fault_manager/report_fault') - client.wait_for_service(timeout_sec=10.0) - request = ReportFault.Request() - request.fault_code = sys.argv[2] - request.event_type = ReportFault.Request.EVENT_FAILED - request.severity = Fault.SEVERITY_ERROR - request.description = 'second source' - request.source_id = sys.argv[3] - print(json.dumps({'accepted': call(client, request).accepted})) -else: - client = node.create_client(ListFaults, '/fault_manager/list_faults') - client.wait_for_service(timeout_sec=10.0) - request = ListFaults.Request() - request.statuses = ['PREFAILED', 'PREPASSED', 'CONFIRMED', 'HEALED', 'CLEARED'] - print(json.dumps([{'code': f.fault_code, 'status': f.status, - 'sources': list(f.reporting_sources)} - for f in call(client, request).faults])) -node.destroy_node() -rclpy.shutdown() -""" + +class FaultManagerClient: + """One node in the test process that talks to the fault manager as a second source.""" + + def __init__(self): + rclpy.init() + self.node = rclpy.create_node('comms_lost_test_client') + self.report = self.node.create_client(ReportFault, '/fault_manager/report_fault') + self.list = self.node.create_client(ListFaults, '/fault_manager/list_faults') + + def close(self): + self.node.destroy_node() + rclpy.shutdown() + + def call(self, client, request): + """Return the answer to , sending it again while the reply is lost.""" + end = time.monotonic() + 60 * SCALE + while time.monotonic() < end: + if not client.wait_for_service(timeout_sec=ANSWER_WAIT_SEC): + continue + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=ANSWER_WAIT_SEC) + if future.done(): + return future.result() + client.remove_pending_request(future) + raise AssertionError(f'{client.srv_name} never answered') def fault_manager_client(run, *args): - """Run the fault manager client script on the scenario's ROS domain and return its JSON.""" - result = subprocess.run([sys.executable, '-c', FAULT_MANAGER_CLIENT, *args], env=run.env, - capture_output=True, text=True, timeout=60 * SCALE) - if result.returncode != 0: - raise AssertionError(f'fault manager client {args} failed:\n{result.stderr}') - return json.loads(result.stdout) + """Report as a second source ('report'), or list every fault ('list').""" + if args[0] == 'report': + request = ReportFault.Request() + request.fault_code = args[1] + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = 'second source' + request.source_id = args[2] + return {'accepted': run.client.call(run.client.report, request).accepted} + request = ListFaults.Request() + request.statuses = ['PREFAILED', 'PREPASSED', 'CONFIRMED', 'HEALED', 'CLEARED'] + return [{'code': f.fault_code, 'status': f.status, 'sources': list(f.reporting_sources)} + for f in run.client.call(run.client.list, request).faults] def comms_lost_records(run): @@ -729,11 +730,12 @@ def main(): env = dict(os.environ, ROS_DOMAIN_ID=ros_domain_id) only = os.environ.get('OPCUA_COMMS_LOST_SCENARIO') failures = 0 + client = FaultManagerClient() for name, scenario in SCENARIOS: if only and only != name: continue workdir = Path(tempfile.mkdtemp(prefix='opcua_comms_lost_')) - run = Run(workdir, server_bin, plugin, env) + run = Run(workdir, server_bin, plugin, env, client) started = time.monotonic() error = None try: @@ -750,6 +752,7 @@ def main(): print(f'FAIL {name} ({elapsed:.0f} s): {error}', file=sys.stderr) for log in run.server_logs: print(f'{log.name}:\n' + alarm_fixture.read_log(log), file=sys.stderr) + client.close() return 1 if failures else 0 From f6b057210094c34ef6784a26df76f5b735462f19 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 18 Nov 2026 20:39:57 +0100 Subject: [PATCH 49/52] test(fault_manager): get an answer and an event before a test acts wait_for_service and count_publishers show that the test found the fault manager, not that the fault manager matched the test. A reply or event sent before that match is lost. The first FaultEvent of test_fault_identity was lost that way: the CRITICAL report was answered and its EVENT_CONFIRMED never arrived. service_round_trip.py sends each client a request that names no fault until one answer comes back, and reports new CRITICAL codes until one's event arrives. The integration tests that start a fault manager call it before they act, after every start. --- .../test/integration/service_round_trip.py | 78 +++++++++++++++++++ .../integration/test_fault_audit_rows.test.py | 15 ++-- .../test_fault_cluster_muting.test.py | 8 +- .../test_fault_correlation_clear.test.py | 8 +- .../test_fault_correlation_owners.test.py | 8 +- .../integration/test_fault_identity.test.py | 14 ++-- .../test_fault_identity_restart.test.py | 25 ++---- .../test_fault_muting_relations.test.py | 8 +- .../test_fault_postgres_migration.test.py | 5 ++ .../test_fault_shared_events.test.py | 12 +-- .../test_fault_snapshot_cap.test.py | 8 +- .../test_fault_snapshot_grouping.test.py | 8 +- .../test_fault_snapshot_queue.test.py | 8 +- .../test/integration/test_integration.test.py | 6 ++ .../test_rosbag_integration.test.py | 5 ++ .../test_severity_consistency.test.py | 22 +++--- 16 files changed, 178 insertions(+), 60 deletions(-) create mode 100644 src/ros2_medkit_fault_manager/test/integration/service_round_trip.py diff --git a/src/ros2_medkit_fault_manager/test/integration/service_round_trip.py b/src/ros2_medkit_fault_manager/test/integration/service_round_trip.py new file mode 100644 index 000000000..17cfd1481 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/service_round_trip.py @@ -0,0 +1,78 @@ +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +First round trips to a fault manager that has just started. + +wait_for_service and count_publishers show that the test found the server. The server +can match the test's reply reader or event subscription later, and a reply or an +event sent before that is lost. +""" + +import time + +import rclpy +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ReportFault + +# A request with no answer after this long is sent again. +ANSWER_WAIT_SEC = 1.0 + + +def call_until_answered(node, client, request, timeout_sec, answer_wait_sec=ANSWER_WAIT_SEC): + """Send until gets an answer and return it; must be repeatable.""" + deadline = time.monotonic() + timeout_sec + while time.monotonic() < deadline: + future = client.call_async(request) + rclpy.spin_until_future_complete(node, future, timeout_sec=answer_wait_sec) + if future.done(): + return future.result() + client.remove_pending_request(future) + raise AssertionError(f'{client.srv_name} did not answer within {timeout_sec} s') + + +def answer_once_each(node, clients, timeout_sec): + """Get one answer on each client. The default request names no fault, so it changes nothing.""" + for client in clients: + call_until_answered(node, client, client.srv_type.Request(), timeout_sec) + + +def wait_for_event_route(node, report_client, events, prefix, timeout_sec): + """ + Report a new CRITICAL code until its event reaches . + + Each code confirms once and publishes one event. Events arrive in order, so a caller + that counts every event can empty afterwards: no earlier probe's event can + come after the one seen. + """ + deadline = time.monotonic() + timeout_sec + index = 0 + while time.monotonic() < deadline: + index += 1 + request = ReportFault.Request() + request.fault_code = f'{prefix}{index}' + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_CRITICAL + request.description = 'event route probe' + request.source_id = '/event_route_probe' + # Sent once: a second report of the code would publish a second event. + client_future = report_client.call_async(request) + settle = time.monotonic() + ANSWER_WAIT_SEC + while time.monotonic() < settle: + if any(e.fault.fault_code == request.fault_code for e in events): + report_client.remove_pending_request(client_future) + return + rclpy.spin_once(node, timeout_sec=0.1) + report_client.remove_pending_request(client_future) + raise AssertionError(f'no fault event reached the test within {timeout_sec} s') diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_audit_rows.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_audit_rows.test.py index f47f25301..6e8b0eaa1 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_audit_rows.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_audit_rows.test.py @@ -29,6 +29,7 @@ import signal import sqlite3 import subprocess +import sys import tempfile import time import unittest @@ -42,6 +43,9 @@ from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import ClearFault, GetFault, ReportFault +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + GENESIS = '0' * 64 HEAL = (' confirmation_threshold: -1\n' ' healing_enabled: true\n' @@ -173,16 +177,7 @@ def _start(self, codes_file): for client in clients.values(): self.assertTrue(client.wait_for_service(timeout_sec=15.0)) # The graph can still list the previous node's services, so wait for an answer. - probe = GetFault.Request() - probe.fault_code = 'NOT.A.FAULT' - deadline = time.monotonic() + 15.0 - while time.monotonic() < deadline: - future = clients['get'].call_async(probe) - rclpy.spin_until_future_complete(self.node, future, timeout_sec=1.0) - if future.result() is not None: - break - else: - self.fail('fault_manager did not answer after start') + answer_once_each(self.node, tuple(clients.values()), 15.0) return proc, clients @staticmethod diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_cluster_muting.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_cluster_muting.test.py index 42fa4c86f..087eacdd8 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_cluster_muting.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_cluster_muting.test.py @@ -25,6 +25,7 @@ """ import os +import sys import tempfile import unittest @@ -37,6 +38,9 @@ from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + TIMEOUT_SEC = 30.0 MODES = ('SH', 'PS') # scenario -> representative policy @@ -110,8 +114,10 @@ def setUpClass(cls): cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') cls.get_client = cls.node.create_client(GetFault, '/fault_manager/get_fault') cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') - for client in (cls.report_client, cls.list_client, cls.get_client, cls.clear_client): + clients = (cls.report_client, cls.list_client, cls.get_client, cls.clear_client) + for client in clients: assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) @classmethod def tearDownClass(cls): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py index ae4da4e18..3c24bc485 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py @@ -27,6 +27,7 @@ """ import os +import sys import tempfile import time import unittest @@ -40,6 +41,9 @@ from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + TIMEOUT_SEC = 30.0 ALL_STATUSES = [ Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, @@ -132,8 +136,10 @@ def setUpClass(cls): cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') cls.get_client = cls.node.create_client(GetFault, '/fault_manager/get_fault') cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') - for client in (cls.report_client, cls.list_client, cls.get_client, cls.clear_client): + clients = (cls.report_client, cls.list_client, cls.get_client, cls.clear_client) + for client in clients: assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) @classmethod def tearDownClass(cls): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py index bd99b3172..ab48d24d2 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py @@ -23,6 +23,7 @@ """ import os +import sys import tempfile import unittest @@ -36,6 +37,9 @@ from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + TIMEOUT_SEC = 30.0 ALL_STATUSES = [ Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, @@ -172,8 +176,10 @@ def setUpClass(cls): cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') cls.get_client = cls.node.create_client(GetFault, '/fault_manager/get_fault') cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') - for client in (cls.report_client, cls.list_client, cls.get_client, cls.clear_client): + clients = (cls.report_client, cls.list_client, cls.get_client, cls.clear_client) + for client in clients: assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) @classmethod def tearDownClass(cls): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py index c031131cc..133a550c1 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py @@ -23,6 +23,7 @@ """ import os +import sys import tempfile import time import unittest @@ -38,6 +39,9 @@ from ros2_medkit_msgs.msg import Fault, FaultEvent from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each, wait_for_event_route # noqa: E402 + TIMEOUT_SEC = 30.0 ALL_STATUSES = [ Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, @@ -112,13 +116,11 @@ def setUpClass(cls): cls.event_sub = cls.node.create_subscription( FaultEvent, '/fault_manager/events', cls.events.append, QoSProfile(depth=100, reliability=ReliabilityPolicy.RELIABLE)) - for client in (cls.report_client, cls.list_client, cls.get_client, cls.clear_client): + clients = (cls.report_client, cls.list_client, cls.get_client, cls.clear_client) + for client in clients: assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) - deadline = time.monotonic() + TIMEOUT_SEC - while (cls.node.count_publishers('/fault_manager/events') == 0 - and time.monotonic() < deadline): - rclpy.spin_once(cls.node, timeout_sec=0.1) - assert cls.node.count_publishers('/fault_manager/events') > 0 + answer_once_each(cls.node, clients, TIMEOUT_SEC) + wait_for_event_route(cls.node, cls.report_client, cls.events, 'ID.PROBE_', TIMEOUT_SEC) @classmethod def tearDownClass(cls): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py index bc4c50633..ebd65a507 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py @@ -32,6 +32,7 @@ import signal import sqlite3 import subprocess +import sys import tempfile import threading import time @@ -46,9 +47,10 @@ from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + TIMEOUT_SEC = 30.0 -# How long one request that changes nothing waits for its answer before it is sent again. -ANSWER_WAIT_SEC = 1.0 ALL_STATUSES = [ Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, Fault.STATUS_HEALED, Fault.STATUS_CLEARED, @@ -201,14 +203,7 @@ def _start(self, db_path, thresholds): self.assertTrue(client.wait_for_service(timeout_sec=TIMEOUT_SEC)) self.assertIsNotNone( process.wait_for_line('FaultManager node started'), 'the node never finished starting') - # The fault manager drops an answer while its reply writer has not matched the - # client's reader, and never sends it again. Each client asks something that changes - # nothing until one answer arrives; after that its answers are delivered. - for client, request in ((self.report, ReportFault.Request()), - (self.get, GetFault.Request()), - (self.list, ListFaults.Request()), - (self.clear, ClearFault.Request())): - self._until_answered(client, request) + answer_once_each(self.node, (self.report, self.get, self.list, self.clear), TIMEOUT_SEC) return process def _stop(self, process): @@ -216,16 +211,6 @@ def _stop(self, process): self.node.destroy_client(client) self.assertEqual(process.stop(), 0) - def _until_answered(self, client, request): - """Send request until one copy is answered. Only for a request that changes nothing.""" - deadline = time.monotonic() + TIMEOUT_SEC - while time.monotonic() < deadline: - future = client.call_async(request) - rclpy.spin_until_future_complete(self.node, future, timeout_sec=ANSWER_WAIT_SEC) - if future.done(): - return - self.fail(f'{client.srv_name} never answered') - def _call(self, client, request): future = client.call_async(request) rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py index d1078230e..8e664a9b8 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py @@ -24,6 +24,7 @@ """ import os +import sys import tempfile import time import unittest @@ -37,6 +38,9 @@ from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import ClearFault, ListFaults, ReportFault +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + TIMEOUT_SEC = 30.0 MODES = ('SH', 'PS') # variant -> mute_symptoms of the hierarchical rule @@ -164,8 +168,10 @@ def setUpClass(cls): cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') - for client in (cls.report_client, cls.list_client, cls.clear_client): + clients = (cls.report_client, cls.list_client, cls.clear_client) + for client in clients: assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) @classmethod def tearDownClass(cls): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py index 32fc97e11..dbb25a401 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py @@ -28,6 +28,7 @@ import queue import signal import subprocess +import sys import tempfile import threading import time @@ -43,6 +44,9 @@ from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import GetFault, GetRosbag, GetSnapshots, ListFaults, ReportFault +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + TIMEOUT_SEC = 30.0 ALL_STATUSES = [ Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, @@ -203,6 +207,7 @@ def _start(self): self.assertTrue(client.wait_for_service(timeout_sec=TIMEOUT_SEC)) self.assertIsNotNone( process.wait_for_line('FaultManager node started'), 'the node never finished starting') + answer_once_each(self.node, tuple(self.clients.values()), TIMEOUT_SEC) return process def _stop(self, process): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py index 6133ff42c..8bfdf490a 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py @@ -22,6 +22,7 @@ import os import shutil +import sys import tempfile import time import unittest @@ -37,6 +38,9 @@ from ros2_medkit_msgs.msg import Fault, FaultEvent from ros2_medkit_msgs.srv import ReportFault +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each, wait_for_event_route # noqa: E402 + TIMEOUT_SEC = 30.0 _temp_dirs = [] @@ -79,11 +83,9 @@ def setUpClass(cls): cls.subscription = cls.node.create_subscription( FaultEvent, '/fault_manager/events', cls.events.append, QoSProfile(depth=100, reliability=ReliabilityPolicy.RELIABLE)) - deadline = time.monotonic() + TIMEOUT_SEC - while (cls.node.count_publishers('/fault_manager/events') == 0 - and time.monotonic() < deadline): - rclpy.spin_once(cls.node, timeout_sec=0.1) - assert cls.node.count_publishers('/fault_manager/events') > 0 + answer_once_each(cls.node, (cls.report_client,), TIMEOUT_SEC) + wait_for_event_route(cls.node, cls.report_client, cls.events, 'SE.PROBE_', TIMEOUT_SEC) + cls.events.clear() @classmethod def tearDownClass(cls): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py index d8d8e64f9..b169bab15 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py @@ -28,6 +28,7 @@ import os import shutil import sqlite3 +import sys import tempfile import threading import time @@ -43,6 +44,9 @@ from ros2_medkit_msgs.srv import ClearFault, GetSnapshots, ReportFault from std_msgs.msg import Int32 +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402, I100 + TIMEOUT_SEC = 40.0 FIRST = '/cap/first' SECOND = '/cap/second' @@ -105,8 +109,10 @@ def setUpClass(cls): cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') cls.snapshots_client = cls.node.create_client( GetSnapshots, '/fault_manager/get_snapshots') - for client in (cls.report_client, cls.clear_client, cls.snapshots_client): + clients = (cls.report_client, cls.clear_client, cls.snapshots_client) + for client in clients: assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) cls.publishers = [cls.node.create_publisher(Int32, topic, 10) for topic in (FIRST, SECOND)] cls.stop = threading.Event() cls.publisher_thread = threading.Thread(target=cls._publish) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py index 8afc169d4..d3d9a2c4b 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py @@ -24,6 +24,7 @@ import json import os +import sys import tempfile import threading import time @@ -39,6 +40,9 @@ from ros2_medkit_msgs.srv import ClearFault, GetSnapshots, ReportFault from std_msgs.msg import Int32 +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402, I100 + TIMEOUT_SEC = 30.0 SOURCES = ['/grp/a', '/grp/b', '/grp/c'] @@ -98,8 +102,10 @@ def setUpClass(cls): cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') cls.snapshots_client = cls.node.create_client( GetSnapshots, '/fault_manager/get_snapshots') - for client in (cls.report_client, cls.clear_client, cls.snapshots_client): + clients = (cls.report_client, cls.clear_client, cls.snapshots_client) + for client in clients: assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) cls.publisher = cls.node.create_publisher(Int32, '/grouping/counter', 10) cls.stop = threading.Event() cls.publisher_thread = threading.Thread(target=cls._publish) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py index e5f7fcd92..9d0666f78 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py @@ -27,6 +27,7 @@ import json import os +import sys import tempfile import threading import time @@ -42,6 +43,9 @@ from ros2_medkit_msgs.srv import ClearFault, GetSnapshots, ReportFault from std_msgs.msg import Int32 +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402, I100 + TIMEOUT_SEC = 40.0 COOLDOWN_SEC = 0.3 FAST = '/queue/fast' @@ -110,8 +114,10 @@ def setUpClass(cls): cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') cls.snapshots_client = cls.node.create_client( GetSnapshots, '/fault_manager/get_snapshots') - for client in (cls.report_client, cls.clear_client, cls.snapshots_client): + clients = (cls.report_client, cls.clear_client, cls.snapshots_client) + for client in clients: assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) # Silent until a test sets release. Created first: the fault manager has processed it # once it has discovered FAST. cls.slow = cls.node.create_publisher(Int32, SLOW, 10) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py index 7ebcef92b..7595511d3 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py @@ -18,6 +18,7 @@ import atexit import json import os +import sys import tempfile import threading import time @@ -34,6 +35,9 @@ from ros2_medkit_msgs.srv import ClearFault, GetSnapshots, ListFaults, ReportFault from sensor_msgs.msg import Temperature +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402, I100 + def get_coverage_env(): """ @@ -160,6 +164,8 @@ def setUpClass(cls): 'clear_fault service not available' assert cls.get_snapshots_client.wait_for_service(timeout_sec=10.0), \ 'get_snapshots service not available' + answer_once_each(cls.node, (cls.report_fault_client, cls.list_faults_client, + cls.clear_fault_client, cls.get_snapshots_client), 30.0) @classmethod def tearDownClass(cls): diff --git a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py index 5ec519eb3..f9f5bdee6 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py @@ -16,6 +16,7 @@ """Integration tests for rosbag capture feature (Issue #120).""" import os +import sys import tempfile import time import unittest @@ -33,6 +34,9 @@ from ros2_medkit_msgs.srv import ClearFault, GetRosbag, ReportFault from sensor_msgs.msg import Temperature +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402, I100 + def get_coverage_env(): """Get environment variables for gcov coverage data collection.""" @@ -284,6 +288,7 @@ def node_is_up(): break time.sleep(0.2) assert not missing, 'fault_manager not ready: ' + ', '.join(missing) + answer_once_each(cls.node, tuple(clients.values()), PRECONDITION_BUDGET_SEC) # Wait for the background publisher to come up before letting the rosbag # ring buffer fill (duration_sec=2.0 configured), so a slow publisher diff --git a/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py b/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py index fa08205c4..4bd301700 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py @@ -28,6 +28,7 @@ import shutil import signal import sqlite3 +import sys import tempfile import time import unittest @@ -42,6 +43,9 @@ from ros2_medkit_msgs.msg import Fault, FaultEvent from ros2_medkit_msgs.srv import ClearFault, ListFaults, ReportFault +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each, wait_for_event_route # noqa: E402 + DB_DIR = tempfile.mkdtemp(prefix='medkit_severity_') DB_PATH = os.path.join(DB_DIR, 'faults.db') EVENT_TOPIC = '/fault_manager/events' @@ -214,18 +218,12 @@ def _wait_for_event(self, code, event_type, occurrence): return None def _prime_events(self): - """Report fresh CRITICAL codes until one event arrives, so the event topic is matched.""" - deadline = time.monotonic() + TIMEOUT_SEC - while time.monotonic() < deadline: - type(self).primes += 1 - code = f'PRIME_{self.primes}' - self._report(code, Fault.SEVERITY_CRITICAL) - settle = time.monotonic() + 1.0 - while time.monotonic() < settle: - if any(e.fault.fault_code == code for e in self.events): - return - rclpy.spin_once(self.node, timeout_sec=0.1) - self.fail('no fault event arrived from the fault manager') + """Get one answer on every client and one event from the fault manager just started.""" + type(self).primes += 1 + answer_once_each( + self.node, (self.report_client, self.list_client, self.clear_client), TIMEOUT_SEC) + wait_for_event_route( + self.node, self.report_client, self.events, f'PRIME_{self.primes}_', TIMEOUT_SEC) def _restart(self, fault_manager_node): old_pid = fault_manager_node.process_details['pid'] From 1a0b408a8efd0505e1ace163d47ab7cddf4af6fb Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 18 Nov 2026 20:40:04 +0100 Subject: [PATCH 50/52] test: the integration tests get one answer per fault manager client first A reply can be lost while the fault manager has not matched the client, and a test that sends one request and waits for its answer then fails. ros2_medkit_test_utils.service_round_trip sends each client a request that names no fault until one answer comes back. The correlation root clear and fault severity tests call it once their services are found. --- src/ros2_medkit_integration_tests/README.md | 1 + .../service_round_trip.py | 55 +++++++++++++++++++ .../test_correlation_root_clear.test.py | 2 + .../test_fault_severity_consistency.test.py | 2 + 4 files changed, 60 insertions(+) create mode 100644 src/ros2_medkit_integration_tests/ros2_medkit_test_utils/service_round_trip.py diff --git a/src/ros2_medkit_integration_tests/README.md b/src/ros2_medkit_integration_tests/README.md index 104326cf2..5b14b364d 100644 --- a/src/ros2_medkit_integration_tests/README.md +++ b/src/ros2_medkit_integration_tests/README.md @@ -28,6 +28,7 @@ ros2_medkit_integration_tests/ coverage.py # GCOV_PREFIX helpers for CI gateway_test_case.py # GatewayTestCase base class launch_helpers.py # create_test_launch(), DEMO_NODE_REGISTRY + service_round_trip.py # answer_once_each(): one answer per client before a test acts test/ features/ # Focused API feature tests scenarios/ # End-to-end workflow tests diff --git a/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/service_round_trip.py b/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/service_round_trip.py new file mode 100644 index 000000000..8f0803fbe --- /dev/null +++ b/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/service_round_trip.py @@ -0,0 +1,55 @@ +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +First round trips to a service server that has just started. + +wait_for_service shows that the client found the server. The server can match the +client's reply reader later, and a reply sent before that is lost. +""" + +import time + +import rclpy + +# A request with no answer after this long is sent again. +ANSWER_WAIT_SEC = 1.0 + + +def call_until_answered(node, client, request, timeout_sec, *, spin=True, + answer_wait_sec=ANSWER_WAIT_SEC): + """ + Send until gets an answer and return it; must be repeatable. + + With spin=False another thread spins , and this only waits for the answer. + """ + deadline = time.monotonic() + timeout_sec + while time.monotonic() < deadline: + future = client.call_async(request) + if spin: + rclpy.spin_until_future_complete(node, future, timeout_sec=answer_wait_sec) + else: + settle = time.monotonic() + answer_wait_sec + while not future.done() and time.monotonic() < settle: + time.sleep(0.02) + if future.done(): + return future.result() + client.remove_pending_request(future) + raise AssertionError(f'{client.srv_name} did not answer within {timeout_sec} s') + + +def answer_once_each(node, clients, timeout_sec, *, spin=True): + """Get one answer on each client. The default request names no fault, so it changes nothing.""" + for client in clients: + call_until_answered(node, client, client.srv_type.Request(), timeout_sec, spin=spin) diff --git a/src/ros2_medkit_integration_tests/test/features/test_correlation_root_clear.test.py b/src/ros2_medkit_integration_tests/test/features/test_correlation_root_clear.test.py index a0d29529d..b13929135 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_correlation_root_clear.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_correlation_root_clear.test.py @@ -36,6 +36,7 @@ from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES from ros2_medkit_test_utils.gateway_test_case import GatewayTestCase from ros2_medkit_test_utils.launch_helpers import create_test_launch +from ros2_medkit_test_utils.service_round_trip import answer_once_each ROOT_CODE = 'LIDAR_RANGE_INVALID' SYMPTOM_CODE = 'ROOT_CLEAR_SYMPTOM' @@ -92,6 +93,7 @@ def setUpClass(cls): cls._list = cls._node.create_client(ListFaults, '/fault_manager/list_faults') assert cls._report.wait_for_service(timeout_sec=30.0) assert cls._list.wait_for_service(timeout_sec=30.0) + answer_once_each(cls._node, (cls._report, cls._list), 30.0) @classmethod def tearDownClass(cls): diff --git a/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py b/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py index a1a2bf025..bdf2d8af4 100644 --- a/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py +++ b/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py @@ -43,6 +43,7 @@ from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES, API_BASE_PATH, FAULT_TIMEOUT from ros2_medkit_test_utils.gateway_test_case import GatewayTestCase from ros2_medkit_test_utils.launch_helpers import create_test_launch +from ros2_medkit_test_utils.service_round_trip import answer_once_each APP_ID = 'temp_sensor' @@ -131,6 +132,7 @@ def setUpClass(cls): cls._report_client = cls._node.create_client(ReportFault, '/fault_manager/report_fault') assert cls._report_client.wait_for_service(timeout_sec=15.0), \ 'report_fault service not available' + answer_once_each(cls._node, (cls._report_client,), 15.0) @classmethod def tearDownClass(cls): From d22b8090611045fb7818891f12632ac7f236927b Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 18 Nov 2026 20:40:06 +0100 Subject: [PATCH 51/52] test(opcua): both fault manager tests use one in-process client test_opcua_per_source_clear started a Python process per call, each one a new client whose first reply can be lost. FaultManagerClient, moved out of the comms-lost test into fault_manager_client.py, keeps one node in the test process and sends a call again while its reply is lost. It is installed next to the test scripts, as alarm_fixture.py is. --- .../ros2_medkit_opcua/CMakeLists.txt | 4 + .../test/integration/fault_manager_client.py | 73 +++++++++++++++++++ .../integration/test_opcua_comms_lost.test.py | 50 +------------ .../test_opcua_per_source_clear.test.py | 64 ++++------------ 4 files changed, 94 insertions(+), 97 deletions(-) create mode 100644 src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/fault_manager_client.py diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/CMakeLists.txt b/src/ros2_medkit_plugins/ros2_medkit_opcua/CMakeLists.txt index b5c3bb48d..c3913e92f 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/CMakeLists.txt +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/CMakeLists.txt @@ -648,6 +648,10 @@ if(BUILD_TESTING) test/integration/test_opcua_per_source_clear.test.py test/integration/test_opcua_startup_sweep_signal.test.py DESTINATION lib/${PROJECT_NAME}) + # The fault manager client the installed scripts import from their directory. + install(FILES + test/integration/fault_manager_client.py + DESTINATION lib/${PROJECT_NAME}) # ---- Read-only / write-capable behaviour over real HTTP ---------------- # The build-inspection test proves the write symbols are gone; this proves diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/fault_manager_client.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/fault_manager_client.py new file mode 100644 index 000000000..ee8446085 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/fault_manager_client.py @@ -0,0 +1,73 @@ +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A second fault source for the OPC-UA tests: one node in the test process.""" + +import time + +import rclpy +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ListFaults, ReportFault + +ALL_STATUSES = ['PREFAILED', 'PREPASSED', 'CONFIRMED', 'HEALED', 'CLEARED'] + + +class FaultManagerClient: + """ + Reports and lists faults. rclpy takes ROS_DOMAIN_ID from the environment. + + A reply can be lost while a fault manager that just started has not matched this + client, so a call with no answer after is sent again. + """ + + def __init__(self, name, answer_wait_sec, timeout_sec): + rclpy.init() + self.node = rclpy.create_node(name) + self.report_client = self.node.create_client(ReportFault, '/fault_manager/report_fault') + self.list_client = self.node.create_client(ListFaults, '/fault_manager/list_faults') + self.answer_wait_sec = answer_wait_sec + self.timeout_sec = timeout_sec + + def close(self): + self.node.destroy_node() + rclpy.shutdown() + + def call(self, client, request): + """Return the answer to , sending it again while the reply is lost.""" + end = time.monotonic() + self.timeout_sec + while time.monotonic() < end: + if not client.wait_for_service(timeout_sec=self.answer_wait_sec): + continue + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=self.answer_wait_sec) + if future.done(): + return future.result() + client.remove_pending_request(future) + raise AssertionError(f'{client.srv_name} never answered') + + def report(self, code, source): + """Report FAILED from . Return whether the fault manager accepted it.""" + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = 'second source' + request.source_id = source + return self.call(self.report_client, request).accepted + + def faults(self): + """Return every fault the fault manager holds, in any status.""" + request = ListFaults.Request() + request.statuses = ALL_STATUSES + return self.call(self.list_client, request).faults diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py index 2e1a68878..d577491bb 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py @@ -61,9 +61,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'fixtures' / 'test_alarm_server')) import alarm_fixture # noqa: E402 -import rclpy # noqa: E402 -from ros2_medkit_msgs.msg import Fault # noqa: E402 -from ros2_medkit_msgs.srv import ListFaults, ReportFault # noqa: E402 +from fault_manager_client import FaultManagerClient # noqa: E402 COMMS_LOST = 'PLC_COMMS_LOST' HELD_BY_OTHERS = f'{COMMS_LOST} is held by sources this bridge did not report under [' @@ -411,52 +409,12 @@ def scenario_late_nameplate(run): 'after the device named itself') -# A reply is asked for again after this long. A reply can be lost while the fault manager has not -# matched this client yet, and every scenario starts a new fault manager. -ANSWER_WAIT_SEC = 1.0 * SCALE - - -class FaultManagerClient: - """One node in the test process that talks to the fault manager as a second source.""" - - def __init__(self): - rclpy.init() - self.node = rclpy.create_node('comms_lost_test_client') - self.report = self.node.create_client(ReportFault, '/fault_manager/report_fault') - self.list = self.node.create_client(ListFaults, '/fault_manager/list_faults') - - def close(self): - self.node.destroy_node() - rclpy.shutdown() - - def call(self, client, request): - """Return the answer to , sending it again while the reply is lost.""" - end = time.monotonic() + 60 * SCALE - while time.monotonic() < end: - if not client.wait_for_service(timeout_sec=ANSWER_WAIT_SEC): - continue - future = client.call_async(request) - rclpy.spin_until_future_complete(self.node, future, timeout_sec=ANSWER_WAIT_SEC) - if future.done(): - return future.result() - client.remove_pending_request(future) - raise AssertionError(f'{client.srv_name} never answered') - - def fault_manager_client(run, *args): """Report as a second source ('report'), or list every fault ('list').""" if args[0] == 'report': - request = ReportFault.Request() - request.fault_code = args[1] - request.event_type = ReportFault.Request.EVENT_FAILED - request.severity = Fault.SEVERITY_ERROR - request.description = 'second source' - request.source_id = args[2] - return {'accepted': run.client.call(run.client.report, request).accepted} - request = ListFaults.Request() - request.statuses = ['PREFAILED', 'PREPASSED', 'CONFIRMED', 'HEALED', 'CLEARED'] + return {'accepted': run.client.report(args[1], args[2])} return [{'code': f.fault_code, 'status': f.status, 'sources': list(f.reporting_sources)} - for f in run.client.call(run.client.list, request).faults] + for f in run.client.faults()] def comms_lost_records(run): @@ -730,7 +688,7 @@ def main(): env = dict(os.environ, ROS_DOMAIN_ID=ros_domain_id) only = os.environ.get('OPCUA_COMMS_LOST_SCENARIO') failures = 0 - client = FaultManagerClient() + client = FaultManagerClient('comms_lost_test_client', 1.0 * SCALE, 60 * SCALE) for name, scenario in SCENARIOS: if only and only != name: continue diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_per_source_clear.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_per_source_clear.test.py index 9a13a12ac..d41062bce 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_per_source_clear.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_per_source_clear.test.py @@ -37,6 +37,8 @@ import urllib.error import urllib.request +from fault_manager_client import FaultManagerClient + COMPONENT = 'per_source_runtime' ENTITY = 'plc_app' CODE = 'PS_STATUS_BIT0' @@ -187,60 +189,18 @@ def start_fault_manager(log_path, env): return proc -FAULT_MANAGER_CLIENT = """ -import json -import sys - -import rclpy -from ros2_medkit_msgs.msg import Fault -from ros2_medkit_msgs.srv import ListFaults, ReportFault - -rclpy.init() -node = rclpy.create_node('per_source_clear_test_client') - - -def call(client, request): - future = client.call_async(request) - rclpy.spin_until_future_complete(node, future, timeout_sec=10.0) - return future.result() - - -if sys.argv[1] == 'report': - client = node.create_client(ReportFault, '/fault_manager/report_fault') - client.wait_for_service(timeout_sec=10.0) - request = ReportFault.Request() - request.fault_code = sys.argv[2] - request.event_type = ReportFault.Request.EVENT_FAILED - request.severity = Fault.SEVERITY_ERROR - request.description = 'second source' - request.source_id = sys.argv[3] - print(json.dumps({'accepted': call(client, request).accepted})) -else: - client = node.create_client(ListFaults, '/fault_manager/list_faults') - client.wait_for_service(timeout_sec=10.0) - request = ListFaults.Request() - request.statuses = ['PREFAILED', 'PREPASSED', 'CONFIRMED', 'HEALED', 'CLEARED'] - print(json.dumps({f.reporting_sources[0]: f.status for f in call(client, request).faults - if f.fault_code == sys.argv[2] and f.reporting_sources})) -node.destroy_node() -rclpy.shutdown() -""" - - -def fault_manager_client(env, *args): - result = subprocess.run([sys.executable, '-c', FAULT_MANAGER_CLIENT, *args], env=env, - capture_output=True, text=True, timeout=60 * SCALE) - if result.returncode != 0: - raise AssertionError(f'fault manager client {args} failed:\n{result.stderr}') - return json.loads(result.stdout) +def gateway_records(client): + """Return {first reporting source: status} of every CODE record.""" + return {f.reporting_sources[0]: f.status for f in client.faults() + if f.fault_code == CODE and f.reporting_sources} -def wait_records(env, accept, deadline): +def wait_records(client, accept, deadline): """Poll a scaled deadline until accept(records) holds. Return the last records.""" end = time.monotonic() + deadline * SCALE records = {} while time.monotonic() < end: - records = fault_manager_client(env, 'list', CODE) + records = gateway_records(client) if accept(records): return records time.sleep(0.3) @@ -266,6 +226,7 @@ def main(): workdir = Path(tempfile.mkdtemp(prefix='opcua_per_source_clear_')) env = dict(os.environ, ROS_DOMAIN_ID=ros_domain_id) procs = [] + client = FaultManagerClient('per_source_clear_test_client', 1.0 * SCALE, 60 * SCALE) try: node_map = workdir / 'nodes.yaml' node_map.write_text( @@ -337,19 +298,19 @@ def main(): return 1 # A second source holds a record of the same code. - if not fault_manager_client(env, 'report', CODE, OTHER)['accepted']: + if not client.report(CODE, OTHER): print('the second source was refused', file=sys.stderr) return 1 set_register(server, server_log, 'StatusWord', 1) records = wait_records( - env, lambda r: r.get(ENTITY) == 'CONFIRMED' and r.get(OTHER) == 'CONFIRMED', 30) + client, lambda r: r.get(ENTITY) == 'CONFIRMED' and r.get(OTHER) == 'CONFIRMED', 30) if records.get(ENTITY) != 'CONFIRMED': print(f'the alarm never raised the gateway record: {records}', file=sys.stderr) return 1 set_register(server, server_log, 'StatusWord', 0) - records = wait_records(env, lambda r: r.get(ENTITY) == 'CLEARED', 30) + records = wait_records(client, lambda r: r.get(ENTITY) == 'CLEARED', 30) if records.get(ENTITY) != 'CLEARED': print(f"the cleared alarm did not clear the gateway's record: {records}\n" + fm_log.read_text(errors='replace')[-3000:], file=sys.stderr) @@ -362,6 +323,7 @@ def main(): finally: for proc in reversed(procs): terminate(proc) + client.close() shutil.rmtree(workdir, ignore_errors=True) From cfefb52e0101d3fac28ebdea8a47a11b90855f97 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Wed, 18 Nov 2026 21:30:04 +0100 Subject: [PATCH 52/52] test(fault_manager): the rosbag tests wait for the history they assert Both rosbag tests slept 3 s before their first fault and counted messages before each later one. A bag holds the messages the ring buffer kept: those received in the last duration_sec, after the end of the last post-fault window. rosbag_ring_buffer.RecordedTopicWatch keeps the receipt times of the recorded topic and of each EVENT_CONFIRMED, and waits until a fault reported now gets the history a test needs. test_06 asks for 1.2 s, as its 1.5 s duration needs 1.0 s of history. test_rosbag_history also gets one answer on each client before it acts. --- .../test/integration/rosbag_ring_buffer.py | 77 +++++++++++++++ .../integration/test_rosbag_history.test.py | 75 +++++--------- .../test_rosbag_integration.test.py | 97 +++++-------------- 3 files changed, 122 insertions(+), 127 deletions(-) create mode 100644 src/ros2_medkit_fault_manager/test/integration/rosbag_ring_buffer.py diff --git a/src/ros2_medkit_fault_manager/test/integration/rosbag_ring_buffer.py b/src/ros2_medkit_fault_manager/test/integration/rosbag_ring_buffer.py new file mode 100644 index 000000000..293207504 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/rosbag_ring_buffer.py @@ -0,0 +1,77 @@ +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The pre-fault history a rosbag capture holds, seen from a second subscriber. + +The capture keeps the messages of the last duration_sec in its ring buffer and stamps +each with its receipt time. A bag starts at the oldest buffered message. A confirmation +empties the buffer and sends the next duration_after_sec of messages to the bag, and a +publisher that stalls leaves the buffer short of old messages. +""" + +import time + +import rclpy +from rclpy.qos import QoSProfile, ReliabilityPolicy +from ros2_medkit_msgs.msg import FaultEvent + +# Timer jitter allowed at the end of a post-fault window. +WINDOW_JITTER_SEC = 0.2 + + +class RecordedTopicWatch: + """Receipt times of a recorded topic, and the post-fault windows that divert it.""" + + def __init__(self, node, topic, msg_type, qos, duration_sec, duration_after_sec): + self.node = node + self.duration_sec = duration_sec + self.duration_after_sec = duration_after_sec + self.receipts = [] + self.buffering_since = 0.0 + node.create_subscription(msg_type, topic, self._on_message, qos) + node.create_subscription( + FaultEvent, '/fault_manager/events', self._on_event, + QoSProfile(depth=100, reliability=ReliabilityPolicy.RELIABLE)) + + def _on_message(self, _msg): + now = time.monotonic() + self.receipts.append(now) + self.receipts = [t for t in self.receipts if now - t <= 2 * self.duration_sec] + + def _on_event(self, msg): + if msg.event_type == FaultEvent.EVENT_CONFIRMED: + self.buffering_since = ( + time.monotonic() + self.duration_after_sec + WINDOW_JITTER_SEC) + + def holds_history(self, history_sec, latency_sec): + """ + Return True when a fault confirmed within gets of history. + + That needs a message received after the last post-fault window, at least + old now and still inside the buffer when the flush runs. + """ + now = time.monotonic() + oldest_kept = now + latency_sec - self.duration_sec + return any(max(self.buffering_since, oldest_kept) <= t <= now - history_sec + for t in self.receipts) + + def wait_for_history(self, history_sec, timeout_sec, latency_sec=0.3): + """Spin until holds_history(); False after .""" + deadline = time.monotonic() + timeout_sec + while time.monotonic() < deadline: + rclpy.spin_once(self.node, timeout_sec=0.05) + if self.holds_history(history_sec, latency_sec): + return True + return False diff --git a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_history.test.py b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_history.test.py index ec6879b8a..e169cd4c8 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_history.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_history.test.py @@ -29,6 +29,7 @@ """ import os +import sys import tempfile import time import unittest @@ -42,10 +43,14 @@ import rclpy from rclpy.node import Node from rclpy.qos import HistoryPolicy, QoSProfile, ReliabilityPolicy -from ros2_medkit_msgs.msg import Fault, FaultEvent +from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import ClearFault, GetRosbag, ListRosbags, ReportFault from sensor_msgs.msg import Temperature +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from rosbag_ring_buffer import RecordedTopicWatch # noqa: E402, I100 +from service_round_trip import answer_once_each # noqa: E402, I100 + def get_coverage_env(): """Get environment variables for gcov coverage data collection.""" @@ -155,8 +160,8 @@ def main(): 'database_path': DATABASE_PATH, 'confirmation_threshold': -1, 'snapshots.rosbag.enabled': True, - 'snapshots.rosbag.duration_sec': 2.0, - 'snapshots.rosbag.duration_after_sec': 0.5, + 'snapshots.rosbag.duration_sec': RING_BUFFER_SEC, + 'snapshots.rosbag.duration_after_sec': POST_FAULT_SEC, 'snapshots.rosbag.topics': '/test/temperature,/test/status', 'snapshots.rosbag.format': 'mcap', 'snapshots.rosbag.storage_path': ROSBAG_STORAGE_PATH, @@ -209,6 +214,9 @@ def start_fault_manager_when_publisher_up(event): # Budget for the fault manager to come up, from the start of setUpClass. The # ctest TIMEOUT is 240 s, so the budget is well inside it. PRECONDITION_BUDGET_SEC = 70.0 +# The capture's ring buffer and post-fault window. +RING_BUFFER_SEC = 2.0 +POST_FAULT_SEC = 0.5 class TestRosbagHistory(unittest.TestCase): @@ -269,12 +277,16 @@ def node_is_up(): break time.sleep(0.2) assert not missing, 'fault_manager not ready: ' + ', '.join(missing) + answer_once_each(cls.node, tuple(clients.values()), PRECONDITION_BUDGET_SEC) - deadline = time.time() + 15.0 - while (not cls.node.get_publishers_info_by_topic('/test/temperature') - and time.time() < deadline): - time.sleep(0.2) - time.sleep(3.0) + # Matches the publisher's BEST_EFFORT sensor QoS, else no messages arrive. + cls.recorded = RecordedTopicWatch( + cls.node, '/test/temperature', Temperature, + QoSProfile(reliability=ReliabilityPolicy.BEST_EFFORT, + history=HistoryPolicy.KEEP_LAST, depth=10), + RING_BUFFER_SEC, POST_FAULT_SEC) + assert cls.recorded.wait_for_history(0.5, 18.0), \ + 'the recorded topic never reached the ring buffer' @classmethod def tearDownClass(cls): @@ -331,50 +343,9 @@ def _get_by_recording(self, recording_id): request.recording_id = recording_id return self._call_service(self.get_rosbag_client, request) - def _wait_for_buffered_data(self, count=5, timeout=8.0): - """ - Wait until the ring buffer holds fresh data before a confirmation. - - A previous fault's post-roll diverts incoming messages straight into the - open bag, so right after that window closes the buffer can be empty and - the next confirmation produces no bag at all. Reset the streak on every - CONFIRMED so the messages counted here provably arrived with no post-roll - newly opened partway through - see the same helper in - test_rosbag_integration. - """ - received = 0 - - def _cb(_msg): - nonlocal received - received += 1 - - def _on_event(msg): - nonlocal received - if msg.event_type == FaultEvent.EVENT_CONFIRMED: - received = 0 - - qos = QoSProfile( - reliability=ReliabilityPolicy.BEST_EFFORT, - history=HistoryPolicy.KEEP_LAST, - depth=10, - ) - events_qos = QoSProfile( - reliability=ReliabilityPolicy.RELIABLE, - history=HistoryPolicy.KEEP_LAST, - depth=100, - ) - sub = self.node.create_subscription(Temperature, '/test/temperature', _cb, qos) - events_sub = self.node.create_subscription( - FaultEvent, '/fault_manager/events', _on_event, events_qos - ) - try: - deadline = time.time() + timeout - while received < count and time.time() < deadline: - rclpy.spin_once(self.node, timeout_sec=0.1) - finally: - self.node.destroy_subscription(sub) - self.node.destroy_subscription(events_sub) - return received >= count + def _wait_for_buffered_data(self, history_sec=0.5, timeout=8.0): + """Wait until a fault confirmed now gets of pre-fault history.""" + return self.recorded.wait_for_history(history_sec, timeout) def _record_occurrence(self, fault_code, known_ids): """Confirm the fault once and return its new recording.""" diff --git a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py index f9f5bdee6..09cc9dc2b 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py @@ -30,11 +30,12 @@ import rclpy from rclpy.node import Node from rclpy.qos import HistoryPolicy, QoSProfile, ReliabilityPolicy -from ros2_medkit_msgs.msg import Fault, FaultEvent +from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import ClearFault, GetRosbag, ReportFault from sensor_msgs.msg import Temperature sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from rosbag_ring_buffer import RecordedTopicWatch # noqa: E402, I100 from service_round_trip import answer_once_each # noqa: E402, I100 @@ -68,6 +69,9 @@ def get_coverage_env(): 'w', suffix='.yaml', prefix='rosbag_per_source_', delete=False) PER_SOURCE_CODES_FILE.write('BURST_PER_SOURCE:\n per_source: true\n') PER_SOURCE_CODES_FILE.close() +# The capture's ring buffer and post-fault window. +RING_BUFFER_SEC = 2.0 +POST_FAULT_SEC = 0.5 def generate_test_description(): @@ -170,8 +174,8 @@ def main(): 'fault_thresholds.config_file': PER_SOURCE_CODES_FILE.name, # Rosbag configuration 'snapshots.rosbag.enabled': True, - 'snapshots.rosbag.duration_sec': 2.0, # 2 second buffer - 'snapshots.rosbag.duration_after_sec': 0.5, # 0.5 second after confirm + 'snapshots.rosbag.duration_sec': RING_BUFFER_SEC, + 'snapshots.rosbag.duration_after_sec': POST_FAULT_SEC, # Use explicit topics - faster than discovery, more reliable for tests 'snapshots.rosbag.topics': '/test/temperature,/test/status', 'snapshots.rosbag.format': 'mcap', @@ -290,14 +294,14 @@ def node_is_up(): assert not missing, 'fault_manager not ready: ' + ', '.join(missing) answer_once_each(cls.node, tuple(clients.values()), PRECONDITION_BUDGET_SEC) - # Wait for the background publisher to come up before letting the rosbag - # ring buffer fill (duration_sec=2.0 configured), so a slow publisher - # start does not leave the buffer empty. - deadline = time.time() + 15.0 - while (not cls.node.get_publishers_info_by_topic('/test/temperature') - and time.time() < deadline): - time.sleep(0.2) - time.sleep(3.0) + # Matches the publisher's BEST_EFFORT sensor QoS, else no messages arrive. + cls.recorded = RecordedTopicWatch( + cls.node, '/test/temperature', Temperature, + QoSProfile(reliability=ReliabilityPolicy.BEST_EFFORT, + history=HistoryPolicy.KEEP_LAST, depth=10), + RING_BUFFER_SEC, POST_FAULT_SEC) + assert cls.recorded.wait_for_history(0.5, 18.0), \ + 'the recorded topic never reached the ring buffer' @classmethod def tearDownClass(cls): @@ -341,59 +345,9 @@ def _wait_for_rosbag(self, fault_code, timeout=12.0): response = self._call_service(self.get_rosbag_client, request) return response - def _wait_for_buffered_data(self, count=5, timeout=8.0): - """ - Wait until the rosbag ring buffer holds fresh data before a confirmation. - - A previous fault's post-fault recording window diverts incoming messages - away from the ring buffer - message_callback() writes straight to the open - bag and returns before it ever touches the buffer - so right after the - window closes the buffer can be empty and flush_to_bag creates no bag. - There is no service to read the buffer, so this counts messages on the same - source the fault manager records instead, as a proxy. Counting alone is not - a safe proxy: a confirmation landing mid-count reopens that same diversion, - so ``count`` messages arriving at this node can still correspond to - near-zero buffered history on the fault manager's side. The broad topic - modes already subscribe to ``/fault_manager/events``, so this does too, and - resets the count on every ``EVENT_CONFIRMED`` - guaranteeing the streak of - messages this function finally returns on arrived with no post-fault window - newly opened partway through it. - """ - received = 0 - - def _cb(_msg): - nonlocal received - received += 1 - - def _on_event(msg): - nonlocal received - if msg.event_type == FaultEvent.EVENT_CONFIRMED: - received = 0 - - # Match the publisher's BEST_EFFORT sensor QoS, else no messages arrive. - qos = QoSProfile( - reliability=ReliabilityPolicy.BEST_EFFORT, - history=HistoryPolicy.KEEP_LAST, - depth=10, - ) - # Matches the fault manager's own reliable, depth-100 events publisher. - events_qos = QoSProfile( - reliability=ReliabilityPolicy.RELIABLE, - history=HistoryPolicy.KEEP_LAST, - depth=100, - ) - sub = self.node.create_subscription(Temperature, '/test/temperature', _cb, qos) - events_sub = self.node.create_subscription( - FaultEvent, '/fault_manager/events', _on_event, events_qos - ) - try: - deadline = time.time() + timeout - while received < count and time.time() < deadline: - rclpy.spin_once(self.node, timeout_sec=0.1) - finally: - self.node.destroy_subscription(sub) - self.node.destroy_subscription(events_sub) - return received >= count + def _wait_for_buffered_data(self, history_sec=0.5, timeout=8.0): + """Wait until a fault confirmed now gets of pre-fault history.""" + return self.recorded.wait_for_history(history_sec, timeout) def test_01_rosbag_created_on_fault_confirmation(self): """Test that rosbag file is created when fault is confirmed.""" @@ -535,20 +489,13 @@ def test_06_rosbag_contains_recorded_duration(self): """ Test that a full-buffer recording reports ~the configured window. - ``duration_sec`` is the recording's real span, not the configured - ``duration_sec + duration_after_sec``. The two only coincide when the - ring buffer actually held a full pre-fault window at confirmation, so - this test waits for that many messages (25 at the publisher's 10 Hz = - 2.5s > the 2.0s buffer) instead of the 5-message default, which only - guarantees ~0.5s of history and would legitimately report ~1.4s. + ``duration_sec`` is the recording's real span: its pre-fault history plus + the post-fault window. 1.5 s needs 1.0 s of history, so the test waits + until the buffer holds 1.2 s. """ fault_code = 'DURATION_TEST' - # Report fault only once the buffer holds a FULL pre-fault window. - # Same refill race as test_01/test_02: the previous test's post-fault window - # diverts messages away from the ring buffer, so reporting straight after it - # closes finds the buffer empty and no bag is created. - self.assertTrue(self._wait_for_buffered_data(count=25), + self.assertTrue(self._wait_for_buffered_data(history_sec=1.2), 'ring buffer never refilled after the previous post-fault window') response = self._report_fault(fault_code, 'Duration test fault') self.assertTrue(response.accepted)