diff --git a/docs/api/messages.rst b/docs/api/messages.rst index 1b796d66e..7e14a432c 100644 --- a/docs/api/messages.rst +++ b/docs/api/messages.rst @@ -15,7 +15,17 @@ Messages Fault.msg ~~~~~~~~~ -Core fault data model representing an aggregated fault condition. +Core fault data model representing one fault record. + +A record is identified by the pair (``fault_code``, owner). A code listed with +``per_source: true`` in ``fault_thresholds.yaml`` has one record per reporting source, owned by +the ``source_id`` of the ``ReportFault`` call. Every other code has one record shared by all +its sources, owned by the reserved id ``*``. Each record has its own status, debounce counter, +``occurrence_count``, severity and timestamps, and is cleared on its own. + +``owner`` names the record a ``Fault`` describes. Passed back as ``source_id``, it addresses exactly +that record, also when a stored shared record and a per-source record of one source carry the same +``reporting_sources`` after the code changed from shared to ``per_source``. .. code-block:: text @@ -49,9 +59,14 @@ Core fault data model representing an aggregated fault condition. # Current fault status (PREFAILED, PREPASSED, CONFIRMED, HEALED, CLEARED) string status - # List of source identifiers that have reported this fault + # Every source that reported this record: one for a per-source record, all of + # them for a shared one. string[] reporting_sources + # Owner of the record this Fault describes: the reporting source of a + # per-source record, "*" for a shared record. + string owner + **Severity Constants:** .. list-table:: @@ -146,7 +161,10 @@ prefix (for example ``/robot1/fault_manager/events``). MutedFaultInfo.msg ~~~~~~~~~~~~~~~~~~ -Information about correlated (muted) symptom faults. +Information about a correlated (muted) symptom record. One entry per muted RECORD: +a root cause mutes the matching symptom records of every owner, so two per-source records +muted on one ``fault_code`` produce two entries carrying that code, told apart by +``source_id``. .. code-block:: text @@ -154,6 +172,7 @@ Information about correlated (muted) symptom faults. string root_cause_code # Root cause that triggered muting string rule_id # Correlation rule ID that matched uint32 delay_ms # Time delay from root cause [ms] + string source_id # Owner of the muted record (``*`` for a shared record) ClusterInfo.msg ~~~~~~~~~~~~~~~ @@ -172,6 +191,12 @@ Auto-detected fault cluster information. uint32 count # Number of faults builtin_interfaces/Time first_at # First fault timestamp builtin_interfaces/Time last_at # Last fault timestamp + string[] fault_owners # Owner of each member, parallel to fault_codes + string representative_owner # Owner of the representative record + +A cluster holds the matching records of every owner, so a member is the record +``(fault_codes[i], fault_owners[i])``. Pass the owner as ``source_id`` to the +single-record services. The owner of a shared record is ``*``. Services -------- @@ -189,7 +214,8 @@ Report a fault event to the FaultManager. uint8 event_type # EVENT_FAILED (0) or EVENT_PASSED (1) uint8 severity # Fault.SEVERITY_* constant (for FAILED events) string description # Human-readable description - string source_id # Fully qualified node name (e.g., "/powertrain/temp_sensor") + string source_id # Fully qualified node name (e.g., "/powertrain/temp_sensor"). + # Owns the record of a per_source code. "*" is reserved and refused. **Response:** @@ -215,7 +241,7 @@ Report a fault event to the FaultManager. ClearFault.srv ~~~~~~~~~~~~~~ -Clear/acknowledge a fault. +Clear/acknowledge one fault record. **Request:** @@ -223,21 +249,35 @@ Clear/acknowledge a fault. string fault_code # Fault code to clear bool skip_correlation_auto_clear # Opt out of correlation cascade clear + string source_id # Owner of the record, or a source that reports into it **Response:** .. code-block:: text - bool success # True if fault was found and cleared + bool success # True if the record was found and cleared string message # Status message or error description string[] auto_cleared_codes # Symptoms auto-cleared with root cause +``source_id`` names the owner of the record to clear (``*`` for a shared record). When no +record has that owner, it names the record that source reports into. Leaving it empty is +unscoped: the call applies only when exactly one record carries the ``fault_code``, and fails +otherwise with ``success=false`` and a ``message`` beginning ``ambiguous:`` that lists the owners. +An ambiguous call clears nothing. ``GetFault``, ``GetSnapshots`` and ``GetRosbag`` carry +the same field and resolve the same way. On ``GetRosbag`` it scopes the ``fault_code`` +lookup only, and the ``recording_id`` path ignores it. + +``GetRosbag`` answers with ``fault_codes`` and ``owners``, parallel arrays with one entry per +fault record the recording is attached to. A burst of several per-source records of one code +lists the code once per owner. A shared record is one entry carrying the owner ``*``. + When ``skip_correlation_auto_clear`` is ``false`` (default), clearing a root-cause fault also clears every symptom that the correlation engine attributes to it via ``auto_clear_with_root`` rules; the cleared symptom codes are returned in ``auto_cleared_codes``. When ``true``, only the requested ``fault_code`` is cleared and ``auto_cleared_codes`` -is empty. The gateway's per-entity ``DELETE +is empty. Either way the cleared root cause ends its relations: the symptoms it +muted are unmuted, and it no longer mutes later reports. The gateway's per-entity ``DELETE /{entity-path}/faults/{fault_code}`` route sets this to ``true`` so that an operator with access to one entity cannot cascade-clear correlated symptoms reported by apps in other entities. The global @@ -246,11 +286,18 @@ cluster-wide clearing still works. .. note:: - Added in ``ros2_medkit_msgs`` post-0.4.0. Adding a request field - changes the service type hash, so out-of-tree callers that invoke - ``/fault_manager/clear_fault`` directly via ``ros2 service call`` or - a generated client must rebuild against the new ``ros2_medkit_msgs`` - release to keep talking to ``fault_manager``. + ``skip_correlation_auto_clear`` was added in ``ros2_medkit_msgs`` post-0.4.0, + and ``source_id`` after it (the same field was added to ``GetFault``, + ``GetSnapshots`` and ``GetRosbag``). Adding a field changes the type hash, and + a field added inside an array element changes the wire layout, so every client + of the fault manager's services and topics must be rebuilt against this + ``ros2_medkit_msgs``. This includes out-of-tree callers that use ``ros2 + service call`` or a generated client. The case that breaks without an error is + ``ListFaults``: ``MutedFaultInfo.source_id`` and ``ClusterInfo.fault_owners`` and + ``representative_owner`` sit inside the elements of the ``muted_faults`` and + ``clusters`` arrays, CDR has no per-element length, and a client built against + an earlier ``ros2_medkit_msgs`` misreads any response that carries a muted or + clustered entry. Trailing fields do not keep old clients working. ListFaults.srv ~~~~~~~~~~~~~~ diff --git a/docs/config/fault-manager.rst b/docs/config/fault-manager.rst index e7590a081..40d42cd68 100644 --- a/docs/config/fault-manager.rst +++ b/docs/config/fault-manager.rst @@ -2,6 +2,8 @@ Fault Manager Configuration =========================== The ``ros2_medkit_fault_manager`` node aggregates and manages faults from multiple sources. +By default one fault code is one record, shared by all the sources that report it. A code +can be switched to one record per reporting source, see `Record Identity per Fault Code`_. This page documents all configuration parameters. .. contents:: Table of Contents @@ -57,10 +59,17 @@ it statically, so it needs network access at configure time. rosdep installs not A build without the option stops at startup when ``storage_type`` is ``postgres``. - Supported servers: PostgreSQL 14 and newer. -- One database belongs to exactly one fault manager. Fault codes are the primary key, so two fault - managers on one database merge their faults and delete each other's rosbag rows. Give each fault - manager its own database, or its own schema with ``options=-csearch_path=`` in the - connection string. +- One database belongs to exactly one fault manager. A record is keyed by ``(fault_code, owner)``, + so two fault managers on one database merge their records and delete each other's rosbag rows. + Give each fault manager its own database, or its own schema with + ``options=-csearch_path=`` in the connection string. +- Tables written before records were keyed by owner (one row per fault code) are migrated in place + on the first connection: ``owner`` is added to ``faults``, ``freeze_frames``, ``snapshots``, + ``near_misses`` and ``rosbag_files``, the primary keys on the fault code are replaced by unique + indexes that include it, and every existing row becomes the shared record of its code (owner + ``*``), keeping its ``reporting_sources``. The same rules as for SQLite apply, see the package + README. The migration runs in one transaction. It is one way: opening a migrated database with + an earlier release is unsupported, and a copy taken before the upgrade is the only way back. - The audit log stays a local SQLite file, see ``audit_log.database_path``. - The node never logs ``database_url``. It logs host, port, database and user, or the service name. Error text from the server is logged and returned with the password from ``database_url`` @@ -199,6 +208,21 @@ The fault manager uses AUTOSAR DEM-style debounce filtering to prevent fault fla which is also the default. ``0`` is rejected: the threshold must be strictly negative, and the node falls back to ``-1`` with a warning. Faults with ``SEVERITY_CRITICAL`` always bypass debounce regardless of this setting. + The bypass (``critical_immediate_confirm`` in the debounce config, on by default and not a ROS + parameter) depends on the record's identity. A record shared by all sources of a code (the + default) bypasses debounce on the severity of the **report**. A ``per_source`` record bypasses it + on its **stored** severity, the highest ever reported for that record, so after a CRITICAL + occurrence is cleared, one WARN report from the same source confirms the next occurrence at once, + and the record shows CRITICAL. The same holds for a FAILED report on a HEALED CRITICAL + ``per_source`` record: the latch does not hold it back. + +.. note:: + + A FAILED report whose severity differs from the stored severity of its ``per_source`` record is + logged as a warning naming the code, both severities and the source. A source should report one code + with one severity, so the warning points at a misconfigured reporter. It is logged once per + code, source and reported severity until the node restarts, so a reporter that repeats the + report at a high rate produces one line. .. important:: @@ -235,25 +259,28 @@ A **near miss** is a FAILED report that moved the debounce counter without the f CONFIRMED - the fault nearly happened. PASSED reports move the counter in the healing direction (the fault receding) and are not near misses. -The fault manager appends one entry per near miss to a per-fault-code series, holding the -timestamp, the counter value after the report, the confirmation threshold, the severity, the -reporting source and the fault status the report left behind. +The fault manager appends one entry per near miss to the series of the fault record it moved (one +series per record: per fault code, or per fault code and reporting source for a ``per_source`` +code), holding the timestamp, the counter value after the report, the confirmation threshold, the +severity the report carried (not the stored severity of the record), the reporting source and the +fault status the report left behind. The status matters when reading the series. The HEALED latch holds the status all the way from the healing threshold down to the confirmation threshold, so reports on the way back into a fault that does confirm are also near misses by the definition above. Entries recording ``PREFAILED`` are approaches from a resting state; entries recording ``HEALED`` are a counter walking back down under the latch. The recorded confirmation threshold belongs to the reporting source, while the counter -is shared by all sources of that fault code, so with per-entity thresholds it is not on its own the -distance to confirmation. The series is **retained when the fault is cleared**, because acknowledging one -fault cycle must not erase how often that code approached confirmation across cycles. +is shared by all sources of a shared record, so with per-entity thresholds it is not on its own the +distance to confirmation (a ``per_source`` record has one source, and it is). The series is +**retained when the fault is cleared**, because acknowledging one fault cycle must not erase how +often that record approached confirmation across cycles. .. code-block:: yaml fault_manager: ros__parameters: near_miss: - max_per_fault: 200 # Entries kept per fault code (0 = unlimited) + max_per_fault: 200 # Entries kept per fault record (0 = unlimited) .. list-table:: :header-rows: 1 @@ -264,7 +291,7 @@ fault cycle must not erase how often that code approached confirmation across cy - Description * - ``near_miss.max_per_fault`` - ``200`` - - Near-miss entries retained per fault code. When the bound is reached the **oldest** + - Near-miss entries retained per fault record. When the bound is reached the **oldest** entries are evicted, the same direction as ``snapshots.max_per_fault`` and the rosbag cap: a series frozen at boot says nothing about whether the rate of near misses is changing. Set to 0 for unlimited, accepting growth with the reporting rate. @@ -343,8 +370,9 @@ threshold overrides: When multiple entities report the same ``fault_code``, each event applies the thresholds resolved from that event's ``source_id``. This means the debounce behavior follows the reporting entity, not the fault. The debounce counter, - however, belongs to the fault code, so the two entities share one counter under - two policies - see `Per-Fault-Code Thresholds`_ for what that does and how to + however, belongs to the record, and the record of a code is shared by all its + sources unless the code is ``per_source``, so the two entities share one counter + under two policies - see `Per-Fault-Code Thresholds`_ for what that does and how to settle it. The node warns when it happens. ``auto_confirm_after_sec`` is global-only and cannot be overridden per-entity. @@ -355,10 +383,10 @@ Per-Fault-Code Thresholds ~~~~~~~~~~~~~~~~~~~~~~~~~ Per-entity thresholds are the right tool when you know which subsystems are noisy but not -which codes they will emit. They have one limit: the debounce counter is kept per -``fault_code``, while the override is chosen per ``source_id``. Two entities reporting one -code therefore share a counter and debounce it under two policies, and the report that -happens to arrive decides the transition: +which codes they will emit. They have one limit: the debounce counter is kept per record, and +a code's record is shared by all its sources, while the override is chosen per ``source_id``. +Two entities reporting one code therefore share a counter and debounce it under two policies, +and the report that happens to arrive decides the transition: .. code-block:: text @@ -377,7 +405,7 @@ that: they are matched on the code itself, so they resolve the same whoever repo config_file: "/etc/ros2_medkit/fault_thresholds.yaml" The file is a map of fault codes to threshold overrides, the same three fields an entity -override carries: +override carries, plus ``per_source`` (see `Record Identity per Fault Code`_): .. code-block:: yaml @@ -428,22 +456,65 @@ override carries: **When one code is debounced two ways** The node resolves the policy for every report, and warns the first time two sources resolve -different policies for one code: +different policies for the shared record of one code: .. code-block:: text [WARN] Fault code 'OVERHEAT' is debounced two ways: '/powertrain/motor/left' resolves confirmation=-5 healing_enabled=false healing=10, '/sensors/lidar/front' resolves - confirmation=-1 healing_enabled=true healing=1. The debounce counter belongs to the - fault code, so whichever source reports decides the transition and the other policy is - bypassed. Pin confirmation_threshold, healing_enabled and healing_threshold for this code - in fault_thresholds.config_file. + confirmation=-1 healing_enabled=true healing=1. The debounce counter belongs to the shared + record of the fault code, so whichever source reports decides the transition and the other + policy is bypassed. Pin confirmation_threshold, healing_enabled and healing_threshold for + this code in fault_thresholds.config_file, or set per_source: true there to give each source + its own record. The warning names both sources and both resolved policies, and is emitted **once per fault code** for the life of the node, so a busy reporter does not turn it into a log storm. It is a diagnostic, not an error: the configuration is legal and the fault is still debounced, just not under a policy an operator chose. Giving the code an entry in -``fault_thresholds.config_file`` that pins all three fields ends it. +``fault_thresholds.config_file`` that pins all three fields, or ``per_source: true``, ends it. +A ``per_source`` code has one source per record and never warns. + +Record Identity per Fault Code +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A fault record is identified by the pair (``fault_code``, owner). By default a code has **one record +shared by all its reporting sources**: a FAILED from any source moves the one debounce counter, a +PASSED from any source moves it back, ``reporting_sources`` lists every source that reported, and a +clear ends the whole record. The shared record is owned by the reserved id ``*``. + +``per_source: true`` on a code's entry in ``fault_thresholds.config_file`` gives each reporting +source **its own record** of that code, owned by the ``source_id`` of its reports. Each record has its +own status, debounce counter, ``occurrence_count``, severity, snapshots, near-miss series and +recordings, and is cleared and healed on its own. + +.. code-block:: yaml + + # fault_thresholds.yaml + BATTERY_LOW: + per_source: true # one record per reporting source + confirmation_threshold: -3 + + ESTOP: + confirmation_threshold: -1 # per_source absent: one record shared by all sources + +- ``per_source`` is a YAML boolean, ``true`` or ``false``. Absent means ``false``, and so does a code + with no entry. Any other value (``yes``, ``3``) rejects the whole file with an error, the same as + any other malformed value in it. Entity overrides do not carry the key. +- ``ReportFault`` refuses ``*`` as a ``source_id``, because it names the shared record. +- The services that act on one record (``ClearFault``, ``GetFault``, ``GetSnapshots``, + ``GetRosbag``) take a ``source_id``. It names the record it owns (``*`` for the shared one). When no + record has that owner, it names the record that source reports into, so a client that knows only a + reporting source still reaches the shared record of a default code. An empty ``source_id`` applies + when exactly one record carries the code, and is refused as ``ambiguous:`` when several do. +- A record belongs to an entity when the entity is one of its ``reporting_sources``. +- The file is read at startup. When a code is switched between the two modes, the records already + stored keep the identity they were written with, and new reports go to the configured one. The + node logs one warning per code that holds active records of the other identity, naming the + code, its configured mode and the owners found. Nothing stored is rewritten. +- A database written when every code was one record opens with each row as the shared record of its + code, keeping its ``reporting_sources``, with its snapshots, freeze frame, recordings and near + misses attached to it. Snapshot Configuration ---------------------- @@ -464,8 +535,8 @@ Basic Snapshot Settings max_message_size: 65536 # Max message size in bytes (64KB) default_topics: [] # Topics to capture for all faults config_file: "" # Path to YAML config file - recapture_cooldown_sec: 60.0 # Min seconds between snapshot captures per fault - max_per_fault: 10 # Max snapshots stored per fault code (0 = unlimited) + recapture_cooldown_sec: 60.0 # Min seconds between snapshot captures per fault record + max_per_fault: 10 # Max snapshots stored per fault record (0 = unlimited) capture_pool_size: 2 # Max concurrent capture threads (1-64) capture_queue_depth: 16 # Max pending captures before policy applies (1-65536) capture_queue_full_policy: reject_newest # reject_newest | drop_oldest @@ -506,11 +577,13 @@ Basic Snapshot Settings - Path to YAML file with fault-specific snapshot configurations. * - ``snapshots.recapture_cooldown_sec`` - ``60.0`` - - Minimum seconds between snapshot captures for the same fault code. + - Minimum seconds between snapshot captures for the same fault record. Prevents snapshot storms when a fault is reported repeatedly. Set to 0 to disable. + Per-source records of one code that confirm inside the cooldown share one capture of + the code's topics: it is sampled once and every record lists it under its own owner. * - ``snapshots.max_per_fault`` - ``10`` - - Maximum number of snapshot rows stored per fault code. One confirmation + - Maximum number of snapshot rows stored per fault record. One confirmation writes one row per configured topic, and those rows are evicted together: past the limit the OLDEST capture set is dropped whole. A capture larger than the cap is kept anyway rather than torn, since half a freeze frame is @@ -563,7 +636,7 @@ Capture continuous rosbag recordings around fault events. max_buffer_mb: 256 # Ring-buffer RAM cap max_bag_size_mb: 50 # Max size per bag file max_total_storage_mb: 500 # Max total storage - max_bags_per_fault: 1 # Recordings kept per fault code + max_bags_per_fault: 1 # Recordings kept per fault record auto_cleanup: true # Auto-delete old bags .. list-table:: @@ -645,7 +718,7 @@ Capture continuous rosbag recordings around fault events. limit until a newer recording replaces it. * - ``rosbag.max_bags_per_fault`` - ``1`` - - How many recordings one fault code keeps. Past the cap the oldest is + - How many recordings one fault record keeps. Past the cap the oldest is unlinked, so the default reproduces the historical behaviour exactly: a new recording replaces the previous one. ``0`` means unlimited, bounded only by ``max_total_storage_mb``. ``3`` is a reasonable value for a fault @@ -784,6 +857,20 @@ An append-only, hash-chained record of fault state transitions, for deployments that have to show afterwards what the fault state was and when it changed. Off by default: with it off there is no table, no file and no write cost. +Every row records the stored severity of the fault record at the transition, +the value ``GET /faults`` and the fault events show. The severity a single +report carried is in the near-miss series and in the severity mismatch warning. + +The ``source`` of a row says what drove the transition. A report drives +``occurred`` and ``confirmed`` rows and its ``source_id`` is the source. An +automatic transition carries its mechanism: ``auto_confirm_timer``, +``auto_heal``, ``startup_reclassify``, ``clear_service`` (an operator clear) and +``correlation_auto_clear`` (a symptom cleared with its root cause). The +``owner`` of a row is the owner of a per-source record and is empty for the +shared record of a code, so a shared record's rows are unchanged and an existing +chain still verifies. A log created before the ``owner`` column existed gets the +column on open. + .. code-block:: yaml fault_manager: diff --git a/docs/tutorials/fault-correlation.rst b/docs/tutorials/fault-correlation.rst index 07357780c..316334509 100644 --- a/docs/tutorials/fault-correlation.rst +++ b/docs/tutorials/fault-correlation.rst @@ -120,6 +120,34 @@ How It Works fm --> src: [SENSOR_003]\nclusters: [{id: "sensor_storm_1",\n fault_codes: [001,002,003,004],\n representative: "SENSOR_003"}] @enduml +**Rules match by fault code across reporting nodes.** A root cause reported by one +node mutes, and with ``auto_clear_with_root`` auto-clears, the matching symptoms +reported by any other node inside the window. An auto-cluster collects the matching +faults of every node. The relations hold fault records, not codes: for a code +configured with ``per_source: true`` each reporting node has its own record. For any +other code there is one shared record, and that record is what is muted or cleared. + +A root cause lists the symptom records reported within ``window_ms`` of its latest +report. A new report of a root cause that is still active renews that window and keeps +its list. A record that is cleared or ends leaves the list of every root cause, so the +list holds only records of the current occurrence. + +- With ``mute_symptoms: true`` the listed records are muted: hidden from the default + listing and shown under ``include_muted``. They stay muted until the root cause ends. + The cascade of ``auto_clear_with_root`` clears exactly the records the root cause muted. +- With ``mute_symptoms: false`` the listed records are not muted. The cascade of + ``auto_clear_with_root`` still clears the records the root cause lists. + +Clearing the root cause ends its relations: with ``skip_correlation_auto_clear`` its +symptoms are not cleared, they keep their status. Every other end of a record does the +same: a root cause that heals, or that a cascade clears, releases the symptoms it +muted. Only a ``ClearFault`` without the flag cascades, and it goes one level. + +A record stays muted while any root cause that lists it under a muting rule, or its +active ``show_as_single`` cluster, still mutes it. When the relation that muted it ends, +a record that no other relation mutes is unmuted and shows in the default listing again. +The reason shown under ``include_muted`` names the relation that mutes it now. + Quick Start ----------- @@ -301,7 +329,15 @@ Auto-cluster mode groups similar faults into clusters. 1. First ``SENSOR_*`` fault starts a potential cluster 2. Additional ``SENSOR_*`` faults within 2000ms join the cluster 3. When 3+ faults accumulate, cluster becomes active -4. Only the representative fault is shown; others are muted +4. Only the representative fault is shown; the other members are muted. They are hidden + from the default listing and show under ``include_muted``. Muting is display only: + the record of a muted member, as ``GetFault`` returns it, is the record it has unmuted. +5. When the representative changes, the current representative is shown and the previous + one is muted. A member that escalates under ``highest_severity``, or a newer member + under ``most_recent``, becomes the shown one. When the shown member is cleared, the + next representative the policy selects is shown. +6. A cluster ends when its last member is cleared. A member that stops being muted keeps + its status: nothing is cleared for the operator. Querying Correlation Data ------------------------- diff --git a/docs/tutorials/snapshots.rst b/docs/tutorials/snapshots.rst index aa2dc16b6..2284b044d 100644 --- a/docs/tutorials/snapshots.rst +++ b/docs/tutorials/snapshots.rst @@ -122,11 +122,13 @@ Configure snapshot capture via fault manager parameters: appears after the fault manager starts is subscribed once it does. * - ``snapshots.recapture_cooldown_sec`` - ``60.0`` - - Minimum seconds between snapshot captures for the same fault code. + - Minimum seconds between snapshot captures for the same fault record. Prevents snapshot storms when a fault is reported repeatedly. Set to 0 to disable. + Per-source records of one code that confirm inside the cooldown share one capture of + the code's topics: it is sampled once and every record lists it under its own owner. * - ``snapshots.max_per_fault`` - ``10`` - - Maximum number of snapshot rows stored per fault code. One confirmation + - Maximum number of snapshot rows stored per fault record. One confirmation writes one row per configured topic, and those rows are evicted together: past the limit the OLDEST capture set is dropped whole. A capture larger than the cap is kept anyway rather than torn, since half a freeze frame @@ -593,7 +595,7 @@ Rosbag Configuration Options burst's bag at a time. * - ``snapshots.rosbag.max_bags_per_fault`` - ``1`` - - Recordings kept per fault code; ``0`` means unlimited. Past the cap the + - Recordings kept per fault record (``0`` means unlimited). Past the cap the fault's oldest recording is dropped, and a bag is deleted only once no fault still references it (a burst shares one recording). ``1`` is the historical behaviour - each re-confirmation replaces the previous bag; diff --git a/scripts/check_cores_without_ros/main.cpp b/scripts/check_cores_without_ros/main.cpp index 57c325552..a162ef9ff 100644 --- a/scripts/check_cores_without_ros/main.cpp +++ b/scripts/check_cores_without_ros/main.cpp @@ -109,13 +109,13 @@ class RecordingEventSink : public ros2_medkit_fault_manager::core::EventSink { class IgnoreCaptureHooks : public ros2_medkit_fault_manager::core::CaptureHooks { public: - ros2_medkit_fault_manager::core::CaptureResult capture_now(const std::string & code) override { - captured.push_back(code); + ros2_medkit_fault_manager::core::CaptureResult capture_now(const ros2_medkit_fault_manager::FaultId & id) override { + captured.push_back(id.fault_code); return {ros2_medkit_fault_manager::core::CaptureEnqueue::kAccepted, {}}; } - void on_prefailed(const std::string & /*code*/) override { + void on_prefailed(const ros2_medkit_fault_manager::FaultId & /*id*/) override { } - void on_cleared(const std::string & /*code*/) override { + void on_cleared(const ros2_medkit_fault_manager::FaultId & /*id*/) override { } std::vector captured; }; @@ -1524,8 +1524,8 @@ static int run() { fm::SqliteFaultStorage storage(db_path.string(), sink); fm::DebounceConfig debounce; debounce.confirmation_threshold = -1; - ok &= check(storage.report_fault_event("NO_ROS_FAULT", fd::ReportKind::kFailed, fd::severity::kError, "d", "/src", - 1000000000LL, debounce), + ok &= check(storage.report_fault_event(fm::FaultId{"NO_ROS_FAULT", fm::kSharedOwner}, fd::ReportKind::kFailed, + fd::severity::kError, "d", "/src", 1000000000LL, debounce), "a FAILED report creates the fault"); const auto faults = storage.list_faults(false, 0, {}); ok &= check(faults.size() == 1, "list_faults returns one record"); @@ -1596,7 +1596,7 @@ static int run() { "core list returns the fault CONFIRMED"); ok &= check(events.types.size() == 1 && events.types[0] == fd::fault_event_type::kConfirmed, "the sink saw exactly one fault_confirmed"); - const auto cleared = fault_core.clear("CORE_FAULT", false); + const auto cleared = fault_core.clear("CORE_FAULT", "", false); ok &= check(cleared.success, "core clear succeeds"); ok &= check(events.types.size() == 2 && events.types[1] == fd::fault_event_type::kCleared, "the sink saw fault_cleared after the clear"); diff --git a/src/ros2_medkit_fault_detection/include/ros2_medkit_fault_detection/fault_types.hpp b/src/ros2_medkit_fault_detection/include/ros2_medkit_fault_detection/fault_types.hpp index 66c378829..3fc0d6588 100644 --- a/src/ros2_medkit_fault_detection/include/ros2_medkit_fault_detection/fault_types.hpp +++ b/src/ros2_medkit_fault_detection/include/ros2_medkit_fault_detection/fault_types.hpp @@ -70,6 +70,8 @@ struct FaultRecord { uint32_t occurrence_count{0}; std::string status; std::vector reporting_sources; + /// Owner of the record: the reporting source of a per-source record, "*" for a shared one. + std::string owner; }; /// FaultEvent.msg field for field. diff --git a/src/ros2_medkit_fault_detection/test/test_fault_detection.cpp b/src/ros2_medkit_fault_detection/test/test_fault_detection.cpp index 6420ed377..5c4fa0313 100644 --- a/src/ros2_medkit_fault_detection/test/test_fault_detection.cpp +++ b/src/ros2_medkit_fault_detection/test/test_fault_detection.cpp @@ -127,10 +127,16 @@ TEST(StatusWordRule, NamedBitDecode) { // 0b1001 -> bit 0 and bit 3 set, bit 7 clear. auto out = fd::evaluate(fd::Value{static_cast(0b1001)}, r); ASSERT_EQ(out.size(), 3u); - EXPECT_TRUE(find(out, "E_STOP")->active); - EXPECT_TRUE(find(out, "PUMP_OVERLOAD")->active); - EXPECT_FALSE(find(out, "FILTER_DIRTY")->active); - EXPECT_EQ(find(out, "PUMP_OVERLOAD")->message, "Pump overload"); + const auto * e_stop = find(out, "E_STOP"); + const auto * pump = find(out, "PUMP_OVERLOAD"); + const auto * filter = find(out, "FILTER_DIRTY"); + ASSERT_NE(e_stop, nullptr); + ASSERT_NE(pump, nullptr); + ASSERT_NE(filter, nullptr); + EXPECT_TRUE(e_stop->active); + EXPECT_TRUE(pump->active); + EXPECT_FALSE(filter->active); + EXPECT_EQ(pump->message, "Pump overload"); } TEST(StatusWordRule, AllClearWhenZero) { @@ -174,13 +180,20 @@ TEST(StatusWordRule, WidthMasksSignExtendedHighBits) { const auto sign_extended = static_cast(0xFFFFFFFFFFFF8000ULL); auto out = fd::evaluate(fd::Value{sign_extended}, r); - EXPECT_TRUE(find(out, "REAL_FAULT")->active); - EXPECT_FALSE(find(out, "SPURIOUS")->active); + const auto * real = find(out, "REAL_FAULT"); + const auto * spurious = find(out, "SPURIOUS"); + ASSERT_NE(real, nullptr); + ASSERT_NE(spurious, nullptr); + EXPECT_TRUE(real->active); + EXPECT_FALSE(spurious->active); // Without a width the same value would spuriously fire bit 20. fd::StatusWordRule wide; wide.bits = {{20u, {"SPURIOUS", "ERROR", "spurious"}}}; - EXPECT_TRUE(find(fd::evaluate(fd::Value{sign_extended}, wide), "SPURIOUS")->active); + const auto wide_out = fd::evaluate(fd::Value{sign_extended}, wide); + const auto * wide_spurious = find(wide_out, "SPURIOUS"); + ASSERT_NE(wide_spurious, nullptr); + EXPECT_TRUE(wide_spurious->active); } // -- Fault-code enum --------------------------------------------------------- @@ -196,10 +209,16 @@ TEST(EnumMapRule, CodeMapsToFaultAndText) { auto out = fd::evaluate(fd::Value{static_cast(11)}, r); ASSERT_EQ(out.size(), 3u); - EXPECT_FALSE(find(out, "VFD_OVERVOLTAGE")->active); - EXPECT_TRUE(find(out, "VFD_OVERCURRENT")->active); - EXPECT_EQ(find(out, "VFD_OVERCURRENT")->message, "Output overcurrent"); - EXPECT_FALSE(find(out, "VFD_OVERTEMP")->active); + const auto * overvoltage = find(out, "VFD_OVERVOLTAGE"); + const auto * overcurrent = find(out, "VFD_OVERCURRENT"); + const auto * overtemp = find(out, "VFD_OVERTEMP"); + ASSERT_NE(overvoltage, nullptr); + ASSERT_NE(overcurrent, nullptr); + ASSERT_NE(overtemp, nullptr); + EXPECT_FALSE(overvoltage->active); + EXPECT_TRUE(overcurrent->active); + EXPECT_EQ(overcurrent->message, "Output overcurrent"); + EXPECT_FALSE(overtemp->active); } TEST(EnumMapRule, OkValueRaisesNothing) { @@ -221,18 +240,29 @@ TEST(EnumMapRule, UnmappedNonOkRaisesCatchAll) { auto out = fd::evaluate(fd::Value{static_cast(99)}, r); ASSERT_EQ(out.size(), 3u); - EXPECT_FALSE(find(out, "OVERVOLT")->active); - EXPECT_FALSE(find(out, "OVERCURR")->active); + const auto * overvolt = find(out, "OVERVOLT"); + const auto * overcurr = find(out, "OVERCURR"); const auto * unk = find(out, "VFD_UNMAPPED"); + ASSERT_NE(overvolt, nullptr); + ASSERT_NE(overcurr, nullptr); ASSERT_NE(unk, nullptr); + EXPECT_FALSE(overvolt->active); + EXPECT_FALSE(overcurr->active); EXPECT_TRUE(unk->active); EXPECT_EQ(unk->message, "unmapped fault code 99"); // ok and mapped values keep the catch-all inactive so the tracker clears it. - EXPECT_FALSE(find(fd::evaluate(fd::Value{static_cast(0)}, r), "VFD_UNMAPPED")->active); + const auto ok = fd::evaluate(fd::Value{static_cast(0)}, r); + const auto * ok_unk = find(ok, "VFD_UNMAPPED"); + ASSERT_NE(ok_unk, nullptr); + EXPECT_FALSE(ok_unk->active); auto mapped = fd::evaluate(fd::Value{static_cast(10)}, r); - EXPECT_TRUE(find(mapped, "OVERVOLT")->active); - EXPECT_FALSE(find(mapped, "VFD_UNMAPPED")->active); + const auto * mapped_overvolt = find(mapped, "OVERVOLT"); + const auto * mapped_unk = find(mapped, "VFD_UNMAPPED"); + ASSERT_NE(mapped_overvolt, nullptr); + ASSERT_NE(mapped_unk, nullptr); + EXPECT_TRUE(mapped_overvolt->active); + EXPECT_FALSE(mapped_unk->active); } TEST(EnumMapRule, NonFiniteValueHoldsState) { @@ -314,8 +344,12 @@ TEST(TransitionTracker, EnumCodeChangeClearsOldRaisesNew) { // Code switches 10 -> 11: clear OVERVOLT, raise OVERCURR in one step. auto changes = tracker.apply(fd::evaluate(fd::Value{static_cast(11)}, r)); ASSERT_EQ(changes.size(), 2u); - EXPECT_FALSE(find(changes, "OVERVOLT")->active); - EXPECT_TRUE(find(changes, "OVERCURR")->active); + const auto * overvolt = find(changes, "OVERVOLT"); + const auto * overcurr = find(changes, "OVERCURR"); + ASSERT_NE(overvolt, nullptr); + ASSERT_NE(overcurr, nullptr); + EXPECT_FALSE(overvolt->active); + EXPECT_TRUE(overcurr->active); } // -- Raise qualifier --------------------------------------------------------- diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index 08649ed0f..6330d9d0a 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -213,6 +213,14 @@ if(POSTGRES_SUPPORT) medkit_add_gtest(test_postgres_storage test/test_postgres_storage.cpp) target_link_libraries(test_postgres_storage fault_manager_lib) medkit_target_dependencies(test_postgres_storage rclcpp ros2_medkit_msgs) + + # SQL runner for the launch test that migrates a PostgreSQL database through the node. + add_executable(pg_exec test/pg_exec.cpp) + target_link_libraries(pg_exec fault_manager_lib) + install(TARGETS pg_exec DESTINATION lib/${PROJECT_NAME}) + + medkit_add_launch_test(test_fault_postgres_migration + test/integration/test_fault_postgres_migration.test.py TIMEOUT 180 LABELS "integration") endif() # Rosbag retention parity: every assertion runs against both storage backends. @@ -220,6 +228,11 @@ endif() target_link_libraries(test_rosbag_storage_parity fault_manager_lib) medkit_target_dependencies(test_rosbag_storage_parity rclcpp ros2_medkit_msgs) + # Severity contract of a fault record, run against both storage backends. + medkit_add_gtest(test_fault_severity test/test_fault_severity.cpp) + target_link_libraries(test_fault_severity fault_manager_lib) + medkit_target_dependencies(test_fault_severity rclcpp ros2_medkit_msgs) + # Fault audit log tests (hash chain, verify, rotation, reopen) medkit_add_gtest(test_fault_audit_log test/test_fault_audit_log.cpp) target_link_libraries(test_fault_audit_log fault_manager_lib) @@ -292,6 +305,39 @@ endif() medkit_add_launch_test(test_fault_code_healing_restart test/integration/test_fault_code_healing_restart.test.py TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_fault_identity + test/integration/test_fault_identity.test.py TIMEOUT 120 LABELS "integration") + + medkit_add_launch_test(test_fault_correlation_owners + test/integration/test_fault_correlation_owners.test.py TIMEOUT 120 LABELS "integration") + + medkit_add_launch_test(test_fault_audit_rows + test/integration/test_fault_audit_rows.test.py TIMEOUT 180 LABELS "integration") + + medkit_add_launch_test(test_fault_snapshot_grouping + test/integration/test_fault_snapshot_grouping.test.py TIMEOUT 120 LABELS "integration") + + medkit_add_launch_test(test_fault_snapshot_queue + test/integration/test_fault_snapshot_queue.test.py TIMEOUT 180 LABELS "integration") + + medkit_add_launch_test(test_fault_shared_events + test/integration/test_fault_shared_events.test.py TIMEOUT 120 LABELS "integration") + + medkit_add_launch_test(test_fault_snapshot_cap + test/integration/test_fault_snapshot_cap.test.py TIMEOUT 120 LABELS "integration") + + medkit_add_launch_test(test_fault_correlation_clear + test/integration/test_fault_correlation_clear.test.py TIMEOUT 120 LABELS "integration") + + medkit_add_launch_test(test_fault_cluster_muting + test/integration/test_fault_cluster_muting.test.py TIMEOUT 120 LABELS "integration") + + medkit_add_launch_test(test_fault_muting_relations + test/integration/test_fault_muting_relations.test.py TIMEOUT 120 LABELS "integration") + + medkit_add_launch_test(test_fault_identity_restart + test/integration/test_fault_identity_restart.test.py TIMEOUT 180 LABELS "integration") + # Drives healing with the event counts a one-event-per-transition reporter # actually sends: one FAILED per raise, one PASSED per clear. Parametrized over # healing_threshold, so the node launches twice, and one case holds a settled @@ -305,6 +351,10 @@ endif() medkit_add_launch_test(test_auto_confirm_visibility test/integration/test_auto_confirm_visibility.test.py TIMEOUT 120 LABELS "integration") + # Restarts the node twice on one SQLite database. + medkit_add_launch_test(test_severity_consistency test/integration/test_severity_consistency.test.py + TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_rosbag_entity_scope test/integration/test_rosbag_entity_scope.test.py TIMEOUT 120 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/README.md b/src/ros2_medkit_fault_manager/README.md index 47d2cc55e..e806cc8c3 100644 --- a/src/ros2_medkit_fault_manager/README.md +++ b/src/ros2_medkit_fault_manager/README.md @@ -5,8 +5,82 @@ Central fault manager node for the ros2_medkit fault management system. ## Overview The FaultManager node provides a central point for fault aggregation and lifecycle management. -It receives fault reports from multiple sources, aggregates them by `fault_code`, and provides -query and clearing interfaces. +It receives fault reports from multiple sources, keeps them as fault records, and provides query +and clearing interfaces. + +A record is identified by its `fault_code` and an **owner**. By default a code has **one record +shared by all its reporting sources**, owned by the reserved id `*`: a FAILED from any source moves +its one debounce counter, a PASSED from any source moves it back, `reporting_sources` lists every +source that reported it, and a clear ends the whole record. A code listed with `per_source: true` in +`fault_thresholds.config_file` instead has **one record per reporting source**, owned by the +`source_id` of its reports. Each record has its own status, debounce counter, occurrence count, +severity, timestamps, freeze frame, snapshots, near-miss series and rosbag links, and is cleared and +healed on its own. `ReportFault` refuses `*` as a `source_id`. + +The services that act on a single record (`~/clear_fault`, `~/get_fault`, `~/get_snapshots`, +`~/get_rosbag`) carry a `source_id` request field. It names the record it owns (`*` for a shared +one). When no record has that owner, it names the record that source reports into, so a client that +knows only a reporting source still reaches the shared record of a default code. Leaving it empty is +unscoped and applies only when exactly one record carries the `fault_code`, otherwise the call fails +with a message beginning `ambiguous:` that lists the owners and changes nothing. + +Every `Fault` the node sends (`~/list_faults`, `~/get_fault`, `~/list_faults_for_entity` and +`~/events`) carries the record's `owner`: the reporting source of a per-source record, `*` for a +shared one. It is the `source_id` that addresses exactly that record. + +### Identity per fault code + +```yaml +# fault_thresholds.config_file +BATTERY_LOW: + per_source: true # one record per reporting source + confirmation_threshold: -3 + +ESTOP: + confirmation_threshold: -1 # per_source absent: one record shared by all sources +``` + +`per_source` is a YAML boolean and defaults to `false`; a code with no entry is shared too. Any other +value (`yes`, `3`) rejects the whole file with an error, like every other malformed value in it. +Entity overrides do not carry the key. The file is read at startup. + +Switching a code between the two modes keeps the records already stored as they were written, and +sends new reports to the configured identity. For each code that holds active records of the other +identity the node logs one warning at startup, naming the code, its configured mode and the owners +found. Nothing stored is rewritten. + +A code reported by several sources and debounced under several policies (per-entity thresholds) has +one counter, so the node warns once per code when two sources resolve different policies for the +shared record. Set `per_source: true` or pin the policy in `fault_thresholds.config_file` to end it. + +### Upgrading a database written before the owner column + +A SQLite store written by an earlier release is keyed by the bare `fault_code` and is rebuilt on the +first open. That store holds one record per code, fed by every source, which is exactly what a +default code is now, so every row becomes the shared record of its code (owner `*`) and keeps its +`reporting_sources` as written. Its freeze frame, snapshots, recording links and near misses attach +to that record, the near misses of every source included. `confirmed_at_ns` is copied as it was, +except on a PREFAILED or PREPASSED row, where it becomes 0 (see Storage Backends below). All of it +runs in one transaction. + +The rebuild is one way, and opening a migrated store with an earlier release is unsupported. That +release runs its own startup cleanup on the file before it serves a request: it deletes the +`rosbag_files` rows that two owners of one code share, so recording links are lost without a +message, and it aborts on the first report of a fault code the store does not hold yet (`NOT NULL +constraint failed: faults.owner`). There is no guard in the code. A copy of `faults.db` taken before +the upgrade is the only way back: restore it instead of pointing the earlier release at the migrated +file. + +A PostgreSQL database written by an earlier release is migrated in place on the first connection, by +the same rules: the primary keys on the fault code become unique indexes over `(fault_code, owner)` +and every row becomes the shared record of its code. It runs in one transaction, so a failure leaves +the tables as they were. This migration is one way too: an earlier release pointed at the migrated +database is unsupported, and a copy of the database taken before the upgrade (`pg_dump`) is the only +way back. + +The table names `faults_new` and `freeze_frames_new` are reserved for the rebuild. Anything found +under those names is debris from a rebuild that did not finish and is dropped on open, with a +warning naming the table. ## Quick Start @@ -46,15 +120,18 @@ ros2 service call /fault_manager/clear_fault ros2_medkit_msgs/srv/ClearFault \ ## Features -- **Multi-source aggregation**: Same `fault_code` from different sources creates a single fault +- **Multi-source aggregation**: Same `fault_code` from different sources creates a single fault, + unless the code is configured `per_source`, which gives each source its own record - **Occurrence tracking**: Counts outages, not reports - the count starts at one and rises only when a cleared fault is raised again - and tracks all reporting sources -- **Severity escalation**: Fault severity is updated if a higher severity is reported +- **Severity escalation**: A record has one severity, the highest ever reported for it, kept + within an occurrence and across occurrences. A report with a different severity is logged (see + [Severity](#severity)) - **Persistent storage**: SQLite (default) or PostgreSQL backend ensures faults survive node restarts - **Debounce filtering** (optional): AUTOSAR DEM-style counter-based fault confirmation with per-entity and per-fault_code threshold overrides - **Snapshot capture**: Captures topic data when faults are confirmed for debugging (the value snapshots are deleted when the fault is cleared, unless `snapshots.retain_on_clear` is set) -- **Near-miss series**: Appends one entry per FAILED report that moved the debounce counter without confirming, bounded per fault code and retained when the fault is cleared -- **Freeze-frame retention**: One compact JSON freeze-frame per fault code, retained across `clear_fault` (see below) +- **Near-miss series**: Appends one entry per FAILED report that moved the debounce counter without confirming, bounded per record and retained when the record is cleared +- **Freeze-frame retention**: One compact JSON freeze-frame per record, retained across `clear_fault` (see below) - **Fault correlation** (optional): Root cause analysis with symptom muting and auto-clear - **Tamper-evident audit log** (optional): Append-only, hash-chained record of fault state transitions for verifiable history @@ -72,13 +149,13 @@ ros2 service call /fault_manager/clear_fault ros2_medkit_msgs/srv/ClearFault \ | `report_fault_queue_depth` | int | `100` | Requests `~/report_fault` holds until read (1-65536, read-only); past it they are dropped without an answer | | `entity_thresholds.config_file` | string | `""` | Path to YAML file with per-entity debounce threshold overrides | | `fault_thresholds.config_file` | string | `""` | Path to YAML file with per-fault_code debounce threshold overrides, applied on top of the entity ones | -| `near_miss.max_per_fault` | int | `200` | Near-miss entries retained per fault code, oldest evicted first (0 = unlimited) | +| `near_miss.max_per_fault` | int | `200` | Near-miss entries retained per fault record, oldest evicted first (0 = unlimited) | ### Snapshot Parameters Snapshots capture topic data when faults are confirmed for post-mortem debugging. -Each confirm also writes a **freeze-frame**: a single compact JSON object mapping every captured topic to its value at confirmation time, keyed by fault code. It differs from per-topic snapshots in two ways: snapshots are deleted when the fault is cleared, while the freeze-frame is retained across `clear_fault` (once the snapshots are gone, `~/get_fault` serves the retained frame so the confirmed-state record stays available after acknowledgement); and a re-confirm that captures nothing (e.g. source publishers down) never overwrites an existing non-empty frame. A fault code with no configured capture set gets no freeze-frame row; a configured capture that samples nothing on its first run records an empty `{}` frame. Freeze-frame storage is bounded by the number of distinct fault codes (one row per code, replaced in place) and rows are never evicted. +Each confirm also writes a **freeze-frame**: a single compact JSON object mapping every captured topic to its value at confirmation time, keyed by the record. Which topics are captured is decided by the fault CODE (`fault_specific` and `patterns` are configuration about what a code means), while what is written belongs to the record, so two `per_source` records of one code capture the same topics into two separate frames. It differs from per-topic snapshots in two ways: snapshots are deleted when the record is cleared, while the freeze-frame is retained across `clear_fault` (once the snapshots are gone, `~/get_fault` serves the retained frame so the confirmed-state record stays available after acknowledgement), and a re-confirm that captures nothing (e.g. source publishers down) never overwrites an existing non-empty frame. A fault code with no configured capture set gets no freeze-frame row. A configured capture that samples nothing on its first run records an empty `{}` frame. Freeze-frame storage is bounded by the number of distinct records (one row per record, replaced in place) and rows are never evicted. Under a fault storm, captures are bounded by a worker pool (`capture_pool_size`) draining a bounded queue (`capture_queue_depth`); excess captures are dropped per `capture_queue_full_policy` and logged (throttled). The pool is shared and is created when snapshots **or** rosbag is enabled, so these parameters bound both. `capture_pool_size` parallelizes freeze-frame snapshot capture only - rosbag stays single-writer regardless of pool size, and correlated faults confirming inside one post-roll window share a single recording. @@ -92,7 +169,7 @@ That single-writer property also shapes what each fault of a burst gets. Nothing | `snapshots.max_message_size` | int | `65536` | Maximum message size in bytes (larger messages skipped) | | `snapshots.default_topics` | string[] | `[]` | Topics to capture for all faults | | `snapshots.config_file` | string | `""` | Path to YAML config for `fault_specific` and `patterns` | -| `snapshots.recapture_cooldown_sec` | double | `60.0` | Min seconds between captures for the same fault code. | +| `snapshots.recapture_cooldown_sec` | double | `60.0` | Min seconds between captures for the same fault record. Per-source records of one code confirming inside it share one capture of the code's topics. | | `snapshots.max_per_fault` | int | `10` | Max snapshots retained per fault. | | `snapshots.capture_pool_size` | int | `2` | Max concurrent capture threads under a fault storm (>= 1). Parallelizes snapshot capture only; rosbag stays single-writer. | | `snapshots.capture_queue_depth` | int | `16` | Max pending captures before the full-queue policy applies (>= 1). | @@ -137,6 +214,12 @@ This is the persisted fault store (`storage_type`), separate from the rosbag sto format used by black-box capture (`snapshots.rosbag.format`, see Rosbag Capture above). **SQLite (default)**: Faults are persisted to disk and survive node restarts. Uses WAL mode for optimal performance. +Each row also records when the current occurrence of its record confirmed (`confirmed_at_ns`, read from the +database file, not by any service). It is 0 until the occurrence confirms: a FAILED report on a CLEARED record +resets it, and the confirmation of that new occurrence sets it again. The owner migration keeps the value of a +CONFIRMED, HEALED or CLEARED row and writes 0 for a PREFAILED or PREPASSED one. A row that confirmed stays +CONFIRMED or HEALED until it is cleared, so a pending row with a value carries it from the occurrence before a +reactivation. **Memory**: Faults are stored in memory only. Useful for testing or when persistence is not required. @@ -152,6 +235,40 @@ The build downloads libpqxx 7.10.7 from GitHub at configure time and links it st A wrong configuration stops the node at startup with exit code 1: a connection string libpq cannot parse, or a server that accepts the connection but does not allow creating the schema, or a table that lacks a column the node uses. A server that cannot be reached (including a wrong password or a missing database) does not stop it: the node runs without storage, services with an error field answer `Fault storage unavailable`, and `ReportFault` answers `accepted=false`. After a failed connection attempt, requests fail at once for 5 s and the next request tries again. An attempt waits at most 2 s per host address unless `connect_timeout` or `PGCONNECT_TIMEOUT` says otherwise, and a request on an open connection to a server that went off the network fails after `tcp_user_timeout` (5 s). With a libpq service, these values come from the service file. See `docs/config/fault-manager.rst` for the full table. +## Severity + +A record has one severity. The stored severity is the highest any FAILED report for it has +carried, within an occurrence and across occurrences: `~/clear_fault` ends an occurrence but does +not lower the severity, and the SQLite store keeps it across restarts. Two reports of one code from +one source with different severities mean a misconfigured reporter, and keeping the maximum is the +conservative reading. Another record of the same code (a `per_source` code has one per source) keeps +its own severity: a CRITICAL report for one record neither escalates nor confirms another. + +What each surface carries: + +| Surface | Severity | +|---|---| +| `~/list_faults`, `~/get_fault`, the gateway's `GET /faults` and fault detail | stored | +| `FaultEvent` on `~/events`, the gateway's `/faults/stream` frames and fault trigger payloads | stored, the same value a read of the record returns after the report | +| Audit rows (`occurred`, `confirmed`, `healed`, `cleared`) | stored, at the moment of the transition | +| Near-miss entries | the severity that report carried | + +The near-miss series is per-report history, so it keeps what the reporter sent. A lower report +stays visible there and in the mismatch warning below. + +`critical_immediate_confirm` depends on the identity of the record. On a record shared by all sources +of a code (the default) the severity of the report decides. On a `per_source` record the stored +severity decides: a FAILED report on a record stored as `CRITICAL` confirms it at once, whatever +severity the report carries, from a cleared record and from the HEALED latch alike. So no report +leaves a `per_source` record `CRITICAL` in `PREFAILED` or `PREPASSED`. + +When a FAILED report on a `per_source` record carries a severity other than the one stored for its record, the node logs a +warning naming the code, the reported and the stored severity and the source. It is logged once per +code, source and reported severity until the node restarts, so a reporter that repeats the same +report many times a second produces one line. The number of lines is bounded by the distinct +combinations, at most four per record, and not by the report rate. Each misconfigured source is +named on its own line. + ## Near-Miss Series A **near miss** is a FAILED report that moved the debounce counter without the fault ending up @@ -160,8 +277,8 @@ to any scoring sense used elsewhere. PASSED reports move the counter too, but in direction (the fault receding), so they are not near misses. The series is **append-only**: one entry per qualifying report, holding the timestamp, the counter -value after the report, the confirmation threshold it was measured against, the severity, the -reporting source, and the fault status the report left behind. Every field describes that one report, so a series spanning a threshold change +value after the report, the confirmation threshold it was measured against, the severity the +report carried, the reporting source, and the fault status the report left behind. Every field describes that one report, so a series spanning a threshold change or a new reporting source stays readable. Nothing is updated in place, so "this code approached confirmation five times this hour" stays answerable. @@ -175,12 +292,13 @@ counter walking back down under the latch. Without the field the two cannot be t rows written before the field existed. With per-entity thresholds the recorded `confirmation_threshold` is the one belonging to the -**reporting source**, while the debounce counter is shared by every source of that fault code. It -is therefore not by itself the distance to confirmation for the fault as a whole. An entry in +**reporting source**, while the debounce counter is shared by every source of a shared record. It +is therefore not by itself the distance to confirmation for the record as a whole. An entry in `fault_thresholds.config_file` that sets `confirmation_threshold` makes the two agree again: it -resolves the same for every source, so the recorded threshold is the fault's own. An entry with -only healing fields keeps the per-source threshold. The node warns -once per code when two sources resolve different policies for it. +resolves the same for every source, so the recorded threshold is the record's own. An entry with +only healing fields keeps the per-source threshold. A `per_source` record has one source, so its +recorded threshold is already the record's own. The node warns once per code when two sources resolve different +policies for the shared record. Entries are kept and evicted in **arrival order**, not by their timestamps. Reporters carry their own clocks, so a report can arrive carrying a timestamp behind one already stored; ordering the @@ -197,11 +315,11 @@ retained snapshots, because it records the most recent confirmation while the sn to earlier ones. `~/get_snapshots` returns one entry per topic and serves the newest capture of that topic, whichever storage backend is in use. -The bound is **per fault code, not per database**. Fault codes are unbounded in cardinality, so a -reporter emitting a stream of distinct codes still grows the table; the bound caps what any single -code costs, not the total. +The bound is **per record, not per database**. Fault codes are unbounded in cardinality (and so is +the set of reporting sources of a `per_source` code), so a reporter emitting a stream of distinct +codes still grows the table. The bound caps what any single record costs, not the total. -Retention is **bounded per fault code** by `near_miss.max_per_fault` (default 200), evicting the +Retention is **bounded per record** by `near_miss.max_per_fault` (default 200), evicting the **oldest** entries first. That is the same direction as `snapshots.max_per_fault` and the rosbag cap, and for the same reason: a series frozen at boot says nothing about whether the rate is changing, and the evidence a technician wants is the evidence from the fault happening now. Set it @@ -214,7 +332,7 @@ storage API or the database file. ## Advanced: Tamper-Evident Audit Log -An optional append-only, hash-chained audit log records every fault state transition (`occurred`, `confirmed`, `healed`, `cleared`) so the fault history is independently verifiable. Auto-recovery (a fault reaching the healing threshold via PASSED events) is recorded as a distinct `healed` row with source `auto_heal`, so the fault's END is in the timeline and is not confused with a manual `cleared`. The manager has no acknowledge action separate from clearing, so `~/clear_fault` is recorded as `cleared` (clear == ack); there is no `ack` kind. The log also records its own lifecycle with `logging_activated` / `logging_deactivated` markers at start and stop. It is **off by default** because it adds a write and storage cost per transition. +An optional append-only, hash-chained audit log records every fault state transition (`occurred`, `confirmed`, `healed`, `cleared`) so the fault history is independently verifiable. Auto-recovery (a record reaching the healing threshold via PASSED events) is recorded as a distinct `healed` row, so the record's END is in the timeline and is not confused with a manual `cleared`. The `source` of a row says what drove it: the reporting source for a row a report drove (`occurred`, `confirmed`), and the mechanism for an automatic one - `auto_confirm_timer` (time-based confirmation), `auto_heal` (healing threshold reached), `startup_reclassify` (a stale HEALED record turned CLEARED at start), `clear_service` (an operator clear) and `correlation_auto_clear` (a symptom cleared with its root cause). The `owner` of a row is the owner of a per-source record, and empty for a shared record, so a row of a shared record is the row it was before the field existed and every existing chain still verifies. The manager has no acknowledge action separate from clearing, so `~/clear_fault` is recorded as `cleared` (clear == ack), and there is no `ack` kind. The log also records its own lifecycle with `logging_activated` / `logging_deactivated` markers at start and stop. Every row records the record's stored severity at the transition (see [Severity](#severity)). It is **off by default** because it adds a write and storage cost per transition. Each transition appends one immutable row holding `record_hash = sha256(prev_hash + canonical(event))` (OpenSSL EVP SHA-256), the `prev_hash` it links to, and a monotonic `seq`. The hash is computed once at insert and never recomputed. A persisted chain head lets the chain resume across restarts. The log is stored in its own SQLite database (separate from the fault store) and is treated as append-only: the manager only ever inserts rows, and `BEFORE UPDATE` / `BEFORE DELETE` triggers reject out-of-band edits (the guarded rotation prune excepted). @@ -318,9 +436,9 @@ never confirms again, and `occurrence_count` does not move either. Prefer leavin `confirmation_threshold` at `-1` for such a reporter, and filter noisy samples in the reporter itself, where the samples are. -Two things ignore the counter. `SEVERITY_CRITICAL` confirms at once unless -`critical_immediate_confirm` is turned off in the debounce config, and that field is not exposed as -a ROS parameter. `auto_confirm_after_sec` promotes on elapsed time since the last FAILED, which is +Two things ignore the counter. A CRITICAL report on a shared record, or a FAILED report on a +`per_source` record stored as `SEVERITY_CRITICAL`, confirms at once unless `critical_immediate_confirm` is turned off in the debounce config, and that field is +not exposed as a ROS parameter. `auto_confirm_after_sec` promotes on elapsed time since the last FAILED, which is not the same as observing that the condition is still there. ### Fault Lifecycle with Debounce @@ -369,7 +487,10 @@ Event types: `0` = EVENT_FAILED, `1` = EVENT_PASSED ### Immediate Confirmation -CRITICAL severity faults bypass debounce and are immediately CONFIRMED, regardless of threshold. +A CRITICAL FAILED report on a shared record confirms it at once, regardless of threshold and of the +HEALED latch. On a `per_source` record the stored severity decides, not the report: after a CRITICAL +occurrence is cleared, one WARN report for the same record confirms the next occurrence, which +shows CRITICAL. This holds under a per-entity threshold override too. ## Advanced: Fault Correlation @@ -384,6 +505,10 @@ subsequent matching symptom faults within a time window are correlated and optio **Auto-Cluster**: Automatically groups related faults that match a pattern within a time window. Useful for detecting "storms" of related faults (e.g., communication errors). +Rules match by fault code across reporting nodes: a root cause reported by one node mutes and +auto-clears the matching symptoms of any other node, and a cluster collects the matching faults of +every node. The relations hold records, so a clear acts on exactly the records the root cause muted. + ### Configuration Enable correlation by providing a YAML configuration file: @@ -464,8 +589,13 @@ Response includes: - `muted_count`: Number of muted symptom faults - `cluster_count`: Number of active fault clusters -- `muted_faults[]`: Details of muted faults (when `include_muted=true`) -- `clusters[]`: Details of active clusters (when `include_clusters=true`) +- `muted_faults[]`: Details of muted records (when `include_muted=true`), one entry per muted + record. Muting is per record: it hides that record, not every record of the code, and each entry + names its owner in `source_id` (`*` for a shared record). +- `clusters[]`: Details of active clusters (when `include_clusters=true`). A member is a record: + `fault_owners[i]` is the owner of `fault_codes[i]`, and `(representative_code, + representative_owner)` names the representative. Pass the owner as `source_id` to GetFault and + ClearFault. ### REST API (via Gateway) @@ -487,7 +617,8 @@ Response fields: "fault_code": "MOTOR_COMM_FL", "root_cause_code": "ESTOP_001", "rule_id": "estop_cascade", - "delay_ms": 50 + "delay_ms": 50, + "source_id": "*" } ], "clusters": [ diff --git a/src/ros2_medkit_fault_manager/config/fault_manager.yaml b/src/ros2_medkit_fault_manager/config/fault_manager.yaml index f6b25f5d6..88e0f6804 100644 --- a/src/ros2_medkit_fault_manager/config/fault_manager.yaml +++ b/src/ros2_medkit_fault_manager/config/fault_manager.yaml @@ -17,7 +17,7 @@ fault_manager: # Near-miss series: one appended entry per FAILED report that moved the # debounce counter without confirming. Retained across clear_fault, bounded - # per fault code, oldest evicted first. 0 = unlimited (grows with the + # per record, oldest evicted first. 0 = unlimited (grows with the # reporting rate). # near_miss.max_per_fault: 200 @@ -36,7 +36,7 @@ fault_manager: # to entity-scoped capture and is crash-safe (falls back / self-disables if no # storage backend is usable). Faults confirming during another fault's # post-roll (duration_after_sec) share that recording, one metadata entry per - # fault; at most 32 attach on top of the first, the rest get no entry (WARN). + # record. At most 32 attach on top of the first, the rest get no entry (WARN). snapshots.rosbag.enabled: false # Capture concurrency bound under fault storms (issue #441). These govern the diff --git a/src/ros2_medkit_fault_manager/config/snapshots.yaml b/src/ros2_medkit_fault_manager/config/snapshots.yaml index 2eba501c9..ef678653f 100644 --- a/src/ros2_medkit_fault_manager/config/snapshots.yaml +++ b/src/ros2_medkit_fault_manager/config/snapshots.yaml @@ -11,6 +11,11 @@ # the reporting source node's own published topics are captured # (parameter snapshots.entity_default, on by default) # +# Resolution is by fault CODE: this file says what a code means, not who reported +# it. What is written belongs to the RECORD, so two per_source records of one code +# (fault_thresholds.config_file) capture the same topics into two separate freeze +# frames. +# # A fault code listed in fault_specific (or matched by a pattern) with an # EMPTY topic list is an explicit per-fault opt-out: nothing is captured and # the entity-default fallback is skipped for that code. @@ -105,7 +110,7 @@ rosbag: # Options: # "entity" - DEFAULT. Subscribe broadly for pre-roll, but on fault confirmation # write only the faulting source node's topics (+ /tf, /tf_static), - # resolved from the fault's reporting source. Useful black box with + # resolved from the record's reporting sources. Useful black box with # zero per-topic config. Falls back to the full buffer if the source # cannot be resolved to a live node. # "config" - Use same topics as JSON snapshots (from this config file) @@ -154,7 +159,9 @@ rosbag: # Storage path for bag files (default: "" = system temp directory) # Empty string uses /tmp/rosbag_snapshots/ # Bag files are named: fault_{fault_code}_{timestamp}/ and that directory name is - # the recording's public id - the last segment of its bulk-data URL. + # the recording's public id - the last segment of its bulk-data URL. The name carries + # the code only: a burst of records sharing one window shares one recording, and each + # of them gets its own metadata row pointing at it. storage_path: "" # Maximum size per bag file in MB (default: 50) @@ -169,10 +176,10 @@ rosbag: # is larger: it is kept and a warning is logged. max_total_storage_mb: 500 - # Recordings kept per fault code (default: 1, 0 = unlimited) - # A fault that keeps re-confirming leaves a trail of black boxes instead of only - # the latest one. Past the cap the fault's OLDEST recording is dropped, and the bag - # is deleted once no fault still references it (a burst shares one recording). + # Recordings kept per record (default: 1, 0 = unlimited) + # A record that keeps re-confirming leaves a trail of black boxes instead of only + # the latest one. Past the cap the record's OLDEST recording is dropped, and the bag + # is deleted once no record still references it (a burst shares one recording). # # 1 is the historical behaviour: each re-confirmation replaces the previous bag. # 3 is a good starting point for an intermittent fault you are chasing. diff --git a/src/ros2_medkit_fault_manager/design/index.rst b/src/ros2_medkit_fault_manager/design/index.rst index 7f4031c32..c631ea165 100644 --- a/src/ros2_medkit_fault_manager/design/index.rst +++ b/src/ros2_medkit_fault_manager/design/index.rst @@ -185,15 +185,15 @@ Main Components - Future implementations can be added in Issue #8: Fault Persistence Options 3. **InMemoryFaultStorage** - Thread-safe in-memory implementation of FaultStorage - - Uses ``std::map`` keyed by ``fault_code`` for O(log n) lookups + - Uses ``std::map`` keyed by ``FaultId`` for O(log n) lookups, ordered by code first - Protected by ``std::mutex`` for concurrent service request handling - - Aggregates reports from multiple sources into single fault entries - - Implements severity escalation (higher severity overwrites lower) - - Tracks occurrence counts and all reporting sources + - Keeps one record per ``FaultId``: one record shared by every source of a code, or one per source for a code configured ``per_source`` + - Implements severity escalation per record (higher severity overwrites lower, across occurrences too) + - Tracks occurrence counts and all reporting sources per record -4. **FaultState** - Internal representation of a fault entry +4. **FaultState** - Internal representation of one fault record - Maps directly to ``ros2_medkit_msgs::msg::Fault`` via ``to_msg()`` - - Uses ``std::set`` for reporting_sources to ensure uniqueness + - Carries its owner and uses ``std::set`` for reporting_sources to ensure uniqueness - Tracks first and last occurrence timestamps - Manages fault status lifecycle with debounce (PREFAILED -> CONFIRMED -> CLEARED) @@ -247,11 +247,15 @@ Services Reports a new fault or updates an existing one. -- **Input validation**: fault_code and source_id cannot be empty, event_type must be valid +- **Input validation**: fault_code and source_id cannot be empty, source_id cannot be the reserved ``*``, event_type must be valid - **Event types**: FAILED (fault detected) or PASSED (fault condition cleared) - **Debounce**: FAILED events decrement counter, PASSED events increment counter -- **Aggregation**: Same fault_code from different sources creates a single fault entry -- **Severity escalation**: Fault severity is updated if a higher severity is reported +- **Record identity**: ``FaultManagerCore::record_id_for`` decides it. A code configured ``per_source`` + has one record per source_id, any other code one record shared by all its sources (owner ``*``, + which ``ReportFault`` refuses as a source_id). Each record is debounced, cleared and healed on its own +- **Severity escalation**: A record's severity is updated if a higher one is reported for it +- **Severity mismatch**: A FAILED report whose severity differs from the stored severity of its + per-source record is logged as a warning, once per code, source and reported severity until restart - **Request queue**: Holds ``report_fault_queue_depth`` requests (default 100) until the executor reads them. The default service QoS holds 10. A reporter has one request in flight per fault code, so one reporter can fill the queue alone. Past the depth the middleware drops requests without an @@ -290,23 +294,53 @@ All ``FaultStorage`` public methods acquire a mutex lock to ensure thread safety when handling concurrent service requests. This is essential since ROS 2 service callbacks may execute on different threads. -Fault Aggregation -~~~~~~~~~~~~~~~~~ +Fault Records +~~~~~~~~~~~~~ -Multiple reports of the same ``fault_code`` (from same or different sources) are -aggregated into a single fault entry. This provides: +A fault record is identified by the pair ``(fault_code, owner)``. A code listed with +``per_source: true`` in ``fault_thresholds.config_file`` has one record per reporting source, +owned by the ``source_id`` of its reports. Every other code has one record shared by all its +sources, owned by the reserved id ``*``. Reports are routed to a record by one function, +``record_id_for``, and the storage backends receive the decided identity. This provides: - **Deduplication**: Prevents fault flooding from repeated reports -- **Source tracking**: Identifies all sources reporting the same fault -- **Occurrence counting**: Tracks how many times a fault was reported +- **Source tracking**: A record lists every source that reported it +- **Occurrence counting**: Tracks how many times a record was raised + +A per-source record keeps the per-reporter state apart: the debounce counter, the status, the +severity, the occurrence count and the timestamps all belong to one reporter, so one reporter +recovering cannot walk another reporter's fault back from confirmation, and one reporter's CRITICAL +cannot escalate another's record. A shared record has one counter fed by every source. + +A request that names a ``source_id`` addresses the record it owns, else the record that source +reports into. Stored records keep the identity they were written with when a code is switched +between the two modes; the node logs a warning per code at startup. Severity Escalation ~~~~~~~~~~~~~~~~~~~ -When a fault is re-reported with a higher severity, the stored severity is updated. -This ensures the fault reflects the worst-case condition. Severity levels are ordered: +When a record is re-reported with a higher severity, the stored severity is updated. This +ensures the record reflects the worst-case condition reported for it. +Severity levels are ordered: ``INFO(0) < WARN(1) < ERROR(2) < CRITICAL(3)``. +A record has one severity. The maximum is kept within an occurrence and across occurrences: a +FAILED report on a CLEARED record starts a new occurrence (``first_occurred`` and +``occurrence_count`` move) but does not lower the severity, and SQLite keeps it across restarts. +Different severities from one source for one code mean a misconfigured reporter, and the maximum +is the conservative choice. + +- **Reads, subscribers and the audit log** carry the stored severity of the record: + ``list_faults``, ``get_fault``, the ``FaultEvent`` payload, and through it the gateway's + ``/faults/stream`` frames and fault trigger payloads. An event therefore carries what a read of + the record returns right after the report. Every audit row records the stored severity at its + transition. +- **Near-miss entries** carry the reported severity: the series is per-report history. +- **Mismatch warning**: when a FAILED report carries another severity than the one stored for its + record, the node logs the code, both severities and the source. A set of + ``(fault_code, source_id, severity)`` keys limits it to one line per key for the process + lifetime, so the log volume does not follow the report rate. + Status Lifecycle (Debounce Model) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -320,7 +354,17 @@ Faults follow an AUTOSAR DEM-style debounce lifecycle: FAILED events decrement the debounce counter (towards confirmation). PASSED events increment the debounce counter (towards healing). -CRITICAL severity bypasses debounce and confirms immediately. +CRITICAL severity bypasses debounce and confirms immediately. On a shared record the severity of +the report decides. On a per-source record the check reads the stored severity after the report is +applied, so any FAILED report on a record stored as CRITICAL confirms it, from CLEARED and from the +HEALED latch alike, and no report leaves a per-source record CRITICAL in PREFAILED or PREPASSED. + +The SQLite store records when the current occurrence of a record confirmed (``confirmed_at_ns``). +A FAILED report on a CLEARED record resets it to 0, and the confirmation of the new occurrence sets +it. The owner migration keeps the value of a CONFIRMED, HEALED or CLEARED row and writes 0 for a +PREFAILED or PREPASSED one: under the latch a row that confirmed stays CONFIRMED or HEALED until a +clear, so a pending row with a value carries it from the occurrence before a reactivation. The +in-memory store keeps no confirmation time. The counter is always clamped to ``[confirmation_threshold, healing_threshold]`` so a long run of one-sided events cannot push it out to the integer limits and delay the opposite transition. diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/capture_thread_pool.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/capture_thread_pool.hpp index 76e70c078..83ea33dd7 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/capture_thread_pool.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/capture_thread_pool.hpp @@ -27,6 +27,7 @@ #include #include "ros2_medkit_fault_manager/core/log_sink.hpp" +#include "ros2_medkit_fault_manager/fault_storage.hpp" namespace ros2_medkit_fault_manager { @@ -46,7 +47,7 @@ enum class EnqueueResult { struct EnqueueOutcome { EnqueueResult result; - std::optional evicted_code; ///< Set only for kEvictedOldest. + std::optional evicted_id; ///< Set only for kEvictedOldest. }; /// Bounded worker pool that runs fault-capture jobs off the service thread. @@ -64,7 +65,7 @@ class CaptureThreadPool { /// for pool_size concurrent calls. Exceptions are caught and logged. /// @param logger_name Logger name of the lines written to @p log. CaptureThreadPool(std::size_t pool_size, std::size_t queue_depth, QueueFullPolicy full_policy, core::LogSink & log, - std::function capture_fn, std::string logger_name = "fault_manager"); + std::function capture_fn, std::string logger_name = "fault_manager"); ~CaptureThreadPool(); CaptureThreadPool(const CaptureThreadPool &) = delete; @@ -72,8 +73,8 @@ class CaptureThreadPool { CaptureThreadPool(CaptureThreadPool &&) = delete; CaptureThreadPool & operator=(CaptureThreadPool &&) = delete; - /// Enqueue a capture job. Non-blocking. Thread-safe. - EnqueueOutcome enqueue(const std::string & fault_code); + /// Enqueue a capture job for one fault record. Non-blocking. Thread-safe. + EnqueueOutcome enqueue(const FaultId & id); /// Stop accepting work, let in-flight jobs finish, discard pending, join all /// workers. Idempotent and noexcept. Called by the destructor. @@ -94,11 +95,11 @@ class CaptureThreadPool { const QueueFullPolicy full_policy_; core::LogSink & log_; std::string logger_name_; - std::function capture_fn_; + std::function capture_fn_; mutable std::mutex queue_mutex_; std::condition_variable cv_; - std::deque queue_; + std::deque queue_; bool stop_{false}; std::atomic dropped_captures_{0}; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp index 80b5e3697..bee980b5d 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_core.hpp @@ -19,10 +19,13 @@ #include #include #include +#include #include #include #include +#include #include +#include #include #include #include @@ -31,6 +34,7 @@ #include "ros2_medkit_fault_manager/core/fault_manager_config.hpp" #include "ros2_medkit_fault_manager/core/fault_manager_startup.hpp" #include "ros2_medkit_fault_manager/core/log_sink.hpp" +#include "ros2_medkit_fault_manager/fault_storage.hpp" namespace ros2_medkit_fault_manager::core { @@ -61,7 +65,7 @@ enum class CaptureEnqueue { struct CaptureResult { CaptureEnqueue result{CaptureEnqueue::kNone}; - std::string evicted_code; ///< Set only for kEvictedOldest. + FaultId evicted_id; ///< Set only for kEvictedOldest. }; /// Snapshot and rosbag capture the core triggers. Called with the core mutex held; must not call back into the core. @@ -74,10 +78,11 @@ class CaptureHooks { CaptureHooks & operator=(CaptureHooks &&) = delete; virtual ~CaptureHooks() = default; - /// Queue the snapshot and rosbag capture of @p code now. The core has already applied the recapture cooldown. - virtual CaptureResult capture_now(const std::string & code) = 0; - virtual void on_prefailed(const std::string & code) = 0; - virtual void on_cleared(const std::string & code) = 0; + /// Queue the snapshot and rosbag capture of the record @p id now. The core has already applied the recapture + /// cooldown. + virtual CaptureResult capture_now(const FaultId & id) = 0; + virtual void on_prefailed(const FaultId & id) = 0; + virtual void on_cleared(const FaultId & id) = 0; }; /// Wall-clock source, nanoseconds since the Unix epoch. @@ -125,6 +130,7 @@ struct ListQuery { /// MutedFaultInfo.msg field for field. struct MutedFault { std::string fault_code; + std::string source_id; ///< Owner of the muted record. std::string root_cause_code; std::string rule_id; uint32_t delay_ms{0}; @@ -137,8 +143,10 @@ struct ClusterSummary { std::string rule_name; std::string label; std::string representative_code; + std::string representative_owner; std::string representative_severity; std::vector fault_codes; + std::vector fault_owners; ///< Parallel to fault_codes uint32_t count{0}; int64_t first_at_ns{0}; int64_t last_at_ns{0}; @@ -196,16 +204,33 @@ class FaultManagerCore { ~FaultManagerCore() = default; ReportOutcome report(const fd::FaultReport & report); - ClearOutcome clear(const std::string & code, bool skip_correlation_auto_clear); + /// Clears one record. An empty @p source_id addresses the only record of @p code and is refused as + /// ambiguous when several records hold it. A set one addresses the record owned by that id, and + /// otherwise the record that source reports into (see resolve_target). + ClearOutcome clear(const std::string & code, const std::string & source_id, bool skip_correlation_auto_clear); ListOutcome list(const ListQuery & query); - FaultDetail get(const std::string & code); + FaultDetail get(const std::string & code, const std::string & source_id); + + /// Resolves the record a single-record request addresses, under the core mutex. Empty @p source_id + /// picks the only record of @p fault_code; several records make it ambiguous. A set one names the + /// record it owns (`*` names the shared one); when no record has that owner, it names the record + /// record_id_for gives that source. On failure returns nullopt and sets @p error. + std::optional resolve_target(const std::string & fault_code, const std::string & source_id, + std::string & error); + /// The record a report from @p source_id for @p fault_code goes into: one record per source when + /// the code is configured per_source, else the one record shared by every source. The only place + /// that builds a record identity from a (code, source) pair. + FaultId record_id_for(const std::string & fault_code, const std::string & source_id) const; + /// Throws FaultStorage::IgnorableConnectionException when the storage is unreachable. std::vector list_for_entity(const std::string & entity_id); /// Appends one audit event. No-op when the log is off, or when only confirmations are logged and /// this is not one. Throws when audit_log.fail_closed is set and the append fails. - void record_transition(const char * transition, const fd::FaultRecord & fault, const std::string & source_id, - int64_t occurred_at_ns); + /// @param id The record that moved: its owner is written to the row unless the record is shared. + /// @param source_id The report's source, or the kAuditSource* mechanism for an automatic transition. + void record_transition(const char * transition, const fd::FaultRecord & fault, const FaultId & id, + const std::string & source_id, int64_t occurred_at_ns); /// Runs the periodic jobs that are due at @p wall_now_ns (ns since the epoch): time-based /// auto-confirmation every kAutoConfirmPeriodSec and correlation cleanup every @@ -251,27 +276,34 @@ class FaultManagerCore { } private: - /// The first debounce policy seen for a fault code, and who reported it. + ReportOutcome report_locked(const fd::FaultReport & report); + ClearOutcome clear_locked(const std::string & code, const std::string & source_id, bool skip_correlation_auto_clear); + ListOutcome list_locked(const ListQuery & query); + /// @p rosbag_paths receives (snapshot index, bag path) for each rosbag entry, for sizing after unlock. + FaultDetail get_locked(const std::string & code, const std::string & source_id, + std::vector> & rosbag_paths); + std::optional resolve_target_locked(const std::string & fault_code, const std::string & source_id, + std::string & error); + void run_auto_confirm(int64_t wall_now_ns); + /// Queues a capture unless @p id is inside its recapture cooldown. + void capture_on_confirm(const FaultId & id); + /// Warns once per (code, source, severity) when a FAILED report disagrees with the stored severity of a + /// per-source record. + void warn_severity_mismatch(const fd::FaultReport & report, uint8_t stored_severity); + struct DebouncePolicyWitness { DebounceConfig config; std::string source_id; bool warned{false}; }; - ReportOutcome report_locked(const fd::FaultReport & report); - ClearOutcome clear_locked(const std::string & code, bool skip_correlation_auto_clear); - ListOutcome list_locked(const ListQuery & query); - /// @p rosbag_paths receives (snapshot index, bag path) for each rosbag entry, for sizing after unlock. - FaultDetail get_locked(const std::string & code, std::vector> & rosbag_paths); - void run_auto_confirm(int64_t wall_now_ns); - /// Queues a capture unless @p code is inside its recapture cooldown. - void capture_on_confirm(const std::string & code); - DebounceConfig resolve_config(const std::string & source_id, const std::string & fault_code) const; - void warn_on_conflicting_debounce_policy(const std::string & fault_code, const std::string & source_id, + /// Warns once per code when two sources feeding the shared record of @p id resolve different debounce + /// policies. A per-source record has one source and never warns. + void warn_on_conflicting_debounce_policy(const FaultId & id, const std::string & source_id, const DebounceConfig & resolved); - void audit_transition(const char * transition, const fd::FaultRecord & fault, const std::string & source_id, - int64_t occurred_at_ns); + void audit_transition(const char * transition, const fd::FaultRecord & fault, const FaultId & id, + const std::string & source_id, int64_t occurred_at_ns); void publish_event(const char * event_type, const fd::FaultRecord & fault, const std::vector & auto_cleared_codes = {}); @@ -290,6 +322,7 @@ class FaultManagerCore { bool audit_fail_closed_{false}; std::mutex mutex_; + std::set> severity_mismatch_logged_; std::unordered_map debounce_policy_witness_; /// Next due time of each tick job, ns since the epoch. int64_t auto_confirm_due_ns_{0}; @@ -297,8 +330,8 @@ class FaultManagerCore { int64_t last_tick_ns_{0}; std::atomic auto_confirm_runs_{0}; std::atomic cleanup_runs_{0}; - /// Recapture cooldown starts, per fault code. Empty when the cooldown is off. - std::unordered_map last_capture_times_; + /// Recapture cooldown starts, per record. Empty when the cooldown is off. + std::map last_capture_times_; std::atomic audit_dropped_writes_{0}; std::atomic audit_healthy_{true}; }; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_startup.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_startup.hpp index f486e92c0..939a946a9 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_startup.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/core/fault_manager_startup.hpp @@ -45,7 +45,7 @@ CoreParts build_core_parts(const FaultManagerConfig & config, LogSink & sink); /// Appends the audit event of one reclassified fault. May throw to fail the startup. using ReclassifyAuditFn = - std::function; + std::function; /// Turns stale HEALED rows into CLEARED for codes whose healing is now disabled. Calls @p audit for /// each flipped fault when @p audit_enabled. Storage that is unreachable skips it with a warning. @@ -53,4 +53,11 @@ void reclassify_stale_healed(FaultStorage & storage, const FaultCodeThresholdRes const DebounceConfig & global_debounce, bool audit_enabled, LogSink & sink, const std::string & logger, const ReclassifyAuditFn & audit); +/// Logs one warning per fault code that holds an active (not CLEARED) record stored under the other +/// identity than the code is configured for: a shared record of a per_source code, or a per-source record +/// of a shared one. Stored records keep the identity they were written with; new reports use the +/// configured one. Storage that is unreachable skips it with a warning. +void warn_on_identity_mode_mismatch(FaultStorage & storage, const FaultCodeThresholdResolver * fault_code_resolver, + LogSink & sink, const std::string & logger); + } // namespace ros2_medkit_fault_manager::core diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp index ccb0810f5..02db12002 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/correlation/correlation_engine.hpp @@ -20,10 +20,12 @@ #include #include #include +#include #include #include "ros2_medkit_fault_manager/correlation/pattern_matcher.hpp" #include "ros2_medkit_fault_manager/correlation/types.hpp" +#include "ros2_medkit_fault_manager/fault_storage.hpp" namespace ros2_medkit_fault_manager { namespace correlation { @@ -48,35 +50,45 @@ struct ProcessFaultResult { /// For auto-cluster: cluster ID if this fault triggered or joined a cluster std::string cluster_id; - /// For auto-cluster: fault codes that should be retroactively muted - /// When cluster reaches min_count threshold, previous non-representative faults + /// For auto-cluster: records that should be retroactively muted + /// When cluster reaches min_count threshold, previous non-representative records /// that were added before threshold was reached need to be muted retroactively - std::vector retroactive_mute_codes; + std::vector retroactive_mute_ids; }; /// Result of clearing a fault struct ProcessClearResult { - /// List of symptom fault codes that should be auto-cleared - std::vector auto_cleared_codes; + /// The symptom RECORDS that should be auto-cleared. Records, not codes: a clear + /// acts on the exact records the root cause muted, whoever owns them. + std::vector auto_cleared_symptoms; }; -/// Information about a muted fault (for ListFaults response) +/// Information about a muted record (for ListFaults response). +/// Carries both halves of the record identity: two owners muted on one code are two +/// entries of that code, told apart by owner. struct MutedFaultData { std::string fault_code; + /// Reporting source that owns the muted record. Filled from the map key, so it + /// cannot drift from the record this entry describes. + std::string owner; std::string root_cause_code; std::string rule_id; uint32_t delay_ms{0}; }; -/// Information about an active cluster (for ListFaults response) +/// Information about an active cluster (for ListFaults response). +/// A member is a record: fault_owners[i] is the owner of fault_codes[i], and +/// (representative_code, representative_owner) is the representative record. struct ClusterData { std::string cluster_id; std::string rule_id; std::string rule_name; std::string label; std::string representative_code; + std::string representative_owner; std::string representative_severity; std::vector fault_codes; + std::vector fault_owners; std::chrono::system_clock::time_point first_at; std::chrono::system_clock::time_point last_at; }; @@ -95,18 +107,22 @@ class CorrelationEngine { /// @param config Correlation configuration (must be enabled and valid) explicit CorrelationEngine(const CorrelationConfig & config); - /// Process an incoming fault - /// @param fault_code The fault code + /// Process an incoming fault record + /// + /// Rules match on the fault CODE, whoever owns the record: a root cause of one owner + /// mutes and auto-clears matching symptom records of any owner. Every RELATION the + /// engine forms (root to symptom, muting, cluster membership) holds exact records. + /// @param id The fault record /// @param severity The fault severity (for representative selection) /// @param timestamp When the fault occurred /// @return Processing result indicating whether to mute, correlations, etc. - ProcessFaultResult process_fault(const std::string & fault_code, const std::string & severity, + ProcessFaultResult process_fault(const FaultId & id, const std::string & severity, std::chrono::steady_clock::time_point timestamp = std::chrono::steady_clock::now()); - /// Process a fault being cleared - /// @param fault_code The fault code being cleared - /// @return Result with list of symptoms to auto-clear - ProcessClearResult process_clear(const std::string & fault_code); + /// Process a fault record being cleared + /// @param id The record being cleared + /// @return Result with the symptom records to auto-clear + ProcessClearResult process_clear(const FaultId & id); /// Get all currently muted faults /// @return List of muted fault data @@ -115,10 +131,10 @@ class CorrelationEngine { /// Get count of muted faults uint32_t get_muted_count() const; - /// Whether a fault code is currently muted as a symptom. - /// @param fault_code Code to test - /// @return True while the code is suppressed by a root cause - bool is_muted(const std::string & fault_code) const; + /// Whether a fault record is currently muted as a symptom. + /// @param id Record to test + /// @return True while the record is suppressed by a root cause + bool is_muted(const FaultId & id) const; /// Get all active clusters /// @return List of cluster data @@ -132,67 +148,99 @@ class CorrelationEngine { void cleanup_expired(); private: - /// Check if fault matches a root cause pattern in any hierarchical rule + /// Check if the record's CODE matches a root cause pattern in any hierarchical rule /// @return Rule ID if matched, empty optional otherwise std::optional try_as_root_cause(const std::string & fault_code); - /// Check if fault is a symptom of any pending root cause + /// Check if the record is a symptom of a pending root cause /// @return ProcessFaultResult with correlation info if matched - std::optional try_as_symptom(const std::string & fault_code, - std::chrono::steady_clock::time_point timestamp); + std::optional try_as_symptom(const FaultId & id, std::chrono::steady_clock::time_point timestamp); - /// Check if fault matches an auto-cluster rule + /// Drops a record from the symptom list of every root cause. Caller holds the mutex. + void forget_symptom(const FaultId & id); + + /// Check if the record's code matches an auto-cluster rule. A cluster holds the records + /// of any owner that match the rule inside its window. /// @return ProcessFaultResult with cluster info if matched - std::optional try_auto_cluster(const std::string & fault_code, const std::string & severity, + std::optional try_auto_cluster(const FaultId & id, const std::string & severity, std::chrono::steady_clock::time_point timestamp); /// Generate unique cluster ID std::string generate_cluster_id(const std::string & rule_id); + /// A cluster as the engine keeps it: members and the representative are records. + struct Cluster { + std::string cluster_id; + std::string rule_id; + std::string rule_name; + std::string label; + FaultId representative; + std::string representative_severity; + std::vector members; + std::chrono::system_clock::time_point first_at; + std::chrono::system_clock::time_point last_at; + }; + /// Record a re-reported member's severity and update the representative if it changes - void update_member_severity(const CorrelationRule & rule, ClusterData & cluster, - std::map & fault_severities, const std::string & fault_code, + void update_member_severity(const CorrelationRule & rule, Cluster & cluster, + std::map & severities, const FaultId & id, const std::string & severity) const; - /// Reassign cluster representative from remaining fault_codes and their severities - void reassign_representative(ClusterData & cluster, - const std::map & fault_severities) const; + /// Reassign cluster representative from the remaining members and their severities + void reassign_representative(Cluster & cluster, const std::map & severities) const; + + /// Mutes every member of a show_as_single cluster except its representative and unmutes the + /// representative. Records muted by another rule are left alone. Caller holds the mutex. + void sync_cluster_muting(const Cluster & cluster); + + /// Sets the muted entry of a record from the relations that are still live: a root cause that + /// lists it under a muting rule, else its active show_as_single cluster. With none, the entry + /// goes. Caller holds the mutex. + void reconcile_muting(const FaultId & id); + + /// Whether a fault code is one of the symptoms of a hierarchical rule. + bool matches_symptom(const CorrelationRule & rule, const std::string & fault_code) const; + + static ClusterData to_cluster_data(const Cluster & cluster); CorrelationConfig config_; std::unique_ptr matcher_; /// Active root causes waiting for symptoms struct PendingRootCause { - std::string fault_code; + FaultId fault_id; std::string rule_id; std::chrono::steady_clock::time_point timestamp; uint32_t window_ms; }; std::vector pending_root_causes_; - /// Mapping from root cause to its symptoms - std::map> root_to_symptoms_; + /// Mapping from a root cause RECORD to its symptom RECORDS + std::map> root_to_symptoms_; + + /// Muted records (record -> data) + std::map muted_faults_; - /// Muted faults (fault_code -> data) - std::map muted_faults_; + /// Delay of a symptom record behind a root cause record, from the last attach ((root, symptom) -> ms) + std::map, uint32_t> relation_delay_ms_; - /// Active clusters (cluster_id -> data) - std::map active_clusters_; + /// Active clusters (cluster_id -> cluster) + std::map active_clusters_; - /// Per-fault severities for active clusters (cluster_id -> (fault_code -> severity)) - std::map> active_cluster_severities_; + /// Per-record severities for active clusters (cluster_id -> (record -> severity)) + std::map> active_cluster_severities_; - /// Mapping from fault code to cluster ID (for faults in clusters) - std::map fault_to_cluster_; + /// Mapping from a record to its cluster ID (for records in clusters) + std::map fault_to_cluster_; /// Pending cluster with steady_clock timestamp for window tracking struct PendingCluster { - ClusterData data; + Cluster data; std::chrono::steady_clock::time_point steady_first_at; - std::map fault_severities; ///< fault_code -> severity + std::map fault_severities; ///< record -> severity }; - /// Pending clusters being formed (rule_id -> cluster data) + /// Pending clusters being formed (rule_id -> cluster data). /// Once min_count is reached, moved to active_clusters_ std::map pending_clusters_; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_audit_log.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_audit_log.hpp index 967090690..b5a7bf92c 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_audit_log.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_audit_log.hpp @@ -49,11 +49,21 @@ struct AuditEvent { std::string transition; ///< one of the kTransition* constants below uint8_t severity{0}; ///< severity at the time of the transition std::string status; ///< resulting fault status (e.g. CONFIRMED) - std::string source_id; ///< reporting source that drove the transition + std::string source_id; ///< the report's source, or a kAuditSource* mechanism for an automatic transition std::string description; ///< human-readable description int64_t occurred_at_ns{0}; ///< wall-clock timestamp of the transition + /// Owner of a per-source record. Empty for a shared record. Hashed only when non-empty, so every + /// row of a shared record hashes as it did before the field existed. + std::string owner; }; +/// `source_id` of the rows no report drives. Stored verbatim, so they are part of the hash. +constexpr const char * kAuditSourceAutoConfirm = "auto_confirm_timer"; +constexpr const char * kAuditSourceAutoHeal = "auto_heal"; +constexpr const char * kAuditSourceClearService = "clear_service"; +constexpr const char * kAuditSourceCorrelationAutoClear = "correlation_auto_clear"; +constexpr const char * kAuditSourceStartupReclassify = "startup_reclassify"; + /// Canonical transition kinds. Stored verbatim, so they are part of the hash. constexpr const char * kTransitionOccurred = "occurred"; constexpr const char * kTransitionConfirmed = "confirmed"; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_manager_node.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_manager_node.hpp index fa1f61e17..167c9e2d2 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_manager_node.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_manager_node.hpp @@ -101,7 +101,7 @@ class FaultManagerNode : public rclcpp::Node { /// service handlers use. Throws when audit_log.fail_closed is set and the append /// fails; otherwise observe audit_dropped_writes()/audit_healthy(). void audit_transition_for_test(const char * transition, const fd::FaultRecord & fault) { - audit_transition(transition, fault, "test", 0); + audit_transition(transition, fault, FaultId{fault.fault_code, kSharedOwner}, "test", 0); } /// Get the storage type being used @@ -179,15 +179,16 @@ class FaultManagerNode : public rclcpp::Node { /// Queue the snapshot and rosbag capture of a confirmed fault on the capture pool. The core calls it /// through the capture hooks after its cooldown check. - core::CaptureResult enqueue_capture(const std::string & fault_code); + core::CaptureResult enqueue_capture(const FaultId & id); /// Append a fault state-transition to the audit log through the core. /// @param transition One of the kTransition* constants. /// @param fault The fault state at the time of the transition. - /// @param source_id The reporting source that drove the transition. + /// @param id The record that moved. + /// @param source_id The report's source, or the kAuditSource* mechanism for an automatic transition. /// @param occurred_at_ns Wall-clock timestamp of the transition. - void audit_transition(const char * transition, const fd::FaultRecord & fault, const std::string & source_id, - int64_t occurred_at_ns); + void audit_transition(const char * transition, const fd::FaultRecord & fault, const FaultId & id, + const std::string & source_id, int64_t occurred_at_ns); core::FaultManagerConfig config_; ///< Validated startup configuration std::string storage_type_; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp index e82c01e9b..4a45b33a0 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/fault_storage.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "ros2_medkit_fault_detection/fault_types.hpp" @@ -60,7 +61,9 @@ struct DebounceConfig { /// Default: 3 (3 more PASSED than FAILED events to heal). int32_t healing_threshold{3}; - /// Whether CRITICAL severity bypasses debounce and confirms immediately. + /// Whether a CRITICAL record confirms immediately on a FAILED report, bypassing debounce and the + /// HEALED latch. A shared record (owner kSharedOwner) goes by the report's severity, any other + /// record by its stored severity. bool critical_immediate_confirm{true}; /// Time-based auto-confirmation. If > 0, PREFAILED faults older than this are auto-confirmed. @@ -89,16 +92,61 @@ bool is_near_miss(bool is_failed_event, const std::string & resulting_status); /// defaults (-1 / 3). Returns true if the config was already valid. bool sanitize_debounce_config(DebounceConfig & config); -/// Internal fault state stored in memory. Times are wall-clock ns >= 0; 0 means never. +/// Owner of a record that every reporting source of its fault code shares. +/// +/// Reserved: ReportFault refuses it as a source_id, so a shared record can never be +/// mistaken for the per-source record of a source that happens to be called `*`. +inline constexpr const char * kSharedOwner = "*"; + +/// Identity of one fault record: a fault code plus the owner of the record. +/// +/// FaultManagerCore::record_id_for decides the owner. A code configured `per_source` gives +/// each reporting source its own record (owner = the report's source_id), any other code has +/// one record shared by every source (owner = kSharedOwner). Every piece of per-record state +/// (status, debounce counter, occurrence count, timestamps, severity, freeze frame, snapshots, +/// near misses, rosbag links) belongs to one record. A clear or a heal of one record never +/// touches another record of the same code. +/// +/// An aggregate on purpose: FaultId{code, owner} is the only way to build one, so no +/// call site can silently pass a bare code where a record identity is required. +struct FaultId { + std::string fault_code; + std::string owner; + + bool operator==(const FaultId & other) const { + return fault_code == other.fault_code && owner == other.owner; + } + + bool operator!=(const FaultId & other) const { + return !(*this == other); + } + + /// Ordered by code first, so the records of one code are contiguous in a map and + /// the ids of one code are a range scan rather than a full sweep. + bool operator<(const FaultId & other) const { + return std::tie(fault_code, owner) < std::tie(other.fault_code, other.owner); + } +}; + +/// A stored record together with the identity it is stored under. +struct FaultEntry { + FaultId id; + fd::FaultRecord record; +}; + +/// Internal fault state of one record, stored in memory. Times are wall-clock ns >= 0; 0 means never. struct FaultState { std::string fault_code; + /// Owner of this record. Half of the record identity. + std::string owner; + /// Every source that reported FAILED for this record. One source for a per-source record. + std::set reporting_sources; uint8_t severity{0}; std::string description; int64_t first_occurred_ns{0}; int64_t last_occurred_ns{0}; uint32_t occurrence_count{0}; ///< Count of genuine occurrences (new fault + each re-raise after CLEARED) std::string status; - std::set reporting_sources; // Debounce state (internal, not exposed in Fault.msg) int32_t debounce_counter{0}; ///< FAILED decrements (-1), PASSED increments (+1) @@ -115,6 +163,7 @@ struct FaultState { /// one moment and only mean anything together. struct SnapshotData { std::string fault_code; + std::string owner; ///< Owner of the record this reading belongs to std::string topic; std::string message_type; std::string data; ///< JSON-encoded message data @@ -126,13 +175,15 @@ struct SnapshotData { /// Compact freeze-frame captured when a fault confirms: a single JSON object mapping /// each captured topic to its latest value at confirmation time. Unlike per-topic -/// snapshots, a freeze-frame is keyed by fault_code (one row per code) and is RETAINED -/// across clear_fault, so the confirmed-state record persists after acknowledgement. -/// A row exists only for fault codes with a configured capture set; a fault code with -/// no capture configured gets no row at all (lookup returns nullopt, never an empty {}). +/// snapshots, a freeze-frame is keyed by the record identity (one row per +/// (fault_code, owner) pair) and is RETAINED across clear_fault, so the confirmed-state +/// record persists after acknowledgement. A row exists only for fault codes with a +/// configured capture set. A fault code with no capture configured gets no row at all +/// (lookup returns nullopt, never an empty {}). struct FreezeFrameData { std::string fault_code; - std::string data; ///< Compact JSON object: {"": , ...} + std::string owner; ///< Owner of the record this frame belongs to + std::string data; ///< Compact JSON object: {"": , ...} int64_t captured_at_ns{0}; }; @@ -146,7 +197,7 @@ struct FreezeFrameData { /// off the wire with its own copy of this rule; keep the two in step. std::string rosbag_recording_id(const std::string & file_path); -/// One entry of the near-miss series for a fault code. +/// One entry of the near-miss series of one fault record. /// /// A near miss is a FAILED report that moved the debounce counter WITHOUT the fault /// ending up CONFIRMED - the fault nearly happened. PASSED reports move the counter @@ -156,21 +207,21 @@ std::string rosbag_recording_id(const std::string & file_path); /// /// The series is append-only: one entry per qualifying report, never updated in place, /// and RETAINED across clear_fault, because acknowledging a fault cycle must not erase -/// the record of how often that code approached confirmation. It is bounded per fault -/// code (see set_max_near_misses_per_fault) and evicts the OLDEST entries first, so a +/// the record of how often that record approached confirmation. It is bounded per +/// record (see set_max_near_misses_per_fault) and evicts the OLDEST entries first, so a /// long-running appliance keeps the recent series rather than freezing it at boot. struct NearMissRecord { std::string fault_code; + std::string owner; ///< Owner of the record the series belongs to int64_t occurred_at_ns{0}; ///< Timestamp of the report that moved the counter int32_t debounce_counter{0}; ///< Counter value AFTER this report - /// Confirmation threshold the report was evaluated against. With per-entity overrides this is - /// the threshold of the REPORTING SOURCE, while the counter is shared by every source of the - /// code, so it is not by itself the distance to confirmation for the fault as a whole. + /// Confirmation threshold the report was evaluated against: the distance to confirmation of + /// the record's counter at that moment. int32_t confirmation_threshold{0}; uint8_t severity{0}; ///< Severity carried by the report - std::string source_id; ///< Reporting source + std::string source_id; ///< Source that sent the report; one record can hold reports of several /// Fault status after the report was applied. Never CONFIRMED - that is what makes the report a /// near miss. It separates a counter climbing from a resting state (PREFAILED) from one walking @@ -186,6 +237,7 @@ struct NearMissRecord { /// only when its last row goes. struct RosbagFileInfo { std::string fault_code; + std::string owner; ///< Owner of the record holding this link std::string recording_id; ///< Basename of file_path; shared by every row of a burst std::string file_path; std::string format; ///< "sqlite3" or "mcap" @@ -222,16 +274,18 @@ class FaultStorage { virtual DebounceConfig get_debounce_config() const = 0; /// Report a fault event (FAILED or PASSED) - /// @param fault_code Global fault identifier + /// @param id The record this event applies to, as FaultManagerCore::record_id_for decided it. + /// The event creates that record when none exists and the event is FAILED. /// @param kind Whether the report is a FAILED or a PASSED event /// @param severity Fault severity level (only used for FAILED events) /// @param description Human-readable description (only used for FAILED events) - /// @param source_id Reporting source identifier + /// @param source_id Source that sent the report. A FAILED event adds it to the record's + /// reporting sources and the near-miss series keeps it per entry. /// @param timestamp_ns Wall-clock time of the event, ns (>= 0) - /// @param config Debounce configuration to apply for this event (resolved per-entity by the node) - /// @return true if this is a new occurrence (new fault or reactivated CLEARED fault), - /// false if existing active fault was updated - virtual bool report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, + /// @param config Debounce configuration to apply for this event (resolved per entity and code by the core) + /// @return true if this is a new occurrence (new record or reactivated CLEARED record), + /// false if an existing active record was updated + virtual bool report_fault_event(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) = 0; @@ -240,37 +294,58 @@ class FaultStorage { /// @param severity Severity level to filter (if filter_by_severity is true) /// @param statuses List of statuses to include (empty = CONFIRMED only) /// @return Vector of matching faults - virtual std::vector list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const = 0; + std::vector list_faults(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const { + std::vector records; + for (auto & entry : list_fault_entries(filter_by_severity, severity, statuses)) { + records.push_back(std::move(entry.record)); + } + return records; + } - /// Get a single fault by fault_code - /// @param fault_code The fault code to look up - /// @return The fault if found, nullopt otherwise - virtual std::optional get_fault(const std::string & fault_code) const = 0; + /// list_faults with the identity each record is stored under. + virtual std::vector list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const = 0; - /// Clear a fault by fault_code (manual acknowledgment). Drops the fault's per-topic snapshots; - /// the freeze-frame and the near-miss series are RETAINED, because they outlive a single fault - /// cycle and cannot be reconstructed afterwards. - /// @param fault_code The fault code to clear - /// @return true if fault was found and cleared, false if not found - virtual bool clear_fault(const std::string & fault_code) = 0; + /// Get a single fault record + /// @param id The record to look up + /// @return The fault if found, nullopt otherwise + virtual std::optional get_fault(const FaultId & id) const = 0; - /// Get total number of stored faults + /// The identity of every record carrying @p fault_code: one shared record, or one per + /// source for a per-source code, or both after the mode of the code changed. + /// + /// The resolution step behind an unscoped service call: with exactly one record the + /// call applies to it, with several it is ambiguous and the owners are the answer. + /// Returns them ordered by owner, so an ambiguity message is stable. + /// @param fault_code The fault code to look up + /// @return Every record id carrying the code, empty when none does + virtual std::vector get_ids_by_code(const std::string & fault_code) const = 0; + + /// Clear one fault record (manual acknowledgment). Drops that record's per-topic snapshots. + /// The freeze-frame and the near-miss series are RETAINED, because they outlive a single fault + /// cycle and cannot be reconstructed afterwards. Another record of the same code is + /// untouched, snapshots included. + /// @param id The record to clear + /// @return true if the record was found and cleared, false if not found + virtual bool clear_fault(const FaultId & id) = 0; + + /// Get total number of stored fault records virtual size_t size() const = 0; - /// Check if a fault exists - virtual bool contains(const std::string & fault_code) const = 0; + /// Check if a fault record exists + virtual bool contains(const FaultId & id) const = 0; - /// Check and confirm PREFAILED faults that have been pending too long (time-based confirmation) + /// Check and confirm PREFAILED records that have been pending too long (time-based confirmation) /// @param now_ns Wall-clock time for age calculation, ns - /// @return Fault codes that were confirmed by this call (so the caller can audit each). - virtual std::vector check_time_based_confirmation(int64_t now_ns) = 0; + /// @return The records that were confirmed by this call (so the caller can audit each). + virtual std::vector check_time_based_confirmation(int64_t now_ns) = 0; - /// Set maximum snapshots per fault code (0 = unlimited) + /// Set maximum snapshots per fault record (0 = unlimited) virtual void set_max_snapshots_per_fault(size_t /*max_count*/) { } - /// Cap on RECORDINGS retained per fault code (0 = unlimited). + /// Cap on RECORDINGS retained per fault record (0 = unlimited). /// /// Enforced inside store_rosbag_file(s), atomically with the insert: past the cap /// the fault's OLDEST recordings lose their row, and a bag whose last referencing @@ -318,16 +393,15 @@ class FaultStorage { /// @param snapshot The snapshot data to store virtual void store_snapshot(const SnapshotData & snapshot) = 0; - /// Get snapshots for a fault, NEWEST capture set first. + /// Get snapshots of one fault record, NEWEST capture set first. /// /// Ordered by capture_id descending, then captured_at_ns descending, in every /// backend. Readers fold rows into a per-topic map, so insertion order would let /// an older capture's values win on one backend and not the other. - /// @param fault_code The fault code to get snapshots for + /// @param id The record to get snapshots for /// @param topic_filter Optional topic filter (empty = all topics) - /// @return Vector of snapshots for the fault - virtual std::vector get_snapshots(const std::string & fault_code, - const std::string & topic_filter = "") const = 0; + /// @return Vector of snapshots of the record + virtual std::vector get_snapshots(const FaultId & id, const std::string & topic_filter = "") const = 0; /// Highest capture_id any stored snapshot holds, across every fault (0 when none). /// @@ -339,22 +413,23 @@ class FaultStorage { return 0; } - /// Store the compact freeze-frame captured for a fault (JSON dict of topic values). - /// Keyed by fault_code: a later capture for the same code replaces the frame. The frame + /// Store the compact freeze-frame captured for a fault record (JSON dict of topic values). + /// Keyed by the record identity carried on @p frame: a later capture for the same record + /// replaces the frame, and another owner's record of the same code keeps its own. The frame /// is retained across clear_fault so the confirmed-state record survives acknowledgement. - /// Storage is bounded by the number of distinct fault codes (one row per code, replaced - /// in place); rows are never evicted. Faults themselves are never deleted (clear_fault - /// only flips status), so there is currently no delete hook to tie eviction to. - /// @param frame The freeze-frame to store + /// Storage is bounded by the number of distinct records (one row per record, replaced in + /// place), and rows are never evicted. Records themselves are never deleted (clear_fault only + /// flips status), so there is currently no delete hook to tie eviction to. + /// @param frame The freeze-frame to store, carrying fault_code and owner virtual void store_freeze_frame(const FreezeFrameData & frame) = 0; - /// Get the freeze-frame captured for a fault, if any. - /// @param fault_code The fault code to look up - /// @return The freeze-frame if one was captured, nullopt otherwise (including fault - /// codes with no capture configured, which never get a row) - virtual std::optional get_freeze_frame(const std::string & fault_code) const = 0; + /// Get the freeze-frame captured for a fault record, if any. + /// @param id The record to look up + /// @return The freeze-frame if one was captured, nullopt otherwise (including records + /// with no capture configured, which never get a row) + virtual std::optional get_freeze_frame(const FaultId & id) const = 0; - /// Set the maximum number of near-miss entries retained per fault code. + /// Set the maximum number of near-miss entries retained per fault record. /// /// Entries beyond the bound are evicted oldest-first, including entries already stored when the /// bound is applied. 0 means unlimited, and so does any bound larger than the storage backend @@ -367,17 +442,18 @@ class FaultStorage { return 0; } - /// Get the near-miss series for a fault code, oldest entry first. - /// The series survives clear_fault; an unknown or never-near-missed code returns empty. - /// @param fault_code The fault code to look up + /// Get the near-miss series of one fault record, oldest entry first. + /// The series survives clear_fault. An unknown or never-near-missed record returns empty. + /// @param id The record to look up /// @return The retained near-miss entries in chronological order - virtual std::vector get_near_misses(const std::string & fault_code) const = 0; + virtual std::vector get_near_misses(const FaultId & id) const = 0; - /// Store rosbag file metadata for a fault - /// @param info The rosbag file info to store (replaces any existing entry for fault_code) + /// Store rosbag file metadata for a fault record + /// @param info The rosbag file info to store, carrying fault_code and owner (replaces any + /// existing link between that record and the same file_path) virtual void store_rosbag_file(const RosbagFileInfo & info) = 0; - /// Store one row per fault of a burst that shares a recording. + /// Store one row per record of a burst that shares a recording. /// /// Implementations MUST be all-or-nothing. The caller treats a throw as "no row /// was written" and discards the recording, so a batch that stored some rows and @@ -396,20 +472,20 @@ class FaultStorage { } } - /// The MOST RECENT recording of a fault, or nullopt. + /// The MOST RECENT recording of a fault record, or nullopt. /// - /// A fault can hold several recordings, so "the" recording is a choice: newest + /// A record can hold several recordings, so "the" recording is a choice: newest /// wins, because a black box is evidence about the machine you are about to /// inspect. Implementations must order deterministically - an unordered pick /// serves an arbitrary recording, which no test catches reliably. - /// @param fault_code The fault code to get rosbag for + /// @param id The record to get rosbag for /// @return Rosbag file info if exists, nullopt otherwise - virtual std::optional get_rosbag_file(const std::string & fault_code) const = 0; + virtual std::optional get_rosbag_file(const FaultId & id) const = 0; - /// Every recording of a fault, newest first. - virtual std::vector get_rosbag_files(const std::string & fault_code) const = 0; + /// Every recording of a fault record, newest first. + virtual std::vector get_rosbag_files(const FaultId & id) const = 0; - /// Every row of one recording - one per fault the recording covers. Backs the + /// Every row of one recording - one per record the recording covers. Backs the /// bulk-data download and the entity authorization scope check, both of which /// start from a recording id and need the faults behind it. virtual std::vector get_rosbag_files_by_recording(const std::string & recording_id) const = 0; @@ -420,23 +496,23 @@ class FaultStorage { /// @return number of rows removed virtual size_t delete_rosbag_recording(const std::string & recording_id) = 0; - /// Delete rosbag file record and the actual file for a fault. Faults from one - /// burst can share a recording; the file is unlinked only with the last record + /// Delete rosbag rows and the actual file for one fault record. Records from one + /// burst can share a recording, and the file is unlinked only with the last row /// that references it. - /// @param fault_code The fault code to delete rosbag for - /// @return true if record was deleted, false if not found - virtual bool delete_rosbag_file(const std::string & fault_code) = 0; + /// @param id The record to delete rosbags for + /// @return true if at least one row was deleted, false if none was found + virtual bool delete_rosbag_file(const FaultId & id) = 0; - /// Delete the records of several faults (typically the whole burst behind one + /// Delete the rows of several records (typically the whole burst behind one /// recording). Backends with real transactions (SQLite) remove the rows /// atomically and unlink the file only after the commit, so a crash mid-delete /// never leaves a row pointing at a removed bag. Default: plain loop. - /// @param fault_codes The fault codes to delete rosbag records for - /// @return Number of records actually deleted - virtual size_t delete_rosbag_files(const std::vector & fault_codes) { + /// @param ids The records to delete rosbag rows for + /// @return Number of records for which rows were actually deleted + virtual size_t delete_rosbag_files(const std::vector & ids) { size_t deleted = 0; - for (const auto & code : fault_codes) { - if (delete_rosbag_file(code)) { + for (const auto & id : ids) { + if (delete_rosbag_file(id)) { ++deleted; } } @@ -452,25 +528,27 @@ class FaultStorage { /// @return Vector of rosbag file info virtual std::vector get_all_rosbag_files() const = 0; - /// Get rosbags for all faults associated with an entity + /// Get rosbags of every record owned by an entity /// @param entity_fqn The entity's fully qualified name to filter by - /// @return Vector of rosbag file info for faults reported by this entity + /// @return Vector of rosbag file info for records this entity owns virtual std::vector list_rosbags_for_entity(const std::string & entity_fqn) const = 0; - /// Get all stored faults regardless of status (for filtering) - /// @return Vector of all faults in storage + /// Get all stored fault records regardless of status (for filtering) + /// @return Vector of all records in storage virtual std::vector get_all_faults() const = 0; - /// Predicate for reclassify_healed_as_cleared: true keeps that fault code HEALED. + /// Predicate for reclassify_healed_as_cleared: true keeps records of that fault code HEALED. using KeepHealedFn = std::function; - /// One-time startup cleanup: reclassify HEALED faults as CLEARED, so a HEALED row left by a + /// One-time startup cleanup: reclassify HEALED records as CLEARED, so a HEALED row left by a /// previous (healing-enabled) run does not behave inconsistently under the latch. /// Default is a no-op (in-memory storage starts empty). /// @param keep_healed Codes for which this returns true stay HEALED (their own healing is still - /// enabled). Empty means reclassify every HEALED fault. - /// @return fault codes of the reclassified faults, so the caller can audit each transition - virtual std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) { + /// enabled). Empty means reclassify every HEALED record. + /// @return the reclassified records, so the caller can audit each transition. Identities, + /// not codes: one code can hold a HEALED record for one owner and an untouched + /// CONFIRMED one for another, and auditing by code would claim both moved. + virtual std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) { (void)keep_healed; return {}; } @@ -491,22 +569,23 @@ class InMemoryFaultStorage : public FaultStorage { void set_debounce_config(const DebounceConfig & config) override; DebounceConfig get_debounce_config() const override; - bool report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, - const std::string & description, const std::string & source_id, int64_t timestamp_ns, - const DebounceConfig & config) override; + bool report_fault_event(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, + const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) override; - std::vector list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const override; + std::vector list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const override; + + std::optional get_fault(const FaultId & id) const override; - std::optional get_fault(const std::string & fault_code) const override; + std::vector get_ids_by_code(const std::string & fault_code) const override; - bool clear_fault(const std::string & fault_code) override; + bool clear_fault(const FaultId & id) override; size_t size() const override; - bool contains(const std::string & fault_code) const override; + bool contains(const FaultId & id) const override; - std::vector check_time_based_confirmation(int64_t now_ns) override; + std::vector check_time_based_confirmation(int64_t now_ns) override; void set_max_snapshots_per_fault(size_t max_count) override; void set_retain_snapshots_on_clear(bool retain) override; @@ -514,32 +593,31 @@ class InMemoryFaultStorage : public FaultStorage { void store_snapshot(const SnapshotData & snapshot) override; void store_snapshots(const std::vector & snapshots) override; - std::vector get_snapshots(const std::string & fault_code, - const std::string & topic_filter = "") const override; + std::vector get_snapshots(const FaultId & id, const std::string & topic_filter = "") const override; int64_t get_max_capture_id() const override; void store_freeze_frame(const FreezeFrameData & frame) override; - std::optional get_freeze_frame(const std::string & fault_code) const override; + std::optional get_freeze_frame(const FaultId & id) const override; void set_max_rosbags_per_fault(size_t max_count) override; size_t set_max_near_misses_per_fault(size_t max_count) override; - std::vector get_near_misses(const std::string & fault_code) const override; + std::vector get_near_misses(const FaultId & id) const override; void store_rosbag_file(const RosbagFileInfo & info) override; /// All-or-nothing, as the base class requires: the batch is built beside the live /// map and swapped in, so a throw leaves the store exactly as it was. void store_rosbag_files(const std::vector & infos) override; - std::optional get_rosbag_file(const std::string & fault_code) const override; - std::vector get_rosbag_files(const std::string & fault_code) const override; + std::optional get_rosbag_file(const FaultId & id) const override; + std::vector get_rosbag_files(const FaultId & id) const override; std::vector get_rosbag_files_by_recording(const std::string & recording_id) const override; - bool delete_rosbag_file(const std::string & fault_code) override; + bool delete_rosbag_file(const FaultId & id) override; size_t delete_rosbag_recording(const std::string & recording_id) override; size_t get_total_rosbag_storage_bytes() const override; std::vector get_all_rosbag_files() const override; std::vector list_rosbags_for_entity(const std::string & entity_fqn) const override; std::vector get_all_faults() const override; - std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) override; + std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) override; private: /// Update fault status based on debounce counter and given config @@ -558,12 +636,12 @@ class InMemoryFaultStorage : public FaultStorage { bool path_referenced(const std::string & file_path) const; mutable std::mutex mutex_; - std::map faults_; + std::map faults_; std::vector snapshots_; - std::map freeze_frames_; ///< fault_code -> freeze-frame (retained across clear) + std::map freeze_frames_; ///< record -> freeze-frame (retained across clear) /// One entry per LINK, mirroring the flat SQLite table rather than a map keyed by - /// fault code - a fault holds several recordings now, and a recording several - /// faults. + /// the record - a record holds several recordings now, and a recording several + /// records. /// /// `seq` is the in-memory twin of SQLite's autoincrement id and is load-bearing, /// not decoration: a burst stamps ONE created_at_ns across every row it writes, so @@ -580,7 +658,7 @@ class InMemoryFaultStorage : public FaultStorage { /// A backend constructed directly (tests, embedders) therefore behaves exactly as /// it always did until someone opts into a history. 0 = unlimited. size_t max_rosbags_per_fault_{1}; - std::map> near_misses_; ///< fault_code -> series (retained across clear) + std::map> near_misses_; ///< record -> series (retained across clear) DebounceConfig config_; size_t max_snapshots_per_fault_{0}; ///< 0 = unlimited bool retain_snapshots_on_clear_{false}; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/postgres_fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/postgres_fault_storage.hpp index 360082c63..0b1205750 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/postgres_fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/postgres_fault_storage.hpp @@ -62,22 +62,23 @@ class PgFaultStorage : public FaultStorage { void set_debounce_config(const DebounceConfig & config) override; DebounceConfig get_debounce_config() const override; - bool report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, - const std::string & description, const std::string & source_id, int64_t timestamp_ns, - const DebounceConfig & config) override; + bool report_fault_event(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, + const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) override; - std::vector list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const override; + std::vector list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const override; - std::optional get_fault(const std::string & fault_code) const override; + std::optional get_fault(const FaultId & id) const override; - bool clear_fault(const std::string & fault_code) override; + std::vector get_ids_by_code(const std::string & fault_code) const override; + + bool clear_fault(const FaultId & id) override; size_t size() const override; - bool contains(const std::string & fault_code) const override; + bool contains(const FaultId & id) const override; - std::vector check_time_based_confirmation(int64_t now_ns) override; + std::vector check_time_based_confirmation(int64_t now_ns) override; void set_max_snapshots_per_fault(size_t max_count) override; void set_retain_snapshots_on_clear(bool retain) override; @@ -87,28 +88,27 @@ class PgFaultStorage : public FaultStorage { void store_snapshot(const SnapshotData & snapshot) override; void store_snapshots(const std::vector & snapshots) override; - std::vector get_snapshots(const std::string & fault_code, - const std::string & topic_filter = "") const override; + std::vector get_snapshots(const FaultId & id, const std::string & topic_filter = "") const override; int64_t get_max_capture_id() const override; void store_freeze_frame(const FreezeFrameData & frame) override; - std::optional get_freeze_frame(const std::string & fault_code) const override; + std::optional get_freeze_frame(const FaultId & id) const override; size_t set_max_near_misses_per_fault(size_t max_count) override; - std::vector get_near_misses(const std::string & fault_code) const override; + std::vector get_near_misses(const FaultId & id) const override; void store_rosbag_file(const RosbagFileInfo & info) override; void store_rosbag_files(const std::vector & infos) override; - std::optional get_rosbag_file(const std::string & fault_code) const override; - std::vector get_rosbag_files(const std::string & fault_code) const override; + std::optional get_rosbag_file(const FaultId & id) const override; + std::vector get_rosbag_files(const FaultId & id) const override; std::vector get_rosbag_files_by_recording(const std::string & recording_id) const override; - bool delete_rosbag_file(const std::string & fault_code) override; + bool delete_rosbag_file(const FaultId & id) override; size_t delete_rosbag_recording(const std::string & recording_id) override; - size_t delete_rosbag_files(const std::vector & fault_codes) override; + size_t delete_rosbag_files(const std::vector & ids) override; size_t get_total_rosbag_storage_bytes() const override; std::vector get_all_rosbag_files() const override; std::vector list_rosbags_for_entity(const std::string & entity_fqn) const override; std::vector get_all_faults() const override; - std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) override; + std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) override; /// Get the connection info string used to initialize the database const std::string & conn_info() const { @@ -137,6 +137,20 @@ class PgFaultStorage : public FaultStorage { /// Create the tables and indexes that do not exist yet, in @p tx, and check the columns of existing ones. void create_schema(pqxx::work & tx) const; + /// Bring tables written before records were keyed by (fault_code, owner) to the current layout, in @p tx. + /// Each legacy row becomes the shared record of its code, with every child row attached to it. Does nothing + /// on a fresh database or one already migrated. + void migrate_to_owner_identity(pqxx::work & tx) const; + + /// Whether @p table exists in the current schema. + bool table_exists(pqxx::work & tx, const char * table) const; + + /// Whether @p table has @p column in the current schema. + bool table_has_column(pqxx::work & tx, const char * table, const char * column) const; + + /// Drop the PRIMARY KEY of @p table, whatever it is named. + void drop_primary_key(pqxx::work & tx, const char * table) const; + /// @p text with every copy of the known password replaced by "***". std::string redact(std::string text) const; @@ -152,24 +166,27 @@ class PgFaultStorage : public FaultStorage { /// report_fault_event body without taking mutex_ or opening a transaction. Caller holds mutex_ /// and supplies the transaction, so the fault row and any near-miss row commit together. - bool report_fault_event_locked(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, + bool report_fault_event_locked(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config, pqxx::work & tx); /// Append one entry to the near-miss series and evict the oldest entries beyond /// max_near_misses_per_fault_. Caller holds mutex_ and has already written the fault row. - /// @param fault_code The fault code that nearly confirmed + /// @param id The record that nearly confirmed; it scopes the series /// @param occurred_at_ns Timestamp of the report /// @param debounce_counter Counter value after the report /// @param config Debounce config the report was evaluated against /// @param severity Severity carried by the report - /// @param source_id Reporting source + /// @param source_id Source that sent the report /// @param resulting_status Fault status after the report was applied /// @param tx Transaction the fault row was written in - void record_near_miss_locked(const std::string & fault_code, int64_t occurred_at_ns, int32_t debounce_counter, + void record_near_miss_locked(const FaultId & id, int64_t occurred_at_ns, int32_t debounce_counter, const DebounceConfig & config, uint8_t severity, const std::string & source_id, const std::string & resulting_status, pqxx::work & tx); + /// One row selected with kFaultColumns, as a record. + fd::FaultRecord read_fault_row(const pqxx::row & r) const; + /// Deserialize JSON array string from PostgreSQL TEXT/JSONB field std::vector parse_json_array(const std::string & json_str) const; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/ros2/ros_capture_hooks.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/ros2/ros_capture_hooks.hpp index 856d3b489..3f2112f4c 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/ros2/ros_capture_hooks.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/ros2/ros_capture_hooks.hpp @@ -26,27 +26,27 @@ namespace ros2_medkit_fault_manager::ros2 { /// the function given at construction; the node reads its capture members when it runs. class RosCaptureHooks : public core::CaptureHooks { public: - using CaptureFn = std::function; - using CodeFn = std::function; + using CaptureFn = std::function; + using RecordFn = std::function; - RosCaptureHooks(CaptureFn capture_now, CodeFn on_prefailed, CodeFn on_cleared) + RosCaptureHooks(CaptureFn capture_now, RecordFn on_prefailed, RecordFn on_cleared) : capture_now_(std::move(capture_now)), on_prefailed_(std::move(on_prefailed)), on_cleared_(std::move(on_cleared)) { } - core::CaptureResult capture_now(const std::string & code) override { - return capture_now_(code); + core::CaptureResult capture_now(const FaultId & id) override { + return capture_now_(id); } - void on_prefailed(const std::string & code) override { - on_prefailed_(code); + void on_prefailed(const FaultId & id) override { + on_prefailed_(id); } - void on_cleared(const std::string & code) override { - on_cleared_(code); + void on_cleared(const FaultId & id) override { + on_cleared_(id); } private: CaptureFn capture_now_; - CodeFn on_prefailed_; - CodeFn on_cleared_; + RecordFn on_prefailed_; + RecordFn on_cleared_; }; } // namespace ros2_medkit_fault_manager::ros2 diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/rosbag_capture.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/rosbag_capture.hpp index f380a8503..afa28afc6 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/rosbag_capture.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/rosbag_capture.hpp @@ -109,22 +109,24 @@ class RosbagCapture { /// Check if ring buffer is currently running bool is_running() const; - /// Called when a fault enters PREFAILED state (for lazy_start mode) - /// @param fault_code The fault code that entered PREFAILED - void on_fault_prefailed(const std::string & fault_code); - - /// Called when a fault is confirmed - flushes buffer to bag file. A fault that - /// confirms while the previous fault's post-roll is still running is attached - /// to that recording (same burst, same window) rather than losing its bag. - /// @param fault_code The fault code that was confirmed - void on_fault_confirmed(const std::string & fault_code); - - /// Called when a fault is cleared - deletes its bag record if auto_cleanup. - /// A shared bag survives until its last referencing fault clears; a fault + /// Called when a fault record enters PREFAILED state (for lazy_start mode) + /// @param id The record that entered PREFAILED + void on_fault_prefailed(const FaultId & id); + + /// Called when a fault record is confirmed - flushes buffer to bag file. A record + /// that confirms while the previous one's post-roll is still running is attached + /// to that recording (same burst, same window) rather than losing its bag. Two + /// owners confirming one code in a burst attach to the one in-flight recording, + /// and each gets its own rosbag_files row. + /// @param id The record that was confirmed + void on_fault_confirmed(const FaultId & id); + + /// Called when a fault record is cleared - deletes its bag record if auto_cleanup. + /// A shared bag survives until its last referencing record clears. A record /// cleared during its burst's post-roll is dropped from the in-flight - /// recording state and never gets a record. - /// @param fault_code The fault code that was cleared - void on_fault_cleared(const std::string & fault_code); + /// recording state and never gets a row. + /// @param id The record that was cleared + void on_fault_cleared(const FaultId & id); /// Get current configuration const RosbagConfig & config() const { @@ -244,10 +246,10 @@ class RosbagCapture { /// (falls back to SensorDataQoS when no publisher is known or qos_match is off) rclcpp::QoS resolve_topic_qos(const std::string & topic) const; - /// Compute the entity topic set for a fault (the faulting source node's + /// Compute the entity topic set for a fault record (its owning source node's /// pub/sub topics + /tf, intersected with the subscribed set). Empty set = /// scope unresolved. Never throws; failures degrade to an empty set. - std::set compute_entity_topics(const std::string & fault_code); + std::set compute_entity_topics(const FaultId & id); /// In "entity" mode, compute the set of topics to write for a confirmed fault /// (the faulting source node's pub/sub topics + /tf). Empty set = write all. @@ -256,7 +258,7 @@ class RosbagCapture { /// In "entity" mode, union an attached fault's entity topics into the active /// capture filter so its data reaches the shared bag from the attach onwards /// (empty resolution widens to all topics). Caller holds post_fault_timer_mutex_. - void widen_capture_filter_for(const std::string & fault_code, const std::set & topics); + void widen_capture_filter_for(const FaultId & id, const std::set & topics); /// Whether a topic should be written to the bag given the active entity filter bool should_capture_topic(const std::string & topic) const; @@ -295,9 +297,9 @@ class RosbagCapture { /// A post-fault recording whose writer is closed, on its way to the store worker. struct ClosedRecording { std::string bag_path; - /// Empty when the primary fault was cleared during the post-roll. - std::string fault_code; - std::set attached_fault_codes; + /// Empty fault_code when the primary record was cleared during the post-roll. + FaultId fault_id; + std::set attached_ids; double duration_sec{0.0}; int64_t created_at_ns{0}; }; @@ -317,7 +319,7 @@ class RosbagCapture { /// Make @p rows durable for the finished bag at @p bag_path, or discard the bag. /// /// Both finalisation paths end here, so a bag that cannot be looked up never - /// survives on disk: retrieval is keyed by fault code and the quota enumerates + /// survives on disk: retrieval goes through the rows and the quota enumerates /// rows, so a directory with no row is unreachable, uncounted, and can never be /// evicted to make room. /// @@ -339,7 +341,7 @@ class RosbagCapture { /// Whether the fault still gets a row: false only under auto_cleanup with no history, /// for a fault the store holds as cleared. Called under clear_rows_mutex_. - bool row_still_wanted(const std::string & code); + bool row_still_wanted(const FaultId & id); /// Whether the operator asked this fault code to keep a recording history. /// @@ -390,9 +392,9 @@ class RosbagCapture { /// Cheap, and checked before the entity scope is resolved: a level-triggered /// reporter re-confirming the same fault would otherwise pay for a fault-store read /// and a full graph enumeration on every repeat, none of which it can use. - bool is_current_recording_primary(const std::string & fault_code) const; + bool is_current_recording_primary(const FaultId & id) const; - bool attach_to_active_recording(const std::string & fault_code, const std::set & entity_topics); + bool attach_to_active_recording(const FaultId & id, const std::set & entity_topics); /// Try to subscribe to a single topic /// @param topic The topic to subscribe to @@ -443,16 +445,17 @@ class RosbagCapture { /// Running state std::atomic running_{false}; - /// Upper bound on how many extra faults one recording is registered for. + /// Upper bound on how many extra records one recording is registered for. static constexpr size_t kMaxAttachedFaults = 32; /// Post-fault recording state - std::string current_fault_code_; + FaultId current_fault_id_; std::string current_bag_path_; - /// Faults confirmed while the post-roll was already running. They share the + /// Records confirmed while the post-roll was already running. They share the /// recording window (one root cause, one burst), so each gets a metadata row - /// pointing at the same bag when it finalises. - std::set attached_fault_codes_; + /// pointing at the same bag when it finalises. Two owners of one code are two + /// entries here and two rows on finalise. + std::set attached_fault_ids_; /// Protects post_fault_timer_, the recording_post_fault_ transitions and the /// state above against concurrent access from on_fault_confirmed() (capture-pool /// thread) and post_fault_timer_callback() / stop() (executor thread). The diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp index c0705c55e..1b02e7f0d 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/snapshot_capture.hpp @@ -15,10 +15,12 @@ #pragma once #include +#include #include #include #include #include +#include #include #include @@ -89,9 +91,9 @@ struct RosbagConfig { /// (reliable/transient-local where offered) instead of forcing best-effort bool qos_match{true}; - /// Recordings retained per fault code (0 = unlimited, bounded only by - /// max_total_storage_mb). Keep-newest: past the cap the fault's OLDEST recording - /// loses its row, and the bag goes once no fault references it. 1 reproduces the + /// Recordings retained per fault record (0 = unlimited, bounded only by + /// max_total_storage_mb). Keep-newest: past the cap the record's OLDEST recording + /// loses its row, and the bag goes once no record references it. 1 reproduces the /// pre-#620 behaviour, where a re-confirm replaced the previous recording. size_t max_bags_per_fault{1}; @@ -132,6 +134,10 @@ struct SnapshotConfig { /// default_topics) always wins when present. bool entity_default{true}; + /// Recapture cooldown in seconds (0 = off). Per-source records of one code that confirm inside it + /// share one capture of the code's topics. + double recapture_cooldown_sec{0.0}; + /// Rosbag recording configuration (optional) RosbagConfig rosbag; }; @@ -165,17 +171,44 @@ class SnapshotCapture { SnapshotCapture(SnapshotCapture &&) = delete; SnapshotCapture & operator=(SnapshotCapture &&) = delete; - /// Capture snapshots for a fault that was just confirmed + /// Capture snapshots for a fault record that was just confirmed + /// + /// Topic resolution stays keyed by the fault CODE (fault_specific and patterns are + /// config about what a code means), while everything written is keyed by the RECORD: + /// the rows, the freeze frame and the mid-capture acknowledgement check. Two owners + /// confirming one code therefore capture the same topics into two separate frames. /// /// If the fault code resolves to no capture set (not in fault_specific, no pattern /// match, no default_topics), the entity-default fallback (when enabled) captures - /// the reporting source node's own published topics instead. A code explicitly + /// the owning source node's own published topics instead. A code explicitly /// listed in fault_specific or matched by a pattern never falls through - a /// present-but-empty topic list is a per-fault opt-out. Only when nothing /// resolves does capture return early: no freeze_frames row is written /// (no empty {} row) and FaultStorage::get_freeze_frame() returns nullopt for it. - /// @param fault_code The fault code that was confirmed - void capture(const std::string & fault_code); + /// @param id The fault record that was confirmed + void capture(const FaultId & id); + + /// What the caller does with the capture of a record. + enum class Admission { + kEnqueue, ///< Queue capture(): the record needs a worker. + kJoined ///< The record shares a capture already running or done: queue nothing for snapshots. + }; + + /// Decide, before capture() is queued, whether the record needs a capture of its own. + /// + /// Per-source records of one code confirming inside the recapture cooldown share ONE capture of the + /// code's topics, and every one of them gets the rows and the freeze frame written under its own + /// owner. A code has at most one queued job for its group: a record joining a queued or running + /// capture queues nothing. A record that confirms again while its old capture still runs gets a + /// capture of its own, taken after that confirmation. Only a code whose topics are resolved by code + /// qualifies: the entity-default topics depend on the owner. A shared record, a disabled cooldown + /// or an unconfigured code is always kEnqueue. + Admission admit(const FaultId & id); + + /// The queued capture job of @p id did not run (the queue refused or evicted it). When it was the + /// job of a group that has not started, the group's capture is lost for every member and the group + /// goes, so that a later confirmation starts a new one. + void withdraw(const FaultId & id); /// Get current configuration const SnapshotConfig & config() const { @@ -196,12 +229,40 @@ class SnapshotCapture { /// entity-default capture. std::vector resolve_topics(const std::string & fault_code, bool & explicit_match) const; - /// Entity-default fallback: topics published by the fault's reporting source - /// node(s), excluding per-node noise (/rosout, /parameter_events). Non-FQN - /// sources (bare plugin entity ids - the gateway covers those) are skipped, - /// and empty is returned when no source resolves to a live node. Never + /// Entity-default fallback: topics published by the record's owning source + /// node, excluding per-node noise (/rosout, /parameter_events). A non-FQN + /// owner (a bare plugin entity id - the gateway covers those) is skipped, + /// and empty is returned when the owner resolves to no live node. Never /// throws; any failure degrades to empty. - std::vector resolve_entity_topics(const std::string & fault_code) const; + std::vector resolve_entity_topics(const FaultId & id) const; + + /// Write the rows and the freeze frame of one capture under the record @p id. + void persist_capture(const FaultId & id, std::vector rows, const nlohmann::json & freeze_frame, + size_t captured_count, bool entity_scoped, int64_t capture_id); + + /// The capture of one code that its per-source records share inside the cooldown. + struct CaptureGroup { + enum class State { kQueued, kRunning, kDone }; + State state{State::kQueued}; + std::string primary; ///< Owner whose job is queued for the group + std::set owners; ///< Records the capture is written for + /// Records that confirmed again while the group ran: each takes a capture of its own. + std::set solo; + std::chrono::steady_clock::time_point started; + /// What the capture sampled, kept while the cooldown runs so a record confirming after it + /// finished gets the same rows. + std::vector rows; + nlohmann::json freeze_frame = nlohmann::json::object(); + size_t captured_count{0}; + int64_t capture_id{0}; + }; + + /// Drop finished groups older than the cooldown. A queued or running group stays until its capture + /// completes. Caller holds groups_mutex_. + void prune_groups_locked(std::chrono::steady_clock::time_point now); + + /// Whether @p id takes part in grouping: a per-source record of a code with by-code topics. + bool groups_by_code(const FaultId & id) const; /// Capture a single topic on-demand (creates temporary subscription) /// On success also records the captured value into @p freeze_frame under the topic key. @@ -209,14 +270,14 @@ class SnapshotCapture { /// Appends to @p rows rather than storing: a capture is persisted as one set, /// so the per-fault cap can drop a whole old capture instead of truncating this /// one topic by topic. - bool capture_topic_on_demand(const std::string & fault_code, const std::string & topic, nlohmann::json & freeze_frame, + bool capture_topic_on_demand(const FaultId & id, const std::string & topic, nlohmann::json & freeze_frame, std::vector & rows); /// Capture a topic from background cache /// On success also records the cached value into @p freeze_frame under the topic key. /// @return true if data was available in cache - bool capture_topic_from_cache(const std::string & fault_code, const std::string & topic, - nlohmann::json & freeze_frame, std::vector & rows); + bool capture_topic_from_cache(const FaultId & id, const std::string & topic, nlohmann::json & freeze_frame, + std::vector & rows); /// Initialize background subscriptions for all configured topics void init_background_subscriptions(); @@ -251,6 +312,10 @@ class SnapshotCapture { std::atomic capture_seq_seeded_{false}; std::mutex capture_seq_seed_mutex_; + /// Captures shared by the per-source records of a code (code -> group), guarded by groups_mutex_. + std::mutex groups_mutex_; + std::map groups_; + /// Compiled regex patterns (cached for performance) std::vector>> compiled_patterns_; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp index 12fcd9b27..e107b8851 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/sqlite_fault_storage.hpp @@ -48,22 +48,23 @@ class SqliteFaultStorage : public FaultStorage { void set_debounce_config(const DebounceConfig & config) override; DebounceConfig get_debounce_config() const override; - bool report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, - const std::string & description, const std::string & source_id, int64_t timestamp_ns, - const DebounceConfig & config) override; + bool report_fault_event(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, + const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) override; - std::vector list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const override; + std::vector list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const override; - std::optional get_fault(const std::string & fault_code) const override; + std::optional get_fault(const FaultId & id) const override; - bool clear_fault(const std::string & fault_code) override; + std::vector get_ids_by_code(const std::string & fault_code) const override; + + bool clear_fault(const FaultId & id) override; size_t size() const override; - bool contains(const std::string & fault_code) const override; + bool contains(const FaultId & id) const override; - std::vector check_time_based_confirmation(int64_t now_ns) override; + std::vector check_time_based_confirmation(int64_t now_ns) override; void set_max_snapshots_per_fault(size_t max_count) override; void set_retain_snapshots_on_clear(bool retain) override; @@ -73,29 +74,28 @@ class SqliteFaultStorage : public FaultStorage { void store_snapshot(const SnapshotData & snapshot) override; void store_snapshots(const std::vector & snapshots) override; - std::vector get_snapshots(const std::string & fault_code, - const std::string & topic_filter = "") const override; + std::vector get_snapshots(const FaultId & id, const std::string & topic_filter = "") const override; int64_t get_max_capture_id() const override; void store_freeze_frame(const FreezeFrameData & frame) override; - std::optional get_freeze_frame(const std::string & fault_code) const override; + std::optional get_freeze_frame(const FaultId & id) const override; size_t set_max_near_misses_per_fault(size_t max_count) override; - std::vector get_near_misses(const std::string & fault_code) const override; + std::vector get_near_misses(const FaultId & id) const override; void store_rosbag_file(const RosbagFileInfo & info) override; void store_rosbag_files(const std::vector & infos) override; - std::optional get_rosbag_file(const std::string & fault_code) const override; - std::vector get_rosbag_files(const std::string & fault_code) const override; + std::optional get_rosbag_file(const FaultId & id) const override; + std::vector get_rosbag_files(const FaultId & id) const override; std::vector get_rosbag_files_by_recording(const std::string & recording_id) const override; - bool delete_rosbag_file(const std::string & fault_code) override; + bool delete_rosbag_file(const FaultId & id) override; size_t delete_rosbag_recording(const std::string & recording_id) override; - size_t delete_rosbag_files(const std::vector & fault_codes) override; + size_t delete_rosbag_files(const std::vector & ids) override; size_t get_total_rosbag_storage_bytes() const override; std::vector get_all_rosbag_files() const override; std::vector list_rosbags_for_entity(const std::string & entity_fqn) const override; std::vector get_all_faults() const override; - std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) override; + std::vector reclassify_healed_as_cleared(const KeepHealedFn & keep_healed = {}) override; /// Get the database path const std::string & db_path() const { @@ -122,46 +122,66 @@ class SqliteFaultStorage : public FaultStorage { /// C++ through rosbag_recording_id() so the basename rule has one implementation. void migrate_rosbag_files_add_recording_id(); - /// Whether a fault other than @p fault_code still references @p file_path. - /// One recording can back several faults of the same burst, so the bag must - /// only be unlinked once the last of them is gone. Caller holds mutex_. - - /// Whether any fault at all still references @p file_path. Caller holds mutex_. + /// Move a database whose fault identity is the bare fault_code onto (fault_code, owner). + /// + /// Detected per table by PRAGMA table_info lacking `owner`, which is what makes it safe + /// to re-run on every open. faults and freeze_frames are REBUILT (their identity is a + /// PRIMARY KEY, which ALTER TABLE cannot change and CREATE TABLE IF NOT EXISTS would + /// silently skip), while snapshots, rosbag_files and near_misses only gain a column. + /// A legacy database holds one record per fault code, fed by every source, so every row + /// becomes the shared record of its code and every child row attaches to it. The rows keep + /// their reporting_sources as written. All of it runs in one transaction. + void migrate_faults_add_owner(); + + /// Whether @p table exists, asked of sqlite_master. Used to tell a leftover scratch table + /// apart from a clean start, which DROP TABLE IF EXISTS on its own cannot report. + bool table_exists(const char * table) const; + + /// Drop one of the migration's scratch tables, warning first when it was really there. + /// `faults_new` and `freeze_frames_new` are reserved for this procedure, so debris under + /// those names is always an unfinished rebuild and never somebody's data. + void drop_scratch_table(const char * table); + + /// Whether @p table already carries the `owner` column. The four tables are created + /// by four independent CREATE TABLE IF NOT EXISTS statements, so a database can hold + /// a legacy one next to one this release just created: each is probed on its own. + bool table_has_owner(const char * table) const; + + /// Whether any row at all still references @p file_path. One recording can back + /// several records of the same burst, so the bag must only be unlinked once the + /// last of them is gone. Caller holds mutex_. bool path_referenced(const std::string & file_path) const; /// store_rosbag_file body without taking mutex_. Caller holds mutex_ and /// unlinks the returned replaced-bag path once the row change is durable. - /// @return file_paths whose last row for this fault the per-fault cap evicted. + /// @return file_paths whose last row for this record the per-record cap evicted. /// The caller unlinks each only after the commit, and only if path_referenced() /// still says nobody holds it. std::vector store_rosbag_file_locked(const RosbagFileInfo & info); /// report_fault_event body without taking mutex_ or opening a transaction. Caller holds mutex_ /// and wraps the call, so the fault row and any near-miss row commit together. - bool report_fault_event_locked(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, + bool report_fault_event_locked(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config); /// Append one entry to the near-miss series and evict the oldest entries beyond /// max_near_misses_per_fault_. Caller holds mutex_ and has already written the fault row. - /// @param fault_code The fault code that nearly confirmed + /// @param id The record that nearly confirmed; it scopes the series /// @param occurred_at_ns Timestamp of the report /// @param debounce_counter Counter value after the report /// @param config Debounce config the report was evaluated against /// @param severity Severity carried by the report - /// @param source_id Reporting source + /// @param source_id Source that sent the report /// @param resulting_status Fault status after the report was applied - void record_near_miss_locked(const std::string & fault_code, int64_t occurred_at_ns, int32_t debounce_counter, + void record_near_miss_locked(const FaultId & id, int64_t occurred_at_ns, int32_t debounce_counter, const DebounceConfig & config, uint8_t severity, const std::string & source_id, const std::string & resulting_status); /// Run a plain SQL statement or throw with the SQLite error. Caller holds mutex_. void exec_or_throw(const char * sql); - /// Parse JSON array string to vector of strings - std::vector parse_json_array(const std::string & json_str) const; - - /// Serialize vector of strings to JSON array string + /// Serialize vector of strings to JSON array string, the form of faults.reporting_sources. static std::string serialize_json_array(const std::vector & vec); std::string db_path_; diff --git a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/threshold_resolver.hpp b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/threshold_resolver.hpp index 80508e6d0..48bf09fb2 100644 --- a/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/threshold_resolver.hpp +++ b/src/ros2_medkit_fault_manager/include/ros2_medkit_fault_manager/threshold_resolver.hpp @@ -40,6 +40,9 @@ struct FaultCodeDebounceOverride { std::optional confirmation_threshold; std::optional healing_enabled; std::optional healing_threshold; + /// Whether each reporting source of the code gets its own record. Unset means false: one record + /// per code, shared by every source. Entity overrides do not carry it. + bool per_source{false}; }; /// Resolves per-entity debounce thresholds using longest-prefix matching. @@ -75,11 +78,12 @@ class EntityThresholdResolver { /// Resolves per-fault_code debounce thresholds by exact match on the fault code. /// -/// The debounce counter lives on the fault code while an entity override is -/// selected by the reporting source, so two entities reporting one code debounce -/// it under two policies. A fault-code override is the layer that removes that: -/// it resolves the same whoever reports, and is applied on top of whatever the -/// entity layer produced. +/// A code reported by several sources has one record and one debounce counter, while +/// an entity override is selected by the reporting source, so two entities reporting +/// one code debounce it under two policies. A fault-code override is the layer that +/// removes that: it resolves the same whoever reports, and is applied on top of +/// whatever the entity layer produced. The same entry carries the per-code choice +/// between one shared record and one record per source. class FaultCodeThresholdResolver { public: FaultCodeThresholdResolver() = default; @@ -94,12 +98,16 @@ class FaultCodeThresholdResolver { /// them. A code with no override returns `base` unchanged. DebounceConfig resolve(const std::string & fault_code, const DebounceConfig & base) const; + /// Whether @p fault_code is configured `per_source`. A code with no entry is not. + bool per_source(const std::string & fault_code) const; + /// Number of configured fault-code entries. size_t size() const; /// Load fault-code threshold overrides from a YAML file. /// Returns an empty vector on parse error (logs warning through @p sink). - /// YAML format: map of fault code -> {confirmation_threshold, healing_enabled, healing_threshold} + /// YAML format: map of fault code -> {confirmation_threshold, healing_enabled, healing_threshold, per_source}. + /// per_source must be the YAML boolean true or false; any other value rejects the whole file. static std::vector load_from_yaml(const std::string & path, core::LogSink & sink); private: diff --git a/src/ros2_medkit_fault_manager/src/capture_thread_pool.cpp b/src/ros2_medkit_fault_manager/src/capture_thread_pool.cpp index 8662855ce..f969c962c 100644 --- a/src/ros2_medkit_fault_manager/src/capture_thread_pool.cpp +++ b/src/ros2_medkit_fault_manager/src/capture_thread_pool.cpp @@ -20,7 +20,7 @@ namespace ros2_medkit_fault_manager { CaptureThreadPool::CaptureThreadPool(std::size_t pool_size, std::size_t queue_depth, QueueFullPolicy full_policy, - core::LogSink & log, std::function capture_fn, + core::LogSink & log, std::function capture_fn, std::string logger_name) : queue_depth_(queue_depth == 0 ? 1 : queue_depth) , full_policy_(full_policy) @@ -44,13 +44,13 @@ CaptureThreadPool::~CaptureThreadPool() { shutdown(); } -EnqueueOutcome CaptureThreadPool::enqueue(const std::string & fault_code) { +EnqueueOutcome CaptureThreadPool::enqueue(const FaultId & id) { std::lock_guard lock(queue_mutex_); if (stop_) { return {EnqueueResult::kRejectedShuttingDown, std::nullopt}; } if (queue_.size() < queue_depth_) { - queue_.push_back(fault_code); + queue_.push_back(id); cv_.notify_one(); return {EnqueueResult::kAccepted, std::nullopt}; } @@ -60,9 +60,9 @@ EnqueueOutcome CaptureThreadPool::enqueue(const std::string & fault_code) { return {EnqueueResult::kDroppedNewest, std::nullopt}; } // kDropOldest: evict the oldest pending job. - std::string evicted = std::move(queue_.front()); + FaultId evicted = std::move(queue_.front()); queue_.pop_front(); - queue_.push_back(fault_code); + queue_.push_back(id); dropped_captures_.fetch_add(1, std::memory_order_relaxed); cv_.notify_one(); return {EnqueueResult::kEvictedOldest, std::move(evicted)}; @@ -103,7 +103,7 @@ std::size_t CaptureThreadPool::pending_size() const { void CaptureThreadPool::worker_loop() { for (;;) { - std::string job; + FaultId job; { std::unique_lock lock(queue_mutex_); cv_.wait(lock, [this] { @@ -120,10 +120,11 @@ void CaptureThreadPool::worker_loop() { capture_fn_(job); } } catch (const std::exception & e) { - core::logf(log_, core::LogLevel::kError, logger_name_, "Capture job for '%s' threw: %s", job.c_str(), e.what()); + core::logf(log_, core::LogLevel::kError, logger_name_, "Capture job for '%s' threw: %s", job.fault_code.c_str(), + e.what()); } catch (...) { core::logf(log_, core::LogLevel::kError, logger_name_, "Capture job for '%s' threw unknown exception", - job.c_str()); + job.fault_code.c_str()); } } } diff --git a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp index e29364da9..e343d4cf1 100644 --- a/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp +++ b/src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include namespace ros2_medkit_fault_manager::core { @@ -60,6 +60,22 @@ int64_t next_due_ns(int64_t due, int64_t now, int64_t period_ns) { return due + k * period_ns; } +/// Name of a Fault.msg SEVERITY_* value, as the constants spell it. +const char * severity_name(uint8_t severity) { + switch (severity) { + case fd::severity::kInfo: + return "INFO"; + case fd::severity::kWarn: + return "WARN"; + case fd::severity::kError: + return "ERROR"; + case fd::severity::kCritical: + return "CRITICAL"; + default: + return "UNKNOWN"; + } +} + std::string extract_topic_name(const std::string & topic_path) { auto pos = topic_path.rfind('/'); if (pos != std::string::npos && pos < topic_path.length() - 1) { @@ -125,9 +141,10 @@ ReportOutcome FaultManagerCore::report(const fd::FaultReport & report) { return report_locked(report); } -ClearOutcome FaultManagerCore::clear(const std::string & code, bool skip_correlation_auto_clear) { +ClearOutcome FaultManagerCore::clear(const std::string & code, const std::string & source_id, + bool skip_correlation_auto_clear) { std::lock_guard lock(mutex_); - return clear_locked(code, skip_correlation_auto_clear); + return clear_locked(code, source_id, skip_correlation_auto_clear); } ListOutcome FaultManagerCore::list(const ListQuery & query) { @@ -135,12 +152,12 @@ ListOutcome FaultManagerCore::list(const ListQuery & query) { return list_locked(query); } -FaultDetail FaultManagerCore::get(const std::string & code) { +FaultDetail FaultManagerCore::get(const std::string & code, const std::string & source_id) { std::vector> rosbag_paths; FaultDetail detail; { std::lock_guard lock(mutex_); - detail = get_locked(code, rosbag_paths); + detail = get_locked(code, source_id, rosbag_paths); } // Reads metadata.yaml per recording; outside the lock so fault reports do not wait on disk. for (const auto & [index, path] : rosbag_paths) { @@ -150,6 +167,68 @@ FaultDetail FaultManagerCore::get(const std::string & code) { return detail; } +std::optional FaultManagerCore::resolve_target(const std::string & fault_code, const std::string & source_id, + std::string & error) { + std::lock_guard lock(mutex_); + return resolve_target_locked(fault_code, source_id, error); +} + +std::optional FaultManagerCore::resolve_target_locked(const std::string & fault_code, + const std::string & source_id, std::string & error) { + error.clear(); + + try { + if (!source_id.empty()) { + // The record the id owns, which is how a shared record (owner `*`) is addressed. Otherwise the + // record this source reports into, so a client that only knows a reporting source reaches the + // shared record of a default code. + FaultId id{fault_code, source_id}; + if (storage_->contains(id)) { + return id; + } + id = record_id_for(fault_code, source_id); + if (!storage_->contains(id)) { + error = "Fault not found: " + fault_code + " (source " + source_id + ")"; + return std::nullopt; + } + return id; + } + + // Unscoped. One record: the call means that record. Several: refuse rather than + // pick, because acting on an arbitrary record is the failure the per-source choice + // exists to avoid, and the caller cannot tell which one it got. + const auto candidates = storage_->get_ids_by_code(fault_code); + if (candidates.empty()) { + error = "Fault not found: " + fault_code; + return std::nullopt; + } + if (candidates.size() > 1) { + std::string owners; + for (const auto & candidate : candidates) { + if (!owners.empty()) { + owners += ", "; + } + owners += candidate.owner; + } + error = "ambiguous: " + fault_code + " has " + std::to_string(candidates.size()) + " records, owned by (" + + owners + "). Set source_id to pick one."; + return std::nullopt; + } + return candidates.front(); + } catch (const FaultStorage::IgnorableConnectionException & e) { + logf(log_, LogLevel::kWarn, config_.logger_name, "Failed to connect with the fault storage server: %s", e.what()); + error = storage_unavailable(e); + return std::nullopt; + } +} + +FaultId FaultManagerCore::record_id_for(const std::string & fault_code, const std::string & source_id) const { + if (fault_code_resolver_ && fault_code_resolver_->per_source(fault_code)) { + return FaultId{fault_code, source_id}; + } + return FaultId{fault_code, kSharedOwner}; +} + std::vector FaultManagerCore::list_for_entity(const std::string & entity_id) { std::lock_guard lock(mutex_); std::vector result; @@ -161,10 +240,10 @@ std::vector FaultManagerCore::list_for_entity(const std::string return result; } -void FaultManagerCore::record_transition(const char * transition, const fd::FaultRecord & fault, +void FaultManagerCore::record_transition(const char * transition, const fd::FaultRecord & fault, const FaultId & id, const std::string & source_id, int64_t occurred_at_ns) { std::lock_guard lock(mutex_); - audit_transition(transition, fault, source_id, occurred_at_ns); + audit_transition(transition, fault, id, source_id, occurred_at_ns); } void FaultManagerCore::tick(int64_t wall_now_ns) { @@ -198,7 +277,7 @@ void FaultManagerCore::run_auto_confirm(int64_t wall_now_ns) { logf(log_, LogLevel::kWarn, logger, "Failed to connect with the fault storage server: %s", e.what()); }; - std::vector confirmed; + std::vector confirmed; try { confirmed = storage_->check_time_based_confirmation(wall_now_ns); } catch (const FaultStorage::IgnorableConnectionException & e) { @@ -209,10 +288,10 @@ void FaultManagerCore::run_auto_confirm(int64_t wall_now_ns) { return; } // Audit every timer-driven PREFAILED->CONFIRMED transition, or the audit log's verify() cannot see it. - for (const auto & fault_code : confirmed) { + for (const auto & id : confirmed) { std::optional fault; try { - fault = storage_->get_fault(fault_code); + fault = storage_->get_fault(id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); return; @@ -220,18 +299,18 @@ void FaultManagerCore::run_auto_confirm(int64_t wall_now_ns) { if (!fault) { continue; } - audit_transition(kTransitionConfirmed, *fault, "auto_confirm_timer", wall_now_ns); + audit_transition(kTransitionConfirmed, *fault, id, kAuditSourceAutoConfirm, wall_now_ns); // Same rules as the report path: a symptom muted by a root cause is not announced, and the // capture is not gated by muting. - if (!correlation_ || !correlation_->is_muted(fault_code)) { + if (!correlation_ || !correlation_->is_muted(id)) { publish_event(fd::fault_event_type::kConfirmed, *fault); } - capture_on_confirm(fault_code); + capture_on_confirm(id); } logf(log_, LogLevel::kInfo, logger, "Auto-confirmed %zu PREFAILED fault(s) due to time threshold", confirmed.size()); } -void FaultManagerCore::capture_on_confirm(const std::string & code) { +void FaultManagerCore::capture_on_confirm(const FaultId & id) { const double cooldown_sec = config_.snapshots_recapture_cooldown_sec; const bool cooldown_enabled = cooldown_sec > 0.0; const auto cooldown = std::chrono::duration(cooldown_sec); @@ -246,23 +325,24 @@ void FaultManagerCore::capture_on_confirm(const std::string & code) { ++it; } } - if (last_capture_times_.find(code) != last_capture_times_.end()) { - logf(log_, LogLevel::kDebug, config_.logger_name, "Skipping capture for '%s' - cooldown active", code.c_str()); + if (last_capture_times_.find(id) != last_capture_times_.end()) { + logf(log_, LogLevel::kDebug, config_.logger_name, "Skipping capture for '%s' (source %s) - cooldown active", + id.fault_code.c_str(), id.owner.c_str()); return; } } - const CaptureResult outcome = hooks_.capture_now(code); + const CaptureResult outcome = hooks_.capture_now(id); if (!cooldown_enabled) { return; } switch (outcome.result) { case CaptureEnqueue::kAccepted: - last_capture_times_[code] = now; + last_capture_times_[id] = now; break; case CaptureEnqueue::kEvictedOldest: - last_capture_times_[code] = now; - last_capture_times_.erase(outcome.evicted_code); // keeps the evicted fault retriable + last_capture_times_[id] = now; + last_capture_times_.erase(outcome.evicted_id); // keeps the evicted record retriable break; case CaptureEnqueue::kDroppedNewest: // not recorded: a later re-confirmation may still capture case CaptureEnqueue::kShuttingDown: @@ -318,14 +398,22 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { return reject("source_id cannot be empty"); } + if (report.source_id == kSharedOwner) { + logf(log_, LogLevel::kWarn, logger, "ReportFault rejected: source_id '%s' is reserved", kSharedOwner); + return reject(std::string("source_id '") + kSharedOwner + "' is reserved"); + } + const auto connection_lost = [&](const FaultStorage::IgnorableConnectionException & e) { logf(log_, LogLevel::kWarn, logger, "Failed to connect with the fault storage server: %s", e.what()); }; - // Get status before update (if fault exists) + // The record this report goes into: the source's own for a per_source code, else the code's shared one. + const FaultId id = record_id_for(report.fault_code, report.source_id); + + // Get status before update (if the record exists) std::optional fault_before; try { - fault_before = storage_->get_fault(report.fault_code); + fault_before = storage_->get_fault(id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); return outcome; @@ -339,8 +427,8 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { const int64_t event_time_ns = clock_.now_ns(); bool is_new = false; try { - is_new = storage_->report_fault_event(report.fault_code, report.kind, report.severity, report.description, - report.source_id, event_time_ns, resolved_config); + is_new = storage_->report_fault_event(id, report.kind, report.severity, report.description, report.source_id, + event_time_ns, resolved_config); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); return outcome; @@ -348,25 +436,29 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { outcome.accepted = true; - // Get updated fault state to publish event + // Only a per-source record confirms on its stored severity, so only there a lower report needs explaining. + if (fault_before && failed && id.owner != kSharedOwner && report.severity != fault_before->severity) { + warn_severity_mismatch(report, fault_before->severity); + } + + // Get updated record state to publish event std::optional fault_after; try { - fault_after = storage_->get_fault(report.fault_code); + fault_after = storage_->get_fault(id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); return outcome; } if (fault_after) { - // Only for codes the store holds: a PASSED for an unknown code writes nothing, so it must + // Only for records the store holds: a PASSED for an unknown record writes nothing, so it must // not add an entry here either (the diagnostic bridge sends PASSED for every OK status). - warn_on_conflicting_debounce_policy(report.fault_code, report.source_id, resolved_config); + warn_on_conflicting_debounce_policy(id, report.source_id, resolved_config); // Process through correlation engine (if enabled). Only FAILED events correlate. bool should_mute = false; if (correlation_ && failed) { - auto correlation_result = - correlation_->process_fault(report.fault_code, correlation::severity_to_string(report.severity)); + auto correlation_result = correlation_->process_fault(id, correlation::severity_to_string(report.severity)); should_mute = correlation_result.should_mute; @@ -381,9 +473,9 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { logf(log_, LogLevel::kDebug, logger, "Fault %s added to cluster %s", report.fault_code.c_str(), correlation_result.cluster_id.c_str()); // Log retroactively muted faults when cluster becomes active - if (!correlation_result.retroactive_mute_codes.empty()) { + if (!correlation_result.retroactive_mute_ids.empty()) { logf(log_, LogLevel::kDebug, logger, "Cluster %s activated: retroactively muting %zu faults", - correlation_result.cluster_id.c_str(), correlation_result.retroactive_mute_codes.size()); + correlation_result.cluster_id.c_str(), correlation_result.retroactive_mute_ids.size()); } } } @@ -404,8 +496,8 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { } just_confirmed = true; } else if (!is_new && fault_after->status == fd::fault_status::kConfirmed) { - // Fault was already CONFIRMED, data updated (last_occurred, severity, sources). - // Not occurrence_count: a re-report inside one occurrence does not touch it. + // Record was already CONFIRMED, data updated (last_occurred, severity, reporting_sources). Not + // occurrence_count: a re-report inside one occurrence does not touch it. if (!should_mute) { publish_event(fd::fault_event_type::kUpdated, *fault_after); } @@ -424,23 +516,27 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { // an acknowledged clear. if (just_healed) { publish_event(fd::fault_event_type::kCleared, *fault_after); + // A healed record has ended: its muted symptoms are unmuted and keep their status. + if (correlation_) { + correlation_->process_clear(id); + } } // Append tamper-evident audit records for the transitions that just happened. // Recorded regardless of correlation muting: muting affects display, not the // fact that the state transition occurred. if (is_new) { - audit_transition(kTransitionOccurred, *fault_after, report.source_id, event_time_ns); + audit_transition(kTransitionOccurred, *fault_after, id, report.source_id, event_time_ns); } if (just_confirmed) { - audit_transition(kTransitionConfirmed, *fault_after, report.source_id, event_time_ns); + audit_transition(kTransitionConfirmed, *fault_after, id, report.source_id, event_time_ns); } if (just_healed) { - audit_transition(kTransitionHealed, *fault_after, "auto_heal", event_time_ns); + audit_transition(kTransitionHealed, *fault_after, id, kAuditSourceAutoHeal, event_time_ns); } if (just_confirmed) { - capture_on_confirm(report.fault_code); + capture_on_confirm(id); } // PREFAILED starts the lazy_start rosbag capture @@ -448,7 +544,7 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { (!is_new && status_before != fd::fault_status::kPrefailed && fault_after->status == fd::fault_status::kPrefailed); if (just_prefailed) { - hooks_.on_prefailed(report.fault_code); + hooks_.on_prefailed(id); } } @@ -467,7 +563,8 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) { return outcome; } -ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_correlation_auto_clear) { +ClearOutcome FaultManagerCore::clear_locked(const std::string & code, const std::string & source_id, + bool skip_correlation_auto_clear) { const std::string & logger = config_.logger_name; ClearOutcome outcome; @@ -477,6 +574,17 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_ return outcome; } + // Which record. Resolved before anything changes, so an ambiguous unscoped call + // clears nothing at all rather than one arbitrary owner's record. + std::string resolve_error; + const auto target = resolve_target_locked(code, source_id, resolve_error); + if (!target) { + outcome.message = resolve_error; + logf(log_, LogLevel::kWarn, logger, "ClearFault rejected: %s", resolve_error.c_str()); + return outcome; + } + const FaultId & id = *target; + const auto connection_lost = [&](const FaultStorage::IgnorableConnectionException & e) { logf(log_, LogLevel::kWarn, logger, "Failed to connect with the fault storage server: %s", e.what()); }; @@ -484,21 +592,25 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_ // Storage first: a clear that the storage refuses must not change the correlation state. bool cleared = false; try { - cleared = storage_->clear_fault(code); + cleared = storage_->clear_fault(id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); outcome.message = storage_unavailable(e); return outcome; } - // Then the correlation engine (to get the auto-clear list). + // Then the correlation engine. A cleared record ends its relations whatever the flag says: its muted + // symptoms are unmuted and it stops muting later reports. Symptoms come back as the exact records the + // root cause muted. // `skip_correlation_auto_clear` lets the caller opt out of cascade-clearing - // correlated symptom fault codes. Per-entity DELETE routes set it to true + // correlated symptom faults. Per-entity DELETE routes set it to true // so they cannot reach across entity boundaries via the correlation graph. - std::vector auto_cleared_codes; - if (correlation_ && !skip_correlation_auto_clear) { - auto clear_result = correlation_->process_clear(code); - auto_cleared_codes = clear_result.auto_cleared_codes; + std::vector auto_cleared; + if (correlation_) { + auto clear_result = correlation_->process_clear(id); + if (!skip_correlation_auto_clear) { + auto_cleared = clear_result.auto_cleared_symptoms; + } } outcome.success = cleared; @@ -508,38 +620,43 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_ return outcome; } - // Evict cooldown tracking for the cleared fault and the auto-cleared symptoms - last_capture_times_.erase(code); - for (const auto & symptom_code : auto_cleared_codes) { - last_capture_times_.erase(symptom_code); + // Evict cooldown tracking for the cleared record and the auto-cleared symptoms + last_capture_times_.erase(id); + for (const auto & symptom_id : auto_cleared) { + last_capture_times_.erase(symptom_id); } // Auto-clear correlated symptoms - for (const auto & symptom_code : auto_cleared_codes) { + std::vector auto_cleared_codes; + auto_cleared_codes.reserve(auto_cleared.size()); + for (const auto & symptom_id : auto_cleared) { try { - storage_->clear_fault(symptom_code); + storage_->clear_fault(symptom_id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); outcome.message = storage_unavailable(e); return outcome; } + auto_cleared_codes.push_back(symptom_id.fault_code); + // The cascade goes one level: a cleared symptom that was itself a root releases its own symptoms. + correlation_->process_clear(symptom_id); if (audit_) { std::optional symptom; try { - symptom = storage_->get_fault(symptom_code); + symptom = storage_->get_fault(symptom_id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); outcome.message = storage_unavailable(e); return outcome; } if (symptom) { - audit_transition(kTransitionCleared, *symptom, "clear_service", clock_.now_ns()); + audit_transition(kTransitionCleared, *symptom, symptom_id, kAuditSourceCorrelationAutoClear, clock_.now_ns()); } } - logf(log_, LogLevel::kDebug, logger, "Auto-cleared symptom: %s (root cause: %s)", symptom_code.c_str(), + logf(log_, LogLevel::kDebug, logger, "Auto-cleared symptom: %s (root cause: %s)", symptom_id.fault_code.c_str(), code.c_str()); - // Also cleanup rosbag for auto-cleared faults - hooks_.on_cleared(symptom_code); + // Also cleanup rosbag for auto-cleared records + hooks_.on_cleared(symptom_id); } outcome.auto_cleared_codes = auto_cleared_codes; @@ -549,16 +666,16 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_ outcome.message = "Fault cleared: " + code + " (auto-cleared " + std::to_string(auto_cleared_codes.size()) + " symptoms)"; } - logf(log_, LogLevel::kInfo, logger, "Fault cleared: %s (auto-cleared %zu symptoms)", code.c_str(), - auto_cleared_codes.size()); + logf(log_, LogLevel::kInfo, logger, "Fault cleared: %s (source=%s, auto-cleared %zu symptoms)", code.c_str(), + id.owner.c_str(), auto_cleared_codes.size()); - // Cleanup rosbag for the main fault (auto_cleanup handled inside RosbagCapture) - hooks_.on_cleared(code); + // Cleanup rosbag for the cleared record (auto_cleanup handled inside RosbagCapture) + hooks_.on_cleared(id); - // Publish the cleared event - get the cleared fault to include in the event + // Publish the cleared event - get the cleared record to include in the event std::optional fault; try { - fault = storage_->get_fault(code); + fault = storage_->get_fault(id); } catch (const FaultStorage::IgnorableConnectionException & e) { connection_lost(e); outcome.message = storage_unavailable(e); @@ -566,7 +683,7 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_ } if (fault) { publish_event(fd::fault_event_type::kCleared, *fault, auto_cleared_codes); - audit_transition(kTransitionCleared, *fault, "clear_service", clock_.now_ns()); + audit_transition(kTransitionCleared, *fault, id, kAuditSourceClearService, clock_.now_ns()); } return outcome; } @@ -574,8 +691,9 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_ ListOutcome FaultManagerCore::list_locked(const ListQuery & query) { const std::string & logger = config_.logger_name; ListOutcome outcome; + std::vector entries; try { - outcome.faults = storage_->list_faults(query.filter_by_severity, query.severity, query.statuses); + entries = storage_->list_fault_entries(query.filter_by_severity, query.severity, query.statuses); } catch (const FaultStorage::IgnorableConnectionException & e) { logf(log_, LogLevel::kWarn, logger, "Failed to connect with the fault storage server: %s", e.what()); return outcome; @@ -587,32 +705,26 @@ ListOutcome FaultManagerCore::list_locked(const ListQuery & query) { outcome.muted_count = correlation_->get_muted_count(); outcome.cluster_count = correlation_->get_cluster_count(); - auto muted_faults = correlation_->get_muted_faults(); - if (query.include_muted) { + const auto muted_faults = correlation_->get_muted_faults(); outcome.muted_faults.reserve(muted_faults.size()); for (const auto & muted : muted_faults) { MutedFault info; info.fault_code = muted.fault_code; + info.source_id = muted.owner; info.root_cause_code = muted.root_cause_code; info.rule_id = muted.rule_id; info.delay_ms = muted.delay_ms; outcome.muted_faults.push_back(info); } } else { - // Build a set of muted codes for O(1) lookup. - std::unordered_set muted_codes; - for (const auto & muted : muted_faults) { - muted_codes.insert(muted.fault_code); - } - - // Remove all faults whose code is muted, in one pass. - auto & faults = outcome.faults; - faults.erase(std::remove_if(faults.begin(), faults.end(), - [&muted_codes](const fd::FaultRecord & fault) { - return muted_codes.count(fault.fault_code) > 0; - }), - faults.end()); + // Hide the muted RECORDS, never every record of a muted code: a record the engine never muted + // stays in the default view. + entries.erase(std::remove_if(entries.begin(), entries.end(), + [this](const FaultEntry & entry) { + return correlation_->is_muted(entry.id); + }), + entries.end()); } if (query.include_clusters) { @@ -625,8 +737,10 @@ ListOutcome FaultManagerCore::list_locked(const ListQuery & query) { info.rule_name = cluster.rule_name; info.label = cluster.label; info.representative_code = cluster.representative_code; + info.representative_owner = cluster.representative_owner; info.representative_severity = cluster.representative_severity; info.fault_codes = cluster.fault_codes; + info.fault_owners = cluster.fault_owners; info.count = static_cast(cluster.fault_codes.size()); info.first_at_ns = std::chrono::duration_cast(cluster.first_at.time_since_epoch()).count(); @@ -637,12 +751,17 @@ ListOutcome FaultManagerCore::list_locked(const ListQuery & query) { } } + outcome.faults.reserve(entries.size()); + for (auto & entry : entries) { + outcome.faults.push_back(std::move(entry.record)); + } + logf(log_, LogLevel::kDebug, logger, "ListFaults returned %zu faults (muted=%u, clusters=%u)", outcome.faults.size(), outcome.muted_count, outcome.cluster_count); return outcome; } -FaultDetail FaultManagerCore::get_locked(const std::string & code, +FaultDetail FaultManagerCore::get_locked(const std::string & code, const std::string & source_id, std::vector> & rosbag_paths) { const std::string & logger = config_.logger_name; FaultDetail detail; @@ -662,9 +781,18 @@ FaultDetail FaultManagerCore::get_locked(const std::string & code, return detail; }; + // Which record + std::string resolve_error; + const auto target = resolve_target_locked(code, source_id, resolve_error); + if (!target) { + detail.error_message = resolve_error; + return detail; + } + const FaultId & id = *target; + std::optional fault; try { - fault = storage_->get_fault(code); + fault = storage_->get_fault(id); } catch (const FaultStorage::IgnorableConnectionException & e) { return unavailable(e); } @@ -681,7 +809,7 @@ FaultDetail FaultManagerCore::get_locked(const std::string & code, // Get freeze frame snapshots from storage std::vector stored_snapshots = {}; try { - stored_snapshots = storage_->get_snapshots(code); + stored_snapshots = storage_->get_snapshots(id); } catch (const FaultStorage::IgnorableConnectionException & e) { return unavailable(e); } @@ -706,7 +834,7 @@ FaultDetail FaultManagerCore::get_locked(const std::string & code, if (stored_snapshots.empty() || storage_->retains_snapshots_on_clear()) { std::optional frame; try { - frame = storage_->get_freeze_frame(code); + frame = storage_->get_freeze_frame(id); } catch (const FaultStorage::IgnorableConnectionException & e) { return unavailable(e); } @@ -726,7 +854,7 @@ FaultDetail FaultManagerCore::get_locked(const std::string & code, // this list is the only place the fault itself advertises them. std::vector> paths; try { - for (const auto & rosbag_info : storage_->get_rosbag_files(code)) { + for (const auto & rosbag_info : storage_->get_rosbag_files(id)) { SnapshotEntry rosbag_snapshot; rosbag_snapshot.type = kSnapshotRosbag; rosbag_snapshot.name = "rosbag_" + rosbag_info.recording_id; @@ -767,11 +895,25 @@ DebounceConfig FaultManagerCore::resolve_config(const std::string & source_id, c return config; } -void FaultManagerCore::warn_on_conflicting_debounce_policy(const std::string & fault_code, - const std::string & source_id, +void FaultManagerCore::warn_severity_mismatch(const fd::FaultReport & report, uint8_t stored_severity) { + if (!severity_mismatch_logged_.emplace(report.fault_code, report.source_id, report.severity).second) { + return; + } + logf(log_, LogLevel::kWarn, config_.logger_name, + "Fault '%s' reported with severity %s by '%s', but its record is stored as %s. A record has one " + "severity and keeps the highest ever reported. Check the reporter's configuration. Logged " + "once per code, source and severity until restart.", + report.fault_code.c_str(), severity_name(report.severity), report.source_id.c_str(), + severity_name(stored_severity)); +} + +void FaultManagerCore::warn_on_conflicting_debounce_policy(const FaultId & id, const std::string & source_id, const DebounceConfig & resolved) { + if (id.owner != kSharedOwner) { + return; + } auto [it, inserted] = - debounce_policy_witness_.try_emplace(fault_code, DebouncePolicyWitness{resolved, source_id, false}); + debounce_policy_witness_.try_emplace(id.fault_code, DebouncePolicyWitness{resolved, source_id, false}); if (inserted || it->second.warned || debounce_policy_equal(it->second.config, resolved)) { return; } @@ -781,16 +923,16 @@ void FaultManagerCore::warn_on_conflicting_debounce_policy(const std::string & f it->second.warned = true; logf(log_, LogLevel::kWarn, config_.logger_name, "Fault code '%s' is debounced two ways: '%s' resolves confirmation=%d healing_enabled=%s healing=%d, " - "'%s' resolves confirmation=%d healing_enabled=%s healing=%d. The debounce counter belongs to the fault " - "code, so whichever source reports decides the transition and the other policy is bypassed. Pin " - "confirmation_threshold, healing_enabled and healing_threshold for this code in " - "fault_thresholds.config_file.", - fault_code.c_str(), it->second.source_id.c_str(), it->second.config.confirmation_threshold, + "'%s' resolves confirmation=%d healing_enabled=%s healing=%d. The debounce counter belongs to the shared " + "record of the fault code, so whichever source reports decides the transition and the other policy is " + "bypassed. Pin confirmation_threshold, healing_enabled and healing_threshold for this code in " + "fault_thresholds.config_file, or set per_source: true there to give each source its own record.", + id.fault_code.c_str(), it->second.source_id.c_str(), it->second.config.confirmation_threshold, it->second.config.healing_enabled ? "true" : "false", it->second.config.healing_threshold, source_id.c_str(), resolved.confirmation_threshold, resolved.healing_enabled ? "true" : "false", resolved.healing_threshold); } -void FaultManagerCore::audit_transition(const char * transition, const fd::FaultRecord & fault, +void FaultManagerCore::audit_transition(const char * transition, const fd::FaultRecord & fault, const FaultId & id, const std::string & source_id, int64_t occurred_at_ns) { if (!audit_) { return; @@ -805,6 +947,7 @@ void FaultManagerCore::audit_transition(const char * transition, const fd::Fault event.severity = fault.severity; event.status = fault.status; event.source_id = source_id; + event.owner = id.owner == kSharedOwner ? std::string() : id.owner; event.description = fault.description; event.occurred_at_ns = occurred_at_ns; diff --git a/src/ros2_medkit_fault_manager/src/core/fault_manager_startup.cpp b/src/ros2_medkit_fault_manager/src/core/fault_manager_startup.cpp index a2f73b250..4d7eb1398 100644 --- a/src/ros2_medkit_fault_manager/src/core/fault_manager_startup.cpp +++ b/src/ros2_medkit_fault_manager/src/core/fault_manager_startup.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -332,10 +333,10 @@ void reclassify_stale_healed(FaultStorage & storage, const FaultCodeThresholdRes if (!reclassified.empty()) { if (audit_enabled) { const int64_t reclassified_at_ns = wall_clock_ns(); - for (const auto & fault_code : reclassified) { - auto fault = storage.get_fault(fault_code); + for (const auto & id : reclassified) { + auto fault = storage.get_fault(id); if (fault) { - audit(*fault, reclassified_at_ns); + audit(id, *fault, reclassified_at_ns); } } } @@ -349,4 +350,49 @@ void reclassify_stale_healed(FaultStorage & storage, const FaultCodeThresholdRes } } +void warn_on_identity_mode_mismatch(FaultStorage & storage, const FaultCodeThresholdResolver * fault_code_resolver, + LogSink & sink, const std::string & logger) { + std::vector entries; + try { + entries = storage.list_fault_entries(false, 0, + {fd::fault_status::kPrefailed, fd::fault_status::kPrepassed, + fd::fault_status::kConfirmed, fd::fault_status::kHealed}); + } catch (const FaultStorage::IgnorableConnectionException & e) { + logf(sink, LogLevel::kWarn, logger, "Startup identity check not done, fault storage unavailable: %s", e.what()); + return; + } + + // Owners of the active records of each code, and whether any of them disagrees with the configured mode. + struct CodeState { + std::vector owners; + bool mismatch{false}; + }; + std::map codes; + for (const auto & entry : entries) { + const bool per_source = fault_code_resolver != nullptr && fault_code_resolver->per_source(entry.id.fault_code); + auto & state = codes[entry.id.fault_code]; + state.owners.push_back(entry.id.owner); + if ((entry.id.owner == kSharedOwner) == per_source) { + state.mismatch = true; + } + } + + for (const auto & [code, state] : codes) { + if (!state.mismatch) { + continue; + } + const bool per_source = fault_code_resolver != nullptr && fault_code_resolver->per_source(code); + std::string owners; + for (const auto & owner : state.owners) { + owners += (owners.empty() ? "" : ", ") + owner; + } + logf(sink, LogLevel::kWarn, logger, + "Fault code '%s' is configured %s but holds active records owned by (%s). Stored records keep the identity " + "they were written with; new reports go into the configured one.", + code.c_str(), + per_source ? "per_source: one record per reporting source" : "shared: one record for all sources", + owners.c_str()); + } +} + } // namespace ros2_medkit_fault_manager::core diff --git a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp index 41bd0b347..0e646f891 100644 --- a/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp +++ b/src/ros2_medkit_fault_manager/src/correlation/correlation_engine.cpp @@ -24,10 +24,12 @@ CorrelationEngine::CorrelationEngine(const CorrelationConfig & config) : config_(config), matcher_(std::make_unique(config.patterns)) { } -ProcessFaultResult CorrelationEngine::process_fault(const std::string & fault_code, const std::string & severity, +ProcessFaultResult CorrelationEngine::process_fault(const FaultId & id, const std::string & severity, std::chrono::steady_clock::time_point timestamp) { std::lock_guard lock(mutex_); + // Rules match on the CODE, whoever owns the record. Relations formed below hold exact records. + const std::string & fault_code = id.fault_code; ProcessFaultResult result; // First, clean up expired entries @@ -42,8 +44,8 @@ ProcessFaultResult CorrelationEngine::process_fault(const std::string & fault_co }), pending_root_causes_.end()); - // Check if this fault is a symptom of an existing root cause - auto symptom_result = try_as_symptom(fault_code, timestamp); + // Check if this record is a symptom of an existing root cause + auto symptom_result = try_as_symptom(id, timestamp); if (symptom_result) { return *symptom_result; } @@ -59,14 +61,14 @@ ProcessFaultResult CorrelationEngine::process_fault(const std::string & fault_co if (rule.id == *root_cause_rule) { // Add to pending root causes PendingRootCause prc; - prc.fault_code = fault_code; + prc.fault_id = id; prc.rule_id = rule.id; prc.timestamp = timestamp; prc.window_ms = rule.window_ms; pending_root_causes_.push_back(prc); - // Initialize symptom list - root_to_symptoms_[fault_code] = {}; + // A root that is already active keeps the symptoms it muted: a re-report only renews its window. + root_to_symptoms_.try_emplace(id); break; } } @@ -74,8 +76,8 @@ ProcessFaultResult CorrelationEngine::process_fault(const std::string & fault_co return result; } - // Check if this fault matches an auto-cluster rule - auto cluster_result = try_auto_cluster(fault_code, severity, timestamp); + // Check if this record matches an auto-cluster rule + auto cluster_result = try_auto_cluster(id, severity, timestamp); if (cluster_result) { return *cluster_result; } @@ -84,20 +86,22 @@ ProcessFaultResult CorrelationEngine::process_fault(const std::string & fault_co return result; } -ProcessClearResult CorrelationEngine::process_clear(const std::string & fault_code) { +ProcessClearResult CorrelationEngine::process_clear(const FaultId & id) { std::lock_guard lock(mutex_); + const std::string & fault_code = id.fault_code; ProcessClearResult result; - // Check if this is a root cause with symptoms - auto it = root_to_symptoms_.find(fault_code); + // Check if this record is a root cause with symptoms. The lookup is by record, so the + // cascade reaches exactly the records this root cause muted. + auto it = root_to_symptoms_.find(id); if (it != root_to_symptoms_.end()) { // Find the rule to check auto_clear_with_root for (const auto & prc : pending_root_causes_) { - if (prc.fault_code == fault_code) { + if (prc.fault_id == id) { for (const auto & rule : config_.rules) { if (rule.id == prc.rule_id && rule.auto_clear_with_root) { - result.auto_cleared_codes = it->second; + result.auto_cleared_symptoms = it->second; break; } } @@ -106,36 +110,38 @@ ProcessClearResult CorrelationEngine::process_clear(const std::string & fault_co } // Also check finalized root causes (not in pending anymore) - if (result.auto_cleared_codes.empty()) { + if (result.auto_cleared_symptoms.empty()) { for (const auto & rule : config_.rules) { if (rule.mode == CorrelationMode::HIERARCHICAL && rule.auto_clear_with_root) { // Check if fault matches this rule's root cause if (matcher_->matches_any(fault_code, rule.root_cause_codes)) { - result.auto_cleared_codes = it->second; + result.auto_cleared_symptoms = it->second; break; } } } } - // Clean up muted faults - for (const auto & symptom_code : it->second) { - muted_faults_.erase(symptom_code); - } - - // Remove from root_to_symptoms + // The symptoms stay muted while another root cause or an active cluster still mutes them + const std::vector released = it->second; root_to_symptoms_.erase(it); + for (auto delay_it = relation_delay_ms_.begin(); delay_it != relation_delay_ms_.end();) { + delay_it = delay_it->first.first == id ? relation_delay_ms_.erase(delay_it) : std::next(delay_it); + } + for (const auto & symptom_id : released) { + reconcile_muting(symptom_id); + } } // Remove from pending root causes pending_root_causes_.erase(std::remove_if(pending_root_causes_.begin(), pending_root_causes_.end(), - [&fault_code](const PendingRootCause & prc) { - return prc.fault_code == fault_code; + [&id](const PendingRootCause & prc) { + return prc.fault_id == id; }), pending_root_causes_.end()); - // Check if this fault is part of a cluster - auto cluster_it = fault_to_cluster_.find(fault_code); + // Check if this record is part of a cluster + auto cluster_it = fault_to_cluster_.find(id); if (cluster_it != fault_to_cluster_.end()) { const std::string cluster_id = cluster_it->second; @@ -147,57 +153,139 @@ ProcessClearResult CorrelationEngine::process_clear(const std::string & fault_co continue; } - auto & codes = pending_cluster.fault_codes; - codes.erase(std::remove(codes.begin(), codes.end(), fault_code), codes.end()); - pending_it->second.fault_severities.erase(fault_code); + auto & members = pending_cluster.members; + members.erase(std::remove(members.begin(), members.end(), id), members.end()); + pending_it->second.fault_severities.erase(id); - if (codes.empty()) { + if (members.empty()) { pending_it = pending_clusters_.erase(pending_it); continue; } - // Reassign representative if the cleared fault was the representative - if (pending_cluster.representative_code == fault_code) { + // Reassign representative if the cleared record was the representative + if (pending_cluster.representative == id) { reassign_representative(pending_cluster, pending_it->second.fault_severities); } ++pending_it; } - // Remove fault from active cluster + // Remove record from active cluster auto active_it = active_clusters_.find(cluster_id); if (active_it != active_clusters_.end()) { auto & active_cluster = active_it->second; - auto & codes = active_cluster.fault_codes; - codes.erase(std::remove(codes.begin(), codes.end(), fault_code), codes.end()); + auto & members = active_cluster.members; + members.erase(std::remove(members.begin(), members.end(), id), members.end()); auto & active_sevs = active_cluster_severities_[cluster_id]; - active_sevs.erase(fault_code); + active_sevs.erase(id); - if (codes.empty()) { + if (members.empty()) { active_clusters_.erase(active_it); active_cluster_severities_.erase(cluster_id); - } else if (active_cluster.representative_code == fault_code) { + } else if (active_cluster.representative == id) { reassign_representative(active_cluster, active_sevs); + sync_cluster_muting(active_cluster); } } fault_to_cluster_.erase(cluster_it); } - // Remove from muted faults if it was a symptom - muted_faults_.erase(fault_code); + // Remove from muted records if it was a symptom + muted_faults_.erase(id); + forget_symptom(id); return result; } +void CorrelationEngine::sync_cluster_muting(const Cluster & cluster) { + for (const auto & rule : config_.rules) { + if (rule.id == cluster.rule_id && rule.show_as_single) { + for (const auto & member : cluster.members) { + reconcile_muting(member); + } + return; + } + } +} + +bool CorrelationEngine::matches_symptom(const CorrelationRule & rule, const std::string & fault_code) const { + for (const auto & pattern_id : rule.symptom_pattern_ids) { + if (matcher_->matches(fault_code, pattern_id)) { + return true; + } + } + return !rule.inline_symptom_codes.empty() && matcher_->matches_any(fault_code, rule.inline_symptom_codes); +} + +void CorrelationEngine::reconcile_muting(const FaultId & id) { + for (const auto & [root_id, symptoms] : root_to_symptoms_) { + if (std::find(symptoms.begin(), symptoms.end(), id) == symptoms.end()) { + continue; + } + // A root is bound to the first rule its code matches. That rule decides whether it mutes. + const auto root_rule_id = try_as_root_cause(root_id.fault_code); + if (!root_rule_id) { + continue; + } + for (const auto & rule : config_.rules) { + if (rule.id != *root_rule_id || !rule.mute_symptoms || !matches_symptom(rule, id.fault_code)) { + continue; + } + MutedFaultData muted; + muted.fault_code = id.fault_code; + muted.owner = id.owner; + muted.root_cause_code = root_id.fault_code; + muted.rule_id = rule.id; + const auto delay_it = relation_delay_ms_.find({root_id, id}); + muted.delay_ms = delay_it != relation_delay_ms_.end() ? delay_it->second : 0; + muted_faults_[id] = muted; + return; + } + } + + const auto cluster_it = fault_to_cluster_.find(id); + if (cluster_it != fault_to_cluster_.end()) { + const auto active_it = active_clusters_.find(cluster_it->second); + if (active_it != active_clusters_.end() && active_it->second.representative != id) { + const auto & cluster = active_it->second; + for (const auto & rule : config_.rules) { + if (rule.id == cluster.rule_id && rule.show_as_single) { + MutedFaultData muted; + muted.fault_code = id.fault_code; + muted.owner = id.owner; + muted.root_cause_code = cluster.representative.fault_code; + muted.rule_id = cluster.rule_id; + muted_faults_[id] = muted; + return; + } + } + } + } + muted_faults_.erase(id); +} + +void CorrelationEngine::forget_symptom(const FaultId & id) { + for (auto delay_it = relation_delay_ms_.begin(); delay_it != relation_delay_ms_.end();) { + delay_it = delay_it->first.second == id ? relation_delay_ms_.erase(delay_it) : std::next(delay_it); + } + for (auto & [root_id, symptoms] : root_to_symptoms_) { + symptoms.erase(std::remove(symptoms.begin(), symptoms.end(), id), symptoms.end()); + } +} + std::vector CorrelationEngine::get_muted_faults() const { std::lock_guard lock(mutex_); std::vector result; result.reserve(muted_faults_.size()); - for (const auto & [code, data] : muted_faults_) { - result.push_back(data); + for (const auto & [muted_id, data] : muted_faults_) { + // Owner taken from the key rather than from the stored copy, so the entry always + // names the record it is filed under. + MutedFaultData entry = data; + entry.owner = muted_id.owner; + result.push_back(entry); } return result; @@ -208,9 +296,9 @@ uint32_t CorrelationEngine::get_muted_count() const { return static_cast(muted_faults_.size()); } -bool CorrelationEngine::is_muted(const std::string & fault_code) const { +bool CorrelationEngine::is_muted(const FaultId & id) const { std::lock_guard lock(mutex_); - return muted_faults_.find(fault_code) != muted_faults_.end(); + return muted_faults_.find(id) != muted_faults_.end(); } std::vector CorrelationEngine::get_clusters() const { @@ -219,8 +307,8 @@ std::vector CorrelationEngine::get_clusters() const { std::vector result; result.reserve(active_clusters_.size()); - for (const auto & [id, data] : active_clusters_) { - result.push_back(data); + for (const auto & [cluster_id, cluster] : active_clusters_) { + result.push_back(to_cluster_data(cluster)); } return result; @@ -261,12 +349,12 @@ void CorrelationEngine::cleanup_expired() { } } - for (const auto & rule_id : expired_pending) { - auto it = pending_clusters_.find(rule_id); + for (const auto & key : expired_pending) { + auto it = pending_clusters_.find(key); if (it != pending_clusters_.end()) { if (active_clusters_.find(it->second.data.cluster_id) == active_clusters_.end()) { - for (const auto & fault_code : it->second.data.fault_codes) { - fault_to_cluster_.erase(fault_code); + for (const auto & member : it->second.data.members) { + fault_to_cluster_.erase(member); } } pending_clusters_.erase(it); @@ -288,8 +376,9 @@ std::optional CorrelationEngine::try_as_root_cause(const std::strin return std::nullopt; } -std::optional CorrelationEngine::try_as_symptom(const std::string & fault_code, +std::optional CorrelationEngine::try_as_symptom(const FaultId & id, std::chrono::steady_clock::time_point timestamp) { + const std::string & fault_code = id.fault_code; for (const auto & prc : pending_root_causes_) { // Find the rule for (const auto & rule : config_.rules) { @@ -297,21 +386,7 @@ std::optional CorrelationEngine::try_as_symptom(const std::s continue; } - // Check if fault matches any symptom pattern - bool matches_symptom = false; - for (const auto & pattern_id : rule.symptom_pattern_ids) { - if (matcher_->matches(fault_code, pattern_id)) { - matches_symptom = true; - break; - } - } - - // Also check inline symptom codes (direct codes with wildcard support) - if (!matches_symptom && !rule.inline_symptom_codes.empty()) { - matches_symptom = matcher_->matches_any(fault_code, rule.inline_symptom_codes); - } - - if (!matches_symptom) { + if (!matches_symptom(rule, fault_code)) { continue; } @@ -324,23 +399,25 @@ std::optional CorrelationEngine::try_as_symptom(const std::s // This fault is a symptom! ProcessFaultResult result; result.should_mute = rule.mute_symptoms; - result.root_cause_code = prc.fault_code; + result.root_cause_code = prc.fault_id.fault_code; result.rule_id = rule.id; result.delay_ms = static_cast(elapsed); // Track the symptom (avoid duplicates) - auto & symptoms = root_to_symptoms_[prc.fault_code]; - if (std::find(symptoms.begin(), symptoms.end(), fault_code) == symptoms.end()) { - symptoms.push_back(fault_code); + auto & symptoms = root_to_symptoms_[prc.fault_id]; + if (std::find(symptoms.begin(), symptoms.end(), id) == symptoms.end()) { + symptoms.push_back(id); } + relation_delay_ms_[{prc.fault_id, id}] = result.delay_ms; if (rule.mute_symptoms) { MutedFaultData muted; muted.fault_code = fault_code; - muted.root_cause_code = prc.fault_code; + muted.owner = id.owner; + muted.root_cause_code = prc.fault_id.fault_code; muted.rule_id = rule.id; muted.delay_ms = result.delay_ms; - muted_faults_[fault_code] = muted; + muted_faults_[id] = muted; } return result; @@ -350,9 +427,9 @@ std::optional CorrelationEngine::try_as_symptom(const std::s return std::nullopt; } -std::optional CorrelationEngine::try_auto_cluster(const std::string & fault_code, - const std::string & severity, +std::optional CorrelationEngine::try_auto_cluster(const FaultId & id, const std::string & severity, std::chrono::steady_clock::time_point timestamp) { + const std::string & fault_code = id.fault_code; for (const auto & rule : config_.rules) { if (rule.mode != CorrelationMode::AUTO_CLUSTER) { continue; @@ -373,7 +450,8 @@ std::optional CorrelationEngine::try_auto_cluster(const std: auto now_system = std::chrono::system_clock::now(); - // Check if we have a pending cluster for this rule + // One pending cluster per rule: records of any owner that match the rule inside the + // window count towards min_count. auto pending_it = pending_clusters_.find(rule.id); if (pending_it != pending_clusters_.end()) { // Check if within time window using steady_clock timestamp @@ -383,8 +461,8 @@ std::optional CorrelationEngine::try_auto_cluster(const std: if (elapsed > static_cast(rule.window_ms)) { // Window expired, start new cluster if (active_clusters_.find(pending_it->second.data.cluster_id) == active_clusters_.end()) { - for (const auto & old_code : pending_it->second.data.fault_codes) { - fault_to_cluster_.erase(old_code); + for (const auto & old_member : pending_it->second.data.members) { + fault_to_cluster_.erase(old_member); } } pending_clusters_.erase(pending_it); @@ -393,22 +471,22 @@ std::optional CorrelationEngine::try_auto_cluster(const std: } // Keep faults that already belong to an active cluster for this rule in that cluster - auto existing_cluster_it = fault_to_cluster_.find(fault_code); + auto existing_cluster_it = fault_to_cluster_.find(id); if (existing_cluster_it != fault_to_cluster_.end()) { auto active_it = active_clusters_.find(existing_cluster_it->second); if (active_it != active_clusters_.end() && active_it->second.rule_id == rule.id) { auto & active_cluster = active_it->second; active_cluster.last_at = now_system; - update_member_severity(rule, active_cluster, active_cluster_severities_[active_cluster.cluster_id], fault_code, + update_member_severity(rule, active_cluster, active_cluster_severities_[active_cluster.cluster_id], id, severity); + sync_cluster_muting(active_cluster); if (pending_it != pending_clusters_.end() && pending_it->second.data.cluster_id == active_cluster.cluster_id) { pending_it->second.data.last_at = now_system; - update_member_severity(rule, pending_it->second.data, pending_it->second.fault_severities, fault_code, - severity); + update_member_severity(rule, pending_it->second.data, pending_it->second.fault_severities, id, severity); } ProcessFaultResult result; result.cluster_id = active_cluster.cluster_id; - if (rule.show_as_single && fault_code != active_cluster.representative_code) { + if (rule.show_as_single && id != active_cluster.representative) { result.should_mute = true; } return result; @@ -423,15 +501,15 @@ std::optional CorrelationEngine::try_auto_cluster(const std: pending.data.rule_id = rule.id; pending.data.rule_name = rule.name; pending.data.label = rule.name; // Use rule name as label - pending.data.representative_code = fault_code; + pending.data.representative = id; pending.data.representative_severity = severity; - pending.data.fault_codes.push_back(fault_code); - pending.fault_severities[fault_code] = severity; + pending.data.members.push_back(id); + pending.fault_severities[id] = severity; pending.data.first_at = now_system; pending.data.last_at = now_system; pending_clusters_[rule.id] = pending; - fault_to_cluster_[fault_code] = pending.data.cluster_id; + fault_to_cluster_[id] = pending.data.cluster_id; // Not enough faults yet for a cluster ProcessFaultResult result; @@ -445,22 +523,21 @@ std::optional CorrelationEngine::try_auto_cluster(const std: auto & cluster = pending.data; // Check for duplicate - if (std::find(cluster.fault_codes.begin(), cluster.fault_codes.end(), fault_code) != cluster.fault_codes.end()) { + if (std::find(cluster.members.begin(), cluster.members.end(), id) != cluster.members.end()) { // Already in cluster - ensure consistent muting for duplicates - update_member_severity(rule, cluster, pending.fault_severities, fault_code, severity); + update_member_severity(rule, cluster, pending.fault_severities, id, severity); ProcessFaultResult result; result.cluster_id = cluster.cluster_id; - if (rule.show_as_single && fault_code != cluster.representative_code && - cluster.fault_codes.size() >= rule.min_count) { + if (rule.show_as_single && id != cluster.representative && cluster.members.size() >= rule.min_count) { result.should_mute = true; } return result; } - cluster.fault_codes.push_back(fault_code); - pending.fault_severities[fault_code] = severity; + cluster.members.push_back(id); + pending.fault_severities[id] = severity; cluster.last_at = now_system; - fault_to_cluster_[fault_code] = cluster.cluster_id; + fault_to_cluster_[id] = cluster.cluster_id; // Update representative based on rule's representative selection bool update_representative = false; @@ -479,7 +556,7 @@ std::optional CorrelationEngine::try_auto_cluster(const std: } if (update_representative) { - cluster.representative_code = fault_code; + cluster.representative = id; cluster.representative_severity = severity; } @@ -488,21 +565,22 @@ std::optional CorrelationEngine::try_auto_cluster(const std: // Sync active cluster if already active or if threshold reached for the first time bool already_active = (active_clusters_.find(cluster.cluster_id) != active_clusters_.end()); - if (already_active || cluster.fault_codes.size() >= rule.min_count) { + if (already_active || cluster.members.size() >= rule.min_count) { active_clusters_[cluster.cluster_id] = cluster; active_cluster_severities_[cluster.cluster_id] = pending.fault_severities; + sync_cluster_muting(active_clusters_[cluster.cluster_id]); // Retroactively mute all non-representative faults added before threshold if (!already_active && rule.show_as_single) { - for (const auto & code : cluster.fault_codes) { - if (code != cluster.representative_code && code != fault_code) { - result.retroactive_mute_codes.push_back(code); + for (const auto & member : cluster.members) { + if (member != cluster.representative && member != id) { + result.retroactive_mute_ids.push_back(member); } } } // Mute non-representative faults - if (rule.show_as_single && fault_code != cluster.representative_code) { + if (rule.show_as_single && id != cluster.representative) { result.should_mute = true; } } @@ -520,29 +598,54 @@ std::string CorrelationEngine::generate_cluster_id(const std::string & rule_id) return oss.str(); } -void CorrelationEngine::update_member_severity(const CorrelationRule & rule, ClusterData & cluster, - std::map & fault_severities, - const std::string & fault_code, const std::string & severity) const { - fault_severities[fault_code] = severity; +ClusterData CorrelationEngine::to_cluster_data(const Cluster & cluster) { + ClusterData data; + data.cluster_id = cluster.cluster_id; + data.rule_id = cluster.rule_id; + data.rule_name = cluster.rule_name; + data.label = cluster.label; + data.representative_code = cluster.representative.fault_code; + data.representative_owner = cluster.representative.owner; + data.representative_severity = cluster.representative_severity; + data.fault_codes.reserve(cluster.members.size()); + data.fault_owners.reserve(cluster.members.size()); + for (const auto & member : cluster.members) { + data.fault_codes.push_back(member.fault_code); + data.fault_owners.push_back(member.owner); + } + data.first_at = cluster.first_at; + data.last_at = cluster.last_at; + return data; +} + +void CorrelationEngine::update_member_severity(const CorrelationRule & rule, Cluster & cluster, + std::map & severities, const FaultId & id, + const std::string & severity) const { + severities[id] = severity; if (rule.representative != Representative::HIGHEST_SEVERITY) { return; } - if (cluster.representative_code == fault_code) { + if (cluster.representative == id) { // A downgraded representative may no longer be the highest - reassign_representative(cluster, fault_severities); + reassign_representative(cluster, severities); } else if (severity_rank(severity) > severity_rank(cluster.representative_severity)) { - cluster.representative_code = fault_code; + cluster.representative = id; cluster.representative_severity = severity; } } -void CorrelationEngine::reassign_representative(ClusterData & cluster, - const std::map & fault_severities) const { - const auto & codes = cluster.fault_codes; - if (codes.empty()) { +void CorrelationEngine::reassign_representative(Cluster & cluster, + const std::map & severities) const { + const auto & members = cluster.members; + if (members.empty()) { return; } + const auto severity_of = [&severities](const FaultId & member) { + auto sev_it = severities.find(member); + return sev_it != severities.end() ? sev_it->second : std::string(); + }; + for (const auto & rule : config_.rules) { if (rule.id != cluster.rule_id) { continue; @@ -550,33 +653,28 @@ void CorrelationEngine::reassign_representative(ClusterData & cluster, switch (rule.representative) { case Representative::FIRST: { - const std::string & first_code = codes.front(); - cluster.representative_code = first_code; - auto sev_it = fault_severities.find(first_code); - cluster.representative_severity = (sev_it != fault_severities.end()) ? sev_it->second : ""; + cluster.representative = members.front(); + cluster.representative_severity = severity_of(members.front()); break; } case Representative::HIGHEST_SEVERITY: { - std::string best_code = codes.front(); + FaultId best = members.front(); int best_rank = -1; - for (const auto & code : codes) { - auto sev_it = fault_severities.find(code); - int rank = (sev_it != fault_severities.end()) ? severity_rank(sev_it->second) : 0; + for (const auto & member : members) { + auto sev_it = severities.find(member); + int rank = (sev_it != severities.end()) ? severity_rank(sev_it->second) : 0; if (rank > best_rank) { best_rank = rank; - best_code = code; + best = member; } } - cluster.representative_code = best_code; - auto best_sev_it = fault_severities.find(best_code); - cluster.representative_severity = (best_sev_it != fault_severities.end()) ? best_sev_it->second : ""; + cluster.representative = best; + cluster.representative_severity = severity_of(best); break; } case Representative::MOST_RECENT: { - const std::string & most_recent_code = codes.back(); - cluster.representative_code = most_recent_code; - auto sev_it = fault_severities.find(most_recent_code); - cluster.representative_severity = (sev_it != fault_severities.end()) ? sev_it->second : ""; + cluster.representative = members.back(); + cluster.representative_severity = severity_of(members.back()); break; } } diff --git a/src/ros2_medkit_fault_manager/src/fault_audit_log.cpp b/src/ros2_medkit_fault_manager/src/fault_audit_log.cpp index fa7d45e27..3332c5d4a 100644 --- a/src/ros2_medkit_fault_manager/src/fault_audit_log.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_audit_log.cpp @@ -215,6 +215,10 @@ std::string FaultAuditLog::canonicalize(int64_t seq, const AuditEvent & event) { out += json_encode(event.source_id); out += ",\"description\":"; out += json_encode(event.description); + if (!event.owner.empty()) { + out += ",\"owner\":"; + out += json_encode(event.owner); + } out += "}"; return out; } @@ -233,12 +237,24 @@ void FaultAuditLog::initialize_schema() { source_id TEXT NOT NULL, description TEXT NOT NULL, prev_hash TEXT NOT NULL, - record_hash TEXT NOT NULL + record_hash TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '' ); CREATE INDEX IF NOT EXISTS idx_audit_log_fault_code ON audit_log(fault_code); )", "create audit_log table"); + // A log created before the owner column existed gets it empty, which is what its rows hash as. + bool has_owner_column = false; + { + Stmt columns(db_.get(), "SELECT 1 FROM pragma_table_info('audit_log') WHERE name = 'owner'"); + has_owner_column = columns.step() == SQLITE_ROW; + } + if (!has_owner_column) { + exec_or_throw(db_.get(), "ALTER TABLE audit_log ADD COLUMN owner TEXT NOT NULL DEFAULT ''", + "add audit_log owner column"); + } + // Single-row persisted chain head, so the chain resumes across restarts. exec_or_throw(db_.get(), R"( @@ -384,7 +400,7 @@ int64_t FaultAuditLog::append(const AuditEvent & event) { try { Stmt insert(db_.get(), "INSERT INTO audit_log (seq, occurred_at_ns, fault_code, transition, severity, status, " - "source_id, description, prev_hash, record_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + "source_id, description, prev_hash, record_hash, owner) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); insert.bind_int64(1, new_seq); insert.bind_int64(2, event.occurred_at_ns); insert.bind_text(3, event.fault_code); @@ -395,6 +411,7 @@ int64_t FaultAuditLog::append(const AuditEvent & event) { insert.bind_text(8, event.description); insert.bind_text(9, prev_hash); insert.bind_text(10, record_hash); + insert.bind_text(11, event.owner); if (insert.step() != SQLITE_DONE) { throw std::runtime_error(std::string("audit: failed to insert record: ") + sqlite3_errmsg(db_.get())); } @@ -509,7 +526,7 @@ std::vector FaultAuditLog::read(int64_t limit, int64_t after_seq) c std::string sql = "SELECT seq, occurred_at_ns, fault_code, transition, severity, status, source_id, description, " - "prev_hash, record_hash FROM audit_log WHERE seq > ? ORDER BY seq ASC"; + "prev_hash, record_hash, owner FROM audit_log WHERE seq > ? ORDER BY seq ASC"; if (limit > 0) { sql += " LIMIT ?"; } @@ -533,6 +550,7 @@ std::vector FaultAuditLog::read(int64_t limit, int64_t after_seq) c rec.event.description = stmt.column_text(7); rec.prev_hash = stmt.column_text(8); rec.record_hash = stmt.column_text(9); + rec.event.owner = stmt.column_text(10); result.push_back(std::move(rec)); } return result; @@ -565,7 +583,7 @@ AuditVerifyResult FaultAuditLog::verify() const { // Walk every retained row oldest-first, recomputing each link. Stmt stmt(db_.get(), "SELECT seq, occurred_at_ns, fault_code, transition, severity, status, source_id, description, " - "prev_hash, record_hash FROM audit_log ORDER BY seq ASC"); + "prev_hash, record_hash, owner FROM audit_log ORDER BY seq ASC"); bool first = true; int64_t expected_seq = 0; @@ -583,6 +601,7 @@ AuditVerifyResult FaultAuditLog::verify() const { rec.event.description = stmt.column_text(7); rec.prev_hash = stmt.column_text(8); rec.record_hash = stmt.column_text(9); + rec.event.owner = stmt.column_text(10); if (first) { first = false; diff --git a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp index 3ea975413..ed44f5b02 100644 --- a/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_manager_node.cpp @@ -117,17 +117,17 @@ FaultManagerNode::FaultManagerNode(const rclcpp::NodeOptions & options) : Node(" event_publisher_ = create_publisher("~/events", rclcpp::QoS(100).reliable()); event_sink_ = std::make_unique(event_publisher_); capture_hooks_ = std::make_unique( - [this](const std::string & code) { - return enqueue_capture(code); + [this](const FaultId & id) { + return enqueue_capture(id); }, - [this](const std::string & code) { + [this](const FaultId & id) { if (rosbag_capture_) { - rosbag_capture_->on_fault_prefailed(code); + rosbag_capture_->on_fault_prefailed(id); } }, - [this](const std::string & code) { + [this](const FaultId & id) { if (rosbag_capture_) { - rosbag_capture_->on_fault_cleared(code); + rosbag_capture_->on_fault_cleared(id); } }); core_ = std::make_unique(config_, std::move(parts), *event_sink_, *capture_hooks_, @@ -136,11 +136,12 @@ FaultManagerNode::FaultManagerNode(const rclcpp::NodeOptions & options) : Node(" // Runs after the override files are loaded, so a code with its own healing_enabled: true keeps // its HEALED status. Each flipped fault is audited, or the audit log's verify() would not see it. - core::reclassify_stale_healed(*storage_, core_->fault_code_resolver(), core_->global_debounce(), - core_->audit_log() != nullptr, log_sink_, config_.logger_name, - [this](const fd::FaultRecord & fault, int64_t reclassified_at_ns) { - audit_transition(kTransitionCleared, fault, "startup_reclassify", reclassified_at_ns); - }); + core::reclassify_stale_healed( + *storage_, core_->fault_code_resolver(), core_->global_debounce(), core_->audit_log() != nullptr, log_sink_, + config_.logger_name, [this](const FaultId & id, const fd::FaultRecord & fault, int64_t reclassified_at_ns) { + audit_transition(kTransitionCleared, fault, id, kAuditSourceStartupReclassify, reclassified_at_ns); + }); + core::warn_on_identity_mode_mismatch(*storage_, core_->fault_code_resolver(), log_sink_, config_.logger_name); // Create service servers report_fault_srv_ = create_service( @@ -231,13 +232,13 @@ FaultManagerNode::FaultManagerNode(const rclcpp::NodeOptions & options) : Node(" capture_pool_ = std::make_unique( static_cast(capture_pool_size_), static_cast(capture_queue_depth_), capture_queue_full_policy_, log_sink_, - [snap, bag, rosbag_mutex](const std::string & fault_code) { + [snap, bag, rosbag_mutex](const FaultId & id) { if (snap) { - snap->capture(fault_code); + snap->capture(id); } if (bag) { std::lock_guard bag_lock(*rosbag_mutex); - bag->on_fault_confirmed(fault_code); + bag->on_fault_confirmed(id); } }, config_.logger_name); @@ -302,11 +303,16 @@ FaultManagerNode::~FaultManagerNode() { } } -core::CaptureResult FaultManagerNode::enqueue_capture(const std::string & fault_code) { +core::CaptureResult FaultManagerNode::enqueue_capture(const FaultId & id) { if (!capture_pool_) { return {}; } - const EnqueueOutcome outcome = capture_pool_->enqueue(fault_code); + // A per-source record whose code is already captured inside the cooldown shares that capture. Its + // snapshots need no worker; only a recording still does. + if (snapshot_capture_ && snapshot_capture_->admit(id) == SnapshotCapture::Admission::kJoined && !rosbag_capture_) { + return {core::CaptureEnqueue::kAccepted, {}}; + } + const EnqueueOutcome outcome = capture_pool_->enqueue(id); // RCLCPP_WARN_THROTTLE needs a non-const Clock lvalue (Humble/Lyrical compat); mirror // rosbag_capture.cpp's local-copy pattern. The counter is cast to unsigned long long for %llu // to avoid -Wuseless-cast (uint64_t == unsigned long on LP64). @@ -315,22 +321,31 @@ core::CaptureResult FaultManagerNode::enqueue_capture(const std::string & fault_ case EnqueueResult::kAccepted: return {core::CaptureEnqueue::kAccepted, {}}; case EnqueueResult::kEvictedOldest: + if (snapshot_capture_ && outcome.evicted_id) { + snapshot_capture_->withdraw(*outcome.evicted_id); + } RCLCPP_WARN_THROTTLE(get_logger(), throttle_clock, 2000, "Capture queue full (drop_oldest): evicted pending '%s' for '%s' " "(pool=%d, queue=%d, total_dropped=%llu)", - outcome.evicted_code ? outcome.evicted_code->c_str() : "?", fault_code.c_str(), + outcome.evicted_id ? outcome.evicted_id->fault_code.c_str() : "?", id.fault_code.c_str(), capture_pool_size_, capture_queue_depth_, static_cast(capture_pool_->dropped_captures())); - return {core::CaptureEnqueue::kEvictedOldest, outcome.evicted_code.value_or(std::string{})}; + return {core::CaptureEnqueue::kEvictedOldest, outcome.evicted_id.value_or(FaultId{})}; case EnqueueResult::kDroppedNewest: + if (snapshot_capture_) { + snapshot_capture_->withdraw(id); + } RCLCPP_WARN_THROTTLE(get_logger(), throttle_clock, 2000, "Capture queue full (reject_newest): dropped capture for '%s' " "(pool=%d, queue=%d, total_dropped=%llu)", - fault_code.c_str(), capture_pool_size_, capture_queue_depth_, + id.fault_code.c_str(), capture_pool_size_, capture_queue_depth_, static_cast(capture_pool_->dropped_captures())); return {core::CaptureEnqueue::kDroppedNewest, {}}; case EnqueueResult::kRejectedShuttingDown: - RCLCPP_DEBUG(get_logger(), "Capture pool shutting down; skipped capture for '%s'", fault_code.c_str()); + if (snapshot_capture_) { + snapshot_capture_->withdraw(id); + } + RCLCPP_DEBUG(get_logger(), "Capture pool shutting down; skipped capture for '%s'", id.fault_code.c_str()); return {core::CaptureEnqueue::kShuttingDown, {}}; } return {}; @@ -369,6 +384,7 @@ void FaultManagerNode::handle_list_faults( for (const auto & muted : outcome.muted_faults) { ros2_medkit_msgs::msg::MutedFaultInfo info; info.fault_code = muted.fault_code; + info.source_id = muted.source_id; info.root_cause_code = muted.root_cause_code; info.rule_id = muted.rule_id; info.delay_ms = muted.delay_ms; @@ -382,8 +398,10 @@ void FaultManagerNode::handle_list_faults( info.rule_name = cluster.rule_name; info.label = cluster.label; info.representative_code = cluster.representative_code; + info.representative_owner = cluster.representative_owner; info.representative_severity = cluster.representative_severity; info.fault_codes = cluster.fault_codes; + info.fault_owners = cluster.fault_owners; info.count = cluster.count; info.first_at = ros2::ns_to_time_msg(cluster.first_at_ns); info.last_at = ros2::ns_to_time_msg(cluster.last_at_ns); @@ -394,7 +412,7 @@ void FaultManagerNode::handle_list_faults( void FaultManagerNode::handle_clear_fault( const std::shared_ptr & request, const std::shared_ptr & response) { - auto outcome = core_->clear(request->fault_code, request->skip_correlation_auto_clear); + auto outcome = core_->clear(request->fault_code, request->source_id, request->skip_correlation_auto_clear); response->success = outcome.success; response->message = std::move(outcome.message); response->auto_cleared_codes = std::move(outcome.auto_cleared_codes); @@ -402,7 +420,7 @@ void FaultManagerNode::handle_clear_fault( void FaultManagerNode::handle_get_fault(const std::shared_ptr & request, const std::shared_ptr & response) { - const auto detail = core_->get(request->fault_code); + const auto detail = core_->get(request->fault_code, request->source_id); response->success = detail.success; response->error_message = detail.error_message; if (detail.fault.fault_code.empty()) { @@ -438,9 +456,9 @@ void FaultManagerNode::publish_fault_event(const std::string & event_type, const fault.fault_code.c_str()); } -void FaultManagerNode::audit_transition(const char * transition, const fd::FaultRecord & fault, +void FaultManagerNode::audit_transition(const char * transition, const fd::FaultRecord & fault, const FaultId & id, const std::string & source_id, int64_t occurred_at_ns) { - core_->record_transition(transition, fault, source_id, occurred_at_ns); + core_->record_transition(transition, fault, id, source_id, occurred_at_ns); } core::FaultManagerConfig FaultManagerNode::declare_config() { @@ -537,6 +555,7 @@ SnapshotConfig FaultManagerNode::create_snapshot_config() { config.default_topics = config_.snapshots_default_topics; config.entity_default = config_.snapshots_entity_default; + config.recapture_cooldown_sec = config_.snapshots_recapture_cooldown_sec; if (!config_.snapshots_config_file.empty()) { load_snapshot_config_from_yaml(config_.snapshots_config_file, config); @@ -644,26 +663,19 @@ void FaultManagerNode::handle_get_snapshots( return; } - // Check if fault exists - std::optional fault; - try { - fault = storage_->get_fault(request->fault_code); - } catch (const FaultStorage::IgnorableConnectionException & e) { - RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); - response->success = false; - response->error_message = storage_unavailable(e); - return; - } - if (!fault) { + // Which record + std::string resolve_error; + const auto target = core_->resolve_target(request->fault_code, request->source_id, resolve_error); + if (!target) { response->success = false; - response->error_message = "Fault not found: " + request->fault_code; + response->error_message = resolve_error; return; } // Get snapshots from storage, newest capture set first. std::vector snapshots = {}; try { - snapshots = storage_->get_snapshots(request->fault_code, request->topic); + snapshots = storage_->get_snapshots(*target, request->topic); } catch (const FaultStorage::IgnorableConnectionException & e) { RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); response->success = false; @@ -733,7 +745,7 @@ void FaultManagerNode::handle_get_snapshots( // No download_url: rosbags are addressed per entity by the gateway. std::optional rosbag_info; try { - rosbag_info = storage_->get_rosbag_file(request->fault_code); + rosbag_info = storage_->get_rosbag_file(*target); } catch (const FaultStorage::IgnorableConnectionException & e) { RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); response->success = false; @@ -760,17 +772,28 @@ void FaultManagerNode::handle_get_snapshots( void FaultManagerNode::handle_get_rosbag(const std::shared_ptr & request, const std::shared_ptr & response) { - // Two ways in. recording_id addresses one specific bag and is tried first; - // fault_code keeps working and means "the newest recording of this fault", which - // is what it has always meant back when a fault could only have one. + // Two ways in. recording_id addresses one specific bag and is tried first, and + // source_id does not scope it: a recording is named directly and is shared by every + // record of its burst. fault_code (with source_id) keeps working and means "the + // newest recording of this record", which is what it has always meant back when a + // fault could only have one. // // A caller that cannot tell the two apart - the gateway, holding one URL segment // that may be either - sets both to the same string and relies on this fallthrough. // An id that names no recording is therefore not an answer yet, only a miss. std::optional rosbag_info; std::vector attached_codes; + std::vector attached_owners; std::string subject; + // fault_codes and owners are parallel: one entry per RECORD the recording is attached to, in the order + // the recording collected them. A burst of several owners of one code therefore lists the code once + // per owner, and a shared record is one entry carrying the shared owner. + const auto attach_row = [&attached_codes, &attached_owners](const RosbagFileInfo & row) { + attached_codes.push_back(row.fault_code); + attached_owners.push_back(row.owner); + }; + if (!request->recording_id.empty()) { // Validation stays a hard failure: a traversal-shaped id is malformed on // either reading, and falling back would quietly re-admit it as a fault code. @@ -794,7 +817,7 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptrrecording_id; rosbag_info = rows.front(); for (const auto & row : rows) { - attached_codes.push_back(row.fault_code); + attach_row(row); } } else if (request->fault_code.empty()) { response->success = false; @@ -812,24 +835,17 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptrfault_code; - // Check if fault exists - std::optional fault; - try { - fault = storage_->get_fault(request->fault_code); - } catch (const FaultStorage::IgnorableConnectionException & e) { - RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); - response->success = false; - response->error_message = storage_unavailable(e); - return; - } - if (!fault) { + // Which record + std::string resolve_error; + const auto target = core_->resolve_target(request->fault_code, request->source_id, resolve_error); + if (!target) { response->success = false; - response->error_message = "Fault not found: " + request->fault_code; + response->error_message = resolve_error; return; } try { - rosbag_info = storage_->get_rosbag_file(request->fault_code); + rosbag_info = storage_->get_rosbag_file(*target); } catch (const FaultStorage::IgnorableConnectionException & e) { RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); response->success = false; @@ -845,7 +861,7 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptrget_rosbag_files_by_recording(rosbag_info->recording_id)) { - attached_codes.push_back(row.fault_code); + attach_row(row); } } catch (const FaultStorage::IgnorableConnectionException & e) { RCLCPP_WARN(get_logger(), "Failed to connect with the fault storage server: %s", e.what()); @@ -854,7 +870,8 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptrfault_code); + attached_codes.push_back(target->fault_code); + attached_owners.push_back(target->owner); } } @@ -877,6 +894,7 @@ void FaultManagerNode::handle_get_rosbag(const std::shared_ptrfile_path = rosbag_info->file_path; response->recording_id = rosbag_info->recording_id; response->fault_codes = attached_codes; + response->owners = attached_owners; response->format = rosbag_info->format; response->duration_sec = rosbag_info->duration_sec; response->size_bytes = core::rosbag_served_bytes(rosbag_info->file_path, rosbag_info->size_bytes); diff --git a/src/ros2_medkit_fault_manager/src/fault_storage.cpp b/src/ros2_medkit_fault_manager/src/fault_storage.cpp index ceddfff19..9e4923204 100644 --- a/src/ros2_medkit_fault_manager/src/fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/fault_storage.cpp @@ -84,17 +84,18 @@ bool sanitize_debounce_config(DebounceConfig & config) { } fd::FaultRecord FaultState::to_record() const { - fd::FaultRecord record; - record.fault_code = fault_code; - record.severity = severity; - record.description = description; - record.first_occurred_ns = first_occurred_ns; - record.last_occurred_ns = last_occurred_ns; - record.last_passed_ns = last_passed_ns; - record.occurrence_count = occurrence_count; - record.status = status; - record.reporting_sources.assign(reporting_sources.begin(), reporting_sources.end()); - return record; + fd::FaultRecord msg; + msg.fault_code = fault_code; + msg.severity = severity; + msg.description = description; + msg.first_occurred_ns = first_occurred_ns; + msg.last_occurred_ns = last_occurred_ns; + msg.last_passed_ns = last_passed_ns; + msg.occurrence_count = occurrence_count; + msg.status = status; + msg.reporting_sources.assign(reporting_sources.begin(), reporting_sources.end()); + msg.owner = owner; + return msg; } void InMemoryFaultStorage::set_debounce_config(const DebounceConfig & config) { @@ -113,22 +114,26 @@ void InMemoryFaultStorage::update_status(FaultState & state, const DebounceConfi state.status = compute_debounce_status(state.debounce_counter, state.status, config); } -bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, +bool InMemoryFaultStorage::report_fault_event(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) { std::lock_guard lock(mutex_); const bool is_failed = (kind == fd::ReportKind::kFailed); + const bool shared = id.owner == kSharedOwner; - auto it = faults_.find(fault_code); + auto it = faults_.find(id); if (it == faults_.end()) { - // New fault - only create entry for FAILED events + // New record - only create one for FAILED events. A PASSED event for a record that + // does not exist has nothing to heal. if (!is_failed) { - return false; // PASSED event for non-existent fault is ignored + return false; // PASSED event for non-existent record is ignored } FaultState state; - state.fault_code = fault_code; + state.fault_code = id.fault_code; + state.owner = id.owner; + state.reporting_sources.insert(source_id); state.severity = severity; state.description = description; state.first_occurred_ns = timestamp_ns; @@ -136,7 +141,6 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd state.last_failed_ns = timestamp_ns; state.occurrence_count = 1; state.debounce_counter = -1; // First FAILED event - state.reporting_sources.insert(source_id); // CRITICAL severity bypasses debounce and confirms immediately if (config.critical_immediate_confirm && severity == fd::severity::kCritical) { @@ -150,11 +154,11 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd record_near_miss(state, config, severity, source_id, timestamp_ns); } - faults_.emplace(fault_code, std::move(state)); + faults_.emplace(id, std::move(state)); return true; } - // Existing fault - update + // Existing record - update auto & state = it->second; // CLEARED faults can be reactivated by FAILED events @@ -179,8 +183,8 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd if (!description.empty()) { state.description = description; } - // Check for immediate CRITICAL confirmation - if (config.critical_immediate_confirm && severity == fd::severity::kCritical) { + // A per-source record confirms on its stored severity, a shared record on the report's own. + if (config.critical_immediate_confirm && (shared ? severity : state.severity) == fd::severity::kCritical) { state.status = fd::fault_status::kConfirmed; } else { update_status(state, config); @@ -192,10 +196,11 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd } // Bring a counter left outside this config's band back into range before applying the report. - // Per-entity threshold overrides mean two sources of the same fault code can be evaluated - // against different bands, so a stored value clamped to one source's ceiling can sit above - // another's. The SQLite backend clamps on read for the same reason, and the two backends have to - // agree on the counter they record and on the status it produces. + // The band a record resolves to can change under it (per-entity overrides when several sources + // share the counter, a reloaded threshold file, a global config that moved between runs), which + // leaves a stored value clamped to the old ceiling sitting above the new one. The SQLite + // backend clamps on read for the same reason, and the two backends have to agree on the counter + // they record and on the status it produces. state.debounce_counter = clamp_debounce_counter(state.debounce_counter, config); if (is_failed) { @@ -227,8 +232,8 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd state.description = description; } - // Check for immediate confirmation of CRITICAL - if (config.critical_immediate_confirm && severity == fd::severity::kCritical) { + // Same rule as on reactivation; it also overrides the HEALED latch. + if (config.critical_immediate_confirm && (shared ? severity : state.severity) == fd::severity::kCritical) { state.debounce_counter = std::min(state.debounce_counter, -1); state.status = fd::fault_status::kConfirmed; return false; @@ -252,8 +257,8 @@ bool InMemoryFaultStorage::report_fault_event(const std::string & fault_code, fd return false; } -std::vector InMemoryFaultStorage::list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const { +std::vector InMemoryFaultStorage::list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const { std::lock_guard lock(mutex_); // Determine which statuses to include @@ -275,10 +280,10 @@ std::vector InMemoryFaultStorage::list_faults(bool filter_by_se } } - std::vector result; + std::vector result; result.reserve(faults_.size()); - for (const auto & [code, state] : faults_) { + for (const auto & [id, state] : faults_) { // Filter by status if (status_filter.find(state.status) == status_filter.end()) { continue; @@ -289,16 +294,16 @@ std::vector InMemoryFaultStorage::list_faults(bool filter_by_se continue; } - result.push_back(state.to_record()); + result.push_back(FaultEntry{id, state.to_record()}); } return result; } -std::optional InMemoryFaultStorage::get_fault(const std::string & fault_code) const { +std::optional InMemoryFaultStorage::get_fault(const FaultId & id) const { std::lock_guard lock(mutex_); - auto it = faults_.find(fault_code); + auto it = faults_.find(id); if (it == faults_.end()) { return std::nullopt; } @@ -306,21 +311,38 @@ std::optional InMemoryFaultStorage::get_fault(const std::string return it->second.to_record(); } -bool InMemoryFaultStorage::clear_fault(const std::string & fault_code) { +std::vector InMemoryFaultStorage::get_ids_by_code(const std::string & fault_code) const { + std::lock_guard lock(mutex_); + + // FaultId orders by code first, so the records of one code are one contiguous range + // and this is a range scan. Owner order inside it is the map's, which is what makes + // an ambiguity message list the owners the same way every time. + std::vector result; + for (auto it = faults_.lower_bound(FaultId{fault_code, ""}); it != faults_.end(); ++it) { + if (it->first.fault_code != fault_code) { + break; + } + result.push_back(it->first); + } + return result; +} + +bool InMemoryFaultStorage::clear_fault(const FaultId & id) { std::lock_guard lock(mutex_); - auto it = faults_.find(fault_code); + auto it = faults_.find(id); if (it == faults_.end()) { return false; } - // Acknowledging a fault drops its value snapshots, unless a history was asked + // Acknowledging a record drops its value snapshots, unless a history was asked // for: with recordings retained past a clear, deleting the readings that go with - // them leaves a fault holding bags whose matching values are gone. + // them leaves a record holding bags whose matching values are gone. Scoped to the + // record, so another owner's readings for the same code stay where they are. if (!retain_snapshots_on_clear_) { snapshots_.erase(std::remove_if(snapshots_.begin(), snapshots_.end(), - [&fault_code](const SnapshotData & s) { - return s.fault_code == fault_code; + [&id](const SnapshotData & s) { + return s.fault_code == id.fault_code && s.owner == id.owner; }), snapshots_.end()); } @@ -334,27 +356,27 @@ size_t InMemoryFaultStorage::size() const { return faults_.size(); } -bool InMemoryFaultStorage::contains(const std::string & fault_code) const { +bool InMemoryFaultStorage::contains(const FaultId & id) const { std::lock_guard lock(mutex_); - return faults_.find(fault_code) != faults_.end(); + return faults_.find(id) != faults_.end(); } -std::vector InMemoryFaultStorage::check_time_based_confirmation(int64_t now_ns) { +std::vector InMemoryFaultStorage::check_time_based_confirmation(int64_t now_ns) { std::lock_guard lock(mutex_); - std::vector confirmed; + std::vector confirmed; if (config_.auto_confirm_after_sec <= 0.0) { return confirmed; // Time-based confirmation disabled } const int64_t threshold_ns = std::llround(config_.auto_confirm_after_sec * 1e9); - for (auto & [code, state] : faults_) { + for (auto & [id, state] : faults_) { if (state.status == fd::fault_status::kPrefailed) { const int64_t age_ns = now_ns - state.last_failed_ns; if (age_ns >= threshold_ns) { state.status = fd::fault_status::kConfirmed; - confirmed.push_back(code); + confirmed.push_back(id); } } } @@ -387,7 +409,7 @@ void InMemoryFaultStorage::store_snapshots(const std::vector & sna } std::lock_guard lock(mutex_); - const std::string fault_code = snapshots.front().fault_code; + const FaultId id{snapshots.front().fault_code, snapshots.front().owner}; // Built beside the live vector and swapped in, the shape store_rosbag_files // uses: a capture is all-or-nothing, so a throw partway must not leave half of @@ -396,15 +418,15 @@ void InMemoryFaultStorage::store_snapshots(const std::vector & sna updated.insert(updated.end(), snapshots.begin(), snapshots.end()); if (max_snapshots_per_fault_ > 0) { - // Evict whole capture sets, oldest first, until this fault fits. + // Evict whole capture sets, oldest first, until this record fits. // // The old rule counted rows and rejected the NEW row once full, so a capture // that straddled the cap was stored in part: some topics present, the rest // silently absent, indistinguishable from "that topic was not publishing". // Keep-newest also stops this cap from opposing the rosbag one. - const auto rows_for_fault = [&updated, &fault_code]() { - return static_cast(std::count_if(updated.begin(), updated.end(), [&fault_code](const SnapshotData & s) { - return s.fault_code == fault_code; + const auto rows_for_fault = [&updated, &id]() { + return static_cast(std::count_if(updated.begin(), updated.end(), [&id](const SnapshotData & s) { + return s.fault_code == id.fault_code && s.owner == id.owner; })); }; @@ -415,7 +437,7 @@ void InMemoryFaultStorage::store_snapshots(const std::vector & sna int64_t newest = 0; bool have_newest = false; for (const auto & s : updated) { - if (s.fault_code == fault_code && (!have_newest || s.capture_id > newest)) { + if (s.fault_code == id.fault_code && s.owner == id.owner && (!have_newest || s.capture_id > newest)) { newest = s.capture_id; have_newest = true; } @@ -425,7 +447,8 @@ void InMemoryFaultStorage::store_snapshots(const std::vector & sna bool found = false; int64_t oldest = 0; for (const auto & s : updated) { - if (s.fault_code == fault_code && s.capture_id != newest && (!found || s.capture_id < oldest)) { + if (s.fault_code == id.fault_code && s.owner == id.owner && s.capture_id != newest && + (!found || s.capture_id < oldest)) { oldest = s.capture_id; found = true; } @@ -434,8 +457,9 @@ void InMemoryFaultStorage::store_snapshots(const std::vector & sna break; // only the newest capture is left and it is over the cap on its own } updated.erase(std::remove_if(updated.begin(), updated.end(), - [&fault_code, oldest](const SnapshotData & s) { - return s.fault_code == fault_code && s.capture_id == oldest; + [&id, oldest](const SnapshotData & s) { + return s.fault_code == id.fault_code && s.owner == id.owner && + s.capture_id == oldest; }), updated.end()); } @@ -444,13 +468,13 @@ void InMemoryFaultStorage::store_snapshots(const std::vector & sna snapshots_.swap(updated); } -std::vector InMemoryFaultStorage::get_snapshots(const std::string & fault_code, +std::vector InMemoryFaultStorage::get_snapshots(const FaultId & id, const std::string & topic_filter) const { std::lock_guard lock(mutex_); std::vector result; for (const auto & snapshot : snapshots_) { - if (snapshot.fault_code == fault_code) { + if (snapshot.fault_code == id.fault_code && snapshot.owner == id.owner) { if (topic_filter.empty() || snapshot.topic == topic_filter) { result.push_back(snapshot); } @@ -477,12 +501,12 @@ int64_t InMemoryFaultStorage::get_max_capture_id() const { void InMemoryFaultStorage::store_freeze_frame(const FreezeFrameData & frame) { std::lock_guard lock(mutex_); - freeze_frames_[frame.fault_code] = frame; + freeze_frames_[FaultId{frame.fault_code, frame.owner}] = frame; } -std::optional InMemoryFaultStorage::get_freeze_frame(const std::string & fault_code) const { +std::optional InMemoryFaultStorage::get_freeze_frame(const FaultId & id) const { std::lock_guard lock(mutex_); - auto it = freeze_frames_.find(fault_code); + auto it = freeze_frames_.find(id); if (it == freeze_frames_.end()) { return std::nullopt; } @@ -491,10 +515,11 @@ std::optional InMemoryFaultStorage::get_freeze_frame(const std: void InMemoryFaultStorage::record_near_miss(const FaultState & state, const DebounceConfig & config, uint8_t severity, const std::string & source_id, int64_t timestamp_ns) { - auto & series = near_misses_[state.fault_code]; + auto & series = near_misses_[FaultId{state.fault_code, state.owner}]; NearMissRecord record; record.fault_code = state.fault_code; + record.owner = state.owner; record.occurred_at_ns = timestamp_ns; record.debounce_counter = state.debounce_counter; record.confirmation_threshold = config.confirmation_threshold; @@ -523,8 +548,8 @@ size_t InMemoryFaultStorage::set_max_near_misses_per_fault(size_t max_count) { // stay over the new one until the next near miss for that code happens to arrive. using DiffType = std::vector::difference_type; size_t evicted = 0; - for (auto & [code, series] : near_misses_) { - (void)code; + for (auto & [id, series] : near_misses_) { + (void)id; if (series.size() > max_count) { const size_t excess = series.size() - max_count; series.erase(series.begin(), series.begin() + static_cast(excess)); @@ -534,9 +559,9 @@ size_t InMemoryFaultStorage::set_max_near_misses_per_fault(size_t max_count) { return evicted; } -std::vector InMemoryFaultStorage::get_near_misses(const std::string & fault_code) const { +std::vector InMemoryFaultStorage::get_near_misses(const FaultId & id) const { std::lock_guard lock(mutex_); - auto it = near_misses_.find(fault_code); + auto it = near_misses_.find(id); if (it == near_misses_.end()) { return {}; } @@ -576,11 +601,11 @@ void InMemoryFaultStorage::store_rosbag_files(const std::vector row.recording_id = rosbag_recording_id(row.file_path); } - // Upsert on the (fault, recording) LINK - the same grain as the SQLite unique + // Upsert on the (record, recording) LINK - the same grain as the SQLite unique // index. Re-storing the same link refreshes it; a link to a different recording // appends, which is the feature. auto it = std::find_if(updated.begin(), updated.end(), [&row](const RosbagRow & r) { - return r.info.fault_code == row.fault_code && r.info.file_path == row.file_path; + return r.info.fault_code == row.fault_code && r.info.owner == row.owner && r.info.file_path == row.file_path; }); if (it != updated.end()) { it->info = row; @@ -592,12 +617,12 @@ void InMemoryFaultStorage::store_rosbag_files(const std::vector continue; } - // Keep the newest N recordings of this fault, oldest evicted first - the same + // Keep the newest N recordings of this record, oldest evicted first - the same // direction as evict_bags_over_quota, so the two eviction owners never need a // tiebreak. At N = 1 this is the pre-#620 behaviour exactly. std::vector mine; for (size_t i = 0; i < updated.size(); ++i) { - if (updated[i].info.fault_code == row.fault_code) { + if (updated[i].info.fault_code == row.fault_code && updated[i].info.owner == row.owner) { mine.push_back(i); } } @@ -623,7 +648,7 @@ void InMemoryFaultStorage::store_rosbag_files(const std::vector // Unlinked only once every row is in, the way the SQLite backend unlinks after its // COMMIT: a throw above must leave the old rows pointing at bags that still exist. // Referencing is decided on the finished batch, not on the state before it - two - // faults of one burst can share the bag being evicted, and checking row by row + // records of one burst can share the bag being evicted, and checking row by row // beforehand would find it still held by a sibling that a later iteration then // evicts, leaking the directory. for (const auto & path : evicted) { @@ -636,13 +661,13 @@ void InMemoryFaultStorage::store_rosbag_files(const std::vector } } -std::optional InMemoryFaultStorage::get_rosbag_file(const std::string & fault_code) const { +std::optional InMemoryFaultStorage::get_rosbag_file(const FaultId & id) const { std::lock_guard lock(mutex_); // Newest, matching the SQLite backend's ORDER BY created_at_ns DESC, id DESC. const RosbagRow * best = nullptr; for (const auto & row : rosbag_files_) { - if (row.info.fault_code != fault_code) { + if (row.info.fault_code != id.fault_code || row.info.owner != id.owner) { continue; } if (best == nullptr || std::tie(best->info.created_at_ns, best->seq) < std::tie(row.info.created_at_ns, row.seq)) { @@ -655,12 +680,12 @@ std::optional InMemoryFaultStorage::get_rosbag_file(const std::s return best->info; } -std::vector InMemoryFaultStorage::get_rosbag_files(const std::string & fault_code) const { +std::vector InMemoryFaultStorage::get_rosbag_files(const FaultId & id) const { std::lock_guard lock(mutex_); std::vector mine; for (const auto & row : rosbag_files_) { - if (row.info.fault_code == fault_code) { + if (row.info.fault_code == id.fault_code && row.info.owner == id.owner) { mine.push_back(&row); } } @@ -687,20 +712,20 @@ InMemoryFaultStorage::get_rosbag_files_by_recording(const std::string & recordin } } std::sort(result.begin(), result.end(), [](const RosbagFileInfo & a, const RosbagFileInfo & b) { - return a.fault_code < b.fault_code; + return std::tie(a.fault_code, a.owner) < std::tie(b.fault_code, b.owner); }); return result; } -bool InMemoryFaultStorage::delete_rosbag_file(const std::string & fault_code) { +bool InMemoryFaultStorage::delete_rosbag_file(const FaultId & id) { std::lock_guard lock(mutex_); - // ALL recordings of this fault. Used by auto_cleanup on clear, where dropping the - // fault's whole black-box history is the intent. + // ALL recordings of this record. Used by auto_cleanup on clear, where dropping the + // record's whole black-box history is the intent. std::set touched; const size_t before = rosbag_files_.size(); for (auto it = rosbag_files_.begin(); it != rosbag_files_.end();) { - if (it->info.fault_code == fault_code) { + if (it->info.fault_code == id.fault_code && it->info.owner == id.owner) { touched.insert(it->info.file_path); it = rosbag_files_.erase(it); } else { @@ -713,7 +738,7 @@ bool InMemoryFaultStorage::delete_rosbag_file(const std::string & fault_code) { for (const auto & path : touched) { if (path_referenced(path)) { - continue; // a sibling fault of the burst still holds it + continue; // a sibling record of the burst still holds it } std::error_code ec; std::filesystem::remove_all(path, ec); @@ -761,7 +786,7 @@ bool InMemoryFaultStorage::path_referenced(const std::string & file_path) const size_t InMemoryFaultStorage::get_total_rosbag_storage_bytes() const { std::lock_guard lock(mutex_); - // Sum per bag, not per fault: one recording can back a burst of correlated + // Sum per bag, not per record: one recording can back a burst of correlated // faults, and double-counting it would evict bags that still fit the quota. // Rows sharing a path can disagree on size while a recording is being // finalised, so take the largest - matching the SQLite backend's MAX() and @@ -807,13 +832,13 @@ std::vector InMemoryFaultStorage::list_rosbags_for_entity(const std::vector rows; for (const auto & row : rosbag_files_) { - // Check if any of the fault's reporting sources contain this entity - auto fault_it = faults_.find(row.info.fault_code); + // The entity owns the row when it is among the reporting sources of the row's record. + auto fault_it = faults_.find(FaultId{row.info.fault_code, row.info.owner}); if (fault_it == faults_.end()) { continue; } - const auto & fault_state = fault_it->second; - if (fault_state.reporting_sources.find(entity_fqn) != fault_state.reporting_sources.end()) { + const auto & sources = fault_it->second.reporting_sources; + if (sources.find(entity_fqn) != sources.end()) { rows.push_back(&row); } } @@ -836,20 +861,20 @@ std::vector InMemoryFaultStorage::get_all_faults() const { std::vector result; result.reserve(faults_.size()); - for (const auto & [code, state] : faults_) { + for (const auto & [id, state] : faults_) { result.push_back(state.to_record()); } return result; } -std::vector InMemoryFaultStorage::reclassify_healed_as_cleared(const KeepHealedFn & keep_healed) { +std::vector InMemoryFaultStorage::reclassify_healed_as_cleared(const KeepHealedFn & keep_healed) { std::lock_guard lock(mutex_); - std::vector reclassified; - for (auto & [code, state] : faults_) { - if (state.status == fd::fault_status::kHealed && !(keep_healed && keep_healed(code))) { + std::vector reclassified; + for (auto & [id, state] : faults_) { + if (state.status == fd::fault_status::kHealed && !(keep_healed && keep_healed(id.fault_code))) { state.status = fd::fault_status::kCleared; - reclassified.push_back(code); + reclassified.push_back(id); } } @@ -857,10 +882,10 @@ std::vector InMemoryFaultStorage::reclassify_healed_as_cleared(cons // them here, and a backend that kept them would answer a snapshot query differently for the // same sequence of calls. The near-miss series is retained, as it is on clear_fault. if (!retain_snapshots_on_clear_ && !reclassified.empty()) { - const std::set affected(reclassified.begin(), reclassified.end()); + const std::set affected(reclassified.begin(), reclassified.end()); snapshots_.erase(std::remove_if(snapshots_.begin(), snapshots_.end(), [&affected](const SnapshotData & s) { - return affected.count(s.fault_code) > 0; + return affected.count(FaultId{s.fault_code, s.owner}) > 0; }), snapshots_.end()); } diff --git a/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp b/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp index e130fbc7b..3da734531 100644 --- a/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/postgres_fault_storage.cpp @@ -50,6 +50,11 @@ constexpr const char * kDefaultKeepalivesIdle = "5"; /// Time after a failed connection round before the next one. Requests in between fail at once. constexpr std::chrono::seconds kReconnectBackoff{5}; +/// Columns every fault read selects; PgFaultStorage::read_fault_row decodes them. +constexpr const char * kFaultColumns = + "fault_code, severity, description, first_occurred_ns, last_occurred_ns, occurrence_count, status, " + "reporting_sources, last_passed_ns, owner"; + /// Parsed connection options by keyword, or nullopt when libpq rejects the string. std::optional> parse_conn_info(const std::string & conn_info) { char * err = nullptr; @@ -305,10 +310,93 @@ DebounceConfig PgFaultStorage::get_debounce_config() const { return config_; } +bool PgFaultStorage::table_exists(pqxx::work & tx, const char * table) const { + return !execute(tx, + "SELECT 1 FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = $1", + std::string(table)) + .empty(); +} + +bool PgFaultStorage::table_has_column(pqxx::work & tx, const char * table, const char * column) const { + return !execute(tx, + "SELECT 1 FROM information_schema.columns WHERE table_schema = current_schema() " + "AND table_name = $1 AND column_name = $2", + std::string(table), std::string(column)) + .empty(); +} + +void PgFaultStorage::drop_primary_key(pqxx::work & tx, const char * table) const { + const auto res = execute(tx, "SELECT conname FROM pg_constraint WHERE conrelid = to_regclass($1) AND contype = 'p'", + std::string(table)); + for (const auto & r : res) { + execute(tx, + std::string("ALTER TABLE ") + table + " DROP CONSTRAINT " + tx.quote_name(r["conname"].as())); + } +} + +void PgFaultStorage::migrate_to_owner_identity(pqxx::work & tx) const { + // Probed per table: a table that does not exist yet is built in the new layout by create_schema. + const auto needs_owner = [&](const char * table) { + return table_exists(tx, table) && !table_has_column(tx, table, "owner"); + }; + const bool faults_needs_owner = needs_owner("faults"); + const bool freeze_frames_needs_owner = needs_owner("freeze_frames"); + const bool snapshots_needs_owner = needs_owner("snapshots"); + const bool rosbag_files_needs_owner = needs_owner("rosbag_files"); + const bool near_misses_needs_owner = needs_owner("near_misses"); + + if (!faults_needs_owner && !freeze_frames_needs_owner && !snapshots_needs_owner && !rosbag_files_needs_owner && + !near_misses_needs_owner) { + return; // fresh database, or already migrated - safe to re-run on every connection + } + + // Every row of a legacy table is a shared record: the old layout held one row per fault code, + // fed by every source. The row keeps its reporting_sources as written, and every child row + // attaches to that shared record. + const std::string shared = kSharedOwner; + + if (faults_needs_owner) { + drop_primary_key(tx, "faults"); + // A pending row carries a confirmation time from the occurrence before a reactivation. + execute(tx, "ALTER TABLE faults ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + execute(tx, + "UPDATE faults SET owner = $1, confirmed_at_ns = CASE WHEN status IN ('PREFAILED', 'PREPASSED') " + "THEN 0 ELSE confirmed_at_ns END", + shared); + } + if (freeze_frames_needs_owner) { + drop_primary_key(tx, "freeze_frames"); + execute(tx, "ALTER TABLE freeze_frames ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + execute(tx, "UPDATE freeze_frames SET owner = $1", shared); + } + if (snapshots_needs_owner) { + execute(tx, "ALTER TABLE snapshots ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + execute(tx, "UPDATE snapshots SET owner = $1", shared); + } + if (rosbag_files_needs_owner) { + execute(tx, "ALTER TABLE rosbag_files ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + execute(tx, "UPDATE rosbag_files SET owner = $1", shared); + // CREATE INDEX IF NOT EXISTS in create_schema keeps an index that exists, so the two indexes + // that carry fault_code are dropped to be rebuilt with owner. + execute(tx, "DROP INDEX IF EXISTS idx_rosbag_files_fault_path"); + execute(tx, "DROP INDEX IF EXISTS idx_rosbag_files_fault_created"); + } + if (near_misses_needs_owner) { + execute(tx, "ALTER TABLE near_misses ADD COLUMN owner TEXT NOT NULL DEFAULT ''"); + execute(tx, "UPDATE near_misses SET owner = $1", shared); + } +} + void PgFaultStorage::create_schema(pqxx::work & tx) const { + migrate_to_owner_identity(tx); + + // A row is one fault RECORD, identified by (fault_code, owner). owner is the shared owner + // for a code every source feeds, or the source_id for a per_source code; reporting_sources + // lists every source that reported the record. Uniqueness through the index below, the + // same shape as SQLite. execute(tx, R"( CREATE TABLE IF NOT EXISTS faults ( - fault_code TEXT PRIMARY KEY, + fault_code TEXT NOT NULL, severity INTEGER NOT NULL, description TEXT NOT NULL, first_occurred_ns BIGINT NOT NULL, @@ -319,15 +407,17 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { debounce_counter INTEGER NOT NULL DEFAULT 0, last_failed_ns BIGINT NOT NULL DEFAULT 0, last_passed_ns BIGINT NOT NULL DEFAULT 0, - confirmed_at_ns BIGINT NOT NULL DEFAULT 0 + confirmed_at_ns BIGINT NOT NULL DEFAULT 0, + owner TEXT NOT NULL DEFAULT '' ); - )"); + CREATE UNIQUE INDEX IF NOT EXISTS idx_faults_code_owner ON faults(fault_code, owner);)"); // Create snapshots table for storing topic data captured when faults are confirmed execute(tx, R"( CREATE TABLE IF NOT EXISTS snapshots ( id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', topic TEXT NOT NULL, message_type TEXT NOT NULL, data TEXT NOT NULL, @@ -335,28 +425,32 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { capture_id BIGINT NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_snapshots_fault_code ON snapshots(fault_code); - CREATE INDEX IF NOT EXISTS idx_snapshots_fault_topic ON snapshots(fault_code, topic);)"); + CREATE INDEX IF NOT EXISTS idx_snapshots_fault_topic ON snapshots(fault_code, topic); + CREATE INDEX IF NOT EXISTS idx_snapshots_record ON snapshots(fault_code, owner, capture_id);)"); // Create freeze_frames table: one compact JSON dict of captured topic values per fault - // code. Unlike snapshots, freeze frames are keyed by fault_code and are NOT removed on - // clear_fault, so the confirmed-state record is retained after acknowledgement. + // RECORD. Unlike snapshots, freeze frames are keyed by the record identity and are NOT + // removed on clear_fault, so the confirmed-state record is retained after acknowledgement. execute(tx, R"( CREATE TABLE IF NOT EXISTS freeze_frames ( - fault_code TEXT PRIMARY KEY, + fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', data TEXT NOT NULL, captured_at_ns BIGINT NOT NULL ); - )"); + CREATE UNIQUE INDEX IF NOT EXISTS idx_freeze_frames_code_owner ON freeze_frames(fault_code, owner);)"); // Create near_misses table: append-only series of FAILED reports that moved the debounce // counter without confirming the fault. One row per qualifying report, never updated in // place, and NOT removed on clear_fault - acknowledging a fault cycle must not erase how - // often that code approached confirmation. Bounded per fault code by the caller-supplied - // limit, evicting the oldest rows first. + // often that code approached confirmation. A row belongs to the record named by + // (fault_code, owner) and keeps the source_id that reported it. Bounded per record by the + // caller-supplied limit, evicting the oldest rows first. execute(tx, R"( CREATE TABLE IF NOT EXISTS near_misses ( id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', occurred_at_ns BIGINT NOT NULL, debounce_counter INTEGER NOT NULL, confirmation_threshold INTEGER NOT NULL, @@ -364,7 +458,8 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { source_id TEXT NOT NULL, resulting_status TEXT NOT NULL DEFAULT '' ); - CREATE INDEX IF NOT EXISTS idx_near_misses_fault_code ON near_misses(fault_code, id); + DROP INDEX IF EXISTS idx_near_misses_fault_code; + CREATE INDEX IF NOT EXISTS idx_near_misses_record ON near_misses(fault_code, owner, id); )"); // Create rosbag_files table. One row = one LINK (a fault claiming a recording): @@ -374,6 +469,7 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { CREATE TABLE IF NOT EXISTS rosbag_files ( id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', recording_id TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL, format TEXT NOT NULL, @@ -383,23 +479,23 @@ void PgFaultStorage::create_schema(pqxx::work & tx) const { ); CREATE INDEX IF NOT EXISTS idx_rosbag_files_fault_code ON rosbag_files(fault_code); CREATE INDEX IF NOT EXISTS idx_rosbag_files_created_at ON rosbag_files(created_at_ns); - CREATE INDEX IF NOT EXISTS idx_rosbag_files_fault_created ON rosbag_files(fault_code, created_at_ns, id); + CREATE INDEX IF NOT EXISTS idx_rosbag_files_fault_created ON rosbag_files(fault_code, owner, created_at_ns, id); CREATE INDEX IF NOT EXISTS idx_rosbag_files_recording ON rosbag_files(recording_id); CREATE INDEX IF NOT EXISTS idx_rosbag_files_path ON rosbag_files(file_path); - CREATE UNIQUE INDEX IF NOT EXISTS idx_rosbag_files_fault_path ON rosbag_files(fault_code, file_path); + CREATE UNIQUE INDEX IF NOT EXISTS idx_rosbag_files_fault_path ON rosbag_files(fault_code, owner, file_path); )"); // CREATE TABLE IF NOT EXISTS keeps a table of another layout. Check the columns so that startup fails. execute(tx, R"( SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, occurrence_count, status, - reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, confirmed_at_ns + reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, confirmed_at_ns, owner FROM faults LIMIT 0; - SELECT id, fault_code, topic, message_type, data, captured_at_ns, capture_id FROM snapshots LIMIT 0; - SELECT fault_code, data, captured_at_ns FROM freeze_frames LIMIT 0; - SELECT id, fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, severity, source_id, + SELECT id, fault_code, owner, topic, message_type, data, captured_at_ns, capture_id FROM snapshots LIMIT 0; + SELECT fault_code, owner, data, captured_at_ns FROM freeze_frames LIMIT 0; + SELECT id, fault_code, owner, occurred_at_ns, debounce_counter, confirmation_threshold, severity, source_id, resulting_status FROM near_misses LIMIT 0; - SELECT id, fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns + SELECT id, fault_code, owner, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns FROM rosbag_files LIMIT 0; )"); } @@ -538,7 +634,22 @@ std::string PgFaultStorage::serialize_json_array(const std::vector return oss.str(); } -bool PgFaultStorage::report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, +fd::FaultRecord PgFaultStorage::read_fault_row(const pqxx::row & r) const { + fd::FaultRecord fault; + fault.fault_code = r["fault_code"].as(); + fault.severity = static_cast(r["severity"].as()); + fault.description = r["description"].as(); + fault.first_occurred_ns = r["first_occurred_ns"].as(); + fault.last_occurred_ns = r["last_occurred_ns"].as(); + fault.occurrence_count = static_cast(r["occurrence_count"].as()); + fault.status = r["status"].as(); + fault.reporting_sources = parse_json_array(r["reporting_sources"].as()); + fault.last_passed_ns = r["last_passed_ns"].as(); + fault.owner = r["owner"].as(); + return fault; +} + +bool PgFaultStorage::report_fault_event(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config) { std::lock_guard lock(mutex_); @@ -549,25 +660,26 @@ bool PgFaultStorage::report_fault_event(const std::string & fault_code, fd::Repo // advance it a second time and the near miss it retried for would still be // missing from the series. run_in_transaction commits both rows together. return run_in_transaction("report_fault_event", [&](pqxx::work & tx) { - return report_fault_event_locked(fault_code, kind, severity, description, source_id, timestamp_ns, config, tx); + return report_fault_event_locked(id, kind, severity, description, source_id, timestamp_ns, config, tx); }); } -bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, +bool PgFaultStorage::report_fault_event_locked(const FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const DebounceConfig & config, pqxx::work & tx) { const bool is_failed = (kind == fd::ReportKind::kFailed); + const std::string & fault_code = id.fault_code; // No row lock needed: mutex_ serializes every call on this connection, and one database belongs to exactly one // fault manager. - // Check if fault exists + // Check if the record exists. Another record of the same code is a different row and is not touched. auto res = execute(tx, - "SELECT severity, occurrence_count, reporting_sources, status, debounce_counter, confirmed_at_ns, " - "first_occurred_ns FROM faults WHERE fault_code = $1", - fault_code); + "SELECT severity, occurrence_count, reporting_sources, status, debounce_counter, " + "confirmed_at_ns, first_occurred_ns FROM faults WHERE fault_code = $1 AND owner = $2", + fault_code, id.owner); if (!res.empty()) { - // Fault exists - update it + // Record exists - update it int existing_severity = res[0]["severity"].as(); int64_t existing_count = res[0]["occurrence_count"].as(); std::string sources_json = res[0]["reporting_sources"].as(); @@ -591,8 +703,10 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f // decrements it to -1, then reuse the existing FAILED logic below. Also // reset first_occurred: this is a new outage cycle, not a continuation // of the one that just cleared. + // The new occurrence has no confirmation time until it confirms. debounce_counter = 0; first_occurred_ns = timestamp_ns; + confirmed_at_ns = 0; is_reactivation = true; } @@ -618,9 +732,11 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f // Decrement towards confirmation, clamped to the thresholds. debounce_counter = clamp_debounce_counter(debounce_counter - 1, config); - // CRITICAL bypasses debounce; otherwise the shared state machine decides (with hysteresis). + // CRITICAL bypasses debounce and the HEALED latch. A per-source record decides on its stored severity, + // a shared record on the report's. Otherwise the shared state machine decides (with hysteresis). std::string new_status; - if (config.critical_immediate_confirm && severity == fd::severity::kCritical) { + if (config.critical_immediate_confirm && + (id.owner == kSharedOwner ? severity : new_severity) == fd::severity::kCritical) { debounce_counter = std::min(debounce_counter, -1); new_status = fd::fault_status::kConfirmed; } else { @@ -638,21 +754,20 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f execute(tx, "UPDATE faults SET severity = $1, last_occurred_ns = $2, last_failed_ns = $3, occurrence_count = $4, " "reporting_sources = $5, status = $6, debounce_counter = $7, confirmed_at_ns = $8, " - "first_occurred_ns = $9 WHERE fault_code = $10", + "first_occurred_ns = $9 WHERE fault_code = $10 AND owner = $11", new_severity, timestamp_ns, timestamp_ns, new_count, serialize_json_array(sources), new_status, - debounce_counter, confirmed_at_ns, first_occurred_ns, fault_code); + debounce_counter, confirmed_at_ns, first_occurred_ns, fault_code, id.owner); } else { execute(tx, "UPDATE faults SET severity = $1, description = $2, last_occurred_ns = $3, last_failed_ns = $4, " "occurrence_count = $5, reporting_sources = $6, status = $7, debounce_counter = $8, " - "confirmed_at_ns = $9, first_occurred_ns = $10 WHERE fault_code = $11", + "confirmed_at_ns = $9, first_occurred_ns = $10 WHERE fault_code = $11 AND owner = $12", new_severity, description, timestamp_ns, timestamp_ns, new_count, serialize_json_array(sources), - new_status, debounce_counter, confirmed_at_ns, first_occurred_ns, fault_code); + new_status, debounce_counter, confirmed_at_ns, first_occurred_ns, fault_code, id.owner); } if (is_near_miss(true, new_status)) { - record_near_miss_locked(fault_code, timestamp_ns, debounce_counter, config, severity, source_id, new_status, - tx); + record_near_miss_locked(id, timestamp_ns, debounce_counter, config, severity, source_id, new_status, tx); } } else { // PASSED event - increment towards healing, clamped to the thresholds. @@ -663,16 +778,18 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f // last_occurred_ns is deliberately NOT touched: a PASSED event is the fault // ENDING, not occurring. Bumping it made a long-stale CONFIRMED fault look // freshly active. The PASSED instant is recorded in last_passed_ns. - execute(tx, "UPDATE faults SET last_passed_ns = $1, status = $2, debounce_counter = $3 WHERE fault_code = $4", - timestamp_ns, new_status, debounce_counter, fault_code); + execute(tx, + "UPDATE faults SET last_passed_ns = $1, status = $2, debounce_counter = $3 " + "WHERE fault_code = $4 AND owner = $5", + timestamp_ns, new_status, debounce_counter, fault_code, id.owner); } return is_reactivation; // Reactivation treated as new occurrence for event publishing } - // New fault - only create for FAILED events + // New record - only create one for FAILED events if (!is_failed) { - return false; // PASSED event for non-existent fault is ignored + return false; // PASSED event for non-existent record is ignored } // Determine initial status based on debounce logic (shared with the in-memory backend). @@ -688,21 +805,21 @@ bool PgFaultStorage::report_fault_event_locked(const std::string & fault_code, f execute( tx, "INSERT INTO faults (fault_code, severity, description, first_occurred_ns, last_occurred_ns, occurrence_count, " - "status, reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, confirmed_at_ns) " - "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", + "status, reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, confirmed_at_ns, owner) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", fault_code, static_cast(severity), description, timestamp_ns, timestamp_ns, 1 /* occurrence_count = 1 */, initial_status, serialize_json_array({source_id}), initial_counter, timestamp_ns, 0, - confirmed_now ? timestamp_ns : 0); + confirmed_now ? timestamp_ns : 0, id.owner); if (is_near_miss(true, initial_status)) { - record_near_miss_locked(fault_code, timestamp_ns, initial_counter, config, severity, source_id, initial_status, tx); + record_near_miss_locked(id, timestamp_ns, initial_counter, config, severity, source_id, initial_status, tx); } return true; // New fault created } -std::vector PgFaultStorage::list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const { +std::vector PgFaultStorage::list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const { std::lock_guard lock(mutex_); // Determine which statuses to include @@ -723,107 +840,94 @@ std::vector PgFaultStorage::list_faults(bool filter_by_severity const std::vector status_filter_vector(status_filter.begin(), status_filter.end()); // Build query - std::string sql = - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, occurrence_count, status, " - "reporting_sources, last_passed_ns FROM faults WHERE status = ANY($1) "; + std::string sql = std::string("SELECT ") + kFaultColumns + " FROM faults WHERE status = ANY($1) "; if (filter_by_severity) { sql += " AND severity = $2"; } - return run_in_transaction("list_faults", [&](pqxx::work & tx) { + return run_in_transaction("list_fault_entries", [&](pqxx::work & tx) { auto res = filter_by_severity ? execute(tx, sql, status_filter_vector, static_cast(severity)) : execute(tx, sql, status_filter_vector); - std::vector result; + std::vector result; for (const auto & r : res) { - fd::FaultRecord fault; - fault.fault_code = r["fault_code"].as(); - fault.severity = static_cast(r["severity"].as()); - fault.description = r["description"].as(); - fault.first_occurred_ns = r["first_occurred_ns"].as(); - fault.last_occurred_ns = r["last_occurred_ns"].as(); - fault.occurrence_count = static_cast(r["occurrence_count"].as()); - fault.status = r["status"].as(); - fault.reporting_sources = parse_json_array(r["reporting_sources"].as()); - fault.last_passed_ns = r["last_passed_ns"].as(); - result.push_back(fault); + result.push_back( + FaultEntry{FaultId{r["fault_code"].as(), r["owner"].as()}, read_fault_row(r)}); } return result; }); } -std::optional PgFaultStorage::get_fault(const std::string & fault_code) const { +std::optional PgFaultStorage::get_fault(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("get_fault", [&](pqxx::work & tx) -> std::optional { - auto res = execute(tx, - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, last_passed_ns FROM faults WHERE fault_code = $1", - fault_code); + auto res = execute(tx, std::string("SELECT ") + kFaultColumns + " FROM faults WHERE fault_code = $1 AND owner = $2", + id.fault_code, id.owner); if (res.empty()) { return std::nullopt; } - auto r = res[0]; - fd::FaultRecord fault; - fault.fault_code = r["fault_code"].as(); - fault.severity = static_cast(r["severity"].as()); - fault.description = r["description"].as(); - fault.first_occurred_ns = r["first_occurred_ns"].as(); - fault.last_occurred_ns = r["last_occurred_ns"].as(); - fault.occurrence_count = static_cast(r["occurrence_count"].as()); - fault.status = r["status"].as(); - fault.reporting_sources = parse_json_array(r["reporting_sources"].as()); - fault.last_passed_ns = r["last_passed_ns"].as(); - return fault; + return read_fault_row(res[0]); + }); +} + +std::vector PgFaultStorage::get_ids_by_code(const std::string & fault_code) const { + std::lock_guard lock(mutex_); + + return run_in_transaction("get_ids_by_code", [&](pqxx::work & tx) { + auto res = execute(tx, "SELECT owner FROM faults WHERE fault_code = $1 ORDER BY owner ASC", fault_code); + std::vector result; + for (const auto & r : res) { + result.push_back(FaultId{fault_code, r["owner"].as()}); + } + return result; }); } -bool PgFaultStorage::clear_fault(const std::string & fault_code) { +bool PgFaultStorage::clear_fault(const FaultId & id) { std::lock_guard lock(mutex_); return run_in_transaction("clear_fault", [&](pqxx::work & tx) { - // The near_misses rows for this code are deliberately left alone. Clearing acknowledges one - // fault cycle; the record of how often the code approached confirmation spans cycles and - // cannot be reconstructed once deleted. + // The near_misses rows for this record are deliberately left alone. Clearing acknowledges + // one fault cycle; the record of how often the code approached confirmation spans cycles. - // Acknowledging a fault drops its value snapshots, unless a history was asked - // for: with recordings retained past a clear, deleting the readings that go with - // them leaves a fault holding bags whose matching values are gone. + // Acknowledging a record drops its value snapshots, scoped to the record so another + // owner's readings for the same code stay. Unless a history was asked for: with + // recordings retained past a clear, deleting the readings leaves bags whose values are gone. if (!retain_snapshots_on_clear_) { - execute(tx, "DELETE FROM snapshots WHERE fault_code = $1", fault_code); + execute(tx, "DELETE FROM snapshots WHERE fault_code = $1 AND owner = $2", id.fault_code, id.owner); } - auto res = - execute(tx, "UPDATE faults SET status = $1 WHERE fault_code = $2", fd::fault_status::kCleared, fault_code); + auto res = execute(tx, "UPDATE faults SET status = $1 WHERE fault_code = $2 AND owner = $3", + fd::fault_status::kCleared, id.fault_code, id.owner); return res.affected_rows() > 0; }); } -std::vector PgFaultStorage::reclassify_healed_as_cleared(const KeepHealedFn & keep_healed) { +std::vector PgFaultStorage::reclassify_healed_as_cleared(const KeepHealedFn & keep_healed) { std::lock_guard lock(mutex_); return run_in_transaction("reclassify_healed_as_cleared", [&](pqxx::work & tx) { - // Collect the codes that will flip first so the caller can audit each one. Codes the + // Collect the records that will flip first so the caller can audit each one. Codes the // caller keeps HEALED are left out. Everything runs in the same transaction, so the // returned list matches the rows actually reclassified below. - auto res = execute(tx, "SELECT fault_code FROM faults WHERE status = $1", fd::fault_status::kHealed); - std::vector reclassified; + auto res = execute(tx, "SELECT fault_code, owner FROM faults WHERE status = $1", fd::fault_status::kHealed); + std::vector reclassified; for (const auto & r : res) { - auto code = r["fault_code"].as(); - if (!(keep_healed && keep_healed(code))) { - reclassified.push_back(std::move(code)); + FaultId id{r["fault_code"].as(), r["owner"].as()}; + if (!(keep_healed && keep_healed(id.fault_code))) { + reclassified.push_back(std::move(id)); } } - for (const auto & code : reclassified) { - // Drop snapshots for the affected faults so a reclassified row matches CLEARED semantics. - // clear_fault is not the only place that takes a fault's readings, so retain_snapshots_on_clear_ - // has to reach here too: otherwise the setting holds until the next restart and then the - // reclassification deletes exactly what it was set to keep. + for (const auto & id : reclassified) { + // Drop snapshots for the affected records so a reclassified row matches CLEARED + // semantics. clear_fault is not the only place that takes a record's readings, so + // retain_snapshots_on_clear_ has to reach here too. if (!retain_snapshots_on_clear_) { - execute(tx, "DELETE FROM snapshots WHERE fault_code = $1", code); + execute(tx, "DELETE FROM snapshots WHERE fault_code = $1 AND owner = $2", id.fault_code, id.owner); } - execute(tx, "UPDATE faults SET status = $1 WHERE fault_code = $2 AND status = $3", fd::fault_status::kCleared, - code, fd::fault_status::kHealed); + execute(tx, "UPDATE faults SET status = $1 WHERE fault_code = $2 AND owner = $3 AND status = $4", + fd::fault_status::kCleared, id.fault_code, id.owner, fd::fault_status::kHealed); } return reclassified; }); @@ -838,16 +942,17 @@ size_t PgFaultStorage::size() const { }); } -bool PgFaultStorage::contains(const std::string & fault_code) const { +bool PgFaultStorage::contains(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("contains", [&](pqxx::work & tx) { - auto res = execute(tx, "SELECT 1 FROM faults WHERE fault_code = $1 LIMIT 1", fault_code); + auto res = + execute(tx, "SELECT 1 FROM faults WHERE fault_code = $1 AND owner = $2 LIMIT 1", id.fault_code, id.owner); return !res.empty(); }); } -std::vector PgFaultStorage::check_time_based_confirmation(int64_t now_ns) { +std::vector PgFaultStorage::check_time_based_confirmation(int64_t now_ns) { std::lock_guard lock(mutex_); if (config_.auto_confirm_after_sec <= 0.0) { @@ -856,16 +961,16 @@ std::vector PgFaultStorage::check_time_based_confirmation(int64_t n const int64_t cutoff_ns = now_ns - std::llround(config_.auto_confirm_after_sec * 1e9); return run_in_transaction("check_time_based_confirmation", [&](pqxx::work & tx) { - // Collect the codes that will flip first so the caller can audit each one. The + // Collect the records that will flip first so the caller can audit each one. The // SELECT predicate mirrors the UPDATE exactly, and both run under the same lock, // so the returned list matches the rows actually confirmed below. - auto res = - execute(tx, "SELECT fault_code FROM faults WHERE status = $1 AND last_failed_ns <= $2 AND last_failed_ns > 0 ", - fd::fault_status::kPrefailed, cutoff_ns); + auto res = execute( + tx, "SELECT fault_code, owner FROM faults WHERE status = $1 AND last_failed_ns <= $2 AND last_failed_ns > 0 ", + fd::fault_status::kPrefailed, cutoff_ns); - std::vector confirmed; + std::vector confirmed; for (const auto & r : res) { - confirmed.push_back(r["fault_code"].as()); + confirmed.push_back(FaultId{r["fault_code"].as(), r["owner"].as()}); } if (!confirmed.empty()) { @@ -904,44 +1009,46 @@ void PgFaultStorage::store_snapshots(const std::vector & snapshots std::lock_guard lock(mutex_); const std::string & fault_code = snapshots.front().fault_code; + const std::string & owner = snapshots.front().owner; // One transaction for the whole capture: a capture is all-or-nothing, and the // old row-at-a-time path could leave a confirmation's values half stored. run_in_transaction("store_snapshots", [&](pqxx::work & tx) { for (const auto & snapshot : snapshots) { - execute( - tx, - "INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns, capture_id) VALUES ($1, $2, " - "$3, $4, $5, $6)", - snapshot.fault_code, snapshot.topic, snapshot.message_type, snapshot.data, snapshot.captured_at_ns, - snapshot.capture_id); + execute(tx, + "INSERT INTO snapshots (fault_code, owner, topic, message_type, data, captured_at_ns, capture_id) " + "VALUES ($1, $2, $3, $4, $5, $6, $7)", + snapshot.fault_code, snapshot.owner, snapshot.topic, snapshot.message_type, snapshot.data, + snapshot.captured_at_ns, snapshot.capture_id); } if (max_snapshots_per_fault_ > 0) { - // Trim whole capture sets, oldest first, until the fault fits. The old rule - // counted rows and rejected the NEW row once full, so a capture straddling + // Trim whole capture sets of THIS record, oldest first, until it fits. The old + // rule counted rows and rejected the NEW row once full, so a capture straddling // the cap was stored in part - some topics present, the rest silently gone, // indistinguishable from "that topic was not publishing". Keep-newest also // stops this cap from opposing the rosbag one. // // The newest capture is never trimmed: if it alone exceeds the cap, the cap - // is smaller than this fault's topic count and tearing it would be the very + // is smaller than this record's topic count and tearing it would be the very // thing being fixed. auto res = - execute(tx, "SELECT MAX(capture_id) AS max_capture_id FROM snapshots WHERE fault_code = $1", fault_code); + execute(tx, "SELECT MAX(capture_id) AS max_capture_id FROM snapshots WHERE fault_code = $1 AND owner = $2", + fault_code, owner); int64_t newest_capture = 0; if (!res.empty() && !res[0]["max_capture_id"].is_null()) { newest_capture = res[0]["max_capture_id"].as(); } while (true) { - auto count_res = execute(tx, "SELECT COUNT(*) AS sz FROM snapshots WHERE fault_code = $1", fault_code); + auto count_res = + execute(tx, "SELECT COUNT(*) AS sz FROM snapshots WHERE fault_code = $1 AND owner = $2", fault_code, owner); if (count_res.empty() || count_res[0]["sz"].as() <= max_snapshots_per_fault_) { break; } auto trim_res = execute(tx, - "DELETE FROM snapshots WHERE fault_code = $1 AND capture_id = " - "(SELECT MIN(capture_id) FROM snapshots WHERE fault_code = $1) " - "AND capture_id <> $2", - fault_code, newest_capture); + "DELETE FROM snapshots WHERE fault_code = $1 AND owner = $2 AND capture_id = " + "(SELECT MIN(capture_id) FROM snapshots WHERE fault_code = $1 AND owner = $2) " + "AND capture_id <> $3", + fault_code, owner, newest_capture); if (trim_res.affected_rows() == 0) { break; } @@ -950,14 +1057,14 @@ void PgFaultStorage::store_snapshots(const std::vector & snapshots }); } -std::vector PgFaultStorage::get_snapshots(const std::string & fault_code, - const std::string & topic_filter) const { +std::vector PgFaultStorage::get_snapshots(const FaultId & id, const std::string & topic_filter) const { std::lock_guard lock(mutex_); std::string sql = - "SELECT fault_code, topic, message_type, data, captured_at_ns, capture_id FROM snapshots WHERE fault_code = $1"; + "SELECT fault_code, owner, topic, message_type, data, captured_at_ns, capture_id FROM snapshots " + "WHERE fault_code = $1 AND owner = $2"; if (!topic_filter.empty()) { - sql += " AND topic = $2"; + sql += " AND topic = $3"; } // capture_id before the timestamp: the rows of one capture are written seconds // apart under load and their timestamps interleave with a neighbouring capture's, @@ -965,12 +1072,14 @@ std::vector PgFaultStorage::get_snapshots(const std::string & faul sql += " ORDER BY capture_id DESC, captured_at_ns DESC"; return run_in_transaction("get_snapshots", [&](pqxx::work & tx) { - auto res = topic_filter.empty() ? execute(tx, sql, fault_code) : execute(tx, sql, fault_code, topic_filter); + auto res = topic_filter.empty() ? execute(tx, sql, id.fault_code, id.owner) + : execute(tx, sql, id.fault_code, id.owner, topic_filter); std::vector result; for (const auto & r : res) { SnapshotData snapshot; snapshot.fault_code = r["fault_code"].as(); + snapshot.owner = r["owner"].as(); snapshot.topic = r["topic"].as(); snapshot.message_type = r["message_type"].as(); snapshot.data = r["data"].as(); @@ -998,26 +1107,28 @@ void PgFaultStorage::store_freeze_frame(const FreezeFrameData & frame) { std::lock_guard lock(mutex_); run_in_transaction("store_freeze_frame", [&](pqxx::work & tx) { - // Keyed by fault_code (PRIMARY KEY): a re-confirm replaces the previous frame. + // Keyed by the record (idx_freeze_frames_code_owner): a re-confirm of the same record + // replaces its frame, and another owner's frame of the same code is its own row. execute(tx, - "INSERT INTO freeze_frames (fault_code, data, captured_at_ns) VALUES ($1, $2, $3) ON CONFLICT(fault_code) " - "DO " - "UPDATE SET data = $2, captured_at_ns = $3 ", - frame.fault_code, frame.data, frame.captured_at_ns); + "INSERT INTO freeze_frames (fault_code, owner, data, captured_at_ns) VALUES ($1, $2, $3, $4) " + "ON CONFLICT(fault_code, owner) DO UPDATE SET data = $3, captured_at_ns = $4 ", + frame.fault_code, frame.owner, frame.data, frame.captured_at_ns); }); } -std::optional PgFaultStorage::get_freeze_frame(const std::string & fault_code) const { +std::optional PgFaultStorage::get_freeze_frame(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("get_freeze_frame", [&](pqxx::work & tx) -> std::optional { - auto res = - execute(tx, "SELECT fault_code, data, captured_at_ns FROM freeze_frames WHERE fault_code = $1", fault_code); + auto res = execute( + tx, "SELECT fault_code, owner, data, captured_at_ns FROM freeze_frames WHERE fault_code = $1 AND owner = $2", + id.fault_code, id.owner); if (res.empty()) { return std::nullopt; } FreezeFrameData frame; frame.fault_code = res[0]["fault_code"].as(); + frame.owner = res[0]["owner"].as(); frame.data = res[0]["data"].as(); frame.captured_at_ns = res[0]["captured_at_ns"].as(); return frame; @@ -1042,7 +1153,7 @@ size_t PgFaultStorage::set_max_near_misses_per_fault(size_t max_count) { auto res = execute(tx, "DELETE FROM near_misses WHERE id IN (" "SELECT id FROM (SELECT id, ROW_NUMBER() OVER " - "(PARTITION BY fault_code ORDER BY id DESC) AS rn FROM near_misses) AS ranked " + "(PARTITION BY fault_code, owner ORDER BY id DESC) AS rn FROM near_misses) AS ranked " "WHERE rn > $1)", static_cast(max_count)); // Returned rather than logged: the storage layer has no logger, and a bound applied by mistake @@ -1051,15 +1162,15 @@ size_t PgFaultStorage::set_max_near_misses_per_fault(size_t max_count) { }); } -void PgFaultStorage::record_near_miss_locked(const std::string & fault_code, int64_t occurred_at_ns, - int32_t debounce_counter, const DebounceConfig & config, uint8_t severity, +void PgFaultStorage::record_near_miss_locked(const FaultId & id, int64_t occurred_at_ns, int32_t debounce_counter, + const DebounceConfig & config, uint8_t severity, const std::string & source_id, const std::string & resulting_status, pqxx::work & tx) { execute(tx, - "INSERT INTO near_misses (fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, severity, " - "source_id, resulting_status) VALUES ($1, $2, $3, $4, $5, $6, $7)", - fault_code, occurred_at_ns, debounce_counter, config.confirmation_threshold, static_cast(severity), - source_id, resulting_status); + "INSERT INTO near_misses (fault_code, owner, occurred_at_ns, debounce_counter, confirmation_threshold, " + "severity, source_id, resulting_status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + id.fault_code, id.owner, occurred_at_ns, debounce_counter, config.confirmation_threshold, + static_cast(severity), source_id, resulting_status); if (max_near_misses_per_fault_ == 0 || max_near_misses_per_fault_ > static_cast(std::numeric_limits::max())) { @@ -1075,25 +1186,26 @@ void PgFaultStorage::record_near_miss_locked(const std::string & fault_code, int // by timestamp would then drop the row that was just appended and make the two backends, which // append in arrival order, disagree on the same input. execute(tx, - "DELETE FROM near_misses WHERE fault_code = $1 AND id NOT IN " - "(SELECT id FROM near_misses WHERE fault_code = $1 ORDER BY id DESC LIMIT $2)", - fault_code, static_cast(max_near_misses_per_fault_)); + "DELETE FROM near_misses WHERE fault_code = $1 AND owner = $2 AND id NOT IN " + "(SELECT id FROM near_misses WHERE fault_code = $1 AND owner = $2 ORDER BY id DESC LIMIT $3)", + id.fault_code, id.owner, static_cast(max_near_misses_per_fault_)); } -std::vector PgFaultStorage::get_near_misses(const std::string & fault_code) const { +std::vector PgFaultStorage::get_near_misses(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("get_near_misses", [&](pqxx::work & tx) { auto res = execute(tx, - "SELECT fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, " - "severity, source_id, resulting_status FROM near_misses WHERE fault_code = $1 " - "ORDER BY id ASC", - fault_code); + "SELECT fault_code, owner, occurred_at_ns, debounce_counter, confirmation_threshold, " + "severity, source_id, resulting_status FROM near_misses " + "WHERE fault_code = $1 AND owner = $2 ORDER BY id ASC", + id.fault_code, id.owner); std::vector result; for (const auto & r : res) { NearMissRecord record; record.fault_code = r["fault_code"].as(); + record.owner = r["owner"].as(); record.occurred_at_ns = r["occurred_at_ns"].as(); record.debounce_counter = r["debounce_counter"].as(); record.confirmation_threshold = r["confirmation_threshold"].as(); @@ -1175,25 +1287,25 @@ std::vector PgFaultStorage::store_rosbag_file_locked(const RosbagFi // re-stored row inside a tie group. execute(tx, "INSERT INTO rosbag_files " - "(fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns) " - "VALUES ($1, $2, $3, $4, $5, $6, $7) " - "ON CONFLICT (fault_code, file_path) DO UPDATE SET " + "(fault_code, owner, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8) " + "ON CONFLICT (fault_code, owner, file_path) DO UPDATE SET " "recording_id = EXCLUDED.recording_id, format = EXCLUDED.format, " "duration_sec = EXCLUDED.duration_sec, size_bytes = EXCLUDED.size_bytes, " "created_at_ns = EXCLUDED.created_at_ns", - row.fault_code, row.recording_id, row.file_path, row.format, row.duration_sec, + row.fault_code, row.owner, row.recording_id, row.file_path, row.format, row.duration_sec, static_cast(row.size_bytes), row.created_at_ns); if (max_rosbags_per_fault_ == 0) { - return {}; // unlimited per fault; only the global byte quota bounds this + return {}; // unlimited per record; only the global byte quota bounds this } - auto res = - execute(tx, - "DELETE FROM rosbag_files WHERE fault_code = $1 AND id NOT IN " - "(SELECT id FROM rosbag_files WHERE fault_code = $1 ORDER BY created_at_ns DESC, id DESC LIMIT $2) " - "RETURNING file_path", - row.fault_code, static_cast(max_rosbags_per_fault_)); + auto res = execute(tx, + "DELETE FROM rosbag_files WHERE fault_code = $1 AND owner = $2 AND id NOT IN " + "(SELECT id FROM rosbag_files WHERE fault_code = $1 AND owner = $2 ORDER BY created_at_ns DESC, " + "id DESC LIMIT $3) " + "RETURNING file_path", + row.fault_code, row.owner, static_cast(max_rosbags_per_fault_)); std::set evicted; for (const auto & r : res) { @@ -1207,6 +1319,7 @@ namespace { RosbagFileInfo read_rosbag_row(const pqxx::row & r) { RosbagFileInfo info; info.fault_code = r["fault_code"].as(); + info.owner = r["owner"].as(); info.recording_id = r["recording_id"].as(); info.file_path = r["file_path"].as(); info.format = r["format"].as(); @@ -1217,18 +1330,19 @@ RosbagFileInfo read_rosbag_row(const pqxx::row & r) { } constexpr const char * kRosbagColumns = - "fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns"; + "fault_code, owner, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns"; } // namespace -std::vector PgFaultStorage::get_rosbag_files(const std::string & fault_code) const { +std::vector PgFaultStorage::get_rosbag_files(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("get_rosbag_files", [&](pqxx::work & tx) { - auto res = execute(tx, - std::string("SELECT ") + kRosbagColumns + - " FROM rosbag_files WHERE fault_code = $1 ORDER BY created_at_ns DESC, id DESC", - fault_code); + auto res = + execute(tx, + std::string("SELECT ") + kRosbagColumns + + " FROM rosbag_files WHERE fault_code = $1 AND owner = $2 ORDER BY created_at_ns DESC, id DESC", + id.fault_code, id.owner); std::vector result; for (const auto & r : res) { @@ -1244,7 +1358,7 @@ std::vector PgFaultStorage::get_rosbag_files_by_recording(const return run_in_transaction("get_rosbag_files_by_recording", [&](pqxx::work & tx) { auto res = execute(tx, std::string("SELECT ") + kRosbagColumns + - " FROM rosbag_files WHERE recording_id = $1 ORDER BY fault_code ASC", + " FROM rosbag_files WHERE recording_id = $1 ORDER BY fault_code ASC, owner ASC", recording_id); std::vector result; @@ -1291,19 +1405,20 @@ size_t PgFaultStorage::delete_rosbag_recording(const std::string & recording_id) return outcome.second; } -std::optional PgFaultStorage::get_rosbag_file(const std::string & fault_code) const { +std::optional PgFaultStorage::get_rosbag_file(const FaultId & id) const { std::lock_guard lock(mutex_); return run_in_transaction("get_rosbag_file", [&](pqxx::work & tx) -> std::optional { - // ORDER BY is load-bearing now that a fault can hold several recordings. Without + // ORDER BY is load-bearing now that a record can hold several recordings. Without // it PostgreSQL may return any matching row, so the fault detail and the download // would serve an arbitrary recording - non-deterministically, which no test // catches reliably. id breaks the tie because a burst stamps one created_at_ns // across all its rows. auto res = execute(tx, - "SELECT fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns " - "FROM rosbag_files WHERE fault_code = $1 ORDER BY created_at_ns DESC, id DESC LIMIT 1", - fault_code); + std::string("SELECT ") + kRosbagColumns + + " FROM rosbag_files WHERE fault_code = $1 AND owner = $2 " + "ORDER BY created_at_ns DESC, id DESC LIMIT 1", + id.fault_code, id.owner); if (res.empty()) { return std::nullopt; } @@ -1311,7 +1426,7 @@ std::optional PgFaultStorage::get_rosbag_file(const std::string }); } -bool PgFaultStorage::delete_rosbag_file(const std::string & fault_code) { +bool PgFaultStorage::delete_rosbag_file(const FaultId & id) { std::lock_guard lock(mutex_); auto outcome = @@ -1320,8 +1435,9 @@ bool PgFaultStorage::delete_rosbag_file(const std::string & fault_code) { // DELETE, leave a surviving row pointing at a directory that is gone - // unreadable for good, and still charged against the storage quota, which sums // rows. This way the worst case is an orphaned directory instead. - auto res = execute(tx, "DELETE FROM rosbag_files WHERE fault_code = $1 RETURNING file_path", fault_code); - // Every path, not the first one: a fault holds as many recordings as its cap + auto res = execute(tx, "DELETE FROM rosbag_files WHERE fault_code = $1 AND owner = $2 RETURNING file_path", + id.fault_code, id.owner); + // Every path, not the first one: a record holds as many recordings as its cap // allows, and reading one row would unlink one bag and leak the rest - rows gone, // directories left behind, uncounted by a quota that sums rows. std::set paths; @@ -1347,8 +1463,8 @@ bool PgFaultStorage::delete_rosbag_file(const std::string & fault_code) { return outcome.second; } -size_t PgFaultStorage::delete_rosbag_files(const std::vector & fault_codes) { - if (fault_codes.empty()) { +size_t PgFaultStorage::delete_rosbag_files(const std::vector & ids) { + if (ids.empty()) { return 0; } std::lock_guard lock(mutex_); @@ -1360,11 +1476,12 @@ size_t PgFaultStorage::delete_rosbag_files(const std::vector & faul run_in_transaction("delete_rosbag_files", [&](pqxx::work & tx) -> std::pair, size_t> { std::set paths; size_t deleted = 0; - for (const auto & code : fault_codes) { - // RETURNING gives every path this code held, not just the first: one fault - // code can name several recordings now, and the sweep must be able to - // reclaim every one of their bags. - auto res = execute(tx, "DELETE FROM rosbag_files WHERE fault_code = $1 RETURNING file_path", code); + for (const auto & id : ids) { + // RETURNING gives every path this record held, not just the first: one record + // can name several recordings now, and the sweep must be able to reclaim every + // one of their bags. + auto res = execute(tx, "DELETE FROM rosbag_files WHERE fault_code = $1 AND owner = $2 RETURNING file_path", + id.fault_code, id.owner); for (const auto & r : res) { paths.insert(r["file_path"].as()); } @@ -1415,21 +1532,12 @@ std::vector PgFaultStorage::get_all_rosbag_files() const { std::lock_guard lock(mutex_); return run_in_transaction("get_all_rosbag_files", [&](pqxx::work & tx) { - auto res = execute(tx, - "SELECT fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns " - "FROM rosbag_files ORDER BY created_at_ns ASC, id ASC"); + auto res = + execute(tx, std::string("SELECT ") + kRosbagColumns + " FROM rosbag_files ORDER BY created_at_ns ASC, id ASC"); std::vector result; for (const auto & r : res) { - RosbagFileInfo info; - info.fault_code = r["fault_code"].as(); - info.recording_id = r["recording_id"].as(); - info.file_path = r["file_path"].as(); - info.format = r["format"].as(); - info.duration_sec = r["duration_sec"].as(); - info.size_bytes = static_cast(r["size_bytes"].as()); - info.created_at_ns = r["created_at_ns"].as(); - result.push_back(info); + result.push_back(read_rosbag_row(r)); } return result; }); @@ -1439,26 +1547,20 @@ std::vector PgFaultStorage::list_rosbags_for_entity(const std::s std::lock_guard lock(mutex_); return run_in_transaction("list_rosbags_for_entity", [&](pqxx::work & tx) { - auto res = execute( - tx, - "SELECT r.fault_code, r.recording_id, r.file_path, r.format, r.duration_sec, r.size_bytes, r.created_at_ns " - "FROM rosbag_files r " - "JOIN faults f ON r.fault_code = f.fault_code " - "JOIN jsonb_array_elements_text(f.reporting_sources::jsonb) AS j(value) ON j.value = $1 " - "ORDER BY r.created_at_ns DESC, r.id DESC", - entity_fqn); + // The rows whose record lists the entity among its reporting sources. + auto res = + execute(tx, + "SELECT r.fault_code, r.owner, r.recording_id, r.file_path, r.format, r.duration_sec, r.size_bytes, " + "r.created_at_ns " + "FROM rosbag_files r " + "JOIN faults f ON r.fault_code = f.fault_code AND r.owner = f.owner " + "JOIN jsonb_array_elements_text(f.reporting_sources::jsonb) AS j(value) ON j.value = $1 " + "ORDER BY r.created_at_ns DESC, r.id DESC", + entity_fqn); std::vector result; for (const auto & r : res) { - RosbagFileInfo info; - info.fault_code = r["fault_code"].as(); - info.recording_id = r["recording_id"].as(); - info.file_path = r["file_path"].as(); - info.format = r["format"].as(); - info.duration_sec = r["duration_sec"].as(); - info.size_bytes = static_cast(r["size_bytes"].as()); - info.created_at_ns = r["created_at_ns"].as(); - result.push_back(info); + result.push_back(read_rosbag_row(r)); } return result; }); @@ -1468,23 +1570,11 @@ std::vector PgFaultStorage::get_all_faults() const { std::lock_guard lock(mutex_); return run_in_transaction("get_all_faults", [&](pqxx::work & tx) { - auto res = execute(tx, - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, last_passed_ns FROM faults"); + auto res = execute(tx, std::string("SELECT ") + kFaultColumns + " FROM faults"); std::vector result; for (const auto & r : res) { - fd::FaultRecord fault; - fault.fault_code = r["fault_code"].as(); - fault.severity = static_cast(r["severity"].as()); - fault.description = r["description"].as(); - fault.first_occurred_ns = r["first_occurred_ns"].as(); - fault.last_occurred_ns = r["last_occurred_ns"].as(); - fault.occurrence_count = static_cast(r["occurrence_count"].as()); - fault.status = r["status"].as(); - fault.reporting_sources = parse_json_array(r["reporting_sources"].as()); - fault.last_passed_ns = r["last_passed_ns"].as(); - result.push_back(fault); + result.push_back(read_fault_row(r)); } return result; }); diff --git a/src/ros2_medkit_fault_manager/src/ros2/fault_msg_conversions.cpp b/src/ros2_medkit_fault_manager/src/ros2/fault_msg_conversions.cpp index c354d7bf8..82d5873ac 100644 --- a/src/ros2_medkit_fault_manager/src/ros2/fault_msg_conversions.cpp +++ b/src/ros2_medkit_fault_manager/src/ros2/fault_msg_conversions.cpp @@ -43,6 +43,7 @@ ros2_medkit_msgs::msg::Fault to_msg(const fd::FaultRecord & record) { msg.occurrence_count = record.occurrence_count; msg.status = record.status; msg.reporting_sources = record.reporting_sources; + msg.owner = record.owner; return msg; } diff --git a/src/ros2_medkit_fault_manager/src/rosbag_capture.cpp b/src/ros2_medkit_fault_manager/src/rosbag_capture.cpp index fb3be1d36..b1168af72 100644 --- a/src/ros2_medkit_fault_manager/src/rosbag_capture.cpp +++ b/src/ros2_medkit_fault_manager/src/rosbag_capture.cpp @@ -422,54 +422,54 @@ void RosbagCapture::store_worker_loop() { } } -void RosbagCapture::on_fault_prefailed(const std::string & fault_code) { +void RosbagCapture::on_fault_prefailed(const FaultId & id) { if (!config_.enabled) { return; } // Start buffer if lazy_start and not already running if (config_.lazy_start && !running_.load()) { - RCLCPP_INFO(node_->get_logger(), "RosbagCapture starting on PREFAILED for fault '%s'", fault_code.c_str()); + RCLCPP_INFO(node_->get_logger(), "RosbagCapture starting on PREFAILED for fault '%s'", id.fault_code.c_str()); start(); } } -bool RosbagCapture::is_current_recording_primary(const std::string & fault_code) const { +bool RosbagCapture::is_current_recording_primary(const FaultId & id) const { std::lock_guard lock(post_fault_timer_mutex_); - return recording_post_fault_.load() && fault_code == current_fault_code_; + return recording_post_fault_.load() && id == current_fault_id_; } -bool RosbagCapture::attach_to_active_recording(const std::string & fault_code, - const std::set & entity_topics) { +bool RosbagCapture::attach_to_active_recording(const FaultId & id, const std::set & entity_topics) { std::lock_guard lock(post_fault_timer_mutex_); if (!recording_post_fault_.load()) { return false; } - if (fault_code == current_fault_code_) { + if (id == current_fault_id_) { return true; } // Cap the attachment set: a flapping detector must not grow it without bound // between two post-roll windows. Past the cap the burst is already recorded, // only the per-fault lookup key is missing. - if (attached_fault_codes_.size() >= kMaxAttachedFaults) { + if (attached_fault_ids_.size() >= kMaxAttachedFaults) { RCLCPP_WARN(node_->get_logger(), "Post-fault recording already covers %zu faults, not attaching '%s'", - attached_fault_codes_.size(), fault_code.c_str()); + attached_fault_ids_.size(), id.fault_code.c_str()); // Widened anyway. What the cap withholds is the lookup key, not the data - and in // entity mode returning here withheld the data too, because the recording never // learned this fault's topics and should_capture_topic() then dropped every one of // its messages. The bag is the burst's black box either way. - widen_capture_filter_for(fault_code, entity_topics); + widen_capture_filter_for(id, entity_topics); return true; } - if (attached_fault_codes_.insert(fault_code).second) { + if (attached_fault_ids_.insert(id).second) { RCLCPP_INFO(node_->get_logger(), "Fault '%s' confirmed during post-fault recording, attaching it to bag for '%s'", - fault_code.c_str(), current_fault_code_.empty() ? "" : current_fault_code_.c_str()); - widen_capture_filter_for(fault_code, entity_topics); + id.fault_code.c_str(), + current_fault_id_.fault_code.empty() ? "" : current_fault_id_.fault_code.c_str()); + widen_capture_filter_for(id, entity_topics); } return true; } -void RosbagCapture::widen_capture_filter_for(const std::string & fault_code, const std::set & topics) { +void RosbagCapture::widen_capture_filter_for(const FaultId & id, const std::set & topics) { // Entity mode scoped the in-flight writes to the first fault's topics; an // attached fault needs its own topics in the bag from the attach onwards, or // its row would serve a recording with none of its data. Union them in, and @@ -490,14 +490,15 @@ void RosbagCapture::widen_capture_filter_for(const std::string & fault_code, con if (topics.empty()) { RCLCPP_WARN(node_->get_logger(), "Entity scope unresolved for attached fault '%s'; widening the recording to all topics", - fault_code.c_str()); + id.fault_code.c_str()); active_capture_topics_.clear(); return; } active_capture_topics_.insert(topics.begin(), topics.end()); } -void RosbagCapture::on_fault_confirmed(const std::string & fault_code) { +void RosbagCapture::on_fault_confirmed(const FaultId & id) { + const std::string & fault_code = id.fault_code; if (!config_.enabled || !running_.load()) { return; } @@ -509,15 +510,15 @@ void RosbagCapture::on_fault_confirmed(const std::string & fault_code) { // guard stays true, messages keep entering the bag, and the recording overruns // duration_after_sec by however long the store stalls. std::set entity_topics; - if (config_.topics == "entity" && !is_current_recording_primary(fault_code)) { - entity_topics = compute_entity_topics(fault_code); + if (config_.topics == "entity" && !is_current_recording_primary(id)) { + entity_topics = compute_entity_topics(id); } // Faults arrive in bursts from one root cause, so a confirmation landing inside // an in-flight post-roll is exactly the correlated fault whose black box matters // most. It shares the recording window, so attach it to the running bag instead // of leaving it with no recording at all. - if (attach_to_active_recording(fault_code, entity_topics)) { + if (attach_to_active_recording(id, entity_topics)) { return; } @@ -575,9 +576,9 @@ void RosbagCapture::on_fault_confirmed(const std::string & fault_code) { // confirmation either attaches to this bag or starts its own - never both. { std::lock_guard lock(post_fault_timer_mutex_); - current_fault_code_ = fault_code; + current_fault_id_ = id; current_bag_path_ = bag_path; - attached_fault_codes_.clear(); + attached_fault_ids_.clear(); recording_post_fault_.store(true); arm_post_fault_timer(); } @@ -619,6 +620,7 @@ void RosbagCapture::on_fault_confirmed(const std::string & fault_code) { RosbagFileInfo info; info.fault_code = fault_code; + info.owner = id.owner; info.recording_id = rosbag_recording_id(bag_path); info.file_path = bag_path; info.format = config_.format; @@ -642,7 +644,8 @@ void RosbagCapture::on_fault_confirmed(const std::string & fault_code) { } } -void RosbagCapture::on_fault_cleared(const std::string & fault_code) { +void RosbagCapture::on_fault_cleared(const FaultId & id) { + const std::string & fault_code = id.fault_code; if (!config_.enabled || !config_.auto_cleanup) { return; } @@ -666,18 +669,18 @@ void RosbagCapture::on_fault_cleared(const std::string & fault_code) { { std::lock_guard lock(post_fault_timer_mutex_); if (recording_post_fault_.load()) { - attached_fault_codes_.erase(fault_code); - if (fault_code == current_fault_code_) { - current_fault_code_.clear(); + attached_fault_ids_.erase(id); + if (id == current_fault_id_) { + current_fault_id_ = FaultId{}; } } } - // Delete the bag file for this fault. Called from the clear service, where a throw terminates the process. + // Delete the bag rows of this record. Called from the clear service, where a throw terminates the process. // Under clear_rows_mutex_: a row the store worker is inserting lands before this delete, never after. try { std::lock_guard rows_lock(clear_rows_mutex_); - if (storage_->delete_rosbag_file(fault_code)) { + if (storage_->delete_rosbag_file(id)) { RCLCPP_INFO(node_->get_logger(), "Auto-cleanup: deleted bag file for fault '%s'", fault_code.c_str()); } } catch (const std::exception & e) { @@ -1022,15 +1025,15 @@ rclcpp::QoS RosbagCapture::resolve_topic_qos(const std::string & topic) const { return qos; } -std::set RosbagCapture::compute_entity_topics(const std::string & fault_code) { +std::set RosbagCapture::compute_entity_topics(const FaultId & id) { std::set topics; // The whole resolution is guarded: this runs on a detached capture thread with - // no outer catch, and storage_->get_fault() can throw on the sqlite backend - // (e.g. SQLITE_BUSY). A throw here would terminate the process - the exact crash - // the crash-safety work removed - so any failure degrades to "write everything". + // no outer catch, and the store can throw on the sqlite backend (e.g. SQLITE_BUSY). + // A throw here would terminate the process - the exact crash the crash-safety work + // removed - so any failure degrades to "write everything". try { - auto fault = storage_->get_fault(fault_code); + auto fault = storage_->get_fault(id); if (fault && !fault->reporting_sources.empty()) { // reporting_sources hold the reporting node's FQN (e.g. "/planner_server"). // Split each into (name, namespace) to match against topic endpoints. @@ -1086,7 +1089,7 @@ std::set RosbagCapture::compute_entity_topics(const std::string & f } } catch (const std::exception & e) { RCLCPP_WARN(node_->get_logger(), "Entity scope resolution failed for fault '%s' (%s); writing full buffer", - fault_code.c_str(), e.what()); + id.fault_code.c_str(), e.what()); topics.clear(); } @@ -1094,13 +1097,14 @@ std::set RosbagCapture::compute_entity_topics(const std::string & f } void RosbagCapture::resolve_entity_topics(const std::string & fault_code, std::set topics) { + // By code only for the log line. The caller already resolved the topics from the record. // The topics arrive resolved, for the same reason widen_capture_filter_for() takes // them resolved: computing them reads the fault store and walks the ROS graph, and // the caller does that once, before it touches any lock. if (config_.topics != "entity") { topics.clear(); } else if (!topics.empty()) { - RCLCPP_INFO(node_->get_logger(), "Entity scope for fault '%s': %zu topic(s) from the faulting node(s) + /tf", + RCLCPP_INFO(node_->get_logger(), "Entity scope for fault '%s': %zu topic(s) from the owning node + /tf", fault_code.c_str(), topics.size()); } else { RCLCPP_WARN(node_->get_logger(), @@ -1536,7 +1540,7 @@ void RosbagCapture::write_manifest(const std::string & bag_path, int64_t created } } -bool RosbagCapture::row_still_wanted(const std::string & code) { +bool RosbagCapture::row_still_wanted(const FaultId & id) { // Same rule as on_fault_cleared: without auto_cleanup, or with a history // configured, a cleared fault keeps its black box. Checking only auto_cleanup // here let an acknowledgement arriving during the post-roll drop every row and @@ -1548,10 +1552,10 @@ bool RosbagCapture::row_still_wanted(const std::string & code) { // the row: the bag stays referenced. std::optional fault; try { - fault = storage_->get_fault(code); + fault = storage_->get_fault(id); } catch (const std::exception & e) { RCLCPP_WARN(node_->get_logger(), "Could not read fault '%s' before storing its bag, keeping its row: %s", - code.c_str(), e.what()); + id.fault_code.c_str(), e.what()); return true; } // Absent is not cleared. A caller driving the capture directly, or a fault the @@ -1575,7 +1579,7 @@ bool RosbagCapture::store_rows_or_discard_bag(std::vector rows, std::lock_guard lock(clear_rows_mutex_); rows.erase(std::remove_if(rows.begin(), rows.end(), [this](const RosbagFileInfo & row) { - return !row_still_wanted(row.fault_code); + return !row_still_wanted(FaultId{row.fault_code, row.owner}); }), rows.end()); if (!rows.empty()) { @@ -1675,8 +1679,8 @@ void RosbagCapture::finalize_post_fault_recording() { // a recording that is already being finalised. Testing the guard under the same // lock also settles a timer firing concurrently with stop() - the loser sees the // recording already claimed and returns. - std::set attached; - std::string fault_code; + std::set attached; + FaultId fault_id; std::string bag_path; int64_t started_at_ns = 0; double recording_span_sec = 0.0; @@ -1692,8 +1696,8 @@ void RosbagCapture::finalize_post_fault_recording() { post_fault_timer_->cancel(); } recording_post_fault_.store(false); - attached.swap(attached_fault_codes_); - fault_code.swap(current_fault_code_); + attached.swap(attached_fault_ids_); + std::swap(fault_id, current_fault_id_); bag_path.swap(current_bag_path_); // Taken here, under the lock that clears the guard: a confirmation racing this // finalise opens its own recording the moment the guard drops, and would @@ -1762,8 +1766,8 @@ void RosbagCapture::finalize_post_fault_recording() { // is never listed before its manifest exists. ClosedRecording recording; recording.bag_path = std::move(bag_path); - recording.fault_code = std::move(fault_code); - recording.attached_fault_codes = std::move(attached); + recording.fault_id = std::move(fault_id); + recording.attached_ids = std::move(attached); recording.duration_sec = recording_span_sec; recording.created_at_ns = get_wall_clock_ns(); enqueue_closed_recording(std::move(recording)); @@ -1785,23 +1789,26 @@ void RosbagCapture::store_closed_recording(const ClosedRecording & recording) { info.duration_sec = recording.duration_sec; info.size_bytes = bag_size; info.created_at_ns = recording.created_at_ns; - // One recording, one row per fault it covers: the correlated faults that - // confirmed inside this window each need their own lookup key to serve it. Which + // One recording, one row per RECORD it covers: the correlated faults that + // confirmed inside this window each need their own lookup key to serve it, and + // two owners of one code in the same burst each get theirs. Which // of them still get one is decided in store_rows_or_discard_bag(). std::vector rows; - if (!recording.fault_code.empty()) { - info.fault_code = recording.fault_code; + if (!recording.fault_id.fault_code.empty()) { + info.fault_code = recording.fault_id.fault_code; + info.owner = recording.fault_id.owner; rows.push_back(info); } - for (const auto & code : recording.attached_fault_codes) { - info.fault_code = code; + for (const auto & attached_id : recording.attached_ids) { + info.fault_code = attached_id.fault_code; + info.owner = attached_id.owner; rows.push_back(info); } if (store_rows_or_discard_bag(std::move(rows), bag_path)) { RCLCPP_INFO(node_->get_logger(), "Bag file completed: %s (%.2f MB, %.1fs, %zu attached fault(s))", bag_path.c_str(), static_cast(bag_size) / (1024.0 * 1024.0), info.duration_sec, - recording.attached_fault_codes.size()); + recording.attached_ids.size()); } } catch (const std::exception & e) { RCLCPP_WARN(node_->get_logger(), "Failed to store recording '%s': %s", recording.bag_path.c_str(), e.what()); diff --git a/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp b/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp index 95de9275e..b6fc94d41 100644 --- a/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp +++ b/src/ros2_medkit_fault_manager/src/snapshot_capture.cpp @@ -13,7 +13,9 @@ // limitations under the License. #include "ros2_medkit_fault_manager/snapshot_capture.hpp" +#include #include +#include #include #include @@ -146,12 +148,91 @@ bool SnapshotCapture::seed_capture_seq() { return true; } -void SnapshotCapture::capture(const std::string & fault_code) { +bool SnapshotCapture::groups_by_code(const FaultId & id) const { + if (id.owner == kSharedOwner || config_.recapture_cooldown_sec <= 0.0) { + return false; + } + bool explicit_match = false; + return !resolve_topics(id.fault_code, explicit_match).empty(); +} + +void SnapshotCapture::prune_groups_locked(std::chrono::steady_clock::time_point now) { + const auto cooldown = std::chrono::duration(config_.recapture_cooldown_sec); + for (auto it = groups_.begin(); it != groups_.end();) { + if (it->second.state == CaptureGroup::State::kDone && now - it->second.started >= cooldown) { + it = groups_.erase(it); + } else { + ++it; + } + } +} + +SnapshotCapture::Admission SnapshotCapture::admit(const FaultId & id) { + if (!config_.enabled || !groups_by_code(id)) { + return Admission::kEnqueue; + } + const auto now = std::chrono::steady_clock::now(); + std::unique_lock lock(groups_mutex_); + prune_groups_locked(now); + auto it = groups_.find(id.fault_code); + if (it == groups_.end()) { + CaptureGroup group; + group.primary = id.owner; + group.owners.insert(id.owner); + group.started = now; + groups_.emplace(id.fault_code, std::move(group)); + return Admission::kEnqueue; + } + CaptureGroup & group = it->second; + const bool is_new = group.owners.insert(id.owner).second; + switch (group.state) { + case CaptureGroup::State::kQueued: + // The group's one job is queued and will sample after this confirmation. + return Admission::kJoined; + case CaptureGroup::State::kRunning: + if (!is_new) { + // The record confirms again while the capture of its previous occurrence still runs: that + // sample is not this occurrence's, so the record leaves the group and is captured alone. + group.owners.erase(id.owner); + group.solo.insert(id.owner); + return Admission::kEnqueue; + } + return Admission::kJoined; + case CaptureGroup::State::kDone: + if (!is_new) { + // The record confirms again (it was acknowledged since): it needs a capture of its own. + groups_.erase(it); + CaptureGroup fresh; + fresh.primary = id.owner; + fresh.owners.insert(id.owner); + fresh.started = now; + groups_.emplace(id.fault_code, std::move(fresh)); + return Admission::kEnqueue; + } + persist_capture(id, group.rows, group.freeze_frame, group.captured_count, false, group.capture_id); + return Admission::kJoined; + } + return Admission::kEnqueue; +} + +void SnapshotCapture::withdraw(const FaultId & id) { + std::lock_guard lock(groups_mutex_); + auto it = groups_.find(id.fault_code); + if (it == groups_.end() || it->second.state != CaptureGroup::State::kQueued || it->second.primary != id.owner) { + return; + } + groups_.erase(it); +} + +void SnapshotCapture::capture(const FaultId & id) { + const std::string & fault_code = id.fault_code; if (!config_.enabled) { RCLCPP_DEBUG(node_->get_logger(), "Snapshot capture disabled, skipping for fault '%s'", fault_code.c_str()); return; } + // Resolution is by CODE: fault_specific and patterns are config about what a code + // means, not about who reported it. bool explicit_match = false; auto topics = resolve_topics(fault_code, explicit_match); // Zero-config fallback: nothing in the explicit config even mentioned this @@ -159,7 +240,7 @@ void SnapshotCapture::capture(const std::string & fault_code) { // per-fault opt-out (explicit_match == true) and must NOT fall through here. bool entity_scoped = false; if (topics.empty() && !explicit_match && config_.entity_default) { - topics = resolve_entity_topics(fault_code); + topics = resolve_entity_topics(id); entity_scoped = !topics.empty(); } if (topics.empty()) { @@ -174,42 +255,128 @@ void SnapshotCapture::capture(const std::string & fault_code) { if (!seed_capture_seq()) { RCLCPP_WARN(node_->get_logger(), "Skipping snapshot capture for fault '%s': fault storage unavailable", fault_code.c_str()); + withdraw(id); return; } - RCLCPP_INFO(node_->get_logger(), "Capturing snapshots for fault '%s' (%zu topics%s)", fault_code.c_str(), - topics.size(), entity_scoped ? ", entity-default" : ""); - - // Accumulate a compact dict of captured topic values alongside the per-topic snapshots. - nlohmann::json freeze_frame = nlohmann::json::object(); - // One id for the whole capture, minted here so every row of this confirmation - // shares it. Monotonic rather than a clock read: the rows are written seconds - // apart under load and a wall clock is not guaranteed to move forward. - const int64_t capture_id = ++capture_seq_; - std::vector rows; - size_t captured_count = 0; - for (const auto & topic : topics) { - bool success = false; - // Entity-default topics are not known at construction, so the background - // cache never holds them - always sample those on demand. - if (config_.background_capture && !entity_scoped) { - success = capture_topic_from_cache(fault_code, topic, freeze_frame, rows); + // Per-source records of one code share one capture: the first worker to start samples, and the + // record of any other worker is written by it, or from what it sampled. + bool grouped = groups_by_code(id); + if (grouped) { + std::lock_guard lock(groups_mutex_); + prune_groups_locked(std::chrono::steady_clock::now()); + auto it = groups_.find(fault_code); + if (it != groups_.end() && it->second.solo.erase(id.owner) > 0) { + grouped = false; // confirmed again while the group ran: sampled and written for this record alone + } else if (it == groups_.end()) { + CaptureGroup group; + group.primary = id.owner; + group.owners.insert(id.owner); + group.state = CaptureGroup::State::kRunning; + group.started = std::chrono::steady_clock::now(); + groups_.emplace(fault_code, std::move(group)); } else { - success = capture_topic_on_demand(fault_code, topic, freeze_frame, rows); + CaptureGroup & group = it->second; + const bool is_new = group.owners.insert(id.owner).second; + if (group.state != CaptureGroup::State::kQueued) { + if (is_new && group.state == CaptureGroup::State::kDone) { + persist_capture(id, group.rows, group.freeze_frame, group.captured_count, false, group.capture_id); + } + RCLCPP_DEBUG(node_->get_logger(), "Fault '%s' (source %s) shares the capture of its code", fault_code.c_str(), + id.owner.c_str()); + return; + } + group.state = CaptureGroup::State::kRunning; + } + } + + // A group whose capture fails is dropped, or its members would wait for a capture that never completes. + try { + RCLCPP_INFO(node_->get_logger(), "Capturing snapshots for fault '%s' (%zu topics%s)", fault_code.c_str(), + topics.size(), entity_scoped ? ", entity-default" : ""); + + // Accumulate a compact dict of captured topic values alongside the per-topic snapshots. + nlohmann::json freeze_frame = nlohmann::json::object(); + // One id for the whole capture, minted here so every row of this confirmation + // shares it. Monotonic rather than a clock read: the rows are written seconds + // apart under load and a wall clock is not guaranteed to move forward. + const int64_t capture_id = ++capture_seq_; + std::vector rows; + size_t captured_count = 0; + for (const auto & topic : topics) { + bool success = false; + // Entity-default topics are not known at construction, so the background + // cache never holds them - always sample those on demand. + if (config_.background_capture && !entity_scoped) { + success = capture_topic_from_cache(id, topic, freeze_frame, rows); + } else { + success = capture_topic_on_demand(id, topic, freeze_frame, rows); + } + + if (success) { + ++captured_count; + } + } + + RCLCPP_INFO(node_->get_logger(), "Captured %zu/%zu snapshots for fault '%s'", captured_count, topics.size(), + fault_code.c_str()); + + if (!grouped) { + persist_capture(id, std::move(rows), freeze_frame, captured_count, entity_scoped, capture_id); + return; } - if (success) { - ++captured_count; + // Write for every record that joined while this ran, then for those that join while it writes. + std::set written; + for (;;) { + std::set todo; + { + std::lock_guard lock(groups_mutex_); + auto it = groups_.find(fault_code); + if (it == groups_.end()) { + // The group aged out while this ran: only the record of this worker is left to write. + if (written.count(id.owner) == 0) { + todo.insert(id.owner); + } + } else { + std::set_difference(it->second.owners.begin(), it->second.owners.end(), written.begin(), written.end(), + std::inserter(todo, todo.end())); + } + if (todo.empty()) { + if (it != groups_.end()) { + it->second.state = CaptureGroup::State::kDone; + it->second.rows = rows; + it->second.freeze_frame = freeze_frame; + it->second.captured_count = captured_count; + it->second.capture_id = capture_id; + } + return; + } + } + for (const auto & owner : todo) { + persist_capture(FaultId{fault_code, owner}, rows, freeze_frame, captured_count, false, capture_id); + written.insert(owner); + } } + } catch (...) { + if (grouped) { + std::lock_guard lock(groups_mutex_); + groups_.erase(fault_code); + } + throw; } +} - RCLCPP_INFO(node_->get_logger(), "Captured %zu/%zu snapshots for fault '%s'", captured_count, topics.size(), - fault_code.c_str()); +void SnapshotCapture::persist_capture(const FaultId & id, std::vector rows, + const nlohmann::json & freeze_frame, size_t captured_count, bool entity_scoped, + int64_t capture_id) { + const std::string & fault_code = id.fault_code; // One write for the whole capture. Storing topic by topic let the per-fault cap // reject the tail of a capture, leaving a freeze frame with holes and nothing // saying which values were dropped. for (auto & row : rows) { + row.owner = id.owner; row.capture_id = capture_id; } @@ -221,7 +388,7 @@ void SnapshotCapture::capture(const std::string & fault_code) { // asked to retain snapshots there is no such promise, so the batch stands. // Absent is not cleared: a caller driving the capture directly still stores. // Mirrors rosbag_capture's wants_a_row. - const auto fault = storage_->get_fault(fault_code); + const auto fault = storage_->get_fault(id); const bool acknowledged_mid_capture = fault.has_value() && fault->status == fd::fault_status::kCleared; if (acknowledged_mid_capture && !storage_->retains_snapshots_on_clear()) { RCLCPP_DEBUG(node_->get_logger(), "Dropping capture for '%s' - acknowledged while it ran", fault_code.c_str()); @@ -229,7 +396,7 @@ void SnapshotCapture::capture(const std::string & fault_code) { storage_->store_snapshots(rows); } - // Persist the compact freeze-frame keyed by fault_code (retained across clear_fault). + // Persist the compact freeze-frame keyed by the RECORD (retained across clear_fault). // Only reached for faults with a configured capture set (unconfigured codes returned // early above and get no row). If nothing was publishing at confirmation time the row // holds an empty object, recording that a configured capture ran but sampled nothing. @@ -244,7 +411,7 @@ void SnapshotCapture::capture(const std::string & fault_code) { fault_code.c_str()); return; } - auto existing = storage_->get_freeze_frame(fault_code); + auto existing = storage_->get_freeze_frame(id); if (existing.has_value() && existing->data != "{}") { RCLCPP_WARN(node_->get_logger(), "Nothing captured for fault '%s'; keeping previously retained freeze-frame", fault_code.c_str()); @@ -254,6 +421,7 @@ void SnapshotCapture::capture(const std::string & fault_code) { FreezeFrameData frame; frame.fault_code = fault_code; + frame.owner = id.owner; frame.data = freeze_frame.dump(); frame.captured_at_ns = get_wall_clock_ns(); storage_->store_freeze_frame(frame); @@ -287,7 +455,8 @@ std::vector SnapshotCapture::resolve_topics(const std::string & fau return {}; } -std::vector SnapshotCapture::resolve_entity_topics(const std::string & fault_code) const { +std::vector SnapshotCapture::resolve_entity_topics(const FaultId & id) const { + const std::string & fault_code = id.fault_code; // Bound the on-demand sweep: each silent-but-published topic costs up to // timeout_sec, so a node with many declared topics must not stall a capture // pool worker indefinitely. @@ -298,7 +467,7 @@ std::vector SnapshotCapture::resolve_entity_topics(const std::strin // pool worker with no outer catch, and storage_->get_fault() can throw on // the sqlite backend. Any failure degrades to "no entity-default capture". try { - auto fault = storage_->get_fault(fault_code); + auto fault = storage_->get_fault(id); if (!fault || fault->reporting_sources.empty()) { return {}; } @@ -308,6 +477,7 @@ std::vector SnapshotCapture::resolve_entity_topics(const std::strin // as (name, ns="/") could match an unrelated root-namespace node of the // same name and freeze-frame its topics. Their zero-config frames come // from the gateway's EntityFreezeFrameCapture instead. + // std::set> wanted; for (const auto & source : fault->reporting_sources) { if (source.size() < 2 || source.front() != '/') { @@ -363,7 +533,7 @@ std::vector SnapshotCapture::resolve_entity_topics(const std::strin return {topics.begin(), topics.end()}; } -bool SnapshotCapture::capture_topic_on_demand(const std::string & fault_code, const std::string & topic, +bool SnapshotCapture::capture_topic_on_demand(const FaultId & id, const std::string & topic, nlohmann::json & freeze_frame, std::vector & rows) { // Get topic type std::string msg_type = get_topic_type(topic); @@ -468,7 +638,8 @@ bool SnapshotCapture::capture_topic_on_demand(const std::string & fault_code, co // Store snapshot (use wall clock time, not sim time, for proper timestamps) SnapshotData snapshot; - snapshot.fault_code = fault_code; + snapshot.fault_code = id.fault_code; + snapshot.owner = id.owner; snapshot.topic = topic; snapshot.message_type = msg_type; snapshot.data = json_data.dump(); @@ -479,7 +650,8 @@ bool SnapshotCapture::capture_topic_on_demand(const std::string & fault_code, co // Record the value into the compact freeze-frame dict under the topic key. freeze_frame[topic] = std::move(json_data); - RCLCPP_DEBUG(node_->get_logger(), "Captured snapshot from '%s' for fault '%s'", topic.c_str(), fault_code.c_str()); + RCLCPP_DEBUG(node_->get_logger(), "Captured snapshot from '%s' for fault '%s'", topic.c_str(), + id.fault_code.c_str()); return true; } catch (const ros2_medkit_serialization::TypeNotFoundError & e) { @@ -493,8 +665,9 @@ bool SnapshotCapture::capture_topic_on_demand(const std::string & fault_code, co return false; } -bool SnapshotCapture::capture_topic_from_cache(const std::string & fault_code, const std::string & topic, +bool SnapshotCapture::capture_topic_from_cache(const FaultId & id, const std::string & topic, nlohmann::json & freeze_frame, std::vector & rows) { + const std::string & fault_code = id.fault_code; std::lock_guard lock(cache_mutex_); auto it = message_cache_.find(topic); @@ -507,7 +680,8 @@ bool SnapshotCapture::capture_topic_from_cache(const std::string & fault_code, c // Store snapshot from cache SnapshotData snapshot; - snapshot.fault_code = fault_code; + snapshot.fault_code = id.fault_code; + snapshot.owner = id.owner; snapshot.topic = topic; snapshot.message_type = cached.message_type; snapshot.data = cached.data; diff --git a/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp b/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp index 29b2a23b1..b343108cc 100644 --- a/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp +++ b/src/ros2_medkit_fault_manager/src/sqlite_fault_storage.cpp @@ -14,12 +14,19 @@ #include "ros2_medkit_fault_manager/sqlite_fault_storage.hpp" +#include +#include #include +#include +#include #include #include #include #include #include +#include +#include +#include namespace ros2_medkit_fault_manager { @@ -103,6 +110,112 @@ class SqliteStatement { sqlite3_stmt * stmt_{nullptr}; }; +/// The strings of a JSON array written by serialize_json_array. A malformed value is logged and +/// yields what was read before the text went wrong. +std::vector parse_json_array(const std::string & json_str, core::LogSink & sink) { + std::vector result; + + if (json_str.size() < 2 || json_str.front() != '[' || json_str.back() != ']') { + if (!json_str.empty()) { + core::logf(sink, core::LogLevel::kWarn, "sqlite_fault_storage", "Malformed JSON array in database: '%s'", + json_str.c_str()); + } + return result; + } + + const std::string content = json_str.substr(1, json_str.size() - 2); + size_t pos = 0; + while (pos < content.size()) { + while (pos < content.size() && std::isspace(static_cast(content[pos])) != 0) { + ++pos; + } + if (pos >= content.size() || content[pos] != '"') { + break; + } + ++pos; // past the opening quote + + std::string value; + while (pos < content.size() && content[pos] != '"') { + if (content[pos] != '\\' || pos + 1 >= content.size()) { + value.push_back(content[pos]); + ++pos; + continue; + } + const char escaped = content[pos + 1]; + pos += 2; + switch (escaped) { + case 'b': + value.push_back('\b'); + break; + case 'f': + value.push_back('\f'); + break; + case 'n': + value.push_back('\n'); + break; + case 'r': + value.push_back('\r'); + break; + case 't': + value.push_back('\t'); + break; + case 'u': { + // serialize_json_array writes only \u00XX, for control bytes. + if (pos + 4 <= content.size() && content.compare(pos, 2, "00") == 0) { + char * end = nullptr; + const std::string digits = content.substr(pos + 2, 2); + const auto code = std::strtol(digits.c_str(), &end, 16); + if (end != nullptr && *end == '\0') { + value.push_back(static_cast(code)); + pos += 4; + break; + } + } + value.push_back('u'); + break; + } + default: + value.push_back(escaped); // covers \" \\ and \/ + break; + } + } + if (pos < content.size()) { + ++pos; // past the closing quote + } + result.push_back(std::move(value)); + + while (pos < content.size() && + (std::isspace(static_cast(content[pos])) != 0 || content[pos] == ',')) { + ++pos; + } + } + return result; +} + +/// Columns every fault read selects, in the order read_fault_row and read_fault_owner decode them. +constexpr const char * kFaultColumns = + "fault_code, severity, description, first_occurred_ns, last_occurred_ns, " + "occurrence_count, status, reporting_sources, last_passed_ns, owner"; + +fd::FaultRecord read_fault_row(SqliteStatement & stmt, core::LogSink & sink) { + fd::FaultRecord fault; + fault.fault_code = stmt.column_text(0); + fault.severity = static_cast(stmt.column_int(1)); + fault.description = stmt.column_text(2); + fault.first_occurred_ns = stmt.column_int64(3); + fault.last_occurred_ns = stmt.column_int64(4); + fault.occurrence_count = static_cast(stmt.column_int64(5)); + fault.status = stmt.column_text(6); + fault.reporting_sources = parse_json_array(stmt.column_text(7), sink); + fault.last_passed_ns = stmt.column_int64(8); + fault.owner = stmt.column_text(9); + return fault; +} + +std::string read_fault_owner(SqliteStatement & stmt) { + return stmt.column_text(9); +} + } // namespace SqliteFaultStorage::SqliteFaultStorage(const std::string & db_path, core::LogSink & log_sink) @@ -150,9 +263,16 @@ DebounceConfig SqliteFaultStorage::get_debounce_config() const { } void SqliteFaultStorage::initialize_schema() { + // A row is one fault RECORD, identified by (fault_code, owner). Uniqueness is the + // index below, never a column constraint, per the house rule spelled out at the + // rosbag_files table: a column constraint cannot be dropped with ALTER TABLE and + // forces a full rebuild the day the identity changes again. + // + // owner is the shared owner for a code every source feeds, or the source_id for a + // per_source code. reporting_sources lists every source that reported the record. const char * create_faults_table_sql = R"( CREATE TABLE IF NOT EXISTS faults ( - fault_code TEXT PRIMARY KEY, + fault_code TEXT NOT NULL, severity INTEGER NOT NULL, description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, @@ -163,7 +283,8 @@ void SqliteFaultStorage::initialize_schema() { debounce_counter INTEGER NOT NULL DEFAULT 0, last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, - confirmed_at_ns INTEGER NOT NULL DEFAULT 0 + confirmed_at_ns INTEGER NOT NULL DEFAULT 0, + owner TEXT NOT NULL ); )"; @@ -214,6 +335,7 @@ void SqliteFaultStorage::initialize_schema() { CREATE TABLE IF NOT EXISTS snapshots ( id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', topic TEXT NOT NULL, message_type TEXT NOT NULL, data TEXT NOT NULL, @@ -230,11 +352,13 @@ void SqliteFaultStorage::initialize_schema() { } // Create freeze_frames table: one compact JSON dict of captured topic values per fault - // code. Unlike snapshots, freeze frames are keyed by fault_code and are NOT removed on - // clear_fault, so the confirmed-state record is retained after acknowledgement. + // RECORD. Unlike snapshots, freeze frames are keyed by the record identity and are NOT + // removed on clear_fault, so the confirmed-state record is retained after acknowledgement. + // Uniqueness through the index, not a column constraint, per the house rule. const char * create_freeze_frames_table_sql = R"( CREATE TABLE IF NOT EXISTS freeze_frames ( - fault_code TEXT PRIMARY KEY, + fault_code TEXT NOT NULL, + owner TEXT NOT NULL, data TEXT NOT NULL, captured_at_ns INTEGER NOT NULL ); @@ -272,12 +396,14 @@ void SqliteFaultStorage::initialize_schema() { // Create near_misses table: append-only series of FAILED reports that moved the debounce // counter without confirming the fault. One row per qualifying report, never updated in // place, and NOT removed on clear_fault - acknowledging a fault cycle must not erase how - // often that code approached confirmation. Bounded per fault code by the caller-supplied - // limit, evicting the oldest rows first. + // often that record approached confirmation. A row belongs to the record named by + // (fault_code, owner) and keeps the source_id that reported it. Bounded per record by the + // caller-supplied limit, evicting the oldest rows first. const char * create_near_misses_table_sql = R"( CREATE TABLE IF NOT EXISTS near_misses ( id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', occurred_at_ns INTEGER NOT NULL, debounce_counter INTEGER NOT NULL, confirmation_threshold INTEGER NOT NULL, @@ -285,7 +411,6 @@ void SqliteFaultStorage::initialize_schema() { source_id TEXT NOT NULL, resulting_status TEXT NOT NULL DEFAULT '' ); - CREATE INDEX IF NOT EXISTS idx_near_misses_fault_code ON near_misses(fault_code, id); )"; if (sqlite3_exec(db_, create_near_misses_table_sql, nullptr, nullptr, &err_msg) != SQLITE_OK) { @@ -315,8 +440,8 @@ void SqliteFaultStorage::initialize_schema() { } } - // Create rosbag_files table. One row = one LINK (a fault claiming a recording): - // several faults of a burst link to one bag, and one fault links to several bags + // Create rosbag_files table. One row = one LINK (a fault record claiming a recording): + // several records of a burst link to one bag, and one record links to several bags // over time. Bytes belong to file_path, not to the row. // // House rule, learned the hard way here: uniqueness is expressed with @@ -328,6 +453,7 @@ void SqliteFaultStorage::initialize_schema() { CREATE TABLE IF NOT EXISTS rosbag_files ( id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, + owner TEXT NOT NULL DEFAULT '', recording_id TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL, format TEXT NOT NULL, @@ -348,6 +474,40 @@ void SqliteFaultStorage::initialize_schema() { migrate_rosbag_files_drop_unique(); migrate_rosbag_files_add_recording_id(); + // After the rosbag rebuilds, so it sees a rosbag_files whose column set is settled: + // the rebuild above copies a fixed column list and would drop an owner column added + // before it. + migrate_faults_add_owner(); + + // After the migration, which adds owner to near_misses and snapshots on a database that predates it. + // The per-record trims and reads of both tables go through these. + if (sqlite3_exec(db_, + "DROP INDEX IF EXISTS idx_near_misses_fault_code;" + "CREATE INDEX IF NOT EXISTS idx_near_misses_record ON near_misses(fault_code, owner, id);" + "CREATE INDEX IF NOT EXISTS idx_snapshots_record ON snapshots(fault_code, owner, capture_id);", + nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + throw std::runtime_error("Failed to create record indexes: " + error); + } + + // The record identity, expressed as an index rather than a PRIMARY KEY so the next + // identity change is a DROP INDEX instead of another table rebuild. + if (sqlite3_exec(db_, "CREATE UNIQUE INDEX IF NOT EXISTS idx_faults_code_owner ON faults(fault_code, owner)", nullptr, + nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + throw std::runtime_error("Failed to create faults unique index: " + error); + } + + if (sqlite3_exec(db_, + "CREATE UNIQUE INDEX IF NOT EXISTS idx_freeze_frames_code_owner ON freeze_frames(fault_code, owner)", + nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + throw std::runtime_error("Failed to create freeze_frames unique index: " + error); + } + // Indexes last, so they serve a fresh table and a rebuilt one alike. // // idx_rosbag_files_path is capability, not tuning: path_referenced() scans on @@ -356,7 +516,7 @@ void SqliteFaultStorage::initialize_schema() { const char * create_rosbag_files_indexes_sql = R"( CREATE INDEX IF NOT EXISTS idx_rosbag_files_fault_code ON rosbag_files(fault_code); CREATE INDEX IF NOT EXISTS idx_rosbag_files_created_at ON rosbag_files(created_at_ns); - CREATE INDEX IF NOT EXISTS idx_rosbag_files_fault_created ON rosbag_files(fault_code, created_at_ns, id); + CREATE INDEX IF NOT EXISTS idx_rosbag_files_fault_created ON rosbag_files(fault_code, owner, created_at_ns, id); CREATE INDEX IF NOT EXISTS idx_rosbag_files_recording ON rosbag_files(recording_id); CREATE INDEX IF NOT EXISTS idx_rosbag_files_path ON rosbag_files(file_path); )"; @@ -376,7 +536,7 @@ void SqliteFaultStorage::initialize_schema() { // rows this index would reject. if (sqlite3_exec(db_, "DELETE FROM rosbag_files WHERE id NOT IN " - "(SELECT MAX(id) FROM rosbag_files GROUP BY fault_code, file_path)", + "(SELECT MAX(id) FROM rosbag_files GROUP BY fault_code, owner, file_path)", nullptr, nullptr, &err_msg) != SQLITE_OK) { std::string error = err_msg ? err_msg : "Unknown error"; sqlite3_free(err_msg); @@ -385,7 +545,7 @@ void SqliteFaultStorage::initialize_schema() { if (sqlite3_exec(db_, "CREATE UNIQUE INDEX IF NOT EXISTS idx_rosbag_files_fault_path " - "ON rosbag_files(fault_code, file_path)", + "ON rosbag_files(fault_code, owner, file_path)", nullptr, nullptr, &err_msg) != SQLITE_OK) { std::string error = err_msg ? err_msg : "Unknown error"; sqlite3_free(err_msg); @@ -463,6 +623,160 @@ void SqliteFaultStorage::migrate_rosbag_files_drop_unique() { } } +bool SqliteFaultStorage::table_has_owner(const char * table) const { + SqliteStatement info(db_, (std::string("PRAGMA table_info(") + table + ")").c_str()); + while (info.step() == SQLITE_ROW) { + if (info.column_text(1) == "owner") { + return true; + } + } + return false; +} + +bool SqliteFaultStorage::table_exists(const char * table) const { + SqliteStatement stmt(db_, "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?"); + stmt.bind_text(1, table); + return stmt.step() == SQLITE_ROW; +} + +void SqliteFaultStorage::drop_scratch_table(const char * table) { + // Announced rather than done quietly. Debris under these names means an earlier + // rebuild did not finish, or someone repaired the database by hand, and an operator + // reading the log after a surprising restart should find out that this open threw + // work away. The names are reserved for this procedure, so nothing else loses data. + if (table_exists(table)) { + core::logf(log_sink_, core::LogLevel::kWarn, "sqlite_fault_storage", + "Dropping leftover migration scratch table '%s' from an unfinished earlier rebuild", table); + } + char * err_msg = nullptr; + const std::string sql = std::string("DROP TABLE IF EXISTS ") + table; + if (sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + throw std::runtime_error(std::string("fault owner migration failed (drop leftover ") + table + "): " + error); + } +} + +void SqliteFaultStorage::migrate_faults_add_owner() { + // Probed PER TABLE, not once for the whole migration: the tables come from independent + // CREATE TABLE IF NOT EXISTS statements, so a database can hold a legacy faults table next to + // a child table this release just created with owner already on it. + const bool faults_needs_owner = !table_has_owner("faults"); + const bool freeze_frames_needs_owner = !table_has_owner("freeze_frames"); + const bool snapshots_needs_owner = !table_has_owner("snapshots"); + const bool rosbag_files_needs_owner = !table_has_owner("rosbag_files"); + const bool near_misses_needs_owner = !table_has_owner("near_misses"); + if (!faults_needs_owner && !freeze_frames_needs_owner && !snapshots_needs_owner && !rosbag_files_needs_owner && + !near_misses_needs_owner) { + return; // fresh database, or already migrated - safe to re-run on every open + } + + // Every row of a legacy database is a shared record: the old schema held one row per fault + // code, fed by every source. The row keeps its reporting_sources as written, and every child + // row (snapshot, freeze frame, rosbag link, near miss) attaches to that shared record. + const std::string shared = std::string("'") + kSharedOwner + "'"; + + char * err_msg = nullptr; + const auto exec = [this, &err_msg](const std::string & sql, const char * what) { + if (sqlite3_exec(db_, sql.c_str(), nullptr, nullptr, &err_msg) != SQLITE_OK) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + err_msg = nullptr; + throw std::runtime_error(std::string("fault owner migration failed (") + what + "): " + error); + } + }; + + exec("BEGIN IMMEDIATE", "begin"); + try { + if (faults_needs_owner) { + // faults: SQLite cannot change a PRIMARY KEY in place and CREATE TABLE IF NOT + // EXISTS is a no-op on an existing database, so the identity change is the + // documented table-rebuild procedure. + // + // The DROP first because a rebuild that was interrupted by a crash, or a manual + // recovery with an external tool, can leave faults_new behind. It is safe because + // faults_new is this procedure's scratch table and nothing else ever reads it. + drop_scratch_table("faults_new"); + exec( + "CREATE TABLE faults_new (" + " fault_code TEXT NOT NULL," + " severity INTEGER NOT NULL," + " description TEXT NOT NULL," + " first_occurred_ns INTEGER NOT NULL," + " last_occurred_ns INTEGER NOT NULL," + " occurrence_count INTEGER NOT NULL," + " status TEXT NOT NULL," + " reporting_sources TEXT NOT NULL," + " debounce_counter INTEGER NOT NULL DEFAULT 0," + " last_failed_ns INTEGER NOT NULL DEFAULT 0," + " last_passed_ns INTEGER NOT NULL DEFAULT 0," + " confirmed_at_ns INTEGER NOT NULL DEFAULT 0," + " owner TEXT NOT NULL)", + "create new faults table"); + + // A pending row carries a confirmation time from the occurrence before a reactivation, + // which earlier releases did not reset. + exec( + "INSERT INTO faults_new (fault_code, severity, description, first_occurred_ns, last_occurred_ns, " + " occurrence_count, status, reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, " + " confirmed_at_ns, owner) " + "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, occurrence_count, status, " + " reporting_sources, debounce_counter, last_failed_ns, last_passed_ns, " + " CASE WHEN status IN ('PREFAILED', 'PREPASSED') THEN 0 ELSE confirmed_at_ns END, " + + shared + " FROM faults", + "copy fault rows"); + + exec("DROP TABLE faults", "drop old faults table"); + exec("ALTER TABLE faults_new RENAME TO faults", "rename faults"); + } + + if (freeze_frames_needs_owner) { + // freeze_frames: same problem, same procedure, same reason for the DROP. + drop_scratch_table("freeze_frames_new"); + exec( + "CREATE TABLE freeze_frames_new (" + " fault_code TEXT NOT NULL," + " owner TEXT NOT NULL," + " data TEXT NOT NULL," + " captured_at_ns INTEGER NOT NULL)", + "create new freeze_frames table"); + exec("INSERT INTO freeze_frames_new (fault_code, owner, data, captured_at_ns) SELECT fault_code, " + shared + + ", data, captured_at_ns FROM freeze_frames", + "copy freeze frames"); + exec("DROP TABLE freeze_frames", "drop old freeze_frames table"); + exec("ALTER TABLE freeze_frames_new RENAME TO freeze_frames", "rename freeze_frames"); + } + + // snapshots, rosbag_files and near_misses only gain a column: their identity is a rowid, + // not the fault code, so ALTER TABLE is enough. + if (snapshots_needs_owner) { + exec("ALTER TABLE snapshots ADD COLUMN owner TEXT NOT NULL DEFAULT ''", "add snapshots.owner"); + exec("UPDATE snapshots SET owner = " + shared, "assign snapshots to shared records"); + } + + if (rosbag_files_needs_owner) { + exec("ALTER TABLE rosbag_files ADD COLUMN owner TEXT NOT NULL DEFAULT ''", "add rosbag_files.owner"); + exec("UPDATE rosbag_files SET owner = " + shared, "assign rosbag links to shared records"); + + // The rosbag indexes that carry fault_code have to widen with it. CREATE INDEX IF + // NOT EXISTS in initialize_schema() is a no-op on an index that already exists, so + // without these drops the unique index would keep its two-column shape. + exec("DROP INDEX IF EXISTS idx_rosbag_files_fault_path", "drop rosbag unique index"); + exec("DROP INDEX IF EXISTS idx_rosbag_files_fault_created", "drop rosbag fault_created index"); + } + + if (near_misses_needs_owner) { + exec("ALTER TABLE near_misses ADD COLUMN owner TEXT NOT NULL DEFAULT ''", "add near_misses.owner"); + exec("UPDATE near_misses SET owner = " + shared, "assign near misses to shared records"); + } + + exec("COMMIT", "commit"); + } catch (...) { + sqlite3_exec(db_, "ROLLBACK", nullptr, nullptr, nullptr); + throw; + } +} + void SqliteFaultStorage::migrate_rosbag_files_add_recording_id() { bool has_recording_id = false; { @@ -529,97 +843,6 @@ void SqliteFaultStorage::migrate_rosbag_files_add_recording_id() { } } -std::vector SqliteFaultStorage::parse_json_array(const std::string & json_str) const { - std::vector result; - - // Simple JSON array parser for ["a", "b", "c"] format - if (json_str.size() < 2 || json_str.front() != '[' || json_str.back() != ']') { - if (!json_str.empty()) { - core::logf(log_sink_, core::LogLevel::kWarn, "sqlite_fault_storage", "Malformed JSON array in database: '%s'", - json_str.c_str()); - } - return result; - } - - std::string content = json_str.substr(1, json_str.size() - 2); - if (content.empty()) { - return result; - } - - size_t pos = 0; - while (pos < content.size()) { - // Skip whitespace - while (pos < content.size() && std::isspace(static_cast(content[pos]))) { - ++pos; - } - if (pos >= content.size()) { - break; - } - - // Expect opening quote - if (content[pos] != '"') { - break; - } - ++pos; - - // Find closing quote (handle escape sequences) - std::string value; - while (pos < content.size() && content[pos] != '"') { - if (content[pos] == '\\' && pos + 1 < content.size()) { - ++pos; - char escaped = content[pos]; - switch (escaped) { - case '"': - value.push_back('"'); - break; - case '\\': - value.push_back('\\'); - break; - case '/': - value.push_back('/'); - break; - case 'b': - value.push_back('\b'); - break; - case 'f': - value.push_back('\f'); - break; - case 'n': - value.push_back('\n'); - break; - case 'r': - value.push_back('\r'); - break; - case 't': - value.push_back('\t'); - break; - default: - // Unknown escape sequence: preserve character as-is - value.push_back(escaped); - break; - } - ++pos; - continue; - } - value.push_back(content[pos]); - ++pos; - } - - if (pos < content.size()) { - ++pos; // Skip closing quote - } - - result.push_back(value); - - // Skip whitespace and comma - while (pos < content.size() && (std::isspace(static_cast(content[pos])) || content[pos] == ',')) { - ++pos; - } - } - - return result; -} - std::string SqliteFaultStorage::serialize_json_array(const std::vector & vec) { std::ostringstream oss; oss << '['; @@ -653,7 +876,19 @@ std::string SqliteFaultStorage::serialize_json_array(const std::vector(c) < 0x20) { + char escape[7]; + std::snprintf(escape, sizeof(escape), "\\u%04x", static_cast(static_cast(c))); + oss << escape; + } else { + oss << c; + } break; } } @@ -663,7 +898,7 @@ std::string SqliteFaultStorage::serialize_json_array(const std::vector lock(mutex_); @@ -678,13 +913,13 @@ bool SqliteFaultStorage::report_fault_event(const std::string & fault_code, fd:: // heartbeat - including one that turns out to write nothing at all - contend for that lock and // fail with SQLITE_BUSY where before it could not. if (kind != fd::ReportKind::kFailed) { - return report_fault_event_locked(fault_code, kind, severity, description, source_id, timestamp_ns, config); + return report_fault_event_locked(id, kind, severity, description, source_id, timestamp_ns, config); } exec_or_throw("BEGIN IMMEDIATE"); try { const bool is_new_occurrence = - report_fault_event_locked(fault_code, kind, severity, description, source_id, timestamp_ns, config); + report_fault_event_locked(id, kind, severity, description, source_id, timestamp_ns, config); exec_or_throw("COMMIT"); return is_new_occurrence; } catch (...) { @@ -693,20 +928,21 @@ bool SqliteFaultStorage::report_fault_event(const std::string & fault_code, fd:: } } -bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_code, fd::ReportKind kind, - uint8_t severity, const std::string & description, - const std::string & source_id, int64_t timestamp_ns, - const DebounceConfig & config) { +bool SqliteFaultStorage::report_fault_event_locked(const FaultId & id, fd::ReportKind kind, uint8_t severity, + const std::string & description, const std::string & source_id, + int64_t timestamp_ns, const DebounceConfig & config) { const bool is_failed = (kind == fd::ReportKind::kFailed); + const std::string & fault_code = id.fault_code; - // Check if fault exists + // Check if the record exists. Another record of the same code is a different row and is not touched. SqliteStatement check_stmt(db_, "SELECT severity, occurrence_count, reporting_sources, status, debounce_counter, " - "confirmed_at_ns, first_occurred_ns FROM faults WHERE fault_code = ?"); + "confirmed_at_ns, first_occurred_ns FROM faults WHERE fault_code = ? AND owner = ?"); check_stmt.bind_text(1, fault_code); + check_stmt.bind_text(2, id.owner); if (check_stmt.step() == SQLITE_ROW) { - // Fault exists - update it + // Record exists - update it int existing_severity = check_stmt.column_int(0); int64_t existing_count = check_stmt.column_int64(1); std::string sources_json = check_stmt.column_text(2); @@ -729,16 +965,18 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod // FAILED event reactivates - reset debounce counter to 0 so FAILED branch // decrements it to -1, then reuse the existing FAILED logic below. Also // reset first_occurred: this is a new outage cycle, not a continuation - // of the one that just cleared. + // of the one that just cleared. The new occurrence has no confirmation + // time until it confirms. debounce_counter = 0; first_occurred_ns = timestamp_ns; + confirmed_at_ns = 0; is_reactivation = true; } if (is_failed) { // FAILED event // Parse existing sources and add new one - std::vector sources = parse_json_array(sources_json); + std::vector sources = parse_json_array(sources_json, log_sink_); std::set sources_set(sources.begin(), sources.end()); sources_set.insert(source_id); sources.assign(sources_set.begin(), sources_set.end()); @@ -757,9 +995,11 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod // Decrement towards confirmation, clamped to the thresholds. debounce_counter = clamp_debounce_counter(debounce_counter - 1, config); - // CRITICAL bypasses debounce; otherwise the shared state machine decides (with hysteresis). + // CRITICAL bypasses debounce and the HEALED latch. A per-source record decides on its stored severity, + // a shared record on the report's. Otherwise the shared state machine decides (with hysteresis). std::string new_status; - if (config.critical_immediate_confirm && severity == fd::severity::kCritical) { + if (config.critical_immediate_confirm && + (id.owner == kSharedOwner ? severity : new_severity) == fd::severity::kCritical) { debounce_counter = std::min(debounce_counter, -1); new_status = fd::fault_status::kConfirmed; } else { @@ -778,11 +1018,11 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod db_, description.empty() ? "UPDATE faults SET severity = ?, last_occurred_ns = ?, last_failed_ns = ?, " "occurrence_count = ?, " "reporting_sources = ?, status = ?, debounce_counter = ?, confirmed_at_ns = ?, " - "first_occurred_ns = ? WHERE fault_code = ?" + "first_occurred_ns = ? WHERE fault_code = ? AND owner = ?" : "UPDATE faults SET severity = ?, description = ?, last_occurred_ns = ?, " "last_failed_ns = ?, " "occurrence_count = ?, reporting_sources = ?, status = ?, debounce_counter = ?, " - "confirmed_at_ns = ?, first_occurred_ns = ? WHERE fault_code = ?"); + "confirmed_at_ns = ?, first_occurred_ns = ? WHERE fault_code = ? AND owner = ?"); if (description.empty()) { update_stmt.bind_int(1, new_severity); @@ -795,6 +1035,7 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod update_stmt.bind_int64(8, confirmed_at_ns); update_stmt.bind_int64(9, first_occurred_ns); update_stmt.bind_text(10, fault_code); + update_stmt.bind_text(11, id.owner); } else { update_stmt.bind_int(1, new_severity); update_stmt.bind_text(2, description); @@ -807,6 +1048,7 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod update_stmt.bind_int64(9, confirmed_at_ns); update_stmt.bind_int64(10, first_occurred_ns); update_stmt.bind_text(11, fault_code); + update_stmt.bind_text(12, id.owner); } if (update_stmt.step() != SQLITE_DONE) { @@ -814,7 +1056,7 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod } if (is_near_miss(true, new_status)) { - record_near_miss_locked(fault_code, timestamp_ns, debounce_counter, config, severity, source_id, new_status); + record_near_miss_locked(id, timestamp_ns, debounce_counter, config, severity, source_id, new_status); } } else { // PASSED event - increment towards healing, clamped to the thresholds. @@ -826,11 +1068,13 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod // ENDING, not occurring. Bumping it made a long-stale CONFIRMED fault look // freshly active. The PASSED instant is recorded in last_passed_ns. SqliteStatement update_stmt( - db_, "UPDATE faults SET last_passed_ns = ?, status = ?, debounce_counter = ? WHERE fault_code = ?"); + db_, + "UPDATE faults SET last_passed_ns = ?, status = ?, debounce_counter = ? WHERE fault_code = ? AND owner = ?"); update_stmt.bind_int64(1, timestamp_ns); update_stmt.bind_text(2, new_status); update_stmt.bind_int(3, debounce_counter); update_stmt.bind_text(4, fault_code); + update_stmt.bind_text(5, id.owner); if (update_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to update fault: ") + sqlite3_errmsg(db_)); @@ -840,9 +1084,10 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod return is_reactivation; // Reactivation treated as new occurrence for event publishing } - // New fault - only create for FAILED events + // New record - only create one for FAILED events. A PASSED event for a record that + // does not exist has nothing to heal. if (!is_failed) { - return false; // PASSED event for non-existent fault is ignored + return false; // PASSED event for non-existent record is ignored } // Determine initial status based on debounce logic (shared with the in-memory backend). @@ -854,12 +1099,12 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod initial_status = compute_debounce_status(initial_counter, "", config); } - // New fault - insert with debounce_counter = -1 + // New record - insert with debounce_counter = -1 SqliteStatement insert_stmt(db_, "INSERT INTO faults (fault_code, severity, description, first_occurred_ns, " "last_occurred_ns, occurrence_count, status, reporting_sources, " - "debounce_counter, last_failed_ns, last_passed_ns, confirmed_at_ns) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + "debounce_counter, last_failed_ns, last_passed_ns, confirmed_at_ns, owner) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); const bool confirmed_now = initial_status == fd::fault_status::kConfirmed; insert_stmt.bind_text(1, fault_code); @@ -874,20 +1119,21 @@ bool SqliteFaultStorage::report_fault_event_locked(const std::string & fault_cod insert_stmt.bind_int64(10, timestamp_ns); // last_failed_ns insert_stmt.bind_int64(11, 0); // last_passed_ns (never passed) insert_stmt.bind_int64(12, confirmed_now ? timestamp_ns : 0); + insert_stmt.bind_text(13, id.owner); if (insert_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to insert fault: ") + sqlite3_errmsg(db_)); } if (is_near_miss(true, initial_status)) { - record_near_miss_locked(fault_code, timestamp_ns, initial_counter, config, severity, source_id, initial_status); + record_near_miss_locked(id, timestamp_ns, initial_counter, config, severity, source_id, initial_status); } return true; // New fault created } -std::vector SqliteFaultStorage::list_faults(bool filter_by_severity, uint8_t severity, - const std::vector & statuses) const { +std::vector SqliteFaultStorage::list_fault_entries(bool filter_by_severity, uint8_t severity, + const std::vector & statuses) const { std::lock_guard lock(mutex_); // Determine which statuses to include @@ -907,9 +1153,7 @@ std::vector SqliteFaultStorage::list_faults(bool filter_by_seve } // Build query - std::string sql = - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, last_passed_ns FROM faults WHERE status IN ("; + std::string sql = std::string("SELECT ") + kFaultColumns + " FROM faults WHERE status IN ("; for (size_t i = 0; i < status_filter.size(); ++i) { if (i > 0) { sql += ", "; @@ -932,81 +1176,72 @@ std::vector SqliteFaultStorage::list_faults(bool filter_by_seve stmt.bind_int(param_index, static_cast(severity)); } - std::vector result; + std::vector result; while (stmt.step() == SQLITE_ROW) { - fd::FaultRecord fault; - fault.fault_code = stmt.column_text(0); - fault.severity = static_cast(stmt.column_int(1)); - fault.description = stmt.column_text(2); - - int64_t first_ns = stmt.column_int64(3); - int64_t last_ns = stmt.column_int64(4); - fault.first_occurred_ns = first_ns; - fault.last_occurred_ns = last_ns; - - fault.occurrence_count = static_cast(stmt.column_int64(5)); - fault.status = stmt.column_text(6); - fault.reporting_sources = parse_json_array(stmt.column_text(7)); - fault.last_passed_ns = stmt.column_int64(8); - - result.push_back(fault); + FaultEntry entry; + entry.record = read_fault_row(stmt, log_sink_); + entry.id = FaultId{entry.record.fault_code, read_fault_owner(stmt)}; + result.push_back(std::move(entry)); } return result; } -std::optional SqliteFaultStorage::get_fault(const std::string & fault_code) const { +std::optional SqliteFaultStorage::get_fault(const FaultId & id) const { std::lock_guard lock(mutex_); - SqliteStatement stmt(db_, - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, last_passed_ns FROM faults WHERE fault_code = ?"); - stmt.bind_text(1, fault_code); + SqliteStatement stmt( + db_, (std::string("SELECT ") + kFaultColumns + " FROM faults WHERE fault_code = ? AND owner = ?").c_str()); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); if (stmt.step() != SQLITE_ROW) { return std::nullopt; } - fd::FaultRecord fault; - fault.fault_code = stmt.column_text(0); - fault.severity = static_cast(stmt.column_int(1)); - fault.description = stmt.column_text(2); + return read_fault_row(stmt, log_sink_); +} - int64_t first_ns = stmt.column_int64(3); - int64_t last_ns = stmt.column_int64(4); - fault.first_occurred_ns = first_ns; - fault.last_occurred_ns = last_ns; +std::vector SqliteFaultStorage::get_ids_by_code(const std::string & fault_code) const { + std::lock_guard lock(mutex_); - fault.occurrence_count = static_cast(stmt.column_int64(5)); - fault.status = stmt.column_text(6); - fault.reporting_sources = parse_json_array(stmt.column_text(7)); - fault.last_passed_ns = stmt.column_int64(8); + // Ordered by owner so an ambiguity message lists the owners the same way every time, + // and so the backends answer in the same order. + SqliteStatement stmt(db_, "SELECT owner FROM faults WHERE fault_code = ? ORDER BY owner ASC"); + stmt.bind_text(1, fault_code); - return fault; + std::vector result; + while (stmt.step() == SQLITE_ROW) { + result.push_back(FaultId{fault_code, stmt.column_text(0)}); + } + return result; } -bool SqliteFaultStorage::clear_fault(const std::string & fault_code) { +bool SqliteFaultStorage::clear_fault(const FaultId & id) { std::lock_guard lock(mutex_); - // The near_misses rows for this code are deliberately left alone. Clearing acknowledges one + // The near_misses rows of this record are deliberately left alone. Clearing acknowledges one // fault cycle; the record of how often the code approached confirmation spans cycles and // cannot be reconstructed once deleted. - // Delete associated snapshots when fault is cleared - // Acknowledging a fault drops its value snapshots, unless a history was asked + // Delete the record's snapshots when it is cleared. + // Acknowledging a record drops its value snapshots, unless a history was asked // for: with recordings retained past a clear, deleting the readings that go with - // them leaves a fault holding bags whose matching values are gone. + // them leaves a record holding bags whose matching values are gone. Scoped to the + // owner, so another owner's readings for the same code stay where they are. if (!retain_snapshots_on_clear_) { - SqliteStatement delete_snapshots(db_, "DELETE FROM snapshots WHERE fault_code = ?"); - delete_snapshots.bind_text(1, fault_code); + SqliteStatement delete_snapshots(db_, "DELETE FROM snapshots WHERE fault_code = ? AND owner = ?"); + delete_snapshots.bind_text(1, id.fault_code); + delete_snapshots.bind_text(2, id.owner); if (delete_snapshots.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to delete snapshots: ") + sqlite3_errmsg(db_)); } } - SqliteStatement stmt(db_, "UPDATE faults SET status = ? WHERE fault_code = ?"); + SqliteStatement stmt(db_, "UPDATE faults SET status = ? WHERE fault_code = ? AND owner = ?"); stmt.bind_text(1, fd::fault_status::kCleared); - stmt.bind_text(2, fault_code); + stmt.bind_text(2, id.fault_code); + stmt.bind_text(3, id.owner); if (stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to clear fault: ") + sqlite3_errmsg(db_)); @@ -1015,20 +1250,22 @@ bool SqliteFaultStorage::clear_fault(const std::string & fault_code) { return sqlite3_changes(db_) > 0; } -std::vector SqliteFaultStorage::reclassify_healed_as_cleared(const KeepHealedFn & keep_healed) { +std::vector SqliteFaultStorage::reclassify_healed_as_cleared(const KeepHealedFn & keep_healed) { std::lock_guard lock(mutex_); - // Collect the codes that will flip first so the caller can audit each one. Codes the + // Collect the records that will flip first so the caller can audit each one. Codes the // caller keeps HEALED are left out. Everything runs under the same lock, so the - // returned list matches the rows actually reclassified below. - std::vector reclassified; + // returned list matches the rows actually reclassified below. Identities, not codes: + // one code can hold a HEALED record for one owner and an untouched CONFIRMED one for + // another. + std::vector reclassified; { - SqliteStatement select_stmt(db_, "SELECT fault_code FROM faults WHERE status = ?"); + SqliteStatement select_stmt(db_, "SELECT fault_code, owner FROM faults WHERE status = ?"); select_stmt.bind_text(1, fd::fault_status::kHealed); while (select_stmt.step() == SQLITE_ROW) { - std::string code = select_stmt.column_text(0); - if (!(keep_healed && keep_healed(code))) { - reclassified.push_back(std::move(code)); + FaultId id{select_stmt.column_text(0), select_stmt.column_text(1)}; + if (!(keep_healed && keep_healed(id.fault_code))) { + reclassified.push_back(std::move(id)); } } } @@ -1037,26 +1274,28 @@ std::vector SqliteFaultStorage::reclassify_healed_as_cleared(const return reclassified; } - // One transaction, so a failure part way does not leave some codes flipped. + // One transaction, so a failure part way does not leave some records flipped. exec_or_throw("BEGIN IMMEDIATE"); try { - for (const auto & code : reclassified) { + for (const auto & id : reclassified) { // Drop snapshots for the affected faults so a reclassified row matches CLEARED semantics. // clear_fault is not the only place that takes a fault's readings, so retain_snapshots_on_clear_ // has to reach here too: otherwise the setting holds until the next restart and then the // reclassification deletes exactly what it was set to keep. if (!retain_snapshots_on_clear_) { - SqliteStatement del(db_, "DELETE FROM snapshots WHERE fault_code = ?"); - del.bind_text(1, code); + SqliteStatement del(db_, "DELETE FROM snapshots WHERE fault_code = ? AND owner = ?"); + del.bind_text(1, id.fault_code); + del.bind_text(2, id.owner); if (del.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to delete snapshots: ") + sqlite3_errmsg(db_)); } } - SqliteStatement stmt(db_, "UPDATE faults SET status = ? WHERE fault_code = ? AND status = ?"); + SqliteStatement stmt(db_, "UPDATE faults SET status = ? WHERE fault_code = ? AND owner = ? AND status = ?"); stmt.bind_text(1, fd::fault_status::kCleared); - stmt.bind_text(2, code); - stmt.bind_text(3, fd::fault_status::kHealed); + stmt.bind_text(2, id.fault_code); + stmt.bind_text(3, id.owner); + stmt.bind_text(4, fd::fault_status::kHealed); if (stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to reclassify HEALED faults: ") + sqlite3_errmsg(db_)); } @@ -1081,35 +1320,37 @@ size_t SqliteFaultStorage::size() const { return static_cast(stmt.column_int64(0)); } -bool SqliteFaultStorage::contains(const std::string & fault_code) const { +bool SqliteFaultStorage::contains(const FaultId & id) const { std::lock_guard lock(mutex_); - SqliteStatement stmt(db_, "SELECT 1 FROM faults WHERE fault_code = ? LIMIT 1"); - stmt.bind_text(1, fault_code); + SqliteStatement stmt(db_, "SELECT 1 FROM faults WHERE fault_code = ? AND owner = ? LIMIT 1"); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); return stmt.step() == SQLITE_ROW; } -std::vector SqliteFaultStorage::check_time_based_confirmation(int64_t now_ns) { +std::vector SqliteFaultStorage::check_time_based_confirmation(int64_t now_ns) { std::lock_guard lock(mutex_); - std::vector confirmed; + std::vector confirmed; if (config_.auto_confirm_after_sec <= 0.0) { return confirmed; // Time-based confirmation disabled } + int64_t threshold_ns = std::llround(config_.auto_confirm_after_sec * 1e9); int64_t cutoff_ns = now_ns - threshold_ns; - // Collect the codes that will flip first so the caller can audit each one. The + // Collect the records that will flip first so the caller can audit each one. The // SELECT predicate mirrors the UPDATE exactly, and both run under the same lock, // so the returned list matches the rows actually confirmed below. { SqliteStatement select_stmt( - db_, "SELECT fault_code FROM faults WHERE status = ? AND last_failed_ns <= ? AND last_failed_ns > 0"); + db_, "SELECT fault_code, owner FROM faults WHERE status = ? AND last_failed_ns <= ? AND last_failed_ns > 0"); select_stmt.bind_text(1, fd::fault_status::kPrefailed); select_stmt.bind_int64(2, cutoff_ns); while (select_stmt.step() == SQLITE_ROW) { - confirmed.push_back(select_stmt.column_text(0)); + confirmed.push_back(FaultId{select_stmt.column_text(0), select_stmt.column_text(1)}); } } @@ -1157,24 +1398,26 @@ void SqliteFaultStorage::store_snapshots(const std::vector & snaps } std::lock_guard lock(mutex_); - const std::string & fault_code = snapshots.front().fault_code; + const FaultId id{snapshots.front().fault_code, snapshots.front().owner}; // One transaction for the whole capture: a capture is all-or-nothing, and the // old row-at-a-time path could leave a confirmation's values half stored. exec_or_throw("BEGIN IMMEDIATE"); try { { - SqliteStatement stmt(db_, - "INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns, capture_id) " - "VALUES (?, ?, ?, ?, ?, ?)"); + SqliteStatement stmt( + db_, + "INSERT INTO snapshots (fault_code, owner, topic, message_type, data, captured_at_ns, capture_id) " + "VALUES (?, ?, ?, ?, ?, ?, ?)"); for (const auto & snapshot : snapshots) { stmt.reset(); stmt.bind_text(1, snapshot.fault_code); - stmt.bind_text(2, snapshot.topic); - stmt.bind_text(3, snapshot.message_type); - stmt.bind_text(4, snapshot.data); - stmt.bind_int64(5, snapshot.captured_at_ns); - stmt.bind_int64(6, snapshot.capture_id); + stmt.bind_text(2, snapshot.owner); + stmt.bind_text(3, snapshot.topic); + stmt.bind_text(4, snapshot.message_type); + stmt.bind_text(5, snapshot.data); + stmt.bind_int64(6, snapshot.captured_at_ns); + stmt.bind_int64(7, snapshot.capture_id); if (stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to store snapshot: ") + sqlite3_errmsg(db_)); } @@ -1182,36 +1425,39 @@ void SqliteFaultStorage::store_snapshots(const std::vector & snaps } if (max_snapshots_per_fault_ > 0) { - // Trim whole capture sets, oldest first, until the fault fits. The old rule + // Trim whole capture sets, oldest first, until the record fits. The old rule // counted rows and rejected the NEW row once full, so a capture straddling // the cap was stored in part - some topics present, the rest silently gone, // indistinguishable from "that topic was not publishing". Keep-newest also // stops this cap from opposing the rosbag one. // // The newest capture is never trimmed: if it alone exceeds the cap, the cap - // is smaller than this fault's topic count and tearing it would be the very + // is smaller than this record's topic count and tearing it would be the very // thing being fixed. - SqliteStatement newest(db_, "SELECT MAX(capture_id) FROM snapshots WHERE fault_code = ?"); - newest.bind_text(1, fault_code); + SqliteStatement newest(db_, "SELECT MAX(capture_id) FROM snapshots WHERE fault_code = ? AND owner = ?"); + newest.bind_text(1, id.fault_code); + newest.bind_text(2, id.owner); int64_t newest_capture = 0; if (newest.step() == SQLITE_ROW) { newest_capture = newest.column_int64(0); } SqliteStatement trim(db_, - "DELETE FROM snapshots WHERE fault_code = ?1 AND capture_id = " - "(SELECT MIN(capture_id) FROM snapshots WHERE fault_code = ?1) " - "AND capture_id <> ?2"); - SqliteStatement count(db_, "SELECT COUNT(*) FROM snapshots WHERE fault_code = ?"); + "DELETE FROM snapshots WHERE fault_code = ?1 AND owner = ?2 AND capture_id = " + "(SELECT MIN(capture_id) FROM snapshots WHERE fault_code = ?1 AND owner = ?2) " + "AND capture_id <> ?3"); + SqliteStatement count(db_, "SELECT COUNT(*) FROM snapshots WHERE fault_code = ? AND owner = ?"); while (true) { count.reset(); - count.bind_text(1, fault_code); + count.bind_text(1, id.fault_code); + count.bind_text(2, id.owner); if (count.step() != SQLITE_ROW || static_cast(count.column_int64(0)) <= max_snapshots_per_fault_) { break; } trim.reset(); - trim.bind_text(1, fault_code); - trim.bind_int64(2, newest_capture); + trim.bind_text(1, id.fault_code); + trim.bind_text(2, id.owner); + trim.bind_int64(3, newest_capture); if (trim.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to trim snapshots: ") + sqlite3_errmsg(db_)); } @@ -1228,15 +1474,15 @@ void SqliteFaultStorage::store_snapshots(const std::vector & snaps } } -std::vector SqliteFaultStorage::get_snapshots(const std::string & fault_code, +std::vector SqliteFaultStorage::get_snapshots(const FaultId & id, const std::string & topic_filter) const { std::lock_guard lock(mutex_); std::vector result; std::string sql = - "SELECT fault_code, topic, message_type, data, captured_at_ns, capture_id FROM snapshots WHERE fault_code " - "= ?"; + "SELECT fault_code, owner, topic, message_type, data, captured_at_ns, capture_id FROM snapshots " + "WHERE fault_code = ? AND owner = ?"; if (!topic_filter.empty()) { sql += " AND topic = ?"; } @@ -1246,19 +1492,21 @@ std::vector SqliteFaultStorage::get_snapshots(const std::string & sql += " ORDER BY capture_id DESC, captured_at_ns DESC"; SqliteStatement stmt(db_, sql.c_str()); - stmt.bind_text(1, fault_code); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); if (!topic_filter.empty()) { - stmt.bind_text(2, topic_filter); + stmt.bind_text(3, topic_filter); } while (stmt.step() == SQLITE_ROW) { SnapshotData snapshot; snapshot.fault_code = stmt.column_text(0); - snapshot.topic = stmt.column_text(1); - snapshot.message_type = stmt.column_text(2); - snapshot.data = stmt.column_text(3); - snapshot.captured_at_ns = stmt.column_int64(4); - snapshot.capture_id = stmt.column_int64(5); + snapshot.owner = stmt.column_text(1); + snapshot.topic = stmt.column_text(2); + snapshot.message_type = stmt.column_text(3); + snapshot.data = stmt.column_text(4); + snapshot.captured_at_ns = stmt.column_int64(5); + snapshot.capture_id = stmt.column_int64(6); result.push_back(snapshot); } @@ -1281,24 +1529,29 @@ int64_t SqliteFaultStorage::get_max_capture_id() const { void SqliteFaultStorage::store_freeze_frame(const FreezeFrameData & frame) { std::lock_guard lock(mutex_); - // Keyed by fault_code (PRIMARY KEY): a re-confirm replaces the previous frame. + // Keyed by (fault_code, owner) through idx_freeze_frames_code_owner: a re-confirm of + // the same record replaces its frame, and another owner's frame for the same code is + // a different row. SqliteStatement stmt(db_, - "INSERT OR REPLACE INTO freeze_frames (fault_code, data, captured_at_ns) " - "VALUES (?, ?, ?)"); + "INSERT OR REPLACE INTO freeze_frames (fault_code, owner, data, captured_at_ns) " + "VALUES (?, ?, ?, ?)"); stmt.bind_text(1, frame.fault_code); - stmt.bind_text(2, frame.data); - stmt.bind_int64(3, frame.captured_at_ns); + stmt.bind_text(2, frame.owner); + stmt.bind_text(3, frame.data); + stmt.bind_int64(4, frame.captured_at_ns); if (stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to store freeze frame: ") + sqlite3_errmsg(db_)); } } -std::optional SqliteFaultStorage::get_freeze_frame(const std::string & fault_code) const { +std::optional SqliteFaultStorage::get_freeze_frame(const FaultId & id) const { std::lock_guard lock(mutex_); - SqliteStatement stmt(db_, "SELECT fault_code, data, captured_at_ns FROM freeze_frames WHERE fault_code = ?"); - stmt.bind_text(1, fault_code); + SqliteStatement stmt( + db_, "SELECT fault_code, owner, data, captured_at_ns FROM freeze_frames WHERE fault_code = ? AND owner = ?"); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); if (stmt.step() != SQLITE_ROW) { return std::nullopt; @@ -1306,8 +1559,9 @@ std::optional SqliteFaultStorage::get_freeze_frame(const std::s FreezeFrameData frame; frame.fault_code = stmt.column_text(0); - frame.data = stmt.column_text(1); - frame.captured_at_ns = stmt.column_int64(2); + frame.owner = stmt.column_text(1); + frame.data = stmt.column_text(2); + frame.captured_at_ns = stmt.column_int64(3); return frame; } @@ -1323,12 +1577,12 @@ size_t SqliteFaultStorage::set_max_near_misses_per_fault(size_t max_count) { } // Apply the bound to what is already in the database. Without this, a database that grew under - // a larger bound (or none) stays over the new bound until each fault code happens to record - // another near miss - and a code that never does keeps its rows for good. + // a larger bound (or none) stays over the new bound until each record happens to log + // another near miss - and a record that never does keeps its rows for good. SqliteStatement trim_stmt(db_, "DELETE FROM near_misses WHERE id IN (" "SELECT id FROM (SELECT id, ROW_NUMBER() OVER " - "(PARTITION BY fault_code ORDER BY id DESC) AS rn FROM near_misses) " + "(PARTITION BY fault_code, owner ORDER BY id DESC) AS rn FROM near_misses) " "WHERE rn > ?)"); trim_stmt.bind_int64(1, static_cast(max_count)); @@ -1342,21 +1596,21 @@ size_t SqliteFaultStorage::set_max_near_misses_per_fault(size_t max_count) { return changed > 0 ? static_cast(changed) : 0; } -void SqliteFaultStorage::record_near_miss_locked(const std::string & fault_code, int64_t occurred_at_ns, - int32_t debounce_counter, const DebounceConfig & config, - uint8_t severity, const std::string & source_id, - const std::string & resulting_status) { +void SqliteFaultStorage::record_near_miss_locked(const FaultId & id, int64_t occurred_at_ns, int32_t debounce_counter, + const DebounceConfig & config, uint8_t severity, + const std::string & source_id, const std::string & resulting_status) { SqliteStatement insert_stmt(db_, - "INSERT INTO near_misses (fault_code, occurred_at_ns, debounce_counter, " + "INSERT INTO near_misses (fault_code, owner, occurred_at_ns, debounce_counter, " "confirmation_threshold, severity, source_id, resulting_status) " - "VALUES (?, ?, ?, ?, ?, ?, ?)"); - insert_stmt.bind_text(1, fault_code); - insert_stmt.bind_int64(2, occurred_at_ns); - insert_stmt.bind_int(3, debounce_counter); - insert_stmt.bind_int(4, config.confirmation_threshold); - insert_stmt.bind_int(5, static_cast(severity)); - insert_stmt.bind_text(6, source_id); - insert_stmt.bind_text(7, resulting_status); + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); + insert_stmt.bind_text(1, id.fault_code); + insert_stmt.bind_text(2, id.owner); + insert_stmt.bind_int64(3, occurred_at_ns); + insert_stmt.bind_int(4, debounce_counter); + insert_stmt.bind_int(5, config.confirmation_threshold); + insert_stmt.bind_int(6, static_cast(severity)); + insert_stmt.bind_text(7, source_id); + insert_stmt.bind_text(8, resulting_status); if (insert_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to store near miss: ") + sqlite3_errmsg(db_)); @@ -1376,27 +1630,29 @@ void SqliteFaultStorage::record_near_miss_locked(const std::string & fault_code, // by timestamp would then drop the row that was just appended and make the two backends, which // append in arrival order, disagree on the same input. SqliteStatement trim_stmt(db_, - "DELETE FROM near_misses WHERE fault_code = ?1 AND id NOT IN " - "(SELECT id FROM near_misses WHERE fault_code = ?1 " - "ORDER BY id DESC LIMIT ?2)"); - trim_stmt.bind_text(1, fault_code); - trim_stmt.bind_int64(2, static_cast(max_near_misses_per_fault_)); + "DELETE FROM near_misses WHERE fault_code = ?1 AND owner = ?2 AND id NOT IN " + "(SELECT id FROM near_misses WHERE fault_code = ?1 AND owner = ?2 " + "ORDER BY id DESC LIMIT ?3)"); + trim_stmt.bind_text(1, id.fault_code); + trim_stmt.bind_text(2, id.owner); + trim_stmt.bind_int64(3, static_cast(max_near_misses_per_fault_)); if (trim_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to trim near-miss series: ") + sqlite3_errmsg(db_)); } } -std::vector SqliteFaultStorage::get_near_misses(const std::string & fault_code) const { +std::vector SqliteFaultStorage::get_near_misses(const FaultId & id) const { std::lock_guard lock(mutex_); std::vector result; SqliteStatement stmt(db_, "SELECT fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, " - "severity, source_id, resulting_status FROM near_misses WHERE fault_code = ? " - "ORDER BY id ASC"); - stmt.bind_text(1, fault_code); + "severity, source_id, resulting_status, owner FROM near_misses " + "WHERE fault_code = ? AND owner = ? ORDER BY id ASC"); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); while (stmt.step() == SQLITE_ROW) { NearMissRecord record; @@ -1407,6 +1663,7 @@ std::vector SqliteFaultStorage::get_near_misses(const std::strin record.severity = static_cast(stmt.column_int(4)); record.source_id = stmt.column_text(5); record.resulting_status = stmt.column_text(6); + record.owner = stmt.column_text(7); result.push_back(std::move(record)); } @@ -1441,7 +1698,7 @@ void SqliteFaultStorage::store_rosbag_files(const std::vector & std::lock_guard lock(mutex_); // One transaction for the whole burst: a crash mid-store must not leave some - // faults of the shared recording without their lookup row. Evicted bags are + // records of the shared recording without their lookup row. Evicted bags are // unlinked only after COMMIT - a ROLLBACK resurrects the rows, which must keep // pointing at bags that still exist. std::vector evicted; @@ -1458,7 +1715,7 @@ void SqliteFaultStorage::store_rosbag_files(const std::vector & } // Referencing is re-checked on the committed state, not on the state each - // eviction saw: two faults of one burst can link the same bag, and a row-by-row + // eviction saw: two records of one burst can link the same bag, and a row-by-row // check inside the loop would find it still held by a sibling that a later // iteration then evicts, leaking the directory. std::set unique_paths(evicted.begin(), evicted.end()); @@ -1477,7 +1734,7 @@ std::vector SqliteFaultStorage::store_rosbag_file_locked(const Rosb row.recording_id = rosbag_recording_id(row.file_path); } - // Upserts on idx_rosbag_files_fault_path, i.e. on the (fault, recording) link. + // Upserts on idx_rosbag_files_fault_path, i.e. on the (record, recording) link. // Re-storing the SAME link refreshes it; a link to a DIFFERENT recording is a new // row now, which is the feature. Nothing is unlinked here - byte lifetime is the // cap's business below, and the caller's, after the commit. @@ -1489,46 +1746,48 @@ std::vector SqliteFaultStorage::store_rosbag_file_locked(const Rosb // two backends in a different order for a re-stored row inside a tie group. SqliteStatement stmt(db_, "INSERT INTO rosbag_files " - "(fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns) " - "VALUES (?, ?, ?, ?, ?, ?, ?) " - "ON CONFLICT(fault_code, file_path) DO UPDATE SET " + "(fault_code, owner, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT(fault_code, owner, file_path) DO UPDATE SET " "recording_id = excluded.recording_id, format = excluded.format, " "duration_sec = excluded.duration_sec, size_bytes = excluded.size_bytes, " "created_at_ns = excluded.created_at_ns"); stmt.bind_text(1, row.fault_code); - stmt.bind_text(2, row.recording_id); - stmt.bind_text(3, row.file_path); - stmt.bind_text(4, row.format); + stmt.bind_text(2, row.owner); + stmt.bind_text(3, row.recording_id); + stmt.bind_text(4, row.file_path); + stmt.bind_text(5, row.format); // Bind duration_sec as a double using sqlite3_bind_double directly - if (sqlite3_bind_double(stmt.get(), 5, row.duration_sec) != SQLITE_OK) { + if (sqlite3_bind_double(stmt.get(), 6, row.duration_sec) != SQLITE_OK) { throw std::runtime_error(std::string("Failed to bind duration_sec: ") + sqlite3_errmsg(db_)); } - stmt.bind_int64(6, static_cast(row.size_bytes)); - stmt.bind_int64(7, row.created_at_ns); + stmt.bind_int64(7, static_cast(row.size_bytes)); + stmt.bind_int64(8, row.created_at_ns); if (stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to store rosbag file: ") + sqlite3_errmsg(db_)); } if (max_rosbags_per_fault_ == 0) { - return {}; // unlimited per fault; only the global byte quota bounds this + return {}; // unlimited per record, only the global byte quota bounds this } - // Keep the newest N recordings of this fault. Oldest-first eviction, the same + // Keep the newest N recordings of this record. Oldest-first eviction, the same // direction as evict_bags_over_quota, so the two eviction owners never need a // tiebreak. At N = 1 this reproduces the pre-#620 behaviour exactly: the new // recording replaces the old and the old bag is unlinked. const char * const doomed_sql = - "FROM rosbag_files WHERE fault_code = ?1 AND id NOT IN " - "(SELECT id FROM rosbag_files WHERE fault_code = ?1 " - " ORDER BY created_at_ns DESC, id DESC LIMIT ?2)"; + "FROM rosbag_files WHERE fault_code = ?1 AND owner = ?2 AND id NOT IN " + "(SELECT id FROM rosbag_files WHERE fault_code = ?1 AND owner = ?2 " + " ORDER BY created_at_ns DESC, id DESC LIMIT ?3)"; std::vector evicted; { SqliteStatement select(db_, (std::string("SELECT DISTINCT file_path ") + doomed_sql).c_str()); select.bind_text(1, row.fault_code); - select.bind_int64(2, static_cast(max_rosbags_per_fault_)); + select.bind_text(2, row.owner); + select.bind_int64(3, static_cast(max_rosbags_per_fault_)); while (select.step() == SQLITE_ROW) { evicted.push_back(select.column_text(0)); } @@ -1539,7 +1798,8 @@ std::vector SqliteFaultStorage::store_rosbag_file_locked(const Rosb SqliteStatement del(db_, (std::string("DELETE ") + doomed_sql).c_str()); del.bind_text(1, row.fault_code); - del.bind_int64(2, static_cast(max_rosbags_per_fault_)); + del.bind_text(2, row.owner); + del.bind_int64(3, static_cast(max_rosbags_per_fault_)); if (del.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to trim rosbag rows: ") + sqlite3_errmsg(db_)); } @@ -1552,27 +1812,30 @@ namespace { RosbagFileInfo read_rosbag_row(SqliteStatement & stmt) { RosbagFileInfo info; info.fault_code = stmt.column_text(0); - info.recording_id = stmt.column_text(1); - info.file_path = stmt.column_text(2); - info.format = stmt.column_text(3); - info.duration_sec = sqlite3_column_double(stmt.get(), 4); - info.size_bytes = static_cast(stmt.column_int64(5)); - info.created_at_ns = stmt.column_int64(6); + info.owner = stmt.column_text(1); + info.recording_id = stmt.column_text(2); + info.file_path = stmt.column_text(3); + info.format = stmt.column_text(4); + info.duration_sec = sqlite3_column_double(stmt.get(), 5); + info.size_bytes = static_cast(stmt.column_int64(6)); + info.created_at_ns = stmt.column_int64(7); return info; } constexpr const char * kRosbagColumns = - "fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns"; + "fault_code, owner, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns"; } // namespace -std::vector SqliteFaultStorage::get_rosbag_files(const std::string & fault_code) const { +std::vector SqliteFaultStorage::get_rosbag_files(const FaultId & id) const { std::lock_guard lock(mutex_); - SqliteStatement stmt(db_, (std::string("SELECT ") + kRosbagColumns + - " FROM rosbag_files WHERE fault_code = ? ORDER BY created_at_ns DESC, id DESC") - .c_str()); - stmt.bind_text(1, fault_code); + SqliteStatement stmt(db_, + (std::string("SELECT ") + kRosbagColumns + + " FROM rosbag_files WHERE fault_code = ? AND owner = ? ORDER BY created_at_ns DESC, id DESC") + .c_str()); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); std::vector result; while (stmt.step() == SQLITE_ROW) { @@ -1585,7 +1848,7 @@ std::vector SqliteFaultStorage::get_rosbag_files_by_recording(co std::lock_guard lock(mutex_); SqliteStatement stmt(db_, (std::string("SELECT ") + kRosbagColumns + - " FROM rosbag_files WHERE recording_id = ? ORDER BY fault_code ASC") + " FROM rosbag_files WHERE recording_id = ? ORDER BY fault_code ASC, owner ASC") .c_str()); stmt.bind_text(1, recording_id); @@ -1644,46 +1907,39 @@ size_t SqliteFaultStorage::delete_rosbag_recording(const std::string & recording return removed; } -std::optional SqliteFaultStorage::get_rosbag_file(const std::string & fault_code) const { +std::optional SqliteFaultStorage::get_rosbag_file(const FaultId & id) const { std::lock_guard lock(mutex_); - // ORDER BY is load-bearing now that a fault can hold several recordings. Without + // ORDER BY is load-bearing now that a record can hold several recordings. Without // it SQLite may return any matching row, so the fault detail and the download // would serve an arbitrary recording - non-deterministically, which no test // catches reliably. id breaks the tie because a burst stamps one created_at_ns // across all its rows. - SqliteStatement stmt(db_, - "SELECT fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns " - "FROM rosbag_files WHERE fault_code = ? " - "ORDER BY created_at_ns DESC, id DESC LIMIT 1"); - stmt.bind_text(1, fault_code); + SqliteStatement stmt(db_, (std::string("SELECT ") + kRosbagColumns + + " FROM rosbag_files WHERE fault_code = ? AND owner = ? " + "ORDER BY created_at_ns DESC, id DESC LIMIT 1") + .c_str()); + stmt.bind_text(1, id.fault_code); + stmt.bind_text(2, id.owner); if (stmt.step() != SQLITE_ROW) { return std::nullopt; } - RosbagFileInfo info; - info.fault_code = stmt.column_text(0); - info.recording_id = stmt.column_text(1); - info.file_path = stmt.column_text(2); - info.format = stmt.column_text(3); - info.duration_sec = sqlite3_column_double(stmt.get(), 4); - info.size_bytes = static_cast(stmt.column_int64(5)); - info.created_at_ns = stmt.column_int64(6); - - return info; + return read_rosbag_row(stmt); } -bool SqliteFaultStorage::delete_rosbag_file(const std::string & fault_code) { +bool SqliteFaultStorage::delete_rosbag_file(const FaultId & id) { std::lock_guard lock(mutex_); - // Every path, not the first one: a fault holds as many recordings as its cap + // Every path, not the first one: a record holds as many recordings as its cap // allows, and stepping once would unlink one bag and leak the rest - rows gone, // directories left behind, uncounted by a quota that sums rows. std::set paths; { - SqliteStatement select_stmt(db_, "SELECT file_path FROM rosbag_files WHERE fault_code = ?"); - select_stmt.bind_text(1, fault_code); + SqliteStatement select_stmt(db_, "SELECT file_path FROM rosbag_files WHERE fault_code = ? AND owner = ?"); + select_stmt.bind_text(1, id.fault_code); + select_stmt.bind_text(2, id.owner); while (select_stmt.step() == SQLITE_ROW) { paths.insert(select_stmt.column_text(0)); } @@ -1694,8 +1950,9 @@ bool SqliteFaultStorage::delete_rosbag_file(const std::string & fault_code) { // done, that would leave a surviving row pointing at a bag that is gone - // unreadable for good, and still charged against the storage quota, which sums // rows. This way the worst case is an orphaned directory instead. - SqliteStatement delete_stmt(db_, "DELETE FROM rosbag_files WHERE fault_code = ?"); - delete_stmt.bind_text(1, fault_code); + SqliteStatement delete_stmt(db_, "DELETE FROM rosbag_files WHERE fault_code = ? AND owner = ?"); + delete_stmt.bind_text(1, id.fault_code); + delete_stmt.bind_text(2, id.owner); if (delete_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to delete rosbag file record: ") + sqlite3_errmsg(db_)); @@ -1703,7 +1960,7 @@ bool SqliteFaultStorage::delete_rosbag_file(const std::string & fault_code) { const bool deleted = sqlite3_changes(db_) > 0; - // Unlink only once no fault references the bag any more. These rows are already + // Unlink only once no row references the bag any more. These rows are already // gone, so path_referenced() sees exactly the siblings of a shared recording. for (const auto & path : paths) { if (!path_referenced(path)) { @@ -1716,8 +1973,8 @@ bool SqliteFaultStorage::delete_rosbag_file(const std::string & fault_code) { return deleted; } -size_t SqliteFaultStorage::delete_rosbag_files(const std::vector & fault_codes) { - if (fault_codes.empty()) { +size_t SqliteFaultStorage::delete_rosbag_files(const std::vector & ids) { + if (ids.empty()) { return 0; } std::lock_guard lock(mutex_); @@ -1729,18 +1986,20 @@ size_t SqliteFaultStorage::delete_rosbag_files(const std::vector & size_t deleted = 0; exec_or_throw("BEGIN IMMEDIATE"); try { - for (const auto & code : fault_codes) { + for (const auto & id : ids) { { - // while, not if: one fault code can name several recordings now, and the + // while, not if: one record can name several recordings now, and the // sweep must be able to reclaim every one of their bags. - SqliteStatement select_stmt(db_, "SELECT file_path FROM rosbag_files WHERE fault_code = ?"); - select_stmt.bind_text(1, code); + SqliteStatement select_stmt(db_, "SELECT file_path FROM rosbag_files WHERE fault_code = ? AND owner = ?"); + select_stmt.bind_text(1, id.fault_code); + select_stmt.bind_text(2, id.owner); while (select_stmt.step() == SQLITE_ROW) { paths.insert(select_stmt.column_text(0)); } } - SqliteStatement delete_stmt(db_, "DELETE FROM rosbag_files WHERE fault_code = ?"); - delete_stmt.bind_text(1, code); + SqliteStatement delete_stmt(db_, "DELETE FROM rosbag_files WHERE fault_code = ? AND owner = ?"); + delete_stmt.bind_text(1, id.fault_code); + delete_stmt.bind_text(2, id.owner); if (delete_stmt.step() != SQLITE_DONE) { throw std::runtime_error(std::string("Failed to delete rosbag file record: ") + sqlite3_errmsg(db_)); } @@ -1773,7 +2032,7 @@ bool SqliteFaultStorage::path_referenced(const std::string & file_path) const { size_t SqliteFaultStorage::get_total_rosbag_storage_bytes() const { std::lock_guard lock(mutex_); - // Sum per bag, not per fault: one recording can back a burst of correlated + // Sum per bag, not per record: one recording can back a burst of correlated // faults, and double-counting it would evict bags that still fit the quota. SqliteStatement stmt( db_, @@ -1792,20 +2051,11 @@ std::vector SqliteFaultStorage::get_all_rosbag_files() const { std::vector result; - SqliteStatement stmt(db_, - "SELECT fault_code, recording_id, file_path, format, duration_sec, size_bytes, created_at_ns " - "FROM rosbag_files ORDER BY created_at_ns ASC, id ASC"); + SqliteStatement stmt( + db_, (std::string("SELECT ") + kRosbagColumns + " FROM rosbag_files ORDER BY created_at_ns ASC, id ASC").c_str()); while (stmt.step() == SQLITE_ROW) { - RosbagFileInfo info; - info.fault_code = stmt.column_text(0); - info.recording_id = stmt.column_text(1); - info.file_path = stmt.column_text(2); - info.format = stmt.column_text(3); - info.duration_sec = sqlite3_column_double(stmt.get(), 4); - info.size_bytes = static_cast(stmt.column_int64(5)); - info.created_at_ns = stmt.column_int64(6); - result.push_back(info); + result.push_back(read_rosbag_row(stmt)); } return result; @@ -1816,29 +2066,21 @@ std::vector SqliteFaultStorage::list_rosbags_for_entity(const st std::vector result; - // Join rosbag_files with faults table and filter by reporting_sources containing entity_fqn. - // Use json_each() for proper JSON array querying instead of LIKE, which treats - // '_' as a single-char wildcard and would produce false positives on ROS names. + // Join rosbag_files with the row's record and keep the rows whose record lists the entity among its + // reporting sources. json_each() rather than LIKE, which treats '_' as a single-char wildcard and + // would produce false positives on ROS names. SqliteStatement stmt(db_, - "SELECT r.fault_code, r.recording_id, r.file_path, r.format, r.duration_sec, r.size_bytes, " - "r.created_at_ns " + "SELECT r.fault_code, r.owner, r.recording_id, r.file_path, r.format, r.duration_sec, " + "r.size_bytes, r.created_at_ns " "FROM rosbag_files r " - "JOIN faults f ON r.fault_code = f.fault_code " + "JOIN faults f ON r.fault_code = f.fault_code AND r.owner = f.owner " "JOIN json_each(f.reporting_sources) j ON j.value = ? " "ORDER BY r.created_at_ns DESC, r.id DESC"); stmt.bind_text(1, entity_fqn); while (stmt.step() == SQLITE_ROW) { - RosbagFileInfo info; - info.fault_code = stmt.column_text(0); - info.recording_id = stmt.column_text(1); - info.file_path = stmt.column_text(2); - info.format = stmt.column_text(3); - info.duration_sec = sqlite3_column_double(stmt.get(), 4); - info.size_bytes = static_cast(stmt.column_int64(5)); - info.created_at_ns = stmt.column_int64(6); - result.push_back(info); + result.push_back(read_rosbag_row(stmt)); } return result; @@ -1847,28 +2089,11 @@ std::vector SqliteFaultStorage::list_rosbags_for_entity(const st std::vector SqliteFaultStorage::get_all_faults() const { std::lock_guard lock(mutex_); - SqliteStatement stmt(db_, - "SELECT fault_code, severity, description, first_occurred_ns, last_occurred_ns, " - "occurrence_count, status, reporting_sources, last_passed_ns FROM faults"); + SqliteStatement stmt(db_, (std::string("SELECT ") + kFaultColumns + " FROM faults").c_str()); std::vector result; while (stmt.step() == SQLITE_ROW) { - fd::FaultRecord fault; - fault.fault_code = stmt.column_text(0); - fault.severity = static_cast(stmt.column_int(1)); - fault.description = stmt.column_text(2); - - int64_t first_ns = stmt.column_int64(3); - int64_t last_ns = stmt.column_int64(4); - fault.first_occurred_ns = first_ns; - fault.last_occurred_ns = last_ns; - - fault.occurrence_count = static_cast(stmt.column_int64(5)); - fault.status = stmt.column_text(6); - fault.reporting_sources = parse_json_array(stmt.column_text(7)); - fault.last_passed_ns = stmt.column_int64(8); - - result.push_back(fault); + result.push_back(read_fault_row(stmt, log_sink_)); } return result; diff --git a/src/ros2_medkit_fault_manager/src/threshold_resolver.cpp b/src/ros2_medkit_fault_manager/src/threshold_resolver.cpp index af3deecd5..1cd5873b9 100644 --- a/src/ros2_medkit_fault_manager/src/threshold_resolver.cpp +++ b/src/ros2_medkit_fault_manager/src/threshold_resolver.cpp @@ -65,6 +65,21 @@ OverrideFields parse_override_fields(const YAML::Node & node, core::LogSink & si return fields; } +/// A YAML boolean, spelled true or false. Throws YAML::BadConversion for any other scalar: yaml-cpp +/// reads "yes", "on" and "y" as booleans too, and a typo must not decide how a code is identified. +bool parse_strict_bool(const YAML::Node & node) { + if (node.IsScalar()) { + const std::string & text = node.Scalar(); + if (text == "true" || text == "True" || text == "TRUE") { + return true; + } + if (text == "false" || text == "False" || text == "FALSE") { + return false; + } + } + throw YAML::BadConversion(node.Mark()); +} + /// Open a threshold config file and hand back its top-level map, or an undefined /// node when the file is missing, unparseable or not a map. Every rejection is /// logged here so both loaders report the same way. @@ -199,6 +214,11 @@ DebounceConfig FaultCodeThresholdResolver::resolve(const std::string & fault_cod return result; } +bool FaultCodeThresholdResolver::per_source(const std::string & fault_code) const { + const auto it = entries_.find(fault_code); + return it != entries_.end() && it->second.per_source; +} + size_t FaultCodeThresholdResolver::size() const { return entries_.size(); } @@ -239,6 +259,9 @@ std::vector FaultCodeThresholdResolver::load_from_yam entry.confirmation_threshold = fields.confirmation_threshold; entry.healing_enabled = fields.healing_enabled; entry.healing_threshold = fields.healing_threshold; + if (item.second["per_source"]) { + entry.per_source = parse_strict_bool(item.second["per_source"]); + } entries.push_back(std::move(entry)); } diff --git a/src/ros2_medkit_fault_manager/test/integration/rosbag_ring_buffer.py b/src/ros2_medkit_fault_manager/test/integration/rosbag_ring_buffer.py new file mode 100644 index 000000000..293207504 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/rosbag_ring_buffer.py @@ -0,0 +1,77 @@ +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The pre-fault history a rosbag capture holds, seen from a second subscriber. + +The capture keeps the messages of the last duration_sec in its ring buffer and stamps +each with its receipt time. A bag starts at the oldest buffered message. A confirmation +empties the buffer and sends the next duration_after_sec of messages to the bag, and a +publisher that stalls leaves the buffer short of old messages. +""" + +import time + +import rclpy +from rclpy.qos import QoSProfile, ReliabilityPolicy +from ros2_medkit_msgs.msg import FaultEvent + +# Timer jitter allowed at the end of a post-fault window. +WINDOW_JITTER_SEC = 0.2 + + +class RecordedTopicWatch: + """Receipt times of a recorded topic, and the post-fault windows that divert it.""" + + def __init__(self, node, topic, msg_type, qos, duration_sec, duration_after_sec): + self.node = node + self.duration_sec = duration_sec + self.duration_after_sec = duration_after_sec + self.receipts = [] + self.buffering_since = 0.0 + node.create_subscription(msg_type, topic, self._on_message, qos) + node.create_subscription( + FaultEvent, '/fault_manager/events', self._on_event, + QoSProfile(depth=100, reliability=ReliabilityPolicy.RELIABLE)) + + def _on_message(self, _msg): + now = time.monotonic() + self.receipts.append(now) + self.receipts = [t for t in self.receipts if now - t <= 2 * self.duration_sec] + + def _on_event(self, msg): + if msg.event_type == FaultEvent.EVENT_CONFIRMED: + self.buffering_since = ( + time.monotonic() + self.duration_after_sec + WINDOW_JITTER_SEC) + + def holds_history(self, history_sec, latency_sec): + """ + Return True when a fault confirmed within gets of history. + + That needs a message received after the last post-fault window, at least + old now and still inside the buffer when the flush runs. + """ + now = time.monotonic() + oldest_kept = now + latency_sec - self.duration_sec + return any(max(self.buffering_since, oldest_kept) <= t <= now - history_sec + for t in self.receipts) + + def wait_for_history(self, history_sec, timeout_sec, latency_sec=0.3): + """Spin until holds_history(); False after .""" + deadline = time.monotonic() + timeout_sec + while time.monotonic() < deadline: + rclpy.spin_once(self.node, timeout_sec=0.05) + if self.holds_history(history_sec, latency_sec): + return True + return False diff --git a/src/ros2_medkit_fault_manager/test/integration/service_round_trip.py b/src/ros2_medkit_fault_manager/test/integration/service_round_trip.py new file mode 100644 index 000000000..17cfd1481 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/service_round_trip.py @@ -0,0 +1,78 @@ +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +First round trips to a fault manager that has just started. + +wait_for_service and count_publishers show that the test found the server. The server +can match the test's reply reader or event subscription later, and a reply or an +event sent before that is lost. +""" + +import time + +import rclpy +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ReportFault + +# A request with no answer after this long is sent again. +ANSWER_WAIT_SEC = 1.0 + + +def call_until_answered(node, client, request, timeout_sec, answer_wait_sec=ANSWER_WAIT_SEC): + """Send until gets an answer and return it; must be repeatable.""" + deadline = time.monotonic() + timeout_sec + while time.monotonic() < deadline: + future = client.call_async(request) + rclpy.spin_until_future_complete(node, future, timeout_sec=answer_wait_sec) + if future.done(): + return future.result() + client.remove_pending_request(future) + raise AssertionError(f'{client.srv_name} did not answer within {timeout_sec} s') + + +def answer_once_each(node, clients, timeout_sec): + """Get one answer on each client. The default request names no fault, so it changes nothing.""" + for client in clients: + call_until_answered(node, client, client.srv_type.Request(), timeout_sec) + + +def wait_for_event_route(node, report_client, events, prefix, timeout_sec): + """ + Report a new CRITICAL code until its event reaches . + + Each code confirms once and publishes one event. Events arrive in order, so a caller + that counts every event can empty afterwards: no earlier probe's event can + come after the one seen. + """ + deadline = time.monotonic() + timeout_sec + index = 0 + while time.monotonic() < deadline: + index += 1 + request = ReportFault.Request() + request.fault_code = f'{prefix}{index}' + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_CRITICAL + request.description = 'event route probe' + request.source_id = '/event_route_probe' + # Sent once: a second report of the code would publish a second event. + client_future = report_client.call_async(request) + settle = time.monotonic() + ANSWER_WAIT_SEC + while time.monotonic() < settle: + if any(e.fault.fault_code == request.fault_code for e in events): + report_client.remove_pending_request(client_future) + return + rclpy.spin_once(node, timeout_sec=0.1) + report_client.remove_pending_request(client_future) + raise AssertionError(f'no fault event reached the test within {timeout_sec} s') diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_audit_rows.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_audit_rows.test.py new file mode 100644 index 000000000..6e8b0eaa1 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_audit_rows.test.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +What the audit rows of a real fault manager say, read from the audit store it wrote. + +A row's source is the report's source for a row a report drove, and the mechanism for an +automatic one (auto_confirm_timer, auto_heal, clear_service, correlation_auto_clear, +startup_reclassify). A row's owner is the owner of a per-source record and empty for a shared +record. The hash chain is recomputed here with the serialization the format had before the +owner existed, so every row of a shared record must still match it byte for byte. +""" + +import hashlib +import json +import os +import signal +import sqlite3 +import subprocess +import sys +import tempfile +import time +import unittest + +from ament_index_python.packages import get_package_prefix +import launch +from launch import LaunchDescription +import launch_testing.actions +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetFault, ReportFault + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + +GENESIS = '0' * 64 +HEAL = (' confirmation_threshold: -1\n' + ' healing_enabled: true\n' + ' healing_threshold: 1\n') +TIMER = ' confirmation_threshold: -5\n' +PER_SOURCE = ' per_source: true\n' + +RUN1_CODES = ( + 'AU.SH.TIMER:\n' + TIMER + + 'AU.PS.TIMER:\n' + PER_SOURCE + TIMER + + 'AU.SH.HEAL:\n' + HEAL + + 'AU.PS.HEAL:\n' + PER_SOURCE + HEAL + + 'AU.PS.OP:\n' + PER_SOURCE + + 'AU.PS.ROOT:\n' + PER_SOURCE + + 'AU.PS.SYM:\n' + PER_SOURCE +) +# Healing is off for the HEAL codes, so a restart turns their HEALED records into CLEARED. +RUN2_CODES = 'AU.PS.HEAL:\n' + PER_SOURCE + +CORRELATION = """\ +correlation: + enabled: true + patterns: + sh_sym: + codes: ["AU.SH.SYM"] + ps_sym: + codes: ["AU.PS.SYM"] + rules: + - id: sh_cascade + name: "Shared cascade" + mode: hierarchical + root_cause: + codes: ["AU.SH.ROOT"] + symptoms: + - pattern: sh_sym + window_ms: 5000 + mute_symptoms: true + auto_clear_with_root: true + - id: ps_cascade + name: "Per-source cascade" + mode: hierarchical + root_cause: + codes: ["AU.PS.ROOT"] + symptoms: + - pattern: ps_sym + window_ms: 5000 + mute_symptoms: true + auto_clear_with_root: true +""" + + +def generate_test_description(): + """Start only a placeholder process, the test starts the fault manager itself.""" + keep_alive = launch.actions.ExecuteProcess(cmd=['sleep', '600']) + return LaunchDescription([keep_alive, launch_testing.actions.ReadyToTest()]) + + +def canonical_before_owner(seq, row): + """Serialize a row exactly as the audit format did before the owner column existed.""" + def enc(value): + return json.dumps(value, ensure_ascii=False) + return ('{"seq":%d,"ts":%d,"code":%s,"transition":%s,"severity":%d,"status":%s,' + '"source":%s,"description":%s}' % ( + seq, row['occurred_at_ns'], enc(row['fault_code']), enc(row['transition']), + row['severity'], enc(row['status']), enc(row['source_id']), + enc(row['description']))) + + +def canonical_with_owner(seq, row): + canonical = canonical_before_owner(seq, row) + if row['owner']: + canonical = canonical[:-1] + ',"owner":%s}' % json.dumps(row['owner'], ensure_ascii=False) + return canonical + + +class TestFaultAuditRows(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + cls.db_path = os.path.join(cls.tmp.name, 'faults.db') + cls.audit_path = os.path.join(cls.tmp.name, 'audit.db') + cls.run1_codes = cls._write('run1_codes.yaml', RUN1_CODES) + cls.run2_codes = cls._write('run2_codes.yaml', RUN2_CODES) + cls.correlation = cls._write('correlation.yaml', CORRELATION) + cls.exe = os.path.join( + get_package_prefix('ros2_medkit_fault_manager'), + 'lib', 'ros2_medkit_fault_manager', 'fault_manager_node') + rclpy.init() + cls.node = Node('test_fault_audit_rows_client') + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + cls.tmp.cleanup() + + @classmethod + def _write(cls, name, text): + path = os.path.join(cls.tmp.name, name) + with open(path, 'w') as f: + f.write(text) + return path + + def _start(self, codes_file): + params = self._write('params.yaml', ( + 'fault_manager:\n' + ' ros__parameters:\n' + ' storage_type: sqlite\n' + f' database_path: {self.db_path}\n' + ' confirmation_threshold: -1\n' + ' healing_enabled: false\n' + ' auto_confirm_after_sec: 1.0\n' + ' snapshots.enabled: false\n' + ' snapshots.rosbag.enabled: false\n' + ' audit_log.enabled: true\n' + f' audit_log.database_path: {self.audit_path}\n' + f' fault_thresholds.config_file: {codes_file}\n' + f' correlation.config_file: {self.correlation}\n' + )) + proc = subprocess.Popen( + [self.exe, '--ros-args', '-r', '__node:=fault_manager', '--params-file', params]) + self.addCleanup(self._kill, proc) + clients = { + 'report': self.node.create_client(ReportFault, '/fault_manager/report_fault'), + 'get': self.node.create_client(GetFault, '/fault_manager/get_fault'), + 'clear': self.node.create_client(ClearFault, '/fault_manager/clear_fault'), + } + for client in clients.values(): + self.assertTrue(client.wait_for_service(timeout_sec=15.0)) + # The graph can still list the previous node's services, so wait for an answer. + answer_once_each(self.node, tuple(clients.values()), 15.0) + return proc, clients + + @staticmethod + def _kill(proc): + if proc.poll() is None: + proc.kill() + proc.wait() + + def _stop(self, proc, clients): + for client in clients.values(): + self.node.destroy_client(client) + proc.send_signal(signal.SIGINT) + self.assertEqual(proc.wait(timeout=30), 0) + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=5.0) + self.assertIsNotNone(future.result(), 'service call timed out') + return future.result() + + def _report(self, clients, code, source, event_type=ReportFault.Request.EVENT_FAILED): + request = ReportFault.Request() + request.fault_code = code + request.event_type = event_type + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(clients['report'], request).accepted) + + def _clear(self, clients, code, source=''): + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + response = self._call(clients['clear'], request) + self.assertTrue(response.success, response.message) + + def _status(self, clients, code, source=''): + request = GetFault.Request() + request.fault_code = code + request.source_id = source + response = self._call(clients['get'], request) + self.assertTrue(response.success, response.error_message) + return response.fault.status + + def _wait_status(self, clients, code, expected): + deadline = time.monotonic() + 20.0 + while time.monotonic() < deadline: + if self._status(clients, code) == expected: + return + self.fail(f'{code} never reached {expected}') + + def _heal(self, clients, code, source): + self._report(clients, code, source) + for _ in range(4): + if self._status(clients, code) == Fault.STATUS_HEALED: + return + self._report(clients, code, source, ReportFault.Request.EVENT_PASSED) + self.assertEqual(self._status(clients, code), Fault.STATUS_HEALED) + + def _rows(self): + connection = sqlite3.connect(f'file:{self.audit_path}?mode=ro', uri=True) + connection.row_factory = sqlite3.Row + try: + return [dict(r) for r in connection.execute('SELECT * FROM audit_log ORDER BY seq')] + finally: + connection.close() + + def _of(self, rows, code): + return [(r['transition'], r['source_id'], r['owner']) for r in rows + if r['fault_code'] == code] + + def test_rows_say_what_drove_them_and_chain_matches_the_old_format(self): + proc, clients = self._start(self.run1_codes) + try: + # Timer confirmation: one FAILED leaves the record PREFAILED, the timer confirms it. + self._report(clients, 'AU.SH.TIMER', '/a') + self._report(clients, 'AU.PS.TIMER', '/a') + self._wait_status(clients, 'AU.SH.TIMER', Fault.STATUS_CONFIRMED) + self._wait_status(clients, 'AU.PS.TIMER', Fault.STATUS_CONFIRMED) + + # Healing. + self._heal(clients, 'AU.SH.HEAL', '/a') + self._heal(clients, 'AU.PS.HEAL', '/a') + + # Operator clear. + self._report(clients, 'AU.SH.OP', '/a') + self._report(clients, 'AU.PS.OP', '/a') + self._clear(clients, 'AU.SH.OP') + self._clear(clients, 'AU.PS.OP') + + # Correlation cascade: the symptom is cleared with its root, from another node. + self._report(clients, 'AU.SH.ROOT', '/root_node') + self._report(clients, 'AU.SH.SYM', '/sym_node') + self._clear(clients, 'AU.SH.ROOT') + self._report(clients, 'AU.PS.ROOT', '/root_node') + self._report(clients, 'AU.PS.SYM', '/sym_node') + self._clear(clients, 'AU.PS.ROOT') + finally: + self._stop(proc, clients) + + # Restart with healing off for both HEAL codes: their HEALED records become CLEARED. + proc, clients = self._start(self.run2_codes) + self._stop(proc, clients) + + rows = self._rows() + + # Shared records: no owner on any row. + self.assertEqual(self._of(rows, 'AU.SH.TIMER'), [ + ('occurred', '/a', ''), ('confirmed', 'auto_confirm_timer', '')]) + self.assertEqual(self._of(rows, 'AU.SH.HEAL'), [ + ('occurred', '/a', ''), ('confirmed', '/a', ''), ('healed', 'auto_heal', ''), + ('cleared', 'startup_reclassify', '')]) + self.assertEqual(self._of(rows, 'AU.SH.OP'), [ + ('occurred', '/a', ''), ('confirmed', '/a', ''), ('cleared', 'clear_service', '')]) + self.assertEqual(self._of(rows, 'AU.SH.ROOT'), [ + ('occurred', '/root_node', ''), ('confirmed', '/root_node', ''), + ('cleared', 'clear_service', '')]) + self.assertEqual(self._of(rows, 'AU.SH.SYM'), [ + ('occurred', '/sym_node', ''), ('confirmed', '/sym_node', ''), + ('cleared', 'correlation_auto_clear', '')]) + + # Per-source records: the same mechanisms, and the owner of the record. + self.assertEqual(self._of(rows, 'AU.PS.TIMER'), [ + ('occurred', '/a', '/a'), ('confirmed', 'auto_confirm_timer', '/a')]) + self.assertEqual(self._of(rows, 'AU.PS.HEAL'), [ + ('occurred', '/a', '/a'), ('confirmed', '/a', '/a'), ('healed', 'auto_heal', '/a'), + ('cleared', 'startup_reclassify', '/a')]) + self.assertEqual(self._of(rows, 'AU.PS.OP'), [ + ('occurred', '/a', '/a'), ('confirmed', '/a', '/a'), + ('cleared', 'clear_service', '/a')]) + self.assertEqual(self._of(rows, 'AU.PS.ROOT'), [ + ('occurred', '/root_node', '/root_node'), ('confirmed', '/root_node', '/root_node'), + ('cleared', 'clear_service', '/root_node')]) + self.assertEqual(self._of(rows, 'AU.PS.SYM'), [ + ('occurred', '/sym_node', '/sym_node'), ('confirmed', '/sym_node', '/sym_node'), + ('cleared', 'correlation_auto_clear', '/sym_node')]) + + # The chain verifies, and every row of a shared record hashes as it did before the + # owner existed. + previous = GENESIS + shared_rows = 0 + for row in rows: + self.assertEqual(row['prev_hash'], previous) + expected = hashlib.sha256( + (previous + canonical_with_owner(row['seq'], row)).encode()).hexdigest() + self.assertEqual(row['record_hash'], expected, row) + if not row['owner']: + shared_rows += 1 + old = hashlib.sha256( + (previous + canonical_before_owner(row['seq'], row)).encode()).hexdigest() + self.assertEqual(row['record_hash'], old, row) + previous = row['record_hash'] + self.assertGreater(shared_rows, 10) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_cluster_muting.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_cluster_muting.test.py new file mode 100644 index 000000000..087eacdd8 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_cluster_muting.test.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A show_as_single cluster shows its current representative and mutes the other members. + +Muting is display only: the default listing hides a muted member, include_muted lists it, and +the record GetFault returns is the one an unmuted member has. The representative the policy +selects is the one shown. When a member leaves, the representative that follows is shown. + +Each scenario runs on shared codes (one code per member) and on a per_source code (one record +per reporting node). +""" + +import os +import sys +import tempfile +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + +TIMEOUT_SEC = 30.0 +MODES = ('SH', 'PS') +# scenario -> representative policy +SCENARIOS = { + 'ONLY': 'first', + 'NEXT': 'first', + 'ESC': 'highest_severity', + 'REC': 'most_recent', + 'GET': 'first', +} + +THRESHOLDS = ''.join( + f'CM_PS_{scenario}_X:\n per_source: true\n' for scenario in SCENARIOS) +CORRELATION = 'correlation:\n enabled: true\n patterns:\n' + ''.join( + f' cm_{mode.lower()}_{scenario.lower()}:\n codes: ["CM_{mode}_{scenario}_*"]\n' + for mode in MODES for scenario in SCENARIOS) + ' rules:\n' + ''.join( + f""" - id: cm_{mode.lower()}_{scenario.lower()} + name: "CM {mode} {scenario}" + mode: auto_cluster + match: + - pattern: cm_{mode.lower()}_{scenario.lower()} + min_count: 3 + window_ms: 30000 + show_as_single: true + representative: {policy} +""" for mode in MODES for scenario, policy in SCENARIOS.items()) + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file('cluster_muting_thresholds_', THRESHOLDS) +CORRELATION_FILE = _temp_file('cluster_muting_rules_', CORRELATION) + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'healing_enabled': False, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'correlation.config_file': CORRELATION_FILE, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestClusterMuting(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_cluster_muting_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + cls.get_client = cls.node.create_client(GetFault, '/fault_manager/get_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + clients = (cls.report_client, cls.list_client, cls.get_client, cls.clear_client) + for client in clients: + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + @staticmethod + def _members(mode, scenario): + """Return [(code, source, owner)] of the three members, in report order.""" + if mode == 'PS': + return [(f'CM_PS_{scenario}_X', f'/n{i}', f'/n{i}') for i in (1, 2, 3)] + return [(f'CM_SH_{scenario}_{i}', '/n', '*') for i in (1, 2, 3)] + + def _report(self, member, severity=Fault.SEVERITY_WARN): + code, source, _ = member + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = severity + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _clear(self, member): + code, _, owner = member + request = ClearFault.Request() + request.fault_code = code + request.source_id = owner + response = self._call(self.clear_client, request) + self.assertTrue(response.success, response.message) + + def _get(self, member): + code, _, owner = member + request = GetFault.Request() + request.fault_code = code + request.source_id = owner + response = self._call(self.get_client, request) + self.assertTrue(response.success, response.error_message) + return response.fault + + def _shown(self, members): + """Return the (code, owner) pairs of the default listing that belong to members.""" + owner_of = {(code, source): owner for code, source, owner in members} + faults = self._call(self.list_client, ListFaults.Request()).faults + return {(f.fault_code, owner_of[(f.fault_code, f.reporting_sources[0])]) + for f in faults if (f.fault_code, f.reporting_sources[0]) in owner_of} + + def _muted(self, members): + """Return the (code, owner) pairs of include_muted that belong to members.""" + request = ListFaults.Request() + request.include_muted = True + response = self._call(self.list_client, request) + ours = {self._key(member) for member in members} + return {(m.fault_code, m.source_id) for m in response.muted_faults} & ours + + def _key(self, member): + return (member[0], member[2]) + + def _each_mode(self, scenario, check): + for mode in MODES: + with self.subTest(mode=mode): + check(self._members(mode, scenario)) + + def test_the_representative_is_shown_and_the_others_are_muted(self): + def check(members): + for member in members: + self._report(member) + first, second, third = members + self.assertEqual(self._shown(members), {self._key(first)}) + self.assertEqual(self._muted(members), {self._key(second), self._key(third)}) + self._each_mode('ONLY', check) + + def test_the_next_representative_is_shown_when_the_shown_one_is_cleared(self): + def check(members): + first, second, third = members + for member in members: + self._report(member) + self._clear(first) + self.assertEqual(self._shown(members), {self._key(second)}) + self.assertEqual(self._muted(members), {self._key(third)}) + self._clear(second) + self.assertEqual(self._shown(members), {self._key(third)}) + self.assertEqual(self._muted(members), set()) + self._clear(third) + self.assertEqual(self._shown(members), set()) + self.assertEqual(self._muted(members), set()) + self._each_mode('NEXT', check) + + def test_a_member_that_escalates_becomes_the_shown_one(self): + def check(members): + first, second, third = members + for member in members: + self._report(member) + self.assertEqual(self._shown(members), {self._key(first)}) + self._report(third, Fault.SEVERITY_CRITICAL) + self.assertEqual(self._shown(members), {self._key(third)}) + self.assertEqual(self._muted(members), {self._key(first), self._key(second)}) + self._each_mode('ESC', check) + + def test_the_most_recent_member_is_shown(self): + def check(members): + first, second, third = members + for member in members: + self._report(member) + self.assertEqual(self._shown(members), {self._key(third)}) + self.assertEqual(self._muted(members), {self._key(first), self._key(second)}) + self._each_mode('REC', check) + + def test_muting_does_not_change_the_record(self): + def check(members): + for member in members: + self._report(member, Fault.SEVERITY_ERROR) + shown, muted = self._get(members[0]), self._get(members[1]) + self.assertIn(self._key(members[1]), self._muted(members)) + self.assertEqual(muted.status, shown.status) + self.assertEqual(muted.status, Fault.STATUS_CONFIRMED) + self.assertEqual(muted.severity, Fault.SEVERITY_ERROR) + self.assertEqual(muted.occurrence_count, shown.occurrence_count) + self._each_mode('GET', check) + + +@launch_testing.post_shutdown_test() +class TestClusterMutingShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(CORRELATION_FILE) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py new file mode 100644 index 000000000..3c24bc485 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_clear.test.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The relations of a root cause across its re-reports and its clear, through a real fault manager. + + * a root cleared with skip_correlation_auto_clear leaves its symptoms standing and unmuted, + and stops muting later reports; + * a root reported again while active keeps the symptoms it already muted, so its clear + cascades to all of them, and only to the ones it still mutes; + * every way a record ends (a cascade clear, a heal) releases the symptoms it muted. + +Each scenario runs on a default (shared) code and on a per_source code whose root and symptom +come from different nodes. +""" + +import os +import sys +import tempfile +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + +TIMEOUT_SEC = 30.0 +ALL_STATUSES = [ + Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, + Fault.STATUS_HEALED, Fault.STATUS_CLEARED, +] +KINDS = ('P7', 'P8', 'P11', 'P12') +MODES = ('SH', 'PS') +# A short window lets a test move past it by reporting until the symptom is no longer muted. +WINDOW_MS = {'P11': 200} + +THRESHOLDS = ''.join( + f'CL_{kind}_PS_{role}:\n per_source: true\n' for kind in KINDS for role in ('ROOT', 'SYM')) +CORRELATION = 'correlation:\n enabled: true\n rules:\n' + ''.join( + f""" - id: {kind.lower()}_{mode.lower()} + name: "{kind} {mode}" + mode: hierarchical + root_cause: + codes: ["CL_{kind}_{mode}_ROOT"] + symptoms: + - codes: ["CL_{kind}_{mode}_SYM"] + window_ms: {WINDOW_MS.get(kind, 30000)} + mute_symptoms: true + auto_clear_with_root: true +""" for kind in KINDS for mode in MODES) + """ - id: p10_outer + name: "P10 outer" + mode: hierarchical + root_cause: + codes: ["CL_P10_OUTER"] + symptoms: + - codes: ["CL_P10_MID"] + window_ms: 30000 + mute_symptoms: true + auto_clear_with_root: true + - id: p10_inner + name: "P10 inner" + mode: hierarchical + root_cause: + codes: ["CL_P10_MID"] + symptoms: + - codes: ["CL_P10_LEAF"] + window_ms: 30000 + mute_symptoms: true + auto_clear_with_root: true +""" + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file('correlation_clear_thresholds_', THRESHOLDS) +CORRELATION_FILE = _temp_file('correlation_clear_rules_', CORRELATION) + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'healing_enabled': True, + 'healing_threshold': 0, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'correlation.config_file': CORRELATION_FILE, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestCorrelationClear(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_correlation_clear_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + cls.get_client = cls.node.create_client(GetFault, '/fault_manager/get_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + clients = (cls.report_client, cls.list_client, cls.get_client, cls.clear_client) + for client in clients: + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source, event=ReportFault.Request.EVENT_FAILED): + request = ReportFault.Request() + request.fault_code = code + request.event_type = event + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _clear(self, code, source, skip): + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + request.skip_correlation_auto_clear = skip + response = self._call(self.clear_client, request) + self.assertTrue(response.success, response.message) + return response + + def _status(self, code, source): + request = GetFault.Request() + request.fault_code = code + request.source_id = source + response = self._call(self.get_client, request) + self.assertTrue(response.success, response.error_message) + return response.fault.status + + def _visible(self): + """Return the codes the default listing shows.""" + return {f.fault_code for f in self._call(self.list_client, ListFaults.Request()).faults} + + def _muted(self): + request = ListFaults.Request() + request.include_muted = True + return {m.fault_code for m in self._call(self.list_client, request).muted_faults} + + def _scenario_sources(self, mode): + """Return (root owner, symptom owner, root source, symptom source).""" + # A per-source record is addressed by its owner, a shared one by '*'. + if mode == 'PS': + return '/root_node', '/sym_node', '/root_node', '/sym_node' + return '*', '*', '/root_node', '/sym_node' + + def _skip_clear_ends_the_relations(self, mode): + root, symptom = f'CL_P7_{mode}_ROOT', f'CL_P7_{mode}_SYM' + root_owner, sym_owner, root_source, sym_source = self._scenario_sources(mode) + self._report(root, root_source) + self._report(symptom, sym_source) + self.assertIn(symptom, self._muted()) + self.assertNotIn(symptom, self._visible()) + + response = self._clear(root, root_owner, skip=True) + self.assertEqual(list(response.auto_cleared_codes), [], 'skip must not cascade') + self.assertEqual(self._status(root, root_owner), Fault.STATUS_CLEARED) + self.assertEqual(self._status(symptom, sym_owner), Fault.STATUS_CONFIRMED) + self.assertNotIn(symptom, self._muted()) + self.assertIn(symptom, self._visible(), 'the symptom stayed muted after its root cleared') + + # The cleared root no longer mutes what is reported after it. + self._report(symptom, sym_source) + self.assertNotIn(symptom, self._muted()) + self.assertIn(symptom, self._visible()) + + def _re_reported_root_keeps_its_symptoms(self, mode): + root, symptom = f'CL_P8_{mode}_ROOT', f'CL_P8_{mode}_SYM' + root_owner, sym_owner, root_source, sym_source = self._scenario_sources(mode) + self._report(root, root_source) + self._report(symptom, sym_source) + self.assertIn(symptom, self._muted()) + self._report(root, root_source) + + response = self._clear(root, root_owner, skip=False) + self.assertIn(symptom, list(response.auto_cleared_codes)) + self.assertEqual(self._status(symptom, sym_owner), Fault.STATUS_CLEARED) + + def _list_exact_symptoms(self, mode): + root, symptom = f'CL_P11_{mode}_ROOT', f'CL_P11_{mode}_SYM' + root_owner, sym_owner, root_source, sym_source = self._scenario_sources(mode) + self._report(root, root_source) + self._report(symptom, sym_source) + self.assertIn(symptom, self._muted()) + self._clear(symptom, sym_owner, skip=False) + self._report(root, root_source) + # Report the symptom until it falls outside the root's window and is no longer muted. + deadline = time.monotonic() + TIMEOUT_SEC + while True: + self._report(symptom, sym_source) + if symptom not in self._muted(): + break + self.assertLess(time.monotonic(), deadline, 'the symptom stayed muted past its window') + self._clear(symptom, sym_owner, skip=False) + + response = self._clear(root, root_owner, skip=False) + self.assertEqual(list(response.auto_cleared_codes), [], + 'the root cleared a symptom it did not mute') + self.assertEqual(self._status(symptom, sym_owner), Fault.STATUS_CONFIRMED) + + def test_p11_default_code_cascade_clears_only_the_muted_symptoms(self): + self._list_exact_symptoms('SH') + + def test_p11_per_source_code_cascade_clears_only_the_muted_symptoms(self): + self._list_exact_symptoms('PS') + + def test_p10_a_cascade_cleared_root_releases_its_own_symptoms(self): + self._report('CL_P10_MID', '/node') + self._report('CL_P10_LEAF', '/node') + self._report('CL_P10_OUTER', '/node') + self._report('CL_P10_MID', '/node') + self.assertIn('CL_P10_MID', self._muted()) + self.assertIn('CL_P10_LEAF', self._muted()) + + response = self._clear('CL_P10_OUTER', '*', skip=False) + self.assertEqual( + list(response.auto_cleared_codes), ['CL_P10_MID'], 'the cascade is one level') + self.assertEqual(self._status('CL_P10_MID', '*'), Fault.STATUS_CLEARED) + self.assertEqual(self._status('CL_P10_LEAF', '*'), Fault.STATUS_CONFIRMED) + self.assertNotIn('CL_P10_LEAF', self._muted()) + self.assertIn('CL_P10_LEAF', self._visible()) + + def _healed_root_releases_its_symptoms(self, mode): + root, symptom = f'CL_P12_{mode}_ROOT', f'CL_P12_{mode}_SYM' + root_owner, sym_owner, root_source, sym_source = self._scenario_sources(mode) + self._report(root, root_source) + self._report(symptom, sym_source) + self.assertIn(symptom, self._muted()) + + self._report(root, root_source, event=ReportFault.Request.EVENT_PASSED) + self.assertEqual(self._status(root, root_owner), Fault.STATUS_HEALED) + self.assertEqual(self._status(symptom, sym_owner), Fault.STATUS_CONFIRMED) + self.assertNotIn(symptom, self._muted()) + self.assertIn(symptom, self._visible(), 'the symptom stayed muted after its root healed') + + def test_p10_default_code_healed_root_releases_its_symptoms(self): + self._healed_root_releases_its_symptoms('SH') + + def test_p10_per_source_code_healed_root_releases_its_symptoms(self): + self._healed_root_releases_its_symptoms('PS') + + def test_p7_default_code_skip_clear_ends_the_relations(self): + self._skip_clear_ends_the_relations('SH') + + def test_p7_per_source_code_skip_clear_ends_the_relations(self): + self._skip_clear_ends_the_relations('PS') + + def test_p8_default_code_re_reported_root_keeps_its_symptoms(self): + self._re_reported_root_keeps_its_symptoms('SH') + + def test_p8_per_source_code_re_reported_root_keeps_its_symptoms(self): + self._re_reported_root_keeps_its_symptoms('PS') + + +@launch_testing.post_shutdown_test() +class TestCorrelationClearShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(CORRELATION_FILE) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py new file mode 100644 index 000000000..ab48d24d2 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_correlation_owners.test.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Correlation across owners, through a real fault manager. + +Rules match by fault code, whoever reports it. A root cause of one node mutes and +auto-clears the matching symptom records of every node; a cluster holds the matching +records of every node. The relations hold records, so they work the same for per_source +codes (one record per node) and for shared codes (one record, owner '*'). +""" + +import os +import sys +import tempfile +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import launch_testing.markers +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + +TIMEOUT_SEC = 30.0 +ALL_STATUSES = [ + Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, + Fault.STATUS_HEALED, Fault.STATUS_CLEARED, +] + +THRESHOLDS = """\ +XO_PS_ESTOP: + per_source: true +XO_PS_MOTOR_A: + per_source: true +XO_PS_MOTOR_B: + per_source: true +XO_PS_STORM_1: + per_source: true +XO_PS_STORM_2: + per_source: true +XO_PS_STORM_3: + per_source: true +XO_PS_CHURN_1: + per_source: true +XO_PS_CHURN_2: + per_source: true +""" + +CORRELATION = """\ +correlation: + enabled: true + default_window_ms: 5000 + patterns: + ps_motor: + codes: ["XO_PS_MOTOR_*"] + sh_motor: + codes: ["XO_SH_MOTOR_*"] + ps_storm: + codes: ["XO_PS_STORM_*"] + sh_storm: + codes: ["XO_SH_STORM_*"] + ps_churn: + codes: ["XO_PS_CHURN_*"] + rules: + - id: ps_estop + name: "Per-source e-stop" + mode: hierarchical + root_cause: + codes: ["XO_PS_ESTOP"] + symptoms: + - pattern: ps_motor + window_ms: 5000 + mute_symptoms: true + auto_clear_with_root: true + - id: sh_estop + name: "Shared e-stop" + mode: hierarchical + root_cause: + codes: ["XO_SH_ESTOP"] + symptoms: + - pattern: sh_motor + window_ms: 5000 + mute_symptoms: true + auto_clear_with_root: true + - id: ps_storm + name: "Per-source storm" + mode: auto_cluster + match: + - pattern: ps_storm + min_count: 3 + window_ms: 5000 + show_as_single: true + representative: first + - id: ps_churn + name: "Per-source churn" + mode: auto_cluster + match: + - pattern: ps_churn + min_count: 3 + window_ms: 30000 + show_as_single: true + representative: first + - id: sh_storm + name: "Shared storm" + mode: auto_cluster + match: + - pattern: sh_storm + min_count: 3 + window_ms: 5000 + show_as_single: true + representative: first +""" + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file('correlation_owners_thresholds_', THRESHOLDS) +CORRELATION_FILE = _temp_file('correlation_owners_rules_', CORRELATION) + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'healing_enabled': False, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'correlation.config_file': CORRELATION_FILE, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestCorrelationAcrossOwners(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_correlation_owners_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + cls.get_client = cls.node.create_client(GetFault, '/fault_manager/get_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + clients = (cls.report_client, cls.list_client, cls.get_client, cls.clear_client) + for client in clients: + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source, severity=Fault.SEVERITY_ERROR): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = severity + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _list(self, statuses=(), include_muted=False, include_clusters=False): + request = ListFaults.Request() + request.statuses = list(statuses) + request.include_muted = include_muted + request.include_clusters = include_clusters + return self._call(self.list_client, request) + + def _clear(self, code, source=''): + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + return self._call(self.clear_client, request) + + def _get(self, code, source=''): + request = GetFault.Request() + request.fault_code = code + request.source_id = source + return self._call(self.get_client, request) + + def _status_of(self, code, owner): + records = [f for f in self._list(ALL_STATUSES, include_muted=True).faults + if f.fault_code == code] + response = self._get(code, owner) + self.assertTrue(response.success, f'{code}/{owner}: {response.error_message} {records}') + return response.fault.status + + def _cascade(self, root, root_node, symptoms, symptom_owner_of_record): + """Report a root and symptoms from other nodes, expect them muted, then cleared with it.""" + self._report(root, root_node) + for code, node in symptoms: + self._report(code, node) + + # Cluster members muted by the storm rules are not this root's symptoms. + muted = [m for m in self._list(include_muted=True).muted_faults + if m.rule_id.endswith('_estop')] + muted_records = sorted((m.fault_code, m.source_id) for m in muted) + expected = sorted((code, symptom_owner_of_record(code, node)) for code, node in symptoms) + self.assertEqual(muted_records, expected) + for m in muted: + self.assertEqual(m.root_cause_code, root) + + visible = {f.fault_code for f in self._list().faults} + self.assertIn(root, visible) + for code, _ in symptoms: + self.assertNotIn(code, visible, 'a muted symptom of another node stays hidden') + + cleared = self._clear(root, root_node) + self.assertTrue(cleared.success, cleared.message) + self.assertEqual(sorted(cleared.auto_cleared_codes), sorted(code for code, _ in symptoms)) + for code, node in symptoms: + self.assertEqual(self._status_of(code, symptom_owner_of_record(code, node)), + Fault.STATUS_CLEARED) + remaining = [m.fault_code for m in self._list(include_muted=True).muted_faults] + for code, _ in symptoms: + self.assertNotIn(code, remaining) + + def _storm(self, rule_id, reports, owner_of_record): + """Report records of different nodes inside one window and expect one cluster.""" + for code, node in reports: + self._report(code, node) + + clusters = [c for c in self._list(include_clusters=True).clusters if c.rule_id == rule_id] + self.assertEqual(len(clusters), 1) + cluster = clusters[0] + members = sorted(zip(cluster.fault_codes, cluster.fault_owners)) + expected = sorted((code, owner_of_record(node)) for code, node in reports) + self.assertEqual(members, expected) + self.assertEqual(len(cluster.fault_owners), len(cluster.fault_codes)) + self.assertEqual(cluster.count, len(reports)) + + # The representative resolves to exactly one record through the single-record services. + response = self._get(cluster.representative_code, cluster.representative_owner) + self.assertTrue(response.success, response.error_message) + self.assertEqual(response.fault.fault_code, cluster.representative_code) + self.assertIn((cluster.representative_code, cluster.representative_owner), members) + return cluster + + def test_per_source_root_mutes_and_clears_symptoms_of_other_nodes(self): + self._cascade( + 'XO_PS_ESTOP', '/estop_node', + [('XO_PS_MOTOR_A', '/motor_node'), ('XO_PS_MOTOR_B', '/drive_node')], + lambda code, node: node) + + def test_shared_root_mutes_and_clears_symptoms_of_other_nodes(self): + self._cascade( + 'XO_SH_ESTOP', '/estop_node', + [('XO_SH_MOTOR_A', '/motor_node'), ('XO_SH_MOTOR_B', '/drive_node')], + lambda code, node: '*') + + def test_per_source_storm_across_nodes_forms_one_cluster(self): + # One code from two nodes is two member records of the cluster. + cluster = self._storm( + 'ps_storm', + [('XO_PS_STORM_1', '/lidar'), ('XO_PS_STORM_1', '/camera'), + ('XO_PS_STORM_2', '/motor_node')], + lambda node: node) + self.assertEqual(cluster.representative_code, 'XO_PS_STORM_1') + self.assertEqual(cluster.representative_owner, '/lidar') + + def test_a_cleared_member_leaves_the_cluster_and_comes_back_once(self): + for code, node in [('XO_PS_CHURN_1', '/lidar'), ('XO_PS_CHURN_1', '/camera'), + ('XO_PS_CHURN_2', '/motor_node')]: + self._report(code, node) + + def cluster(): + clusters = [c for c in self._list(include_clusters=True).clusters + if c.rule_id == 'ps_churn'] + self.assertEqual(len(clusters), 1) + return clusters[0] + + before = cluster() + self.assertEqual((before.representative_code, before.representative_owner), + ('XO_PS_CHURN_1', '/lidar')) + + self._clear('XO_PS_CHURN_1', '/lidar') + after = cluster() + self.assertEqual(sorted(zip(after.fault_codes, after.fault_owners)), + [('XO_PS_CHURN_1', '/camera'), ('XO_PS_CHURN_2', '/motor_node')]) + self.assertNotEqual((after.representative_code, after.representative_owner), + ('XO_PS_CHURN_1', '/lidar'), 'a cleared record is representative') + self.assertIn((after.representative_code, after.representative_owner), + zip(after.fault_codes, after.fault_owners)) + + self._report('XO_PS_CHURN_1', '/lidar') + again = cluster() + members = list(zip(again.fault_codes, again.fault_owners)) + self.assertEqual(members.count(('XO_PS_CHURN_1', '/lidar')), 1) + self.assertEqual(len(members), 3) + + def test_shared_storm_across_nodes_forms_one_cluster(self): + cluster = self._storm( + 'sh_storm', + [('XO_SH_STORM_1', '/lidar'), ('XO_SH_STORM_2', '/camera'), + ('XO_SH_STORM_3', '/motor_node')], + lambda node: '*') + self.assertEqual(cluster.representative_owner, '*') + + +@launch_testing.post_shutdown_test() +class TestCorrelationOwnersShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(CORRELATION_FILE) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py new file mode 100644 index 000000000..133a550c1 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity.test.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Record identity per fault code, through a real fault manager. + +A code listed with per_source: true in fault_thresholds.yaml gets one record per +reporting source. Every other code, listed or not, has one record shared by all +its sources, with one debounce counter. The shared record is owned by the +reserved id '*', which no reporter may use. +""" + +import os +import sys +import tempfile +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import launch_testing.markers +import rclpy +from rclpy.node import Node +from rclpy.qos import QoSProfile, ReliabilityPolicy +from ros2_medkit_msgs.msg import Fault, FaultEvent +from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each, wait_for_event_route # noqa: E402 + +TIMEOUT_SEC = 30.0 +ALL_STATUSES = [ + Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, + Fault.STATUS_HEALED, Fault.STATUS_CLEARED, +] + +# Codes with an entry: per_source absent, false and true. A code with no entry is shared too. +THRESHOLDS = """\ +ID.ABSENT: + confirmation_threshold: -3 +ID.FALSE: + per_source: false + confirmation_threshold: -3 +ID.PER: + per_source: true + confirmation_threshold: -1 + healing_enabled: true + healing_threshold: 1 +ID.PER_LIST: + per_source: true + confirmation_threshold: -1 +ID.CRIT_SHARED: + confirmation_threshold: -3 +ID.CRIT_PER: + per_source: true + confirmation_threshold: -3 +ID.PER_OWNER: + per_source: true + confirmation_threshold: -1 +""" + +THRESHOLDS_FILE = tempfile.NamedTemporaryFile( + 'w', suffix='.yaml', prefix='fault_identity_', delete=False) +THRESHOLDS_FILE.write(THRESHOLDS) +THRESHOLDS_FILE.close() + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'healing_enabled': False, + 'fault_thresholds.config_file': THRESHOLDS_FILE.name, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestFaultIdentity(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_fault_identity_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + cls.get_client = cls.node.create_client(GetFault, '/fault_manager/get_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + cls.events = [] + cls.event_sub = cls.node.create_subscription( + FaultEvent, '/fault_manager/events', cls.events.append, + QoSProfile(depth=100, reliability=ReliabilityPolicy.RELIABLE)) + clients = (cls.report_client, cls.list_client, cls.get_client, cls.clear_client) + for client in clients: + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) + wait_for_event_route(cls.node, cls.report_client, cls.events, 'ID.PROBE_', TIMEOUT_SEC) + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source, event_type=ReportFault.Request.EVENT_FAILED, + severity=Fault.SEVERITY_ERROR): + request = ReportFault.Request() + request.fault_code = code + request.event_type = event_type + request.severity = severity + request.description = f'{code} from {source}' + request.source_id = source + return self._call(self.report_client, request).accepted + + def _get(self, code, source=''): + request = GetFault.Request() + request.fault_code = code + request.source_id = source + return self._call(self.get_client, request) + + def _clear(self, code, source=''): + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + return self._call(self.clear_client, request) + + def _records(self, code): + request = ListFaults.Request() + request.statuses = ALL_STATUSES + return [f for f in self._call(self.list_client, request).faults if f.fault_code == code] + + def test_default_code_has_one_counter_for_all_sources(self): + """Three sources, one FAILED each: one record that confirms on the third.""" + sources = ['/debounce/a', '/debounce/b', '/debounce/c'] + expected = [Fault.STATUS_PREFAILED, Fault.STATUS_PREFAILED, Fault.STATUS_CONFIRMED] + for code in ('ID.ABSENT', 'ID.FALSE'): + with self.subTest(code=code): + for source, status in zip(sources, expected): + self.assertTrue(self._report(code, source)) + records = self._records(code) + self.assertEqual(len(records), 1) + self.assertEqual(records[0].status, status) + self.assertEqual(list(records[0].reporting_sources), sources) + + def test_default_code_is_one_record_that_any_source_reaches(self): + """Any reporting source and the unscoped call reach the one record; a clear ends it.""" + code = 'ID.PLAIN' + sources = ['/plain/a', '/plain/b', '/plain/c'] + for source in sources: + self.assertTrue(self._report(code, source)) + + for source in ['', '*'] + sources: + with self.subTest(get_source=source): + response = self._get(code, source) + self.assertTrue(response.success, response.error_message) + self.assertEqual(response.fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(list(response.fault.reporting_sources), sources) + self.assertEqual(response.fault.occurrence_count, 1) + + cleared = self._clear(code, '/plain/b') + self.assertTrue(cleared.success, cleared.message) + self.assertEqual(self._get(code).fault.status, Fault.STATUS_CLEARED) + self.assertEqual(len(self._records(code)), 1) + + # A later FAILED reactivates the same record as a new occurrence. + self.assertTrue(self._report(code, '/plain/a')) + response = self._get(code) + self.assertEqual(response.fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(response.fault.occurrence_count, 2) + + unscoped = self._clear(code) + self.assertTrue(unscoped.success, unscoped.message) + self.assertEqual(self._get(code).fault.status, Fault.STATUS_CLEARED) + + def test_per_source_code_has_a_record_per_source(self): + """Three records; an unscoped call is refused; clear and heal touch one record.""" + code = 'ID.PER' + sources = ['/per/a', '/per/b', '/per/c'] + for source in sources: + self.assertTrue(self._report(code, source)) + + records = self._records(code) + owners = sorted(list(r.reporting_sources) for r in records) + self.assertEqual(owners, [[s] for s in sources]) + self.assertTrue(all(r.status == Fault.STATUS_CONFIRMED for r in records)) + + ambiguous_get = self._get(code) + self.assertFalse(ambiguous_get.success) + self.assertTrue(ambiguous_get.error_message.startswith('ambiguous:'), + ambiguous_get.error_message) + ambiguous_clear = self._clear(code) + self.assertFalse(ambiguous_clear.success) + self.assertTrue(ambiguous_clear.message.startswith('ambiguous:'), ambiguous_clear.message) + + # A clear of /per/a leaves /per/b and /per/c CONFIRMED. + self.assertTrue(self._clear(code, '/per/a').success) + self.assertEqual(self._get(code, '/per/a').fault.status, Fault.STATUS_CLEARED) + self.assertEqual(self._get(code, '/per/b').fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(self._get(code, '/per/c').fault.status, Fault.STATUS_CONFIRMED) + + # Two PASSED from /per/b heal /per/b only. + for _ in range(2): + self.assertTrue(self._report(code, '/per/b', ReportFault.Request.EVENT_PASSED)) + self.assertEqual(self._get(code, '/per/b').fault.status, Fault.STATUS_HEALED) + self.assertEqual(self._get(code, '/per/c').fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(self._get(code, '/per/a').fault.status, Fault.STATUS_CLEARED) + + def test_default_and_per_source_codes_side_by_side(self): + """The same three sources: one record for a default code, three for a per_source one.""" + sources = ['/side/a', '/side/b', '/side/c'] + for source in sources: + self.assertTrue(self._report('ID.PLAIN_LIST', source)) + self.assertTrue(self._report('ID.PER_LIST', source)) + self.assertEqual(len(self._records('ID.PLAIN_LIST')), 1) + self.assertEqual(len(self._records('ID.PER_LIST')), 3) + + def _confirmed_events(self, code): + return [e for e in self.events + if e.fault.fault_code == code and e.event_type == FaultEvent.EVENT_CONFIRMED] + + def _wait_for_confirmed_event(self, code): + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline and not self._confirmed_events(code): + rclpy.spin_once(self.node, timeout_sec=0.1) + self.assertTrue(self._confirmed_events(code), f'no fault_confirmed event for {code}') + + def _critical_clear_warn(self, code, first_source, second_source): + """CRITICAL, clear the record, then a WARN: the sequence that tells the two rules apart.""" + self.assertTrue(self._report(code, first_source, severity=Fault.SEVERITY_CRITICAL)) + self._wait_for_confirmed_event(code) + cleared = self._clear(code) + self.assertTrue(cleared.success, cleared.message) + self.assertTrue(self._report(code, second_source, severity=Fault.SEVERITY_WARN)) + # The events arrive in order, so once a marker's confirmation is here, any event the WARN + # report produced is too. + marker = f'{code}.MARKER' + self.assertTrue(self._report(marker, '/crit/marker', severity=Fault.SEVERITY_CRITICAL)) + self._wait_for_confirmed_event(marker) + + def test_a_shared_record_confirms_on_the_severity_of_the_report(self): + """CRITICAL from /a, clear, WARN from /b on a default code stays pending.""" + code = 'ID.CRIT_SHARED' + self._critical_clear_warn(code, '/crit/a', '/crit/b') + record = self._get(code).fault + self.assertEqual(record.status, Fault.STATUS_PREFAILED) + self.assertEqual(record.occurrence_count, 2) + request = ListFaults.Request() + listed = self._call(self.list_client, request).faults + self.assertEqual([f for f in listed if f.fault_code == code], []) + self.assertEqual(len(self._confirmed_events(code)), 1) + + def test_a_per_source_record_confirms_on_its_stored_severity(self): + """CRITICAL from /a, clear, WARN from /a on a per_source code confirms at once.""" + code = 'ID.CRIT_PER' + self._critical_clear_warn(code, '/crit/a', '/crit/a') + record = self._get(code).fault + self.assertEqual(record.status, Fault.STATUS_CONFIRMED) + self.assertEqual(record.severity, Fault.SEVERITY_CRITICAL) + self.assertEqual(record.occurrence_count, 2) + self.assertEqual(len(self._confirmed_events(code)), 2) + + def _wait_for_confirmed_events(self, code, count): + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline and len(self._confirmed_events(code)) < count: + rclpy.spin_once(self.node, timeout_sec=0.1) + self.assertEqual( + len(self._confirmed_events(code)), count, f'fault_confirmed events of {code}') + + def test_every_fault_names_the_record_it_describes(self): + """ListFaults, GetFault and the confirmation event carry the owner of the record.""" + sources = ['/owner/a', '/owner/b'] + for source in sources: + self.assertTrue(self._report('ID.PER_OWNER', source)) + self.assertTrue(self._report('ID.SHARED_OWNER', source)) + + # A per_source code: one record per source, each owned by its source. + listed = sorted( + (r.owner, list(r.reporting_sources)) for r in self._records('ID.PER_OWNER')) + self.assertEqual(listed, [(s, [s]) for s in sources]) + for source in sources: + with self.subTest(per_source_get=source): + response = self._get('ID.PER_OWNER', source) + self.assertTrue(response.success, response.error_message) + self.assertEqual(response.fault.owner, source) + self._wait_for_confirmed_events('ID.PER_OWNER', 2) + self.assertEqual( + sorted(e.fault.owner for e in self._confirmed_events('ID.PER_OWNER')), sources) + + # A shared code: one record owned by '*', whichever source addresses it. + listed = [(r.owner, list(r.reporting_sources)) for r in self._records('ID.SHARED_OWNER')] + self.assertEqual(listed, [('*', sources)]) + for source in ['', '*'] + sources: + with self.subTest(shared_get=source): + response = self._get('ID.SHARED_OWNER', source) + self.assertTrue(response.success, response.error_message) + self.assertEqual(response.fault.owner, '*') + self._wait_for_confirmed_events('ID.SHARED_OWNER', 1) + self.assertEqual(self._confirmed_events('ID.SHARED_OWNER')[0].fault.owner, '*') + + def test_the_shared_owner_is_not_a_source_id(self): + """A report claiming the reserved owner is refused and creates nothing.""" + for code in ('ID.RESERVED_SHARED', 'ID.PER'): + with self.subTest(code=code): + before = len(self._records(code)) + self.assertFalse(self._report(code, '*')) + self.assertEqual(len(self._records(code)), before) + + +@launch_testing.post_shutdown_test() +class TestFaultIdentityShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE.name) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py new file mode 100644 index 000000000..ebd65a507 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_identity_restart.test.py @@ -0,0 +1,427 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Record identity across restarts on one SQLite file. + +- Changing a code between shared and per_source keeps the stored records as they + were written and sends new reports to the configured identity. The fault + manager says so once per code at startup. +- A database written when every code was one shared record opens as shared + records: each row keeps its reporting sources and every child row (near + misses, snapshots, freeze frame, recording link) attaches to it. + +The test starts the fault manager itself, so a restart is a process exit and a +new process, not a respawn. +""" + +import os +import queue +import signal +import sqlite3 +import subprocess +import sys +import tempfile +import threading +import time +import unittest + +from ament_index_python.packages import get_package_prefix +import launch +from launch import LaunchDescription +import launch_testing.actions +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetFault, ListFaults, ReportFault + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + +TIMEOUT_SEC = 30.0 +ALL_STATUSES = [ + Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, + Fault.STATUS_HEALED, Fault.STATUS_CLEARED, +] +SERVICE = '/fault_manager/report_fault' + +# The fault manager's tables as the release before record identity created them: one row per +# fault code, no owner column anywhere. +MAIN_SCHEMA = """ +CREATE TABLE faults ( + fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, description TEXT NOT NULL, + first_occurred_ns INTEGER NOT NULL, last_occurred_ns INTEGER NOT NULL, + occurrence_count INTEGER NOT NULL, status TEXT NOT NULL, reporting_sources TEXT NOT NULL, + debounce_counter INTEGER NOT NULL DEFAULT 0, last_failed_ns INTEGER NOT NULL DEFAULT 0, + last_passed_ns INTEGER NOT NULL DEFAULT 0, confirmed_at_ns INTEGER NOT NULL DEFAULT 0); +CREATE TABLE snapshots ( + id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, topic TEXT NOT NULL, + message_type TEXT NOT NULL, data TEXT NOT NULL, captured_at_ns INTEGER NOT NULL, + capture_id INTEGER NOT NULL DEFAULT 0); +CREATE INDEX idx_snapshots_fault_code ON snapshots(fault_code); +CREATE INDEX idx_snapshots_fault_topic ON snapshots(fault_code, topic); +CREATE TABLE freeze_frames ( + fault_code TEXT PRIMARY KEY, data TEXT NOT NULL, captured_at_ns INTEGER NOT NULL); +CREATE TABLE near_misses ( + id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, occurred_at_ns INTEGER NOT NULL, + debounce_counter INTEGER NOT NULL, confirmation_threshold INTEGER NOT NULL, + severity INTEGER NOT NULL, source_id TEXT NOT NULL, resulting_status TEXT NOT NULL DEFAULT ''); +CREATE INDEX idx_near_misses_fault_code ON near_misses(fault_code, id); +CREATE TABLE rosbag_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, + recording_id TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL, format TEXT NOT NULL, + duration_sec REAL NOT NULL, size_bytes INTEGER NOT NULL, created_at_ns INTEGER NOT NULL); +CREATE INDEX idx_rosbag_files_fault_code ON rosbag_files(fault_code); +CREATE INDEX idx_rosbag_files_created_at ON rosbag_files(created_at_ns); +CREATE INDEX idx_rosbag_files_fault_created ON rosbag_files(fault_code, created_at_ns, id); +CREATE INDEX idx_rosbag_files_recording ON rosbag_files(recording_id); +CREATE INDEX idx_rosbag_files_path ON rosbag_files(file_path); +CREATE UNIQUE INDEX idx_rosbag_files_fault_path ON rosbag_files(fault_code, file_path); +""" + + +def generate_test_description(): + """Start only a placeholder process, so launch keeps the test running.""" + keep_alive = launch.actions.ExecuteProcess(cmd=['sleep', '600']) + return LaunchDescription([keep_alive, launch_testing.actions.ReadyToTest()]) + + +class FaultManagerProcess: + """One fault_manager_node process with its output readable line by line.""" + + def __init__(self, exe, params_file): + self.proc = subprocess.Popen( + [exe, '--ros-args', '-r', '__node:=fault_manager', '--params-file', params_file], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, errors='replace') + self.lines = queue.Queue() + self.seen = [] + threading.Thread(target=self._pump, daemon=True).start() + + def _pump(self): + for line in self.proc.stdout: + self.lines.put(line) + self.lines.put(None) + + def wait_for_line(self, text): + """Return the first output line containing text, or None at the end or the timeout.""" + for line in self.seen: + if text in line: + return line + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + try: + line = self.lines.get(timeout=1.0) + except queue.Empty: + continue + if line is None: + return None + self.seen.append(line) + if text in line: + return line + return None + + def stop(self): + if self.proc.poll() is None: + self.proc.send_signal(signal.SIGINT) + return self.proc.wait(timeout=TIMEOUT_SEC) + + def kill(self): + if self.proc.poll() is None: + self.proc.kill() + self.proc.wait() + + +class TestFaultIdentityRestart(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + cls.exe = os.path.join( + get_package_prefix('ros2_medkit_fault_manager'), + 'lib', 'ros2_medkit_fault_manager', 'fault_manager_node') + rclpy.init() + cls.node = Node('test_fault_identity_restart_client') + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + cls.tmp.cleanup() + + def _path(self, name): + return os.path.join(self.tmp.name, name) + + def _write(self, name, text): + path = self._path(name) + with open(path, 'w') as f: + f.write(text) + return path + + def _wait_until_service_is_gone(self): + """Wait until the graph no longer lists the previous process's services.""" + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + names = [name for name, _ in self.node.get_service_names_and_types()] + if SERVICE not in names: + return + rclpy.spin_once(self.node, timeout_sec=0.1) + self.fail('the stopped fault manager is still listed in the graph') + + def _start(self, db_path, thresholds): + """Start the fault manager on db_path with a thresholds file, and wait until it answers.""" + self._wait_until_service_is_gone() + thresholds_file = self._write('thresholds.yaml', thresholds) + params = self._write('params.yaml', ( + 'fault_manager:\n' + ' ros__parameters:\n' + ' storage_type: sqlite\n' + f' database_path: {db_path}\n' + ' confirmation_threshold: -1\n' + ' snapshots.enabled: false\n' + ' snapshots.rosbag.enabled: false\n' + f' fault_thresholds.config_file: {thresholds_file}\n' + )) + process = FaultManagerProcess(self.exe, params) + self.addCleanup(process.kill) + self.report = self.node.create_client(ReportFault, SERVICE) + self.get = self.node.create_client(GetFault, '/fault_manager/get_fault') + self.list = self.node.create_client(ListFaults, '/fault_manager/list_faults') + self.clear = self.node.create_client(ClearFault, '/fault_manager/clear_fault') + for client in (self.report, self.get, self.list, self.clear): + self.assertTrue(client.wait_for_service(timeout_sec=TIMEOUT_SEC)) + self.assertIsNotNone( + process.wait_for_line('FaultManager node started'), 'the node never finished starting') + answer_once_each(self.node, (self.report, self.get, self.list, self.clear), TIMEOUT_SEC) + return process + + def _stop(self, process): + for client in (self.report, self.get, self.list, self.clear): + self.node.destroy_client(client) + self.assertEqual(process.stop(), 0) + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report, request).accepted) + + def _get(self, code, source=''): + request = GetFault.Request() + request.fault_code = code + request.source_id = source + return self._call(self.get, request) + + def _records(self, code): + request = ListFaults.Request() + request.statuses = ALL_STATUSES + return [f for f in self._call(self.list, request).faults if f.fault_code == code] + + def _sources_of_records(self, code): + return sorted(list(f.reporting_sources) for f in self._records(code)) + + def test_changing_the_mode_keeps_stored_records_and_routes_new_reports(self): + db_path = self._path('mode_change.db') + shared_then_per_source = 'M.SHARED:\n confirmation_threshold: -1\n' \ + 'N.PER:\n per_source: true\n confirmation_threshold: -1\n' + per_source_then_shared = 'M.SHARED:\n per_source: true\n confirmation_threshold: -1\n' \ + 'N.PER:\n confirmation_threshold: -1\n' + + process = self._start(db_path, shared_then_per_source) + try: + for code in ('M.SHARED', 'N.PER'): + for source in ('/a', '/b'): + self._report(code, source) + self.assertEqual(self._sources_of_records('M.SHARED'), [['/a', '/b']]) + self.assertEqual(self._sources_of_records('N.PER'), [['/a'], ['/b']]) + # The startup check runs before the node says it started, and _start waited for that. + self.assertFalse( + [line for line in process.seen if 'is configured' in line], + 'a store that matches its configuration must not warn') + finally: + self._stop(process) + + process = self._start(db_path, per_source_then_shared) + try: + self.assertIsNotNone( + process.wait_for_line("Fault code 'M.SHARED' is configured per_source"), + 'no startup warning for the shared record of a per_source code') + self.assertIsNotNone( + process.wait_for_line("Fault code 'N.PER' is configured shared"), + 'no startup warning for the per-source records of a shared code') + + # The stored records are as they were written, and addressable by their owner. + self.assertEqual(self._sources_of_records('M.SHARED'), [['/a', '/b']]) + self.assertEqual(self._sources_of_records('N.PER'), [['/a'], ['/b']]) + shared = self._get('M.SHARED', '*') + self.assertTrue(shared.success, shared.error_message) + self.assertEqual(list(shared.fault.reporting_sources), ['/a', '/b']) + for source in ('/a', '/b'): + owned = self._get('N.PER', source) + self.assertTrue(owned.success, owned.error_message) + self.assertEqual(list(owned.fault.reporting_sources), [source]) + + # New reports go to the configured identity. + self._report('M.SHARED', '/c') + self._report('N.PER', '/c') + self.assertEqual(self._sources_of_records('M.SHARED'), [['/a', '/b'], ['/c']]) + self.assertEqual(self._sources_of_records('N.PER'), [['/a'], ['/b'], ['/c']]) + self.assertEqual( + list(self._get('M.SHARED', '*').fault.reporting_sources), ['/a', '/b']) + self.assertEqual(list(self._get('N.PER', '*').fault.reporting_sources), ['/c']) + finally: + self._stop(process) + + with sqlite3.connect(db_path) as db: + rows = db.execute( + 'SELECT fault_code, owner FROM faults ORDER BY fault_code, owner').fetchall() + self.assertEqual(rows, [ + ('M.SHARED', '*'), ('M.SHARED', '/c'), + ('N.PER', '*'), ('N.PER', '/a'), ('N.PER', '/b'), + ]) + + def test_the_owner_tells_a_kept_shared_record_from_a_new_per_source_one(self): + """After the switch, the kept shared record of /s and the new one differ by owner.""" + db_path = self._path('owner_switch.db') + process = self._start(db_path, 'O.SWITCH:\n confirmation_threshold: -1\n') + try: + self._report('O.SWITCH', '/s') + request = ClearFault.Request() + request.fault_code = 'O.SWITCH' + self.assertTrue(self._call(self.clear, request).success) + self.assertEqual( + [(r.owner, list(r.reporting_sources)) for r in self._records('O.SWITCH')], + [('*', ['/s'])]) + finally: + self._stop(process) + + process = self._start( + db_path, 'O.SWITCH:\n per_source: true\n confirmation_threshold: -1\n') + try: + self._report('O.SWITCH', '/s') + expected = [('*', Fault.STATUS_CLEARED), ('/s', Fault.STATUS_CONFIRMED)] + listed = sorted( + (r.owner, r.status) for r in self._records('O.SWITCH') + if list(r.reporting_sources) == ['/s']) + self.assertEqual(listed, expected) + for owner, status in expected: + with self.subTest(owner=owner): + record = self._get('O.SWITCH', owner) + self.assertTrue(record.success, record.error_message) + self.assertEqual(record.fault.owner, owner) + self.assertEqual(record.fault.status, status) + finally: + self._stop(process) + + def test_a_malformed_per_source_value_rejects_the_whole_file(self): + thresholds = 'A:\n per_source: true\nB:\n per_source: "yes"\n' + process = self._start(self._path('malformed.db'), thresholds) + try: + self.assertIsNotNone( + process.wait_for_line('Failed to parse fault thresholds config'), + 'the malformed per_source value was not reported') + # The file is dropped as a whole, so A is not per_source either: one shared record. + self._report('A', '/a') + self._report('A', '/b') + records = self._records('A') + self.assertEqual(len(records), 1) + self.assertEqual(list(records[0].reporting_sources), ['/a', '/b']) + self.assertTrue(self._get('A', '*').success) + finally: + self._stop(process) + + def test_a_store_written_with_one_record_per_code_opens_as_shared_records(self): + db_path = self._path('legacy.db') + bag_path = self._path('fault_MIG.CODE_1') + os.makedirs(bag_path) + with sqlite3.connect(db_path) as db: + db.executescript(MAIN_SCHEMA) + db.execute( + "INSERT INTO faults VALUES ('MIG.CODE', 2, 'from the old release', 1, 1, 1, " + "'CONFIRMED', '[\"/s1\",\"/s2\"]', -1, 1, 0, 0)") + db.execute( + "INSERT INTO faults VALUES ('MIG.OLD', 1, 'cleared long ago', 1, 1, 1, " + "'CLEARED', '[\"/s3\"]', 0, 1, 0, 0)") + db.execute( + "INSERT INTO faults VALUES ('MIG.TWO', 2, 'second code', 1, 1, 1, " + "'CONFIRMED', '[\"/s5\"]', -1, 1, 0, 0)") + for second, source in ((5, '/s1'), (6, '/s1'), (7, '/s2')): + db.execute( + 'INSERT INTO near_misses (fault_code, occurred_at_ns, debounce_counter, ' + 'confirmation_threshold, severity, source_id, resulting_status) ' + "VALUES ('MIG.CODE', ?, -1, -3, 2, ?, 'PREFAILED')", (second, source)) + db.execute( + 'INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) ' + "VALUES ('MIG.CODE', '/t', 'std_msgs/msg/Float64', '{\"data\": 1.5}', 9)") + db.execute("INSERT INTO freeze_frames VALUES ('MIG.CODE', '{\"/t\": 1.5}', 9)") + db.execute( + 'INSERT INTO rosbag_files (fault_code, recording_id, file_path, format, ' + 'duration_sec, size_bytes, created_at_ns) ' + "VALUES ('MIG.CODE', 'fault_MIG.CODE_1', ?, 'mcap', 5.0, 100, 11)", (bag_path,)) + + process = self._start(db_path, 'UNRELATED.CODE:\n confirmation_threshold: -1\n') + try: + # Each code is one record, with every source it recorded. + self.assertEqual(self._sources_of_records('MIG.CODE'), [['/s1', '/s2']]) + self.assertEqual(self._sources_of_records('MIG.OLD'), [['/s3']]) + for source in ('', '*', '/s1', '/s2'): + record = self._get('MIG.CODE', source) + self.assertTrue(record.success, f'source {source!r}: {record.error_message}') + self.assertEqual(record.fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(list(record.fault.reporting_sources), ['/s1', '/s2']) + detail = self._get('MIG.CODE').environment_data.snapshots + self.assertEqual(sorted(s.type for s in detail), ['freeze_frame', 'rosbag']) + + # It then behaves as any shared record: a new source joins it, a clear ends it, a later + # FAILED starts a new occurrence. + self._report('MIG.CODE', '/s4') + self.assertEqual(self._sources_of_records('MIG.CODE'), [['/s1', '/s2', '/s4']]) + self.assertEqual(self._get('MIG.CODE').fault.occurrence_count, 1) + request = ClearFault.Request() + request.fault_code = 'MIG.TWO' + self.assertTrue(self._call(self.clear, request).success) + self.assertEqual(self._get('MIG.TWO').fault.status, Fault.STATUS_CLEARED) + self._report('MIG.TWO', '/s1') + self.assertEqual(self._get('MIG.TWO').fault.occurrence_count, 2) + finally: + self._stop(process) + + with sqlite3.connect(db_path) as db: + self.assertEqual( + db.execute('SELECT fault_code, owner FROM faults ORDER BY fault_code').fetchall(), + [('MIG.CODE', '*'), ('MIG.OLD', '*'), ('MIG.TWO', '*')]) + for table in ('snapshots', 'freeze_frames', 'rosbag_files'): + self.assertEqual( + db.execute(f'SELECT owner FROM {table}').fetchall(), [('*',)], table) + # Every near miss of both sources belongs to the one shared record. + self.assertEqual( + db.execute( + "SELECT source_id FROM near_misses WHERE fault_code = 'MIG.CODE' " + "AND owner = '*' ORDER BY id").fetchall(), + [('/s1',), ('/s1',), ('/s2',)]) + self.assertEqual( + db.execute( + 'SELECT COUNT(*) FROM near_misses n WHERE NOT EXISTS ' + '(SELECT 1 FROM faults f WHERE f.fault_code = n.fault_code ' + 'AND f.owner = n.owner)').fetchone(), + (0,)) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py new file mode 100644 index 000000000..8e664a9b8 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_muting_relations.test.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A record stays muted while any live relation still mutes it. + +Two relations can mute one record: a hierarchical root cause that lists it, and an active +show_as_single cluster. Ending one relation, by a clear with or without the skip flag or by a +heal, must not unmute a record the other relation still mutes. The same holds for two root +causes that list one symptom. Each scenario runs on shared codes (one code per record) and on +per_source codes (one record per reporting node). +""" + +import os +import sys +import tempfile +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, ListFaults, ReportFault + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + +TIMEOUT_SEC = 30.0 +MODES = ('SH', 'PS') +# variant -> mute_symptoms of the hierarchical rule +VARIANTS = {'CLR': 'true', 'HEAL': 'true', 'NOMUTE': 'false'} + + +def _code(variant, mode, role): + return f'CL_MX_{variant}_{mode}_{role}' + + +THRESHOLDS = ''.join( + f'{_code(v, "PS", role)}:\n per_source: true\n' + for v in VARIANTS for role in ('ROOT', 'T_X')) + ''.join( + f'CL_TR_PS_{role}:\n per_source: true\n' for role in ('R1', 'R2', 'S')) + ''.join( + f'CL_F5_PS_{role}:\n per_source: true\n' for role in ('X', 'Y', 'S_1')) +CORRELATION = 'correlation:\n enabled: true\n patterns:\n' + ''.join( + f' mx_{v.lower()}_{m.lower()}:\n codes: ["{_code(v, m, "T")}_*"]\n' + for v in VARIANTS for m in MODES) + ' rules:\n' + ''.join( + f""" - id: mx_h_{v.lower()}_{m.lower()} + name: "MX hierarchical {v} {m}" + mode: hierarchical + root_cause: + codes: ["{_code(v, m, 'ROOT')}"] + symptoms: + - codes: ["{_code(v, m, 'T')}_*"] + window_ms: 30000 + mute_symptoms: {mute} + auto_clear_with_root: true + - id: mx_c_{v.lower()}_{m.lower()} + name: "MX cluster {v} {m}" + mode: auto_cluster + match: + - pattern: mx_{v.lower()}_{m.lower()} + min_count: 3 + window_ms: 30000 + show_as_single: true + representative: first +""" for v, mute in VARIANTS.items() for m in MODES) + ''.join( + f""" - id: tr_{m.lower()} + name: "TR {m}" + mode: hierarchical + root_cause: + codes: ["CL_TR_{m}_R1", "CL_TR_{m}_R2"] + symptoms: + - codes: ["CL_TR_{m}_S", "CL_TR_{m}_SENT_*"] + window_ms: 300 + mute_symptoms: true + auto_clear_with_root: true +""" for m in MODES) + ''.join( + f""" - id: f5_a_{m.lower()} + name: "F5 A {m}" + mode: hierarchical + root_cause: + codes: ["CL_F5_{m}_X"] + symptoms: + - codes: ["CL_F5_{m}_S_*"] + window_ms: 30000 + mute_symptoms: false + auto_clear_with_root: true + - id: f5_b_{m.lower()} + name: "F5 B {m}" + mode: hierarchical + root_cause: + codes: ["CL_F5_{m}_X"] + symptoms: + - codes: ["CL_F5_{m}_S_*"] + window_ms: 30000 + mute_symptoms: true + auto_clear_with_root: true + - id: f5_c_{m.lower()} + name: "F5 C {m}" + mode: hierarchical + root_cause: + codes: ["CL_F5_{m}_Y"] + symptoms: + - codes: ["CL_F5_{m}_S_*", "CL_F5_{m}_Z_*"] + window_ms: 300 + mute_symptoms: true + auto_clear_with_root: true +""" for m in MODES) + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file('muting_relations_thresholds_', THRESHOLDS) +CORRELATION_FILE = _temp_file('muting_relations_rules_', CORRELATION) + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'healing_enabled': True, + 'healing_threshold': 0, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'correlation.config_file': CORRELATION_FILE, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestMutingRelations(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_muting_relations_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + clients = (cls.report_client, cls.list_client, cls.clear_client) + for client in clients: + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source, event=ReportFault.Request.EVENT_FAILED): + request = ReportFault.Request() + request.fault_code = code + request.event_type = event + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _clear(self, code, owner, skip): + request = ClearFault.Request() + request.fault_code = code + request.source_id = owner + request.skip_correlation_auto_clear = skip + response = self._call(self.clear_client, request) + self.assertTrue(response.success, response.message) + + def _shown(self, codes): + """Return the (code, first source) pairs of the default listing among codes.""" + faults = self._call(self.list_client, ListFaults.Request()).faults + return {(f.fault_code, f.reporting_sources[0]) for f in faults if f.fault_code in codes} + + def _muted(self, codes): + """Return {(code, owner): root_cause_code} of include_muted among codes.""" + request = ListFaults.Request() + request.include_muted = True + muted = self._call(self.list_client, request).muted_faults + return {(m.fault_code, m.source_id): m.root_cause_code for m in muted + if m.fault_code in codes} + + @staticmethod + def _records(variant, mode): + """Return (root, root source, root owner, [(code, source, owner)] of three records).""" + root = _code(variant, mode, 'ROOT') + if mode == 'PS': + return root, '/r', '/r', [(_code(variant, mode, 'T_X'), f'/n{i}', f'/n{i}') + for i in (1, 2, 3)] + return root, '/r', '*', [(f'{_code(variant, mode, "T")}_{i}', '/n', '*') + for i in (1, 2, 3)] + + def _cluster_and_root(self, variant, mode, end_root): + root, root_source, root_owner, records = self._records(variant, mode) + codes = {code for code, _, _ in records} + for code, source, _ in records: + self._report(code, source) + first, second, third = records + self.assertEqual(self._shown(codes), {(first[0], first[1])}) + self._report(root, root_source) + self._report(second[0], second[1]) + end_root(root, root_source, root_owner) + self.assertEqual( + self._shown(codes), {(first[0], first[1])}, + 'a record its active cluster mutes was shown when the root cause ended') + self.assertEqual(set(self._muted(codes)), {(second[0], second[2]), (third[0], third[2])}) + + def _each_mode(self, check): + for mode in MODES: + with self.subTest(mode=mode): + check(mode) + + def test_a_cleared_root_leaves_the_cluster_mute(self): + def end_root(root, source, owner): + self._clear(root, owner, skip=True) + self._each_mode(lambda mode: self._cluster_and_root('CLR', mode, end_root)) + + def test_a_healed_root_leaves_the_cluster_mute(self): + def end_root(root, source, owner): + self._report(root, source, event=ReportFault.Request.EVENT_PASSED) + self._each_mode(lambda mode: self._cluster_and_root('HEAL', mode, end_root)) + + def test_a_root_that_does_not_mute_leaves_the_cluster_mute(self): + def end_root(root, source, owner): + self._clear(root, owner, skip=True) + self._each_mode(lambda mode: self._cluster_and_root('NOMUTE', mode, end_root)) + + def _two_roots(self, mode): + r1, r2 = f'CL_TR_{mode}_R1', f'CL_TR_{mode}_R2' + symptom = f'CL_TR_{mode}_S' + owner = lambda source: source if mode == 'PS' else '*' # noqa: E731 + self._report(r1, '/r1') + self._report(symptom, '/s') + self.assertIn((symptom, owner('/s')), self._muted({symptom})) + # Report fresh symptoms until one comes back unmuted: R1's window has passed. + deadline = time.monotonic() + TIMEOUT_SEC + index = 0 + while True: + index += 1 + sentinel = f'CL_TR_{mode}_SENT_{index}' + self._report(sentinel, '/s') + if not self._muted({sentinel}): + break + self.assertLess(time.monotonic(), deadline, 'the window of the first root never ended') + self._report(r2, '/r2') + self._report(symptom, '/s') + self.assertEqual(self._muted({symptom}), {(symptom, owner('/s')): r2}) + + self._clear(r1, owner('/r1'), skip=True) + self.assertEqual(self._shown({symptom}), set(), 'the second root still mutes the symptom') + self.assertEqual(self._muted({symptom}), {(symptom, owner('/s')): r2}) + + def test_a_cleared_root_leaves_the_mute_of_another_root(self): + self._each_mode(self._two_roots) + + def _muted_info(self, codes): + """Return {(code, owner): (root_cause_code, delay_ms)} of include_muted among codes.""" + request = ListFaults.Request() + request.include_muted = True + muted = self._call(self.list_client, request).muted_faults + return {(m.fault_code, m.source_id): (m.root_cause_code, m.delay_ms) for m in muted + if m.fault_code in codes} + + def _cross_window(self, prefix, source): + """Report fresh symptoms until one is no longer muted: the window of the root passed.""" + deadline = time.monotonic() + TIMEOUT_SEC + index = 0 + while True: + index += 1 + sentinel = f'{prefix}_{index}' + self._report(sentinel, source) + if not self._muted({sentinel}): + return + self.assertLess(time.monotonic(), deadline, 'the window of the root never ended') + + def _root_bound_to_its_own_rule(self, mode): + x, y = f'CL_F5_{mode}_X', f'CL_F5_{mode}_Y' + symptom = f'CL_F5_{mode}_S_1' + owner = '/s' if mode == 'PS' else '*' + self._report(y, '/y') + self._report(symptom, '/s') + self.assertEqual(self._muted({symptom}), {(symptom, owner): y}) + self._cross_window(f'CL_F5_{mode}_Z', '/s') + # X is bound to its first rule, which does not mute. The later rule for X never applies. + self._report(x, '/x') + self._report(symptom, '/s') + + self._clear(y, '/y' if mode == 'PS' else '*', skip=True) + self.assertEqual(self._muted({symptom}), {}, 'a rule the root is not bound to muted it') + self.assertEqual(self._shown({symptom}), {(symptom, '/s')}) + + def test_a_root_mutes_only_through_its_own_rule(self): + self._each_mode(self._root_bound_to_its_own_rule) + + def _delay_follows_the_relation(self, mode): + r1, r2 = f'CL_TR_{mode}_R1', f'CL_TR_{mode}_R2' + symptom = f'CL_TR_{mode}_S' + owner = '/s' if mode == 'PS' else '*' + self._report(r1, '/r1') + # Report again until the delay behind the first root is more than a millisecond. + deadline = time.monotonic() + TIMEOUT_SEC + while True: + self._report(symptom, '/s') + root, delay = self._muted_info({symptom})[(symptom, owner)] + if delay > 0: + break + self.assertLess(time.monotonic(), deadline, 'the delay behind the first root stayed 0') + self.assertEqual(root, r1) + self._cross_window(f'CL_TR_{mode}_SENT', '/s') + self._report(r2, '/r2') + self._report(symptom, '/s') + self.assertEqual(self._muted({symptom}), {(symptom, owner): r2}) + + self._clear(r2, '/r2' if mode == 'PS' else '*', skip=True) + self.assertEqual(self._muted_info({symptom}), {(symptom, owner): (r1, delay)}) + + def test_the_delay_moves_with_the_mute(self): + self._each_mode(self._delay_follows_the_relation) + + +@launch_testing.post_shutdown_test() +class TestMutingRelationsShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(CORRELATION_FILE) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py new file mode 100644 index 000000000..dbb25a401 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_postgres_migration.test.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A PostgreSQL database written by the release before record identity, migrated through the node. + +The tables are created with that release's layout and rows, the fault manager is started on them +with storage_type: postgres, and the migrated records are read over its services: every code is one +shared record (owner '*') with its reporting sources, and its snapshot, freeze frame and recording +link attach to it. A report from a new source joins the record, and a restart changes nothing. +Every record names its owner: '*' for the shared ones, the source for a per_source code. +Needs a server: ROS2_MEDKIT_TEST_PG_CONN, as for the storage tests. +""" + +import os +import queue +import signal +import subprocess +import sys +import tempfile +import threading +import time +import unittest +import uuid + +from ament_index_python.packages import get_package_prefix +import launch +from launch import LaunchDescription +import launch_testing.actions +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import GetFault, GetRosbag, GetSnapshots, ListFaults, ReportFault + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402 + +TIMEOUT_SEC = 30.0 +ALL_STATUSES = [ + Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, + Fault.STATUS_HEALED, Fault.STATUS_CLEARED, +] +SERVICE = '/fault_manager/report_fault' + +# The tables as the release before record identity created them: one row per fault code. +MAIN_SCHEMA = """ +CREATE TABLE faults ( + fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, description TEXT NOT NULL, + first_occurred_ns BIGINT NOT NULL, last_occurred_ns BIGINT NOT NULL, + occurrence_count BIGINT NOT NULL, status TEXT NOT NULL, reporting_sources TEXT NOT NULL, + debounce_counter INTEGER NOT NULL DEFAULT 0, last_failed_ns BIGINT NOT NULL DEFAULT 0, + last_passed_ns BIGINT NOT NULL DEFAULT 0, confirmed_at_ns BIGINT NOT NULL DEFAULT 0); +CREATE TABLE snapshots ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + topic TEXT NOT NULL, message_type TEXT NOT NULL, data TEXT NOT NULL, + captured_at_ns BIGINT NOT NULL, capture_id BIGINT NOT NULL DEFAULT 0); +CREATE INDEX idx_snapshots_fault_code ON snapshots(fault_code); +CREATE INDEX idx_snapshots_fault_topic ON snapshots(fault_code, topic); +CREATE TABLE freeze_frames ( + fault_code TEXT PRIMARY KEY, data TEXT NOT NULL, captured_at_ns BIGINT NOT NULL); +CREATE TABLE near_misses ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + occurred_at_ns BIGINT NOT NULL, debounce_counter INTEGER NOT NULL, + confirmation_threshold INTEGER NOT NULL, severity INTEGER NOT NULL, + source_id TEXT NOT NULL, resulting_status TEXT NOT NULL DEFAULT ''); +CREATE INDEX idx_near_misses_fault_code ON near_misses(fault_code, id); +CREATE TABLE rosbag_files ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + recording_id TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL, format TEXT NOT NULL, + duration_sec DOUBLE PRECISION NOT NULL, size_bytes BIGINT NOT NULL, + created_at_ns BIGINT NOT NULL); +CREATE INDEX idx_rosbag_files_fault_code ON rosbag_files(fault_code); +CREATE INDEX idx_rosbag_files_created_at ON rosbag_files(created_at_ns); +CREATE INDEX idx_rosbag_files_fault_created ON rosbag_files(fault_code, created_at_ns, id); +CREATE INDEX idx_rosbag_files_recording ON rosbag_files(recording_id); +CREATE INDEX idx_rosbag_files_path ON rosbag_files(file_path); +CREATE UNIQUE INDEX idx_rosbag_files_fault_path ON rosbag_files(fault_code, file_path); +""" + + +def generate_test_description(): + """Start only a placeholder process, so launch keeps the test running.""" + keep_alive = launch.actions.ExecuteProcess(cmd=['sleep', '600']) + return LaunchDescription([keep_alive, launch_testing.actions.ReadyToTest()]) + + +class FaultManagerProcess: + """One fault_manager_node process with its output readable line by line.""" + + def __init__(self, exe, params_file): + self.proc = subprocess.Popen( + [exe, '--ros-args', '-r', '__node:=fault_manager', '--params-file', params_file], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, errors='replace') + self.lines = queue.Queue() + self.seen = [] + threading.Thread(target=self._pump, daemon=True).start() + + def _pump(self): + for line in self.proc.stdout: + self.lines.put(line) + self.lines.put(None) + + def wait_for_line(self, text): + """Return the first output line containing text, or None at the end or the timeout.""" + for line in self.seen: + if text in line: + return line + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + try: + line = self.lines.get(timeout=1.0) + except queue.Empty: + continue + if line is None: + return None + self.seen.append(line) + if text in line: + return line + return None + + def stop(self): + if self.proc.poll() is None: + self.proc.send_signal(signal.SIGINT) + return self.proc.wait(timeout=TIMEOUT_SEC) + + def kill(self): + if self.proc.poll() is None: + self.proc.kill() + self.proc.wait() + + +class TestPostgresMigration(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + prefix = get_package_prefix('ros2_medkit_fault_manager') + cls.exe = os.path.join(prefix, 'lib', 'ros2_medkit_fault_manager', 'fault_manager_node') + cls.pg_exec = os.path.join(prefix, 'lib', 'ros2_medkit_fault_manager', 'pg_exec') + cls.base = os.environ.get('ROS2_MEDKIT_TEST_PG_CONN') + assert cls.base, 'set ROS2_MEDKIT_TEST_PG_CONN to a PostgreSQL server' + cls.schema = 'medkit_migrate_' + uuid.uuid4().hex[:12] + cls.conn = cls.base + '?options=-csearch_path%3D' + cls.schema + cls._sql(cls.base, f'CREATE SCHEMA {cls.schema}') + rclpy.init() + cls.node = Node('test_postgres_migration_client') + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + cls._sql(cls.base, f'DROP SCHEMA IF EXISTS {cls.schema} CASCADE') + cls.tmp.cleanup() + + @classmethod + def _sql(cls, conn, statements): + result = subprocess.run( + [cls.pg_exec, conn], input=statements, text=True, + capture_output=True) + assert result.returncode == 0, result.stderr + + def _wait_until_service_is_gone(self): + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + names = [name for name, _ in self.node.get_service_names_and_types()] + if SERVICE not in names: + return + rclpy.spin_once(self.node, timeout_sec=0.1) + self.fail('the stopped fault manager is still listed in the graph') + + def _start(self): + self._wait_until_service_is_gone() + thresholds = os.path.join(self.tmp.name, 'thresholds.yaml') + with open(thresholds, 'w') as f: + f.write('PG.PER:\n per_source: true\n') + params = os.path.join(self.tmp.name, 'params.yaml') + with open(params, 'w') as f: + f.write('fault_manager:\n ros__parameters:\n' + ' storage_type: postgres\n' + f' database_url: "{self.conn}"\n' + ' confirmation_threshold: -1\n' + ' snapshots.enabled: false\n' + ' snapshots.rosbag.enabled: false\n' + f' fault_thresholds.config_file: {thresholds}\n') + process = FaultManagerProcess(self.exe, params) + self.addCleanup(process.kill) + self.clients = { + 'report': self.node.create_client(ReportFault, SERVICE), + 'get': self.node.create_client(GetFault, '/fault_manager/get_fault'), + 'list': self.node.create_client(ListFaults, '/fault_manager/list_faults'), + 'snapshots': self.node.create_client(GetSnapshots, '/fault_manager/get_snapshots'), + 'rosbag': self.node.create_client(GetRosbag, '/fault_manager/get_rosbag'), + } + for client in self.clients.values(): + self.assertTrue(client.wait_for_service(timeout_sec=TIMEOUT_SEC)) + self.assertIsNotNone( + process.wait_for_line('FaultManager node started'), 'the node never finished starting') + answer_once_each(self.node, tuple(self.clients.values()), TIMEOUT_SEC) + return process + + def _stop(self, process): + for client in self.clients.values(): + self.node.destroy_client(client) + self.assertEqual(process.stop(), 0) + + def _call(self, name, request): + future = self.clients[name].call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call('report', request).accepted) + + def _records(self, code): + request = ListFaults.Request() + request.statuses = ALL_STATUSES + return [f for f in self._call('list', request).faults if f.fault_code == code] + + def _check_migrated(self, sources): + records = self._records('MIG.CODE') + self.assertEqual([list(r.reporting_sources) for r in records], [sources]) + self.assertEqual([r.owner for r in records], ['*']) + for source in ('', '*', '/s1'): + request = GetFault.Request() + request.fault_code = 'MIG.CODE' + request.source_id = source + record = self._call('get', request) + self.assertTrue(record.success, f'source {source!r}: {record.error_message}') + self.assertEqual(record.fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(list(record.fault.reporting_sources), sources) + self.assertEqual(record.fault.owner, '*') + self.assertEqual([list(r.reporting_sources) for r in self._records('MIG.OLD')], [['/s3']]) + + snapshots = GetSnapshots.Request() + snapshots.fault_code = 'MIG.CODE' + snapshots.source_id = '*' + response = self._call('snapshots', snapshots) + self.assertTrue(response.success, response.error_message) + self.assertIn('/t', response.data) + + rosbag = GetRosbag.Request() + rosbag.fault_code = 'MIG.CODE' + rosbag.source_id = '*' + bag = self._call('rosbag', rosbag) + self.assertTrue(bag.success, bag.error_message) + self.assertEqual(list(bag.fault_codes), ['MIG.CODE']) + self.assertEqual(list(bag.owners), ['*']) + + def _check_per_source_owners(self, sources): + listed = sorted((r.owner, list(r.reporting_sources)) for r in self._records('PG.PER')) + self.assertEqual(listed, [(s, [s]) for s in sources]) + for source in sources: + request = GetFault.Request() + request.fault_code = 'PG.PER' + request.source_id = source + record = self._call('get', request) + self.assertTrue(record.success, f'source {source!r}: {record.error_message}') + self.assertEqual(record.fault.owner, source) + + def test_a_database_written_with_one_record_per_code_is_migrated_by_the_node(self): + bag_path = os.path.join(self.tmp.name, 'fault_MIG.CODE_1') + os.makedirs(bag_path) + self._sql(self.conn, MAIN_SCHEMA + f""" +INSERT INTO faults VALUES ('MIG.CODE', 2, 'from the old release', 1, 1, 1, 'CONFIRMED', + '["/s1","/s2"]', -1, 1, 0, 0); +INSERT INTO faults VALUES ('MIG.OLD', 1, 'cleared long ago', 1, 1, 1, 'CLEARED', + '["/s3"]', 0, 1, 0, 0); +INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) + VALUES ('MIG.CODE', '/t', 'std_msgs/msg/Float64', '{{"data": 1.5}}', 9); +INSERT INTO freeze_frames VALUES ('MIG.CODE', '{{"/t": 1.5}}', 9); +INSERT INTO rosbag_files (fault_code, recording_id, file_path, format, duration_sec, + size_bytes, created_at_ns) + VALUES ('MIG.CODE', 'fault_MIG.CODE_1', '{bag_path}', 'mcap', 5.0, 100, 11); +""") + + process = self._start() + try: + self._check_migrated(['/s1', '/s2']) + # A new source joins the shared record. + self._report('MIG.CODE', '/s4') + self._check_migrated(['/s1', '/s2', '/s4']) + # A per_source code gets one record per source, each owned by its source. + for source in ('/p1', '/p2'): + self._report('PG.PER', source) + self._check_per_source_owners(['/p1', '/p2']) + finally: + self._stop(process) + + process = self._start() + try: + self._check_migrated(['/s1', '/s2', '/s4']) + self._check_per_source_owners(['/p1', '/p2']) + finally: + self._stop(process) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py new file mode 100644 index 000000000..8bfdf490a --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_shared_events.test.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +The shared record of a default code, through a real fault manager on both stores. + +Two sources reporting one default code feed one record: the first report confirms it, the +second updates it, and the update names both sources with the occurrence count still 1. +""" + +import os +import shutil +import sys +import tempfile +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import launch_testing.markers +import rclpy +from rclpy.node import Node +from rclpy.qos import QoSProfile, ReliabilityPolicy +from ros2_medkit_msgs.msg import Fault, FaultEvent +from ros2_medkit_msgs.srv import ReportFault + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each, wait_for_event_route # noqa: E402 + +TIMEOUT_SEC = 30.0 +_temp_dirs = [] + + +@launch_testing.parametrize('storage_type', ['memory', 'sqlite']) +@launch_testing.markers.keep_alive +def generate_test_description(storage_type): + temp_dir = tempfile.mkdtemp(prefix='shared_events_') + _temp_dirs.append(temp_dir) + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': storage_type, + 'database_path': os.path.join(temp_dir, 'faults.db'), + 'confirmation_threshold': -1, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node, 'storage_type': storage_type}, + ) + + +class TestSharedRecordEvents(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_shared_events_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + assert cls.report_client.wait_for_service(timeout_sec=TIMEOUT_SEC) + cls.events = [] + cls.subscription = cls.node.create_subscription( + FaultEvent, '/fault_manager/events', cls.events.append, + QoSProfile(depth=100, reliability=ReliabilityPolicy.RELIABLE)) + answer_once_each(cls.node, (cls.report_client,), TIMEOUT_SEC) + wait_for_event_route(cls.node, cls.report_client, cls.events, 'SE.PROBE_', TIMEOUT_SEC) + cls.events.clear() + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = f'{code} from {source}' + request.source_id = source + future = self.report_client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.result().accepted) + + def _wait_events(self, count): + deadline = time.monotonic() + TIMEOUT_SEC + while len(self.events) < count and time.monotonic() < deadline: + rclpy.spin_once(self.node, timeout_sec=0.1) + self.assertGreaterEqual(len(self.events), count) + + def test_a_second_source_updates_the_one_shared_record(self, storage_type): + self._report('SE.DEFAULT', '/a') + self._report('SE.DEFAULT', '/b') + self._wait_events(2) + + events = [(e.event_type, e.fault) for e in self.events + if e.fault.fault_code == 'SE.DEFAULT'] + self.assertEqual([t for t, _ in events], + [FaultEvent.EVENT_CONFIRMED, FaultEvent.EVENT_UPDATED], storage_type) + updated = events[1][1] + self.assertEqual(list(updated.reporting_sources), ['/a', '/b']) + self.assertEqual(updated.occurrence_count, 1) + + +@launch_testing.post_shutdown_test() +class TestSharedRecordEventsShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info, fault_manager_node): + launch_testing.asserts.assertExitCodes(proc_info, process=fault_manager_node) + + def test_temp_dirs_removed(self): + for path in _temp_dirs: + shutil.rmtree(path, ignore_errors=True) + self.assertFalse(os.path.exists(path)) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py new file mode 100644 index 000000000..b169bab15 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_cap.test.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Snapshots of per-source records with the recapture cooldown off, through a real fault manager. + + * a cooldown of 0 turns grouping off: two owners confirmed back to back are two captures; + * the per-record cap keeps the newest capture of each record, both topics of it, and one + owner's captures never trim another owner's. + +The cap is read from the store the fault manager writes, because GetSnapshots only ever +returns the newest capture of a record. +""" + +import json +import os +import shutil +import sqlite3 +import sys +import tempfile +import threading +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetSnapshots, ReportFault +from std_msgs.msg import Int32 + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402, I100 + +TIMEOUT_SEC = 40.0 +FIRST = '/cap/first' +SECOND = '/cap/second' +TEMP_DIR = tempfile.mkdtemp(prefix='snapshot_cap_') +DB_PATH = os.path.join(TEMP_DIR, 'faults.db') + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file( + 'snapshot_cap_codes_', ''.join(f'{c}:\n per_source: true\n' for c in ('XC.TWO', 'XC.CAP'))) +SNAPSHOTS_FILE = _temp_file( + 'snapshot_cap_topics_', + 'fault_specific:\n' + f' XC.TWO:\n - {FIRST}\n' + f' XC.CAP:\n - {FIRST}\n - {SECOND}\n') + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'sqlite', + 'database_path': DB_PATH, + 'confirmation_threshold': -1, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'snapshots.enabled': True, + 'snapshots.config_file': SNAPSHOTS_FILE, + 'snapshots.timeout_sec': 3.0, + 'snapshots.capture_pool_size': 3, + 'snapshots.recapture_cooldown_sec': 0.0, + 'snapshots.max_per_fault': 1, + 'snapshots.retain_on_clear': True, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestSnapshotCap(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_snapshot_cap_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + cls.snapshots_client = cls.node.create_client( + GetSnapshots, '/fault_manager/get_snapshots') + clients = (cls.report_client, cls.clear_client, cls.snapshots_client) + for client in clients: + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) + cls.publishers = [cls.node.create_publisher(Int32, topic, 10) for topic in (FIRST, SECOND)] + cls.stop = threading.Event() + cls.publisher_thread = threading.Thread(target=cls._publish) + cls.publisher_thread.start() + + @classmethod + def _publish(cls): + value = 0 + while not cls.stop.wait(0.1): + value += 1 + for publisher in cls.publishers: + publisher.publish(Int32(data=value)) + + warm = False + + def setUp(self): + # The node samples only topics it has discovered. Raise a throw-away fault until a + # capture of it holds both topics, so the first test does not race the discovery. + if TestSnapshotCap.warm: + return + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + self._report('XC.CAP', '/cap/warmup') + request = GetSnapshots.Request() + request.fault_code = 'XC.CAP' + request.source_id = '/cap/warmup' + for _ in range(20): + response = self._call(self.snapshots_client, request) + if response.success and response.data and json.loads(response.data).get('topics'): + break + rclpy.spin_once(self.node, timeout_sec=0.1) + data = json.loads(response.data) if response.success and response.data else {} + if sorted(data.get('topics', {})) == [FIRST, SECOND]: + TestSnapshotCap.warm = True + return + self._clear('XC.CAP', '/cap/warmup') + self.fail('the fault manager never captured both topics') + + @classmethod + def tearDownClass(cls): + cls.stop.set() + cls.publisher_thread.join() + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_CRITICAL + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _clear(self, code, source): + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + self.assertTrue(self._call(self.clear_client, request).success) + + def _wait_snapshot(self, code, source, newer_than=0.0): + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + request = GetSnapshots.Request() + request.fault_code = code + request.source_id = source + response = self._call(self.snapshots_client, request) + if response.success and response.data: + data = json.loads(response.data) + if data.get('topics') and data['captured_at'] > newer_than: + return data + rclpy.spin_once(self.node, timeout_sec=0.1) + self.fail(f'{code}/{source} never listed a snapshot newer than {newer_than}') + + def _stored(self, code): + """Return {owner: {capture_id: [topics]}} as the store holds it.""" + with sqlite3.connect(f'file:{DB_PATH}?mode=ro', uri=True) as db: + rows = db.execute( + 'SELECT owner, capture_id, topic FROM snapshots WHERE fault_code = ?', + (code,)).fetchall() + stored = {} + for owner, capture_id, topic in rows: + stored.setdefault(owner, {}).setdefault(capture_id, []).append(topic) + return stored + + def test_a_cooldown_of_zero_turns_grouping_off(self): + self._report('XC.TWO', '/cap/a') + self._report('XC.TWO', '/cap/b') + self._wait_snapshot('XC.TWO', '/cap/a') + self._wait_snapshot('XC.TWO', '/cap/b') + stored = self._stored('XC.TWO') + self.assertEqual(sorted(stored), ['/cap/a', '/cap/b']) + capture_ids = {cid for captures in stored.values() for cid in captures} + self.assertEqual(len(capture_ids), 2, 'two owners must be two captures') + + def test_the_cap_keeps_the_newest_capture_of_each_record(self): + self._report('XC.CAP', '/cap/a') + first = self._wait_snapshot('XC.CAP', '/cap/a') + self._clear('XC.CAP', '/cap/a') + self._report('XC.CAP', '/cap/a') + second = self._wait_snapshot('XC.CAP', '/cap/a', newer_than=first['captured_at']) + self._report('XC.CAP', '/cap/b') + own = self._wait_snapshot('XC.CAP', '/cap/b') + + self.assertEqual(sorted(second['topics']), [FIRST, SECOND]) + self.assertEqual(sorted(own['topics']), [FIRST, SECOND]) + stored = self._stored('XC.CAP') + self.assertEqual(len(stored['/cap/a']), 1, 'the cap keeps one capture per record') + self.assertEqual(sorted(next(iter(stored['/cap/a'].values()))), [FIRST, SECOND]) + self.assertEqual(len(stored['/cap/b']), 1) + self.assertEqual(sorted(next(iter(stored['/cap/b'].values()))), [FIRST, SECOND]) + self.assertNotEqual(stored['/cap/a'].keys(), stored['/cap/b'].keys()) + + +@launch_testing.post_shutdown_test() +class TestSnapshotCapShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(SNAPSHOTS_FILE) + shutil.rmtree(TEMP_DIR, ignore_errors=True) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py new file mode 100644 index 000000000..d3d9a2c4b --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_grouping.test.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Snapshot capture of per-source records, through a real fault manager. + +Per-source records of one code that confirm inside the recapture cooldown share ONE capture of +the code's topics, and every record lists those snapshots under its own owner. The rows of one +capture all carry the same captured_at, so two captures of one code cannot be mistaken for one. +A record that confirms again after it was acknowledged gets a capture of its own. +""" + +import json +import os +import sys +import tempfile +import threading +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetSnapshots, ReportFault +from std_msgs.msg import Int32 + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402, I100 + +TIMEOUT_SEC = 30.0 +SOURCES = ['/grp/a', '/grp/b', '/grp/c'] + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file( + 'snapshot_grouping_codes_', + 'XG.GROUP:\n per_source: true\nXG.AGAIN:\n per_source: true\n') +SNAPSHOTS_FILE = _temp_file( + 'snapshot_grouping_topics_', + 'fault_specific:\n' + ' XG.GROUP:\n - /grouping/counter\n' + ' XG.AGAIN:\n - /grouping/counter\n' + ' XG.SHARED:\n - /grouping/counter\n' + 'patterns:\n' + ' "XG_PROBE_.*":\n - /grouping/counter\n') + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'snapshots.enabled': True, + 'snapshots.config_file': SNAPSHOTS_FILE, + 'snapshots.timeout_sec': 5.0, + 'snapshots.capture_pool_size': 3, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestSnapshotGrouping(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_snapshot_grouping_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + cls.snapshots_client = cls.node.create_client( + GetSnapshots, '/fault_manager/get_snapshots') + clients = (cls.report_client, cls.clear_client, cls.snapshots_client) + for client in clients: + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) + cls.publisher = cls.node.create_publisher(Int32, '/grouping/counter', 10) + cls.stop = threading.Event() + cls.publisher_thread = threading.Thread(target=cls._publish) + cls.publisher_thread.start() + cls._wait_for_discovery('XG_PROBE_') + + @classmethod + def _wait_for_discovery(cls, topic_probe_prefix): + """Report fresh probe codes until the fault manager has discovered the probed topic.""" + deadline = time.monotonic() + TIMEOUT_SEC + index = 0 + while time.monotonic() < deadline: + index += 1 + code = f'{topic_probe_prefix}{index}' + report = ReportFault.Request() + report.fault_code = code + report.event_type = ReportFault.Request.EVENT_FAILED + report.severity = Fault.SEVERITY_CRITICAL + report.description = 'discovery probe' + report.source_id = '/probe' + future = cls.report_client.call_async(report) + rclpy.spin_until_future_complete(cls.node, future, timeout_sec=10.0) + request = GetSnapshots.Request() + request.fault_code = code + attempt_end = time.monotonic() + 1.5 + while time.monotonic() < attempt_end: + future = cls.snapshots_client.call_async(request) + rclpy.spin_until_future_complete(cls.node, future, timeout_sec=10.0) + response = future.result() + if response is not None and response.success and response.data: + if json.loads(response.data).get('topics'): + return + rclpy.spin_once(cls.node, timeout_sec=0.1) + raise AssertionError('the fault manager never discovered the published topic') + + @classmethod + def _publish(cls): + value = 0 + while not cls.stop.wait(0.2): + value += 1 + cls.publisher.publish(Int32(data=value)) + + @classmethod + def tearDownClass(cls): + cls.stop.set() + cls.publisher_thread.join() + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_CRITICAL + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _snapshot(self, code, source): + request = GetSnapshots.Request() + request.fault_code = code + request.source_id = source + return self._call(self.snapshots_client, request) + + def _wait_snapshot(self, code, source, newer_than=0.0): + """Poll GetSnapshots until the record lists a capture newer than @p newer_than.""" + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + response = self._snapshot(code, source) + if response.success and response.data: + data = json.loads(response.data) + if data.get('topics') and data['captured_at'] > newer_than: + return data + rclpy.spin_once(self.node, timeout_sec=0.1) + self.fail(f'{code}/{source} never listed a snapshot') + + def test_per_source_records_share_one_capture(self): + for source in SOURCES: + self._report('XG.GROUP', source) + + captures = [self._wait_snapshot('XG.GROUP', source) for source in SOURCES] + for data in captures: + self.assertIn('/grouping/counter', data['topics']) + # One capture: every owner lists the very same sampled values and the same moment. + self.assertEqual({d['captured_at'] for d in captures}, {captures[0]['captured_at']}) + self.assertEqual( + {json.dumps(d['topics'], sort_keys=True) for d in captures}, + {json.dumps(captures[0]['topics'], sort_keys=True)}) + + # Three records carry the code, so an unscoped call is refused and returns no evidence. + unscoped = self._snapshot('XG.GROUP', '') + self.assertFalse(unscoped.success) + self.assertTrue(unscoped.error_message.startswith('ambiguous:'), unscoped.error_message) + self.assertEqual(unscoped.data, '') + + def test_a_record_that_confirms_again_gets_a_capture_of_its_own(self): + code = 'XG.AGAIN' + source = '/grp/again' + self._report(code, source) + first = self._wait_snapshot(code, source) + + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + self.assertTrue(self._call(self.clear_client, request).success) + + self._report(code, source) + second = self._wait_snapshot(code, source, newer_than=first['captured_at']) + self.assertGreater(second['captured_at'], first['captured_at']) + + def test_a_shared_record_is_captured_once(self): + for source in SOURCES: + self._report('XG.SHARED', source) + data = self._wait_snapshot('XG.SHARED', '') + self.assertIn('/grouping/counter', data['topics']) + + +@launch_testing.post_shutdown_test() +class TestSnapshotGroupingShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(SNAPSHOTS_FILE) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py new file mode 100644 index 000000000..9d0666f78 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_fault_snapshot_queue.test.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Shared snapshot capture of per-source records under queue pressure, through a real fault manager. + +One worker, a queue of two and drop_oldest, so every extra queued job evicts another fault's +capture. A capture of the slow topic runs until the test releases it: the topic is silent until +then, and the capture timeout is longer than any test. + + * a record joining the queued capture of its code queues no job of its own; + * a capture that outlives the recapture cooldown keeps every record that joined it; + * a record that confirms again while its old capture runs gets a sample taken afterwards. +""" + +import json +import os +import sys +import tempfile +import threading +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ClearFault, GetSnapshots, ReportFault +from std_msgs.msg import Int32 + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402, I100 + +TIMEOUT_SEC = 40.0 +COOLDOWN_SEC = 0.3 +FAST = '/queue/fast' +SLOW = '/queue/slow' + + +def _temp_file(prefix, content): + handle = tempfile.NamedTemporaryFile('w', suffix='.yaml', prefix=prefix, delete=False) + handle.write(content) + handle.close() + return handle.name + + +THRESHOLDS_FILE = _temp_file( + 'snapshot_queue_codes_', + ''.join(f'{code}:\n per_source: true\n' + for code in ('XQ.QUEUE', 'XQ.PRUNE', 'XQ.PRUNE.TRIGGER', 'XQ.AGAIN'))) +SNAPSHOTS_FILE = _temp_file( + 'snapshot_queue_topics_', + 'fault_specific:\n' + f' XQ.BUSY:\n - {SLOW}\n' + f' XQ.OTHER:\n - {FAST}\n' + f' XQ.QUEUE:\n - {FAST}\n' + f' XQ.PRUNE:\n - {FAST}\n - {SLOW}\n' + f' XQ.PRUNE.TRIGGER:\n - {FAST}\n' + f' XQ.AGAIN:\n - {FAST}\n - {SLOW}\n' + 'patterns:\n' + f' "XQ_PROBE_.*":\n - {FAST}\n') + + +def generate_test_description(): + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'memory', + 'confirmation_threshold': -1, + 'fault_thresholds.config_file': THRESHOLDS_FILE, + 'snapshots.enabled': True, + 'snapshots.config_file': SNAPSHOTS_FILE, + 'snapshots.timeout_sec': TIMEOUT_SEC, + 'snapshots.capture_pool_size': 1, + 'snapshots.capture_queue_depth': 2, + 'snapshots.capture_queue_full_policy': 'drop_oldest', + 'snapshots.recapture_cooldown_sec': COOLDOWN_SEC, + 'snapshots.rosbag.enabled': False, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestSnapshotQueue(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_snapshot_queue_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + cls.snapshots_client = cls.node.create_client( + GetSnapshots, '/fault_manager/get_snapshots') + clients = (cls.report_client, cls.clear_client, cls.snapshots_client) + for client in clients: + assert client.wait_for_service(timeout_sec=TIMEOUT_SEC) + answer_once_each(cls.node, clients, TIMEOUT_SEC) + # Silent until a test sets release. Created first: the fault manager has processed it + # once it has discovered FAST. + cls.slow = cls.node.create_publisher(Int32, SLOW, 10) + cls.fast = cls.node.create_publisher(Int32, FAST, 10) + cls.stop = threading.Event() + cls.release = threading.Event() + cls.publisher_thread = threading.Thread(target=cls._publish) + cls.publisher_thread.start() + cls._wait_for_discovery('XQ_PROBE_') + + @classmethod + def _wait_for_discovery(cls, topic_probe_prefix): + """Report fresh probe codes until the fault manager has discovered the probed topic.""" + deadline = time.monotonic() + TIMEOUT_SEC + index = 0 + while time.monotonic() < deadline: + index += 1 + code = f'{topic_probe_prefix}{index}' + report = ReportFault.Request() + report.fault_code = code + report.event_type = ReportFault.Request.EVENT_FAILED + report.severity = Fault.SEVERITY_CRITICAL + report.description = 'discovery probe' + report.source_id = '/probe' + future = cls.report_client.call_async(report) + rclpy.spin_until_future_complete(cls.node, future, timeout_sec=10.0) + request = GetSnapshots.Request() + request.fault_code = code + attempt_end = time.monotonic() + 1.5 + while time.monotonic() < attempt_end: + future = cls.snapshots_client.call_async(request) + rclpy.spin_until_future_complete(cls.node, future, timeout_sec=10.0) + response = future.result() + if response is not None and response.success and response.data: + if json.loads(response.data).get('topics'): + return + rclpy.spin_once(cls.node, timeout_sec=0.1) + raise AssertionError('the fault manager never discovered the published topic') + + @classmethod + def _publish(cls): + value = 0 + while not cls.stop.wait(0.1): + value += 1 + cls.fast.publish(Int32(data=value)) + if cls.release.is_set(): + cls.slow.publish(Int32(data=value)) + + @classmethod + def tearDownClass(cls): + cls.stop.set() + cls.publisher_thread.join() + cls.node.destroy_node() + rclpy.shutdown() + + def setUp(self): + # A capture of SLOW subscribes when it starts, so it gets nothing published before. + self.release.clear() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _report(self, code, source): + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_CRITICAL + request.description = f'{code} from {source}' + request.source_id = source + self.assertTrue(self._call(self.report_client, request).accepted) + + def _clear(self, code, source): + request = ClearFault.Request() + request.fault_code = code + request.source_id = source + self.assertTrue(self._call(self.clear_client, request).success) + + def _snapshot(self, code, source): + """Return the record's snapshot data, or None while it lists none.""" + request = GetSnapshots.Request() + request.fault_code = code + request.source_id = source + response = self._call(self.snapshots_client, request) + if response.success and response.data: + data = json.loads(response.data) + if data.get('topics'): + return data + return None + + def _wait_snapshot(self, code, source, between=None): + """Poll until the record lists a snapshot; run @p between after every miss.""" + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + data = self._snapshot(code, source) + if data: + return data + if between: + between() + rclpy.spin_once(self.node, timeout_sec=0.1) + self.fail(f'{code}/{source} never listed a snapshot') + + def _wait_capture_started(self, proc_output, fault_manager_node, code): + proc_output.assertWaitFor( + f"Capturing snapshots for fault '{code}'", + process=fault_manager_node, timeout=TIMEOUT_SEC) + + def test_a_joining_record_does_not_evict_another_faults_capture( + self, proc_output, fault_manager_node): + # The worker is busy with a capture that waits on a silent topic. + self._report('XQ.BUSY', '/q/busy') + self._wait_capture_started(proc_output, fault_manager_node, 'XQ.BUSY') + # The oldest queued job belongs to another fault. + self._report('XQ.OTHER', '/q/other') + # Two owners of one code: one queued job for both. A second job would fill the queue and + # evict the oldest one, which is the other fault's. + self._report('XQ.QUEUE', '/q/a') + self._report('XQ.QUEUE', '/q/b') + self.release.set() + + self._wait_snapshot('XQ.OTHER', '/q/other') + first = self._wait_snapshot('XQ.QUEUE', '/q/a') + second = self._wait_snapshot('XQ.QUEUE', '/q/b') + self.assertEqual(first['captured_at'], second['captured_at']) + + def test_a_capture_that_outlives_the_cooldown_keeps_its_records( + self, proc_output, fault_manager_node): + self._report('XQ.PRUNE', '/q/a') + admitted = time.monotonic() + self._wait_capture_started(proc_output, fault_manager_node, 'XQ.PRUNE') + self._report('XQ.PRUNE', '/q/b') + + # Every admission of another grouped code is a chance to drop an aged-out group. The + # capture runs until a trigger is admitted after the cooldown. + triggers = iter(range(10000)) + + def admit_trigger(): + sent = time.monotonic() + self._report('XQ.PRUNE.TRIGGER', f'/q/trigger{next(triggers)}') + if sent - admitted > COOLDOWN_SEC: + self.release.set() + + first = self._wait_snapshot('XQ.PRUNE', '/q/a', between=admit_trigger) + self.assertTrue(self.release.is_set(), 'the capture ended before it was released') + second = self._wait_snapshot('XQ.PRUNE', '/q/b') + self.assertEqual(first['captured_at'], second['captured_at']) + + def test_a_record_confirming_again_is_not_given_the_old_sample( + self, proc_output, fault_manager_node): + self._report('XQ.AGAIN', '/q/a') + self._wait_capture_started(proc_output, fault_manager_node, 'XQ.AGAIN') + self._report('XQ.AGAIN', '/q/b') + + # The record is acknowledged and confirms again while the first capture still waits on + # the silent topic. That capture sampled the previous occurrence. + self._clear('XQ.AGAIN', '/q/a') + confirmed_again = time.time() + self._report('XQ.AGAIN', '/q/a') + self.release.set() + + old = self._wait_snapshot('XQ.AGAIN', '/q/b') + again = self._wait_snapshot('XQ.AGAIN', '/q/a') + self.assertNotEqual(again['captured_at'], old['captured_at'], + 'the record was given the sample of its previous occurrence') + self.assertGreater(again['captured_at'], confirmed_again) + + +@launch_testing.post_shutdown_test() +class TestSnapshotQueueShutdown(unittest.TestCase): + + def test_the_slow_topic_was_discovered(self, proc_output, fault_manager_node): + for text in proc_output[fault_manager_node]: + self.assertNotIn( + f"Topic '{SLOW}' not found", text.text.decode(errors='replace'), + 'a capture skipped the slow topic, so nothing was kept busy') + + def test_exit_code(self, proc_info): + launch_testing.asserts.assertExitCodes(proc_info) + os.unlink(THRESHOLDS_FILE) + os.unlink(SNAPSHOTS_FILE) diff --git a/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py index 1743b40be..7595511d3 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_integration.test.py @@ -15,8 +15,11 @@ """Integration tests for ros2_medkit_fault_manager services.""" +import atexit import json import os +import sys +import tempfile import threading import time import unittest @@ -32,6 +35,9 @@ from ros2_medkit_msgs.srv import ClearFault, GetSnapshots, ListFaults, ReportFault from sensor_msgs.msg import Temperature +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each # noqa: E402, I100 + def get_coverage_env(): """ @@ -72,6 +78,13 @@ def get_coverage_env(): def generate_test_description(): """Generate launch description with fault_manager node.""" + # One code gets a record per source, for the scoped-clear test. Every other code is shared. + per_source_codes = tempfile.NamedTemporaryFile( + 'w', suffix='.yaml', prefix='per_source_codes_', delete=False) + per_source_codes.write('TEST_SCOPED_CLEAR:\n per_source: true\n') + per_source_codes.close() + atexit.register(os.unlink, per_source_codes.name) + # Get path to test snapshot config pkg_share = get_package_share_directory('ros2_medkit_fault_manager') snapshot_config = os.path.join(pkg_share, 'test', 'test_snapshots.yaml') @@ -96,6 +109,7 @@ def generate_test_description(): 'snapshots.capture_queue_depth': 2, 'snapshots.capture_queue_full_policy': 'reject_newest', 'correlation.config_file': correlation_config, # Enable correlation + 'fault_thresholds.config_file': per_source_codes.name, }], # Give the node room to flush coverage data at shutdown before SIGKILL. sigterm_timeout='30', @@ -150,6 +164,8 @@ def setUpClass(cls): 'clear_fault service not available' assert cls.get_snapshots_client.wait_for_service(timeout_sec=10.0), \ 'get_snapshots service not available' + answer_once_each(cls.node, (cls.report_fault_client, cls.list_faults_client, + cls.clear_fault_client, cls.get_snapshots_client), 30.0) @classmethod def tearDownClass(cls): @@ -976,6 +992,56 @@ def test_26_cleared_fault_can_be_reactivated(self): self.assertEqual(len(fault.reporting_sources), 2) print(f'Cleared fault reactivated: occurrence_count={fault.occurrence_count}') + def test_26b_scoped_clear_leaves_the_other_owners_record(self): + """A scoped clear takes one record, an unscoped clear of two is refused.""" + fault_code = 'TEST_SCOPED_CLEAR' + owners = ['/owner_a', '/owner_b'] + + for owner in owners: + request = ReportFault.Request() + request.fault_code = fault_code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_CRITICAL + request.description = 'Two owners' + request.source_id = owner + self.assertTrue(self._call_service(self.report_fault_client, request).accepted) + + # Unscoped: two records carry the code, so the call is refused outright. + unscoped = ClearFault.Request() + unscoped.fault_code = fault_code + unscoped_response = self._call_service(self.clear_fault_client, unscoped) + self.assertFalse(unscoped_response.success) + self.assertTrue(unscoped_response.message.startswith('ambiguous:'), + unscoped_response.message) + for owner in owners: + self.assertIn(owner, unscoped_response.message) + + # Nothing was cleared by the refusal. + confirmed = ListFaults.Request() + confirmed.statuses = [Fault.STATUS_CONFIRMED] + still = [f for f in self._call_service(self.list_faults_client, confirmed).faults + if f.fault_code == fault_code] + self.assertEqual(len(still), 2) + + # Scoped: exactly one record moves. + scoped = ClearFault.Request() + scoped.fault_code = fault_code + scoped.source_id = '/owner_a' + self.assertTrue(self._call_service(self.clear_fault_client, scoped).success) + + after = self._call_service(self.list_faults_client, confirmed).faults + remaining = [f for f in after if f.fault_code == fault_code] + self.assertEqual(len(remaining), 1) + self.assertEqual(remaining[0].reporting_sources, ['/owner_b']) + + cleared_request = ListFaults.Request() + cleared_request.statuses = [Fault.STATUS_CLEARED] + cleared = [f for f in self._call_service(self.list_faults_client, cleared_request).faults + if f.fault_code == fault_code] + self.assertEqual(len(cleared), 1) + self.assertEqual(cleared[0].reporting_sources, ['/owner_a']) + print('Scoped clear took one record and left the other CONFIRMED') + def test_27_capture_pool_bounds_fault_storm(self, proc_output): """ Burst of distinct CRITICAL faults that overflows the bounded pool (issue #441). diff --git a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_history.test.py b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_history.test.py index 7187b9dc1..e169cd4c8 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_history.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_history.test.py @@ -14,7 +14,7 @@ # limitations under the License. """ -End-to-end tests for keeping several recordings per fault code (Issue #620). +End-to-end tests for keeping several recordings per fault record (Issue #620). The other rosbag suites all run at the shipped default of one recording per fault, which is what every release before this one could store. This one raises @@ -29,6 +29,7 @@ """ import os +import sys import tempfile import time import unittest @@ -42,10 +43,14 @@ import rclpy from rclpy.node import Node from rclpy.qos import HistoryPolicy, QoSProfile, ReliabilityPolicy -from ros2_medkit_msgs.msg import Fault, FaultEvent +from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import ClearFault, GetRosbag, ListRosbags, ReportFault from sensor_msgs.msg import Temperature +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from rosbag_ring_buffer import RecordedTopicWatch # noqa: E402, I100 +from service_round_trip import answer_once_each # noqa: E402, I100 + def get_coverage_env(): """Get environment variables for gcov coverage data collection.""" @@ -74,7 +79,7 @@ def get_coverage_env(): def generate_test_description(): - """Launch a fault_manager keeping several recordings per fault code.""" + """Launch a fault_manager keeping several recordings per fault record.""" publisher_script = """ import rclpy from rclpy.node import Node @@ -155,8 +160,8 @@ def main(): 'database_path': DATABASE_PATH, 'confirmation_threshold': -1, 'snapshots.rosbag.enabled': True, - 'snapshots.rosbag.duration_sec': 2.0, - 'snapshots.rosbag.duration_after_sec': 0.5, + 'snapshots.rosbag.duration_sec': RING_BUFFER_SEC, + 'snapshots.rosbag.duration_after_sec': POST_FAULT_SEC, 'snapshots.rosbag.topics': '/test/temperature,/test/status', 'snapshots.rosbag.format': 'mcap', 'snapshots.rosbag.storage_path': ROSBAG_STORAGE_PATH, @@ -209,10 +214,13 @@ def start_fault_manager_when_publisher_up(event): # Budget for the fault manager to come up, from the start of setUpClass. The # ctest TIMEOUT is 240 s, so the budget is well inside it. PRECONDITION_BUDGET_SEC = 70.0 +# The capture's ring buffer and post-fault window. +RING_BUFFER_SEC = 2.0 +POST_FAULT_SEC = 0.5 class TestRosbagHistory(unittest.TestCase): - """Several recordings per fault code, addressed by recording id.""" + """Several recordings per fault record, addressed by recording id.""" @classmethod def setUpClass(cls): @@ -269,12 +277,16 @@ def node_is_up(): break time.sleep(0.2) assert not missing, 'fault_manager not ready: ' + ', '.join(missing) + answer_once_each(cls.node, tuple(clients.values()), PRECONDITION_BUDGET_SEC) - deadline = time.time() + 15.0 - while (not cls.node.get_publishers_info_by_topic('/test/temperature') - and time.time() < deadline): - time.sleep(0.2) - time.sleep(3.0) + # Matches the publisher's BEST_EFFORT sensor QoS, else no messages arrive. + cls.recorded = RecordedTopicWatch( + cls.node, '/test/temperature', Temperature, + QoSProfile(reliability=ReliabilityPolicy.BEST_EFFORT, + history=HistoryPolicy.KEEP_LAST, depth=10), + RING_BUFFER_SEC, POST_FAULT_SEC) + assert cls.recorded.wait_for_history(0.5, 18.0), \ + 'the recorded topic never reached the ring buffer' @classmethod def tearDownClass(cls): @@ -331,50 +343,9 @@ def _get_by_recording(self, recording_id): request.recording_id = recording_id return self._call_service(self.get_rosbag_client, request) - def _wait_for_buffered_data(self, count=5, timeout=8.0): - """ - Wait until the ring buffer holds fresh data before a confirmation. - - A previous fault's post-roll diverts incoming messages straight into the - open bag, so right after that window closes the buffer can be empty and - the next confirmation produces no bag at all. Reset the streak on every - CONFIRMED so the messages counted here provably arrived with no post-roll - newly opened partway through - see the same helper in - test_rosbag_integration. - """ - received = 0 - - def _cb(_msg): - nonlocal received - received += 1 - - def _on_event(msg): - nonlocal received - if msg.event_type == FaultEvent.EVENT_CONFIRMED: - received = 0 - - qos = QoSProfile( - reliability=ReliabilityPolicy.BEST_EFFORT, - history=HistoryPolicy.KEEP_LAST, - depth=10, - ) - events_qos = QoSProfile( - reliability=ReliabilityPolicy.RELIABLE, - history=HistoryPolicy.KEEP_LAST, - depth=100, - ) - sub = self.node.create_subscription(Temperature, '/test/temperature', _cb, qos) - events_sub = self.node.create_subscription( - FaultEvent, '/fault_manager/events', _on_event, events_qos - ) - try: - deadline = time.time() + timeout - while received < count and time.time() < deadline: - rclpy.spin_once(self.node, timeout_sec=0.1) - finally: - self.node.destroy_subscription(sub) - self.node.destroy_subscription(events_sub) - return received >= count + def _wait_for_buffered_data(self, history_sec=0.5, timeout=8.0): + """Wait until a fault confirmed now gets of pre-fault history.""" + return self.recorded.wait_for_history(history_sec, timeout) def _record_occurrence(self, fault_code, known_ids): """Confirm the fault once and return its new recording.""" diff --git a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py index c1e41db99..09cc9dc2b 100644 --- a/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py +++ b/src/ros2_medkit_fault_manager/test/integration/test_rosbag_integration.test.py @@ -16,6 +16,7 @@ """Integration tests for rosbag capture feature (Issue #120).""" import os +import sys import tempfile import time import unittest @@ -29,10 +30,14 @@ import rclpy from rclpy.node import Node from rclpy.qos import HistoryPolicy, QoSProfile, ReliabilityPolicy -from ros2_medkit_msgs.msg import Fault, FaultEvent +from ros2_medkit_msgs.msg import Fault from ros2_medkit_msgs.srv import ClearFault, GetRosbag, ReportFault from sensor_msgs.msg import Temperature +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from rosbag_ring_buffer import RecordedTopicWatch # noqa: E402, I100 +from service_round_trip import answer_once_each # noqa: E402, I100 + def get_coverage_env(): """Get environment variables for gcov coverage data collection.""" @@ -59,6 +64,15 @@ def get_coverage_env(): # Path to temp publisher script (set in generate_test_description, cleaned up in shutdown test) PUBLISHER_SCRIPT_PATH = None +# One code gets a record per reporting source; every other code is shared. +PER_SOURCE_CODES_FILE = tempfile.NamedTemporaryFile( + 'w', suffix='.yaml', prefix='rosbag_per_source_', delete=False) +PER_SOURCE_CODES_FILE.write('BURST_PER_SOURCE:\n per_source: true\n') +PER_SOURCE_CODES_FILE.close() +# The capture's ring buffer and post-fault window. +RING_BUFFER_SEC = 2.0 +POST_FAULT_SEC = 0.5 + def generate_test_description(): """Generate launch description with fault_manager node with rosbag enabled.""" @@ -157,10 +171,11 @@ def main(): parameters=[{ 'storage_type': 'memory', 'confirmation_threshold': -1, # Single report confirms immediately + 'fault_thresholds.config_file': PER_SOURCE_CODES_FILE.name, # Rosbag configuration 'snapshots.rosbag.enabled': True, - 'snapshots.rosbag.duration_sec': 2.0, # 2 second buffer - 'snapshots.rosbag.duration_after_sec': 0.5, # 0.5 second after confirm + 'snapshots.rosbag.duration_sec': RING_BUFFER_SEC, + 'snapshots.rosbag.duration_after_sec': POST_FAULT_SEC, # Use explicit topics - faster than discovery, more reliable for tests 'snapshots.rosbag.topics': '/test/temperature,/test/status', 'snapshots.rosbag.format': 'mcap', @@ -277,15 +292,16 @@ def node_is_up(): break time.sleep(0.2) assert not missing, 'fault_manager not ready: ' + ', '.join(missing) + answer_once_each(cls.node, tuple(clients.values()), PRECONDITION_BUDGET_SEC) - # Wait for the background publisher to come up before letting the rosbag - # ring buffer fill (duration_sec=2.0 configured), so a slow publisher - # start does not leave the buffer empty. - deadline = time.time() + 15.0 - while (not cls.node.get_publishers_info_by_topic('/test/temperature') - and time.time() < deadline): - time.sleep(0.2) - time.sleep(3.0) + # Matches the publisher's BEST_EFFORT sensor QoS, else no messages arrive. + cls.recorded = RecordedTopicWatch( + cls.node, '/test/temperature', Temperature, + QoSProfile(reliability=ReliabilityPolicy.BEST_EFFORT, + history=HistoryPolicy.KEEP_LAST, depth=10), + RING_BUFFER_SEC, POST_FAULT_SEC) + assert cls.recorded.wait_for_history(0.5, 18.0), \ + 'the recorded topic never reached the ring buffer' @classmethod def tearDownClass(cls): @@ -329,59 +345,9 @@ def _wait_for_rosbag(self, fault_code, timeout=12.0): response = self._call_service(self.get_rosbag_client, request) return response - def _wait_for_buffered_data(self, count=5, timeout=8.0): - """ - Wait until the rosbag ring buffer holds fresh data before a confirmation. - - A previous fault's post-fault recording window diverts incoming messages - away from the ring buffer - message_callback() writes straight to the open - bag and returns before it ever touches the buffer - so right after the - window closes the buffer can be empty and flush_to_bag creates no bag. - There is no service to read the buffer, so this counts messages on the same - source the fault manager records instead, as a proxy. Counting alone is not - a safe proxy: a confirmation landing mid-count reopens that same diversion, - so ``count`` messages arriving at this node can still correspond to - near-zero buffered history on the fault manager's side. The broad topic - modes already subscribe to ``/fault_manager/events``, so this does too, and - resets the count on every ``EVENT_CONFIRMED`` - guaranteeing the streak of - messages this function finally returns on arrived with no post-fault window - newly opened partway through it. - """ - received = 0 - - def _cb(_msg): - nonlocal received - received += 1 - - def _on_event(msg): - nonlocal received - if msg.event_type == FaultEvent.EVENT_CONFIRMED: - received = 0 - - # Match the publisher's BEST_EFFORT sensor QoS, else no messages arrive. - qos = QoSProfile( - reliability=ReliabilityPolicy.BEST_EFFORT, - history=HistoryPolicy.KEEP_LAST, - depth=10, - ) - # Matches the fault manager's own reliable, depth-100 events publisher. - events_qos = QoSProfile( - reliability=ReliabilityPolicy.RELIABLE, - history=HistoryPolicy.KEEP_LAST, - depth=100, - ) - sub = self.node.create_subscription(Temperature, '/test/temperature', _cb, qos) - events_sub = self.node.create_subscription( - FaultEvent, '/fault_manager/events', _on_event, events_qos - ) - try: - deadline = time.time() + timeout - while received < count and time.time() < deadline: - rclpy.spin_once(self.node, timeout_sec=0.1) - finally: - self.node.destroy_subscription(sub) - self.node.destroy_subscription(events_sub) - return received >= count + def _wait_for_buffered_data(self, history_sec=0.5, timeout=8.0): + """Wait until a fault confirmed now gets of pre-fault history.""" + return self.recorded.wait_for_history(history_sec, timeout) def test_01_rosbag_created_on_fault_confirmation(self): """Test that rosbag file is created when fault is confirmed.""" @@ -523,20 +489,13 @@ def test_06_rosbag_contains_recorded_duration(self): """ Test that a full-buffer recording reports ~the configured window. - ``duration_sec`` is the recording's real span, not the configured - ``duration_sec + duration_after_sec``. The two only coincide when the - ring buffer actually held a full pre-fault window at confirmation, so - this test waits for that many messages (25 at the publisher's 10 Hz = - 2.5s > the 2.0s buffer) instead of the 5-message default, which only - guarantees ~0.5s of history and would legitimately report ~1.4s. + ``duration_sec`` is the recording's real span: its pre-fault history plus + the post-fault window. 1.5 s needs 1.0 s of history, so the test waits + until the buffer holds 1.2 s. """ fault_code = 'DURATION_TEST' - # Report fault only once the buffer holds a FULL pre-fault window. - # Same refill race as test_01/test_02: the previous test's post-fault window - # diverts messages away from the ring buffer, so reporting straight after it - # closes finds the buffer empty and no bag is created. - self.assertTrue(self._wait_for_buffered_data(count=25), + self.assertTrue(self._wait_for_buffered_data(history_sec=1.2), 'ring buffer never refilled after the previous post-fault window') response = self._report_fault(fault_code, 'Duration test fault') self.assertTrue(response.accepted) @@ -586,6 +545,58 @@ def test_07_burst_faults_share_one_recording(self): 'faults of one burst must share a single recording') print(f'Shared recording: {responses["BURST_SHARED_A"].file_path}') + # Default codes have one shared record each: one entry per code, owned by '*'. + for response in responses.values(): + self.assertEqual(sorted(response.fault_codes), fault_codes) + self.assertEqual(list(response.owners), ['*'] * len(fault_codes)) + self.assertEqual(len(response.owners), len(response.fault_codes)) + + def test_08_per_source_burst_lists_every_owner_of_the_recording(self): + """Two per-source records of one code on one recording are two parallel entries.""" + code = 'BURST_PER_SOURCE' + owners = ['/burst_node_a', '/burst_node_b'] + + self.assertTrue(self._wait_for_buffered_data(), + 'ring buffer did not refill before reporting faults') + + for owner in owners: + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = f'Per-source burst from {owner}' + request.source_id = owner + self.assertTrue(self._call_service(self.report_fault_client, request).accepted) + + responses = {} + for owner in owners: + deadline = time.time() + 12.0 + while True: + request = GetRosbag.Request() + request.fault_code = code + request.source_id = owner + response = self._call_service(self.get_rosbag_client, request) + if response.success or time.time() > deadline: + break + rclpy.spin_once(self.node, timeout_sec=0.5) + self.assertTrue(response.success, + f'GetRosbag failed for {owner}: {response.error_message}') + responses[owner] = response + + self.assertEqual(responses[owners[0]].file_path, responses[owners[1]].file_path, + 'records of one burst must share a single recording') + + # Two records carry the code, so an unscoped call is refused and returns no recording. + request = GetRosbag.Request() + request.fault_code = code + unscoped = self._call_service(self.get_rosbag_client, request) + self.assertFalse(unscoped.success) + self.assertTrue(unscoped.error_message.startswith('ambiguous:'), unscoped.error_message) + self.assertEqual(unscoped.file_path, '') + for response in responses.values(): + self.assertEqual(list(response.fault_codes), [code, code]) + self.assertEqual(sorted(response.owners), owners) + @launch_testing.post_shutdown_test() class TestRosbagCaptureShutdown(unittest.TestCase): @@ -597,6 +608,7 @@ def test_exit_code(self, proc_info): proc_info, process='fault_manager_node' ) + os.unlink(PER_SOURCE_CODES_FILE.name) def test_cleanup_temp_directory(self): """Clean up temporary rosbag storage directory and publisher script.""" diff --git a/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py b/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py new file mode 100644 index 000000000..4bd301700 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_severity_consistency.test.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Severity contract of a fault record through a real fault manager on SQLite. + +The database starts as one written before records had an owner, so the first +start migrates it. The fault manager is then restarted twice on it. Every +record here is the shared record of its code. A record keeps the highest +severity ever reported for it. A report on a CRITICAL record confirms it. A reactivated occurrence, +migrated or not, has no confirmation time until it confirms. A report whose +severity differs from the stored one is logged once per process. +""" + +import os +import shutil +import signal +import sqlite3 +import sys +import tempfile +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.asserts +import rclpy +from rclpy.node import Node +from rclpy.qos import QoSProfile, ReliabilityPolicy +from ros2_medkit_msgs.msg import Fault, FaultEvent +from ros2_medkit_msgs.srv import ClearFault, ListFaults, ReportFault + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from service_round_trip import answer_once_each, wait_for_event_route # noqa: E402 + +DB_DIR = tempfile.mkdtemp(prefix='medkit_severity_') +DB_PATH = os.path.join(DB_DIR, 'faults.db') +EVENT_TOPIC = '/fault_manager/events' +SOURCE = '/robot/reporter' +SHARED_OWNER = '*' +ALL_STATUSES = [ + Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED, Fault.STATUS_CONFIRMED, + Fault.STATUS_HEALED, Fault.STATUS_CLEARED, +] +MISMATCH_LINE = "Fault 'SEV_CRIT' reported with severity WARN" +TIMEOUT_SEC = 30.0 +LEGACY_CONFIRMED_AT_NS = 1_000_000_000 + +# The faults table as releases before record identity created it: one row per code. +LEGACY_FAULTS_DDL = ( + 'CREATE TABLE faults (fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, ' + 'description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, ' + 'last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, ' + 'status TEXT NOT NULL, reporting_sources TEXT NOT NULL, ' + 'debounce_counter INTEGER NOT NULL DEFAULT 0, ' + 'last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, ' + 'confirmed_at_ns INTEGER NOT NULL DEFAULT 0)') + + +# Every legacy row carries LEGACY_CONFIRMED_AT_NS. The pending ones were left so by a reactivation. +LEGACY_ROWS = ( + ('LEGACY_CONF_AT', Fault.SEVERITY_ERROR, Fault.STATUS_CLEARED), + ('LEGACY_CRIT', Fault.SEVERITY_CRITICAL, Fault.STATUS_CLEARED), + ('LEGACY_PREFAILED', Fault.SEVERITY_ERROR, Fault.STATUS_PREFAILED), + ('LEGACY_PREPASSED', Fault.SEVERITY_WARN, Fault.STATUS_PREPASSED), + ('LEGACY_CONFIRMED', Fault.SEVERITY_ERROR, Fault.STATUS_CONFIRMED), + ('LEGACY_HEALED', Fault.SEVERITY_ERROR, Fault.STATUS_HEALED), +) + + +def seed_legacy_database(): + """Write the LEGACY_ROWS into a faults table that has no owner column.""" + with sqlite3.connect(DB_PATH) as db: + db.execute(LEGACY_FAULTS_DDL) + for code, severity, status in LEGACY_ROWS: + db.execute( + 'INSERT INTO faults VALUES (?, ?, ?, ?, ?, 1, ?, ?, -3, ?, 0, ?)', + (code, severity, 'legacy', 500_000_000, 500_000_000, status, + f'["{SOURCE}"]', 500_000_000, LEGACY_CONFIRMED_AT_NS)) + + +# Codes whose records are per source, so the stored severity decides immediate confirmation. The +# migrated rows and every other code are shared records, where the report's severity decides. +PER_SOURCE_CODES = ('SEV_CRIT', 'HEAL_CRIT') + tuple(f'SEV_MARK_{i}' for i in range(1, 9)) +THRESHOLDS_PATH = os.path.join(DB_DIR, 'fault_thresholds.yaml') + + +def generate_test_description(): + seed_legacy_database() + with open(THRESHOLDS_PATH, 'w') as f: + for code in PER_SOURCE_CODES: + f.write(f'{code}:\n per_source: true\n') + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'sqlite', + 'database_path': DB_PATH, + 'confirmation_threshold': -3, + 'healing_enabled': True, + 'healing_threshold': 1, + 'fault_thresholds.config_file': THRESHOLDS_PATH, + 'snapshots.enabled': False, + 'snapshots.rosbag.enabled': False, + }], + # The test restarts the node by PID; launch brings it back on the same database. + respawn=True, + respawn_delay=1.0, + sigterm_timeout='30', + sigkill_timeout='15', + ) + return ( + LaunchDescription([fault_manager_node, launch_testing.actions.ReadyToTest()]), + {'fault_manager_node': fault_manager_node}, + ) + + +class TestSeverityConsistency(unittest.TestCase): + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_severity_consistency_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + cls.clear_client = cls.node.create_client(ClearFault, '/fault_manager/clear_fault') + cls.events = [] + qos = QoSProfile(depth=100, reliability=ReliabilityPolicy.RELIABLE) + cls.event_sub = cls.node.create_subscription( + FaultEvent, EVENT_TOPIC, cls.events.append, qos) + assert cls.report_client.wait_for_service(timeout_sec=TIMEOUT_SEC) + assert cls.list_client.wait_for_service(timeout_sec=TIMEOUT_SEC) + assert cls.clear_client.wait_for_service(timeout_sec=TIMEOUT_SEC) + cls.primes = 0 + cls.markers = 0 + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request, timeout=10.0): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=timeout) + if not future.done(): + client.remove_pending_request(future) + return None + return future.result() + + def _report(self, code, severity, event_type=ReportFault.Request.EVENT_FAILED): + request = ReportFault.Request() + request.fault_code = code + request.event_type = event_type + request.severity = severity + request.description = f'{code} report' + request.source_id = SOURCE + response = self._call(self.report_client, request) + self.assertIsNotNone(response, 'report_fault timed out') + self.assertTrue(response.accepted) + + def _passed(self, code): + self._report(code, Fault.SEVERITY_INFO, ReportFault.Request.EVENT_PASSED) + + def _clear(self, code): + request = ClearFault.Request() + request.fault_code = code + request.source_id = SOURCE + response = self._call(self.clear_client, request) + self.assertIsNotNone(response, 'clear_fault timed out') + self.assertTrue(response.success) + + def _list(self, statuses=None, severity=None): + request = ListFaults.Request() + request.statuses = statuses if statuses is not None else ALL_STATUSES + if severity is not None: + request.filter_by_severity = True + request.severity = severity + response = self._call(self.list_client, request) + self.assertIsNotNone(response, 'list_faults timed out') + return {f.fault_code: f for f in response.faults} + + def _fault(self, code): + fault = self._list().get(code) + self.assertIsNotNone(fault, f'{code} is not stored') + return fault + + def _confirmed_at(self, code): + with sqlite3.connect(DB_PATH, timeout=10.0) as db: + row = db.execute( + 'SELECT confirmed_at_ns FROM faults WHERE fault_code = ? AND owner = ?', + (code, SHARED_OWNER)).fetchone() + self.assertIsNotNone(row, f'{code} has no row in the database') + return row[0] + + def _wait_for_event(self, code, event_type, occurrence): + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + for event in list(self.events): + if (event.fault.fault_code, event.event_type, event.fault.occurrence_count) == \ + (code, event_type, occurrence): + return event + rclpy.spin_once(self.node, timeout_sec=0.1) + return None + + def _prime_events(self): + """Get one answer on every client and one event from the fault manager just started.""" + type(self).primes += 1 + answer_once_each( + self.node, (self.report_client, self.list_client, self.clear_client), TIMEOUT_SEC) + wait_for_event_route( + self.node, self.report_client, self.events, f'PRIME_{self.primes}_', TIMEOUT_SEC) + + def _restart(self, fault_manager_node): + old_pid = fault_manager_node.process_details['pid'] + os.kill(old_pid, signal.SIGTERM) + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline and fault_manager_node.process_details['pid'] == old_pid: + time.sleep(0.2) + self.assertNotEqual( + fault_manager_node.process_details['pid'], old_pid, + 'the fault manager was not restarted') + + probe = ListFaults.Request() + probe.statuses = ALL_STATUSES + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline: + if self.list_client.wait_for_service(timeout_sec=1.0): + if self._call(self.list_client, probe, timeout=2.0) is not None: + self._prime_events() + return + self.fail('the restarted fault manager never answered list_faults') + + def _mismatch_lines(self, proc_output): + """ + Count the SEV_CRIT mismatch warnings logged so far. + + A marker code mismatches once, after everything reported before it. The output is one + ordered stream, so once the marker's line has arrived every earlier line has too. + """ + type(self).markers += 1 + marker = f'SEV_MARK_{self.markers}' + self._report(marker, Fault.SEVERITY_CRITICAL) + self._report(marker, Fault.SEVERITY_WARN) + marker_line = f"Fault '{marker}' reported with severity WARN" + + def text(): + return ''.join(io.text.decode(errors='replace') for io in proc_output) + deadline = time.monotonic() + TIMEOUT_SEC + while time.monotonic() < deadline and marker_line not in text(): + time.sleep(0.1) + self.assertIn(marker_line, text(), 'the marker mismatch was never logged') + return text().count(MISMATCH_LINE) + + def test_severity_contract_across_restarts(self, fault_manager_node, proc_output): + self._prime_events() + + # Process 1 migrated the legacy rows. Only a row that confirmed keeps its time. + for code, _, status in LEGACY_ROWS: + pending = status in (Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED) + with self.subTest(case='migrated row', code=code): + self.assertEqual(self._fault(code).status, status) + self.assertEqual(self._confirmed_at(code), + 0 if pending else LEGACY_CONFIRMED_AT_NS) + + # A cleared row's next FAILED starts a new occurrence. + self._report('LEGACY_CONF_AT', Fault.SEVERITY_ERROR) + self._report('LEGACY_CRIT', Fault.SEVERITY_WARN) + with self.subTest(case='migrated, reactivated, not yet confirmed'): + fault = self._fault('LEGACY_CONF_AT') + self.assertEqual(fault.status, Fault.STATUS_PREFAILED) + self.assertEqual(fault.occurrence_count, 2) + self.assertEqual(self._confirmed_at('LEGACY_CONF_AT'), 0) + with self.subTest(case='migrated shared CRITICAL, WARN: the report decides'): + fault = self._fault('LEGACY_CRIT') + self.assertEqual(fault.severity, Fault.SEVERITY_CRITICAL) + self.assertEqual(fault.status, Fault.STATUS_PREFAILED) + + # Process 1: CRITICAL, cleared; CRITICAL, healed; ERROR confirmed, cleared. + self._report('SEV_CRIT', Fault.SEVERITY_CRITICAL) + self.assertEqual(self._fault('SEV_CRIT').status, Fault.STATUS_CONFIRMED) + self._clear('SEV_CRIT') + + self._report('HEAL_CRIT', Fault.SEVERITY_CRITICAL) + self._passed('HEAL_CRIT') + self._passed('HEAL_CRIT') + self.assertEqual(self._fault('HEAL_CRIT').status, Fault.STATUS_HEALED) + + for _ in range(3): + self._report('CONF_AT', Fault.SEVERITY_ERROR) + self.assertEqual(self._fault('CONF_AT').status, Fault.STATUS_CONFIRMED) + first_confirmed_at = self._confirmed_at('CONF_AT') + self.assertGreater(first_confirmed_at, 0) + self._clear('CONF_AT') + + self._restart(fault_manager_node) + + # Process 2: one WARN report on a CRITICAL code confirms it, after a clear or a heal. + self._report('SEV_CRIT', Fault.SEVERITY_WARN) + self._report('HEAL_CRIT', Fault.SEVERITY_WARN) + with self.subTest(case='CRITICAL, clear, restart, WARN'): + fault = self._fault('SEV_CRIT') + self.assertEqual(fault.severity, Fault.SEVERITY_CRITICAL) + self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(fault.occurrence_count, 2) + event = self._wait_for_event('SEV_CRIT', FaultEvent.EVENT_CONFIRMED, 2) + self.assertIsNotNone(event, 'no fault_confirmed event for occurrence 2') + self.assertEqual(event.fault.severity, Fault.SEVERITY_CRITICAL) + + with self.subTest(case='CRITICAL, heal, restart, WARN'): + fault = self._fault('HEAL_CRIT') + self.assertEqual(fault.severity, Fault.SEVERITY_CRITICAL) + self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) + self.assertEqual(fault.occurrence_count, 1) + + with self.subTest(case='no pending CRITICAL'): + pending = self._list(statuses=[Fault.STATUS_PREFAILED, Fault.STATUS_PREPASSED], + severity=Fault.SEVERITY_CRITICAL) + # LEGACY_CRIT is a shared record, which a WARN report leaves pending. + self.assertEqual(sorted(pending), ['LEGACY_CRIT']) + + for _ in range(9): + self._report('SEV_CRIT', Fault.SEVERITY_WARN) + + self._report('CONF_AT', Fault.SEVERITY_ERROR) + with self.subTest(case='reactivated, not yet confirmed'): + self.assertEqual(self._fault('CONF_AT').status, Fault.STATUS_PREFAILED) + self.assertEqual(self._confirmed_at('CONF_AT'), 0) + + with self.subTest(case='mismatch logged once in this process'): + self.assertEqual(self._mismatch_lines(proc_output), 1) + + self._restart(fault_manager_node) + + # Process 3: the reactivated occurrences still have no confirmation time, then confirm. + with self.subTest(case='migrated rows after two restarts'): + self.assertEqual(self._confirmed_at('LEGACY_PREPASSED'), 0) + self.assertEqual(self._confirmed_at('LEGACY_CONFIRMED'), LEGACY_CONFIRMED_AT_NS) + self.assertEqual(self._confirmed_at('LEGACY_HEALED'), LEGACY_CONFIRMED_AT_NS) + with self.subTest(case='reactivated, restarted, not yet confirmed'): + self.assertEqual(self._confirmed_at('CONF_AT'), 0) + self.assertEqual(self._confirmed_at('LEGACY_CONF_AT'), 0) + self._report('LEGACY_CONF_AT', Fault.SEVERITY_ERROR) + self._report('LEGACY_CONF_AT', Fault.SEVERITY_ERROR) + self.assertEqual(self._fault('LEGACY_CONF_AT').status, Fault.STATUS_CONFIRMED) + with self.subTest(case='migrated record confirms its new occurrence'): + self.assertGreater(self._confirmed_at('LEGACY_CONF_AT'), LEGACY_CONFIRMED_AT_NS) + self._report('CONF_AT', Fault.SEVERITY_ERROR) + self._report('CONF_AT', Fault.SEVERITY_ERROR) + fault = self._fault('CONF_AT') + self.assertEqual(fault.status, Fault.STATUS_CONFIRMED) + started = fault.first_occurred + occurrence_start_ns = started.sec * 1_000_000_000 + started.nanosec + with self.subTest(case='confirmation time belongs to this occurrence'): + self.assertGreater(self._confirmed_at('CONF_AT'), occurrence_start_ns) + self.assertGreater(occurrence_start_ns, first_confirmed_at) + + for _ in range(10): + self._report('SEV_CRIT', Fault.SEVERITY_WARN) + with self.subTest(case='the bound restarts with the process'): + self.assertEqual(self._mismatch_lines(proc_output), 2) + + +@launch_testing.post_shutdown_test() +class TestSeverityConsistencyShutdown(unittest.TestCase): + + def test_exit_code(self, proc_info): + shutil.rmtree(DB_DIR, ignore_errors=True) + launch_testing.asserts.assertExitCodes(proc_info) diff --git a/src/ros2_medkit_fault_manager/test/pg_exec.cpp b/src/ros2_medkit_fault_manager/test/pg_exec.cpp new file mode 100644 index 000000000..91aaf1fd5 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/pg_exec.cpp @@ -0,0 +1,40 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Runs the SQL on stdin against the PostgreSQL server of the connection string in argv[1]. +// A launch test uses it to write and drop tables, since the test has no PostgreSQL client of its own. + +#include + +#include +#include +#include + +int main(int argc, char ** argv) { + if (argc != 2) { + std::cerr << "usage: pg_exec < statements.sql\n"; + return 2; + } + try { + std::ostringstream sql; + sql << std::cin.rdbuf(); + pqxx::connection connection(argv[1]); + pqxx::nontransaction tx(connection); + tx.exec(sql.str()); + } catch (const std::exception & e) { + std::cerr << "pg_exec failed: " << e.what() << "\n"; + return 1; + } + return 0; +} diff --git a/src/ros2_medkit_fault_manager/test/test_capture_thread_pool.cpp b/src/ros2_medkit_fault_manager/test/test_capture_thread_pool.cpp index defba8395..8fce2c335 100644 --- a/src/ros2_medkit_fault_manager/test/test_capture_thread_pool.cpp +++ b/src/ros2_medkit_fault_manager/test/test_capture_thread_pool.cpp @@ -33,6 +33,7 @@ using ros2_medkit_fault_manager::CaptureThreadPool; using ros2_medkit_fault_manager::EnqueueResult; +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::QueueFullPolicy; namespace { @@ -42,13 +43,21 @@ ros2_medkit_fault_manager::ros2::RosLogSink & test_logger() { return sink; } +// The pool queues fault RECORDS. These tests are about queue mechanics, so every +// job here belongs to one owner and the code alone tells the jobs apart. +constexpr const char * kOwner = "/reporter"; + +FaultId job(const std::string & code) { + return FaultId{code, kOwner}; +} + // Capture_fn that records every invocation and blocks each job until released, // so tests can deterministically hold workers busy and inspect concurrency. class GatedCallback { public: - std::function fn() { - return [this](const std::string & code) { - run(code); + std::function fn() { + return [this](const FaultId & id) { + run(id.fault_code); }; } @@ -132,7 +141,7 @@ TEST_F(CaptureThreadPoolTest, BoundsAndReachesPoolSize) { GatedCallback gc; CaptureThreadPool pool(2, 8, QueueFullPolicy::kRejectNewest, test_logger(), gc.fn()); for (int i = 0; i < 6; ++i) { - pool.enqueue("j" + std::to_string(i)); + pool.enqueue(job("j" + std::to_string(i))); } gc.wait_until_active(2); // prove 2 ran concurrently EXPECT_EQ(gc.max_active(), 2u); // never more than pool_size @@ -144,8 +153,8 @@ TEST_F(CaptureThreadPoolTest, BoundsAndReachesPoolSize) { TEST_F(CaptureThreadPoolTest, PoolSizeOneSerializes) { GatedCallback gc; CaptureThreadPool pool(1, 8, QueueFullPolicy::kRejectNewest, test_logger(), gc.fn()); - pool.enqueue("a"); - pool.enqueue("b"); + pool.enqueue(job("a")); + pool.enqueue(job("b")); gc.wait_until_active(1); EXPECT_EQ(gc.max_active(), 1u); gc.release(); @@ -159,13 +168,13 @@ TEST_F(CaptureThreadPoolTest, PoolSizeOneSerializes) { TEST_F(CaptureThreadPoolTest, ZeroSizesClampToOne) { GatedCallback gc; CaptureThreadPool pool(0, 0, QueueFullPolicy::kDropOldest, test_logger(), gc.fn()); - EXPECT_EQ(pool.enqueue("a").result, EnqueueResult::kAccepted); - gc.wait_until_active(1); // a worker exists (pool_size clamped to 1) - EXPECT_EQ(pool.enqueue("b").result, EnqueueResult::kAccepted); // queue_depth clamped to 1 - auto outcome = pool.enqueue("c"); // full -> evict oldest, no UB + EXPECT_EQ(pool.enqueue(job("a")).result, EnqueueResult::kAccepted); + gc.wait_until_active(1); // a worker exists (pool_size clamped to 1) + EXPECT_EQ(pool.enqueue(job("b")).result, EnqueueResult::kAccepted); // queue_depth clamped to 1 + auto outcome = pool.enqueue(job("c")); // full -> evict oldest, no UB EXPECT_EQ(outcome.result, EnqueueResult::kEvictedOldest); - ASSERT_TRUE(outcome.evicted_code.has_value()); - EXPECT_EQ(*outcome.evicted_code, "b"); + ASSERT_TRUE(outcome.evicted_id.has_value()); + EXPECT_EQ(outcome.evicted_id->fault_code, "b"); gc.release(); pool.shutdown(); } @@ -173,14 +182,14 @@ TEST_F(CaptureThreadPoolTest, ZeroSizesClampToOne) { TEST_F(CaptureThreadPoolTest, RejectNewestDropsExcessExactly) { GatedCallback gc; CaptureThreadPool pool(2, 8, QueueFullPolicy::kRejectNewest, test_logger(), gc.fn()); - pool.enqueue("blk0"); - pool.enqueue("blk1"); + pool.enqueue(job("blk0")); + pool.enqueue(job("blk1")); gc.wait_until_active(2); // both workers busy; queue empties for (int i = 0; i < 8; ++i) { // fill pending queue (queue_depth) - EXPECT_EQ(pool.enqueue("f" + std::to_string(i)).result, EnqueueResult::kAccepted); + EXPECT_EQ(pool.enqueue(job("f" + std::to_string(i))).result, EnqueueResult::kAccepted); } for (int i = 0; i < 5; ++i) { // overflow - EXPECT_EQ(pool.enqueue("x" + std::to_string(i)).result, EnqueueResult::kDroppedNewest); + EXPECT_EQ(pool.enqueue(job("x" + std::to_string(i))).result, EnqueueResult::kDroppedNewest); } EXPECT_EQ(pool.dropped_captures(), 5u); gc.release(); @@ -193,17 +202,17 @@ TEST_F(CaptureThreadPoolTest, RejectNewestDropsExcessExactly) { TEST_F(CaptureThreadPoolTest, DropOldestEvictsOldestPending) { GatedCallback gc; CaptureThreadPool pool(2, 8, QueueFullPolicy::kDropOldest, test_logger(), gc.fn()); - pool.enqueue("blk0"); - pool.enqueue("blk1"); + pool.enqueue(job("blk0")); + pool.enqueue(job("blk1")); gc.wait_until_active(2); for (int i = 0; i < 8; ++i) { - EXPECT_EQ(pool.enqueue("f" + std::to_string(i)).result, EnqueueResult::kAccepted); + EXPECT_EQ(pool.enqueue(job("f" + std::to_string(i))).result, EnqueueResult::kAccepted); } for (int i = 0; i < 5; ++i) { // each overflow evicts current oldest f0..f4 - auto outcome = pool.enqueue("x" + std::to_string(i)); + auto outcome = pool.enqueue(job("x" + std::to_string(i))); EXPECT_EQ(outcome.result, EnqueueResult::kEvictedOldest); - ASSERT_TRUE(outcome.evicted_code.has_value()); - EXPECT_EQ(*outcome.evicted_code, "f" + std::to_string(i)); + ASSERT_TRUE(outcome.evicted_id.has_value()); + EXPECT_EQ(outcome.evicted_id->fault_code, "f" + std::to_string(i)); } EXPECT_EQ(pool.dropped_captures(), 5u); gc.release(); @@ -221,10 +230,10 @@ TEST_F(CaptureThreadPoolTest, DropOldestEvictsOldestPending) { TEST_F(CaptureThreadPoolTest, FifoOrder) { GatedCallback gc; CaptureThreadPool pool(1, 8, QueueFullPolicy::kRejectNewest, test_logger(), gc.fn()); - pool.enqueue("A"); + pool.enqueue(job("A")); gc.wait_until_active(1); - pool.enqueue("B"); - pool.enqueue("C"); + pool.enqueue(job("B")); + pool.enqueue(job("C")); gc.release(); gc.wait_until_started(3); // A, B, C must all run before shutdown discards queue pool.shutdown(); @@ -236,8 +245,8 @@ TEST_F(CaptureThreadPoolTest, CallbackThrowKeepsWorkerAlive) { std::mutex m; std::condition_variable cv; int ran = 0; - auto fn = [&](const std::string & code) { - if (code == "boom") { + auto fn = [&](const FaultId & id) { + if (id.fault_code == "boom") { throw std::runtime_error("boom"); } { @@ -247,9 +256,9 @@ TEST_F(CaptureThreadPoolTest, CallbackThrowKeepsWorkerAlive) { cv.notify_all(); }; CaptureThreadPool pool(1, 8, QueueFullPolicy::kRejectNewest, test_logger(), fn); - pool.enqueue("boom"); - pool.enqueue("ok1"); - pool.enqueue("ok2"); + pool.enqueue(job("boom")); + pool.enqueue(job("ok1")); + pool.enqueue(job("ok2")); { std::unique_lock lock(m); cv.wait(lock, [&] { @@ -261,9 +270,9 @@ TEST_F(CaptureThreadPoolTest, CallbackThrowKeepsWorkerAlive) { } TEST_F(CaptureThreadPoolTest, EnqueueAfterShutdownRejected) { - CaptureThreadPool pool(1, 8, QueueFullPolicy::kRejectNewest, test_logger(), [](const std::string &) {}); + CaptureThreadPool pool(1, 8, QueueFullPolicy::kRejectNewest, test_logger(), [](const FaultId &) {}); pool.shutdown(); - auto outcome = pool.enqueue("x"); + auto outcome = pool.enqueue(job("x")); EXPECT_EQ(outcome.result, EnqueueResult::kRejectedShuttingDown); EXPECT_EQ(pool.dropped_captures(), 0u); } @@ -271,9 +280,9 @@ TEST_F(CaptureThreadPoolTest, EnqueueAfterShutdownRejected) { TEST_F(CaptureThreadPoolTest, ShutdownDiscardsPendingCompletesInFlight) { GatedCallback gc; CaptureThreadPool pool(1, 8, QueueFullPolicy::kRejectNewest, test_logger(), gc.fn()); - EXPECT_EQ(pool.enqueue("inflight").result, EnqueueResult::kAccepted); + EXPECT_EQ(pool.enqueue(job("inflight")).result, EnqueueResult::kAccepted); gc.wait_until_active(1); - EXPECT_EQ(pool.enqueue("pending").result, EnqueueResult::kAccepted); + EXPECT_EQ(pool.enqueue(job("pending")).result, EnqueueResult::kAccepted); EXPECT_EQ(pool.pending_size(), 1u); std::thread shut([&] { @@ -291,7 +300,7 @@ TEST_F(CaptureThreadPoolTest, ShutdownDiscardsPendingCompletesInFlight) { } TEST_F(CaptureThreadPoolTest, ShutdownIsIdempotent) { - CaptureThreadPool pool(2, 8, QueueFullPolicy::kRejectNewest, test_logger(), [](const std::string &) {}); + CaptureThreadPool pool(2, 8, QueueFullPolicy::kRejectNewest, test_logger(), [](const FaultId &) {}); pool.shutdown(); pool.shutdown(); // must not throw, hang, or double-join SUCCEED(); @@ -305,14 +314,14 @@ TEST_F(CaptureThreadPoolTest, ShutdownIsIdempotent) { TEST_F(CaptureThreadPoolTest, ConcurrentEnqueueDuringShutdownIsClean) { std::atomic ran{0}; std::atomic accepted{0}; - CaptureThreadPool pool(4, 64, QueueFullPolicy::kRejectNewest, test_logger(), [&ran](const std::string &) { + CaptureThreadPool pool(4, 64, QueueFullPolicy::kRejectNewest, test_logger(), [&ran](const FaultId &) { ran.fetch_add(1, std::memory_order_relaxed); }); std::atomic saw_shutdown{false}; std::thread producer([&] { for (int i = 0; i < 100000; ++i) { - const EnqueueResult r = pool.enqueue("j" + std::to_string(i)).result; + const EnqueueResult r = pool.enqueue(job("j" + std::to_string(i))).result; if (r == EnqueueResult::kAccepted) { accepted.fetch_add(1, std::memory_order_relaxed); } else if (r == EnqueueResult::kRejectedShuttingDown) { @@ -329,7 +338,7 @@ TEST_F(CaptureThreadPoolTest, ConcurrentEnqueueDuringShutdownIsClean) { // The producer must have crossed the shutdown transition (else the race was not // exercised), and every post-shutdown enqueue must reject cleanly. EXPECT_TRUE(saw_shutdown.load()); - EXPECT_EQ(pool.enqueue("after").result, EnqueueResult::kRejectedShuttingDown); + EXPECT_EQ(pool.enqueue(job("after")).result, EnqueueResult::kRejectedShuttingDown); // shutdown() discards pending work, so some accepted jobs may not run - but no // job may run that was never accepted. EXPECT_LE(ran.load(), accepted.load()); @@ -341,7 +350,7 @@ TEST_F(CaptureThreadPoolTest, ReleasesCaptureFnAfterShutdown) { auto sentinel = std::make_shared(42); { std::weak_ptr weak = sentinel; - CaptureThreadPool pool(2, 8, QueueFullPolicy::kRejectNewest, test_logger(), [held = sentinel](const std::string &) { + CaptureThreadPool pool(2, 8, QueueFullPolicy::kRejectNewest, test_logger(), [held = sentinel](const FaultId &) { (void)held; }); sentinel.reset(); // pool's capture_fn now holds the only strong ref diff --git a/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp b/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp index 3d803825b..68f2f5f12 100644 --- a/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp +++ b/src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp @@ -14,7 +14,9 @@ #include +#include #include +#include #include #include "ros2_medkit_fault_manager/correlation/config_parser.hpp" @@ -23,6 +25,26 @@ using namespace ros2_medkit_fault_manager::correlation; using namespace std::chrono_literals; +using ros2_medkit_fault_manager::FaultId; + +namespace { + +// The engine relates fault RECORDS. Rules match by code whoever owns the record, so most +// of this suite reports everything from one owner; the tests that name a second owner +// check that a relation crosses owners. +constexpr const char * kOwner = "/reporter_a"; +constexpr const char * kOtherOwner = "/reporter_b"; + +FaultId rec(const std::string & code) { + return FaultId{code, kOwner}; +} + +FaultId other(const std::string & code) { + return FaultId{code, kOtherOwner}; +} + +} // namespace + class CorrelationEngineTest : public ::testing::Test { protected: void SetUp() override { @@ -114,7 +136,7 @@ TEST_F(CorrelationEngineTest, RootCauseRecognized) { auto config = create_hierarchical_config(); CorrelationEngine engine(config); - auto result = engine.process_fault("ESTOP_001", "CRITICAL"); + auto result = engine.process_fault(rec("ESTOP_001"), "CRITICAL"); EXPECT_FALSE(result.should_mute); EXPECT_TRUE(result.is_root_cause); @@ -126,11 +148,11 @@ TEST_F(CorrelationEngineTest, SymptomMuted) { CorrelationEngine engine(config); // First, report root cause - auto root_result = engine.process_fault("ESTOP_001", "CRITICAL"); + auto root_result = engine.process_fault(rec("ESTOP_001"), "CRITICAL"); EXPECT_TRUE(root_result.is_root_cause); // Then report symptom - auto symptom_result = engine.process_fault("MOTOR_COMM_FL", "ERROR"); + auto symptom_result = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); EXPECT_TRUE(symptom_result.should_mute); EXPECT_FALSE(symptom_result.is_root_cause); @@ -151,10 +173,10 @@ TEST_F(CorrelationEngineTest, MultipleSymptomsMuted) { auto config = create_hierarchical_config(); CorrelationEngine engine(config); - engine.process_fault("ESTOP_001", "CRITICAL"); - engine.process_fault("MOTOR_COMM_FL", "ERROR"); - engine.process_fault("MOTOR_COMM_FR", "ERROR"); - engine.process_fault("DRIVE_FAULT", "ERROR"); + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); + engine.process_fault(rec("MOTOR_COMM_FR"), "ERROR"); + engine.process_fault(rec("DRIVE_FAULT"), "ERROR"); EXPECT_EQ(3u, engine.get_muted_count()); } @@ -164,7 +186,7 @@ TEST_F(CorrelationEngineTest, SymptomBeforeRootCauseNotCorrelated) { CorrelationEngine engine(config); // Report symptom BEFORE root cause - auto symptom_result = engine.process_fault("MOTOR_COMM_FL", "ERROR"); + auto symptom_result = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); // Should NOT be muted (no root cause yet) EXPECT_FALSE(symptom_result.should_mute); @@ -177,11 +199,11 @@ TEST_F(CorrelationEngineTest, SymptomAfterWindowNotCorrelated) { CorrelationEngine engine(config); auto start = std::chrono::steady_clock::now(); - engine.process_fault("ESTOP_001", "CRITICAL", start); + engine.process_fault(rec("ESTOP_001"), "CRITICAL", start); // Report symptom AFTER window (window is 1000ms) auto after_window = start + 1500ms; - auto symptom_result = engine.process_fault("MOTOR_COMM_FL", "ERROR", after_window); + auto symptom_result = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", after_window); // Should NOT be muted (outside window) EXPECT_FALSE(symptom_result.should_mute); @@ -192,22 +214,56 @@ TEST_F(CorrelationEngineTest, ClearRootCauseClearsSymptoms) { auto config = create_hierarchical_config(); CorrelationEngine engine(config); - engine.process_fault("ESTOP_001", "CRITICAL"); - engine.process_fault("MOTOR_COMM_FL", "ERROR"); - engine.process_fault("MOTOR_COMM_FR", "ERROR"); + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); + engine.process_fault(rec("MOTOR_COMM_FR"), "ERROR"); EXPECT_EQ(2u, engine.get_muted_count()); // Clear root cause - auto clear_result = engine.process_clear("ESTOP_001"); + auto clear_result = engine.process_clear(rec("ESTOP_001")); - EXPECT_EQ(2u, clear_result.auto_cleared_codes.size()); - EXPECT_NE(std::find(clear_result.auto_cleared_codes.begin(), clear_result.auto_cleared_codes.end(), "MOTOR_COMM_FL"), - clear_result.auto_cleared_codes.end()); - EXPECT_NE(std::find(clear_result.auto_cleared_codes.begin(), clear_result.auto_cleared_codes.end(), "MOTOR_COMM_FR"), - clear_result.auto_cleared_codes.end()); + EXPECT_EQ(2u, clear_result.auto_cleared_symptoms.size()); + const auto & symptoms = clear_result.auto_cleared_symptoms; + EXPECT_NE(std::find(symptoms.begin(), symptoms.end(), rec("MOTOR_COMM_FL")), symptoms.end()); + EXPECT_NE(std::find(symptoms.begin(), symptoms.end(), rec("MOTOR_COMM_FR")), symptoms.end()); - // Muted faults should be cleared + // Muted records should be cleared + EXPECT_EQ(0u, engine.get_muted_count()); +} + +// Rules match by code across owners; the relation then holds the exact records. +TEST_F(CorrelationEngineTest, RootCauseMutesMatchingSymptomsOfAnyOwner) { + auto config = create_hierarchical_config(); + CorrelationEngine engine(config); + + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + + const auto same_owner = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); + const auto other_owner = engine.process_fault(other("MOTOR_COMM_FL"), "ERROR"); + + EXPECT_TRUE(same_owner.should_mute); + EXPECT_TRUE(other_owner.should_mute); + EXPECT_TRUE(engine.is_muted(rec("MOTOR_COMM_FL"))); + EXPECT_TRUE(engine.is_muted(other("MOTOR_COMM_FL"))); + EXPECT_EQ(2u, engine.get_muted_count()); +} + +// The cascade acts on exactly the records the root cause muted. +TEST_F(CorrelationEngineTest, ClearRootCauseCascadesToTheRecordsItMuted) { + auto config = create_hierarchical_config(); + CorrelationEngine engine(config); + + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + engine.process_fault(other("MOTOR_COMM_FL"), "ERROR"); + engine.process_fault(rec("MOTOR_COMM_FR"), "ERROR"); + + const auto clear_result = engine.process_clear(rec("ESTOP_001")); + + const auto & symptoms = clear_result.auto_cleared_symptoms; + ASSERT_EQ(2u, symptoms.size()); + EXPECT_NE(std::find(symptoms.begin(), symptoms.end(), other("MOTOR_COMM_FL")), symptoms.end()); + EXPECT_NE(std::find(symptoms.begin(), symptoms.end(), rec("MOTOR_COMM_FR")), symptoms.end()); EXPECT_EQ(0u, engine.get_muted_count()); } @@ -215,8 +271,8 @@ TEST_F(CorrelationEngineTest, UnrelatedFaultNotCorrelated) { auto config = create_hierarchical_config(); CorrelationEngine engine(config); - engine.process_fault("ESTOP_001", "CRITICAL"); - auto result = engine.process_fault("SENSOR_TIMEOUT", "ERROR"); + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + auto result = engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR"); // SENSOR_TIMEOUT doesn't match motor_errors or drive_faults patterns EXPECT_FALSE(result.should_mute); @@ -242,19 +298,19 @@ TEST_F(CorrelationEngineTest, InlineSymptomCodesMuted) { CorrelationEngine engine(config); // Report root cause - auto root_result = engine.process_fault("ESTOP_001", "CRITICAL"); + auto root_result = engine.process_fault(rec("ESTOP_001"), "CRITICAL"); EXPECT_TRUE(root_result.is_root_cause); // Report symptoms matching inline codes - auto motor_result = engine.process_fault("MOTOR_COMM_FL", "ERROR"); + auto motor_result = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); EXPECT_TRUE(motor_result.should_mute); EXPECT_EQ("ESTOP_001", motor_result.root_cause_code); - auto drive_result = engine.process_fault("DRIVE_FAULT", "ERROR"); + auto drive_result = engine.process_fault(rec("DRIVE_FAULT"), "ERROR"); EXPECT_TRUE(drive_result.should_mute); // Unrelated fault should not be muted - auto sensor_result = engine.process_fault("SENSOR_ERROR", "ERROR"); + auto sensor_result = engine.process_fault(rec("SENSOR_ERROR"), "ERROR"); EXPECT_FALSE(sensor_result.should_mute); EXPECT_EQ(2u, engine.get_muted_count()); @@ -282,13 +338,13 @@ TEST_F(CorrelationEngineTest, MixedPatternAndInlineSymptoms) { auto config = parse_config_string(yaml); CorrelationEngine engine(config); - engine.process_fault("ESTOP_001", "CRITICAL"); + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); // Both pattern-matched and inline-matched faults should be muted - auto sensor_result = engine.process_fault("SENSOR_TIMEOUT", "ERROR"); + auto sensor_result = engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR"); EXPECT_TRUE(sensor_result.should_mute); - auto motor_result = engine.process_fault("MOTOR_FAULT", "ERROR"); + auto motor_result = engine.process_fault(rec("MOTOR_FAULT"), "ERROR"); EXPECT_TRUE(motor_result.should_mute); EXPECT_EQ(2u, engine.get_muted_count()); @@ -303,16 +359,16 @@ TEST_F(CorrelationEngineTest, AutoClusterTriggersAtMinCount) { CorrelationEngine engine(config); // Report faults - need 3 for cluster - auto r1 = engine.process_fault("MOTOR_COMM_FL", "ERROR"); + auto r1 = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); EXPECT_FALSE(r1.should_mute); EXPECT_EQ(0u, engine.get_cluster_count()); - auto r2 = engine.process_fault("SENSOR_TIMEOUT", "ERROR"); + auto r2 = engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR"); EXPECT_FALSE(r2.should_mute); EXPECT_EQ(0u, engine.get_cluster_count()); // Third fault triggers cluster - auto r3 = engine.process_fault("DRIVE_COMM_ERROR", "WARNING"); + auto r3 = engine.process_fault(rec("DRIVE_COMM_ERROR"), "WARNING"); // Third fault should be muted (show_as_single=true) EXPECT_TRUE(r3.should_mute); EXPECT_EQ(1u, engine.get_cluster_count()); @@ -322,8 +378,8 @@ TEST_F(CorrelationEngineTest, AutoClusterNotTriggeredBelowMinCount) { auto config = create_auto_cluster_config(); CorrelationEngine engine(config); - engine.process_fault("MOTOR_COMM_FL", "ERROR"); - engine.process_fault("SENSOR_TIMEOUT", "ERROR"); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR"); // Only 2 faults, need 3 EXPECT_EQ(0u, engine.get_cluster_count()); @@ -333,9 +389,9 @@ TEST_F(CorrelationEngineTest, AutoClusterHighestSeverityRepresentative) { auto config = create_auto_cluster_config(); CorrelationEngine engine(config); - engine.process_fault("MOTOR_COMM_FL", "WARNING"); - engine.process_fault("SENSOR_TIMEOUT", "CRITICAL"); // Higher severity - engine.process_fault("DRIVE_COMM_ERROR", "ERROR"); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING"); + engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL"); // Higher severity + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR"); auto clusters = engine.get_clusters(); ASSERT_EQ(1u, clusters.size()); @@ -361,8 +417,8 @@ TEST_F(CorrelationEngineTest, AutoClusterFirstRepresentative) { auto config = parse_config_string(yaml); CorrelationEngine engine(config); - engine.process_fault("FIRST_ERROR", "WARNING"); - engine.process_fault("SECOND_ERROR", "CRITICAL"); + engine.process_fault(rec("FIRST_ERROR"), "WARNING"); + engine.process_fault(rec("SECOND_ERROR"), "CRITICAL"); auto clusters = engine.get_clusters(); ASSERT_EQ(1u, clusters.size()); @@ -375,11 +431,11 @@ TEST_F(CorrelationEngineTest, AutoClusterWindowExpires) { auto start = std::chrono::steady_clock::now(); - engine.process_fault("MOTOR_COMM_FL", "ERROR", start); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", start + 100ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", start); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", start + 100ms); // Third fault after window (500ms) - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", start + 600ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", start + 600ms); // Cluster should have been reset, so still not at min_count // Actually the third fault starts a new pending cluster @@ -394,10 +450,10 @@ TEST_F(CorrelationEngineTest, CleanupExpiredRemovesPendingRootCauses) { auto past = std::chrono::steady_clock::now() - std::chrono::milliseconds(2000); // Report root cause with old timestamp - engine.process_fault("ESTOP_001", "CRITICAL", past); + engine.process_fault(rec("ESTOP_001"), "CRITICAL", past); // Symptom reported NOW should NOT be correlated (root cause expired) - auto result = engine.process_fault("MOTOR_COMM_FL", "ERROR"); + auto result = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); EXPECT_FALSE(result.should_mute); // Not muted - root cause window expired EXPECT_EQ(0u, engine.get_muted_count()); @@ -405,7 +461,7 @@ TEST_F(CorrelationEngineTest, CleanupExpiredRemovesPendingRootCauses) { engine.cleanup_expired(); // Another symptom should also not be correlated - auto result2 = engine.process_fault("MOTOR_TIMEOUT_RR", "ERROR"); + auto result2 = engine.process_fault(rec("MOTOR_TIMEOUT_RR"), "ERROR"); EXPECT_FALSE(result2.should_mute); EXPECT_EQ(0u, engine.get_muted_count()); } @@ -416,8 +472,8 @@ TEST_F(CorrelationEngineTest, CleanupExpiredRemovesPendingClusters) { // Create pending cluster with old timestamp auto past = std::chrono::steady_clock::now() - std::chrono::milliseconds(1000); - engine.process_fault("MOTOR_COMM_FL", "ERROR", past); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", past + 10ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", past); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", past + 10ms); // Cluster should not be active (only 2 faults, need 3) EXPECT_EQ(0u, engine.get_cluster_count()); @@ -426,7 +482,7 @@ TEST_F(CorrelationEngineTest, CleanupExpiredRemovesPendingClusters) { engine.cleanup_expired(); // New fault should start fresh pending cluster, not join expired one - auto result = engine.process_fault("DRIVE_COMM_ERROR", "ERROR"); + auto result = engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR"); EXPECT_FALSE(result.should_mute); // First in new cluster EXPECT_EQ(0u, engine.get_cluster_count()); // Still not enough } @@ -437,9 +493,9 @@ TEST_F(CorrelationEngineTest, CleanupExpiredDoesNotBreakActiveCluster) { // Create an active cluster with old timestamps (past the 500ms window) auto past = std::chrono::steady_clock::now() - std::chrono::milliseconds(1000); - engine.process_fault("MOTOR_COMM_FL", "ERROR", past); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", past + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "WARNING", past + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", past); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", past + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "WARNING", past + 20ms); // Cluster should be active (3 faults >= min_count=3) EXPECT_EQ(1u, engine.get_cluster_count()); @@ -452,7 +508,7 @@ TEST_F(CorrelationEngineTest, CleanupExpiredDoesNotBreakActiveCluster) { EXPECT_EQ(1u, engine.get_cluster_count()); // process_clear must still find the active cluster via fault_to_cluster_ - engine.process_clear("MOTOR_COMM_FL"); + engine.process_clear(rec("MOTOR_COMM_FL")); // Cluster should still exist with 2 remaining fault codes and updated representative EXPECT_EQ(1u, engine.get_cluster_count()); @@ -472,18 +528,19 @@ TEST_F(CorrelationEngineTest, HierarchicalAndClusterCoexist) { CorrelationEngine engine(config); // Hierarchical: ESTOP_001 -> MOTOR_* - engine.process_fault("ESTOP_001", "CRITICAL"); - auto motor_result = engine.process_fault("MOTOR_COMM_FL", "ERROR"); + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + auto motor_result = engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); EXPECT_TRUE(motor_result.should_mute); EXPECT_EQ(1u, engine.get_muted_count()); // Auto-cluster: SENSOR_* (need 2) - engine.process_fault("SENSOR_LIDAR", "ERROR"); - engine.process_fault("SENSOR_IMU", "ERROR"); + engine.process_fault(rec("SENSOR_LIDAR"), "ERROR"); + engine.process_fault(rec("SENSOR_IMU"), "ERROR"); EXPECT_EQ(1u, engine.get_cluster_count()); - // Both should coexist - EXPECT_EQ(1u, engine.get_muted_count()); // MOTOR_COMM_FL + // Both should coexist: the symptom and the cluster member that is not the representative are muted + EXPECT_EQ(2u, engine.get_muted_count()); + EXPECT_TRUE(engine.is_muted(rec("MOTOR_COMM_FL"))); EXPECT_EQ(1u, engine.get_cluster_count()); // SENSOR cluster } @@ -495,9 +552,9 @@ TEST_F(CorrelationEngineTest, DuplicateFaultCode) { auto config = create_hierarchical_config(); CorrelationEngine engine(config); - engine.process_fault("ESTOP_001", "CRITICAL"); - engine.process_fault("MOTOR_COMM_FL", "ERROR"); - engine.process_fault("MOTOR_COMM_FL", "ERROR"); // Duplicate + engine.process_fault(rec("ESTOP_001"), "CRITICAL"); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR"); // Duplicate // Should still only have 1 muted fault (no duplicate) // Note: current implementation allows duplicates in symptoms list @@ -521,9 +578,9 @@ TEST_F(CorrelationEngineTest, ClearNonExistentFault) { CorrelationEngine engine(config); // Clear fault that was never reported - auto result = engine.process_clear("NONEXISTENT"); + auto result = engine.process_clear(rec("NONEXISTENT")); - EXPECT_TRUE(result.auto_cleared_codes.empty()); + EXPECT_TRUE(result.auto_cleared_symptoms.empty()); } TEST_F(CorrelationEngineTest, AutoClusterRetroactiveMuting) { @@ -550,21 +607,21 @@ TEST_F(CorrelationEngineTest, AutoClusterRetroactiveMuting) { auto t0 = std::chrono::steady_clock::now(); // Fault #1 - representative (FIRST policy) - auto result1 = engine.process_fault("SENSOR_001", "ERROR", t0); + auto result1 = engine.process_fault(rec("SENSOR_001"), "ERROR", t0); EXPECT_FALSE(result1.should_mute); // First fault is representative EXPECT_FALSE(result1.cluster_id.empty()); - EXPECT_TRUE(result1.retroactive_mute_codes.empty()); // Cluster not active yet + EXPECT_TRUE(result1.retroactive_mute_ids.empty()); // Cluster not active yet // Fault #2 - not muted because cluster not active - auto result2 = engine.process_fault("SENSOR_002", "ERROR", t0 + std::chrono::milliseconds(10)); + auto result2 = engine.process_fault(rec("SENSOR_002"), "ERROR", t0 + std::chrono::milliseconds(10)); EXPECT_FALSE(result2.should_mute); // Cluster still not active - EXPECT_TRUE(result2.retroactive_mute_codes.empty()); + EXPECT_TRUE(result2.retroactive_mute_ids.empty()); // Fault #3 - triggers cluster activation (min_count=3) - auto result3 = engine.process_fault("SENSOR_003", "ERROR", t0 + std::chrono::milliseconds(20)); - EXPECT_TRUE(result3.should_mute); // #3 is muted (not representative) - EXPECT_EQ(1u, result3.retroactive_mute_codes.size()); // #2 should be retroactively muted - EXPECT_EQ("SENSOR_002", result3.retroactive_mute_codes[0]); + auto result3 = engine.process_fault(rec("SENSOR_003"), "ERROR", t0 + std::chrono::milliseconds(20)); + EXPECT_TRUE(result3.should_mute); // #3 is muted (not representative) + EXPECT_EQ(1u, result3.retroactive_mute_ids.size()); // #2 should be retroactively muted + EXPECT_EQ(rec("SENSOR_002"), result3.retroactive_mute_ids[0]); // Cluster should be active with 3 faults EXPECT_EQ(1u, engine.get_cluster_count()); @@ -584,16 +641,16 @@ TEST_F(CorrelationEngineTest, ClearFaultRemovesFromPendingCluster) { auto t0 = std::chrono::steady_clock::now(); // Add 2 faults (below min_count=3), creating a pending cluster - engine.process_fault("MOTOR_COMM_FL", "ERROR", t0); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", t0 + 10ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", t0 + 10ms); EXPECT_EQ(0u, engine.get_cluster_count()); // Still pending // Clear one of the faults - engine.process_clear("MOTOR_COMM_FL"); + engine.process_clear(rec("MOTOR_COMM_FL")); // Now add a third fault - should NOT activate the cluster because // the cleared fault was removed from pending, so only 2 faults total - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", t0 + 20ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", t0 + 20ms); EXPECT_EQ(0u, engine.get_cluster_count()); // Still not enough } @@ -620,15 +677,15 @@ TEST_F(CorrelationEngineTest, ClearRepresentativeReassignsPendingCluster) { auto t0 = std::chrono::steady_clock::now(); // Add 2 faults, creating a pending cluster - engine.process_fault("FIRST_ERROR", "CRITICAL", t0); - engine.process_fault("SECOND_ERROR", "ERROR", t0 + 10ms); + engine.process_fault(rec("FIRST_ERROR"), "CRITICAL", t0); + engine.process_fault(rec("SECOND_ERROR"), "ERROR", t0 + 10ms); // Clear the representative (first fault) - engine.process_clear("FIRST_ERROR"); + engine.process_clear(rec("FIRST_ERROR")); // Add 2 more faults to reach min_count (SECOND_ERROR + 2 new = 3) - engine.process_fault("THIRD_ERROR", "ERROR", t0 + 20ms); - engine.process_fault("FOURTH_ERROR", "ERROR", t0 + 30ms); + engine.process_fault(rec("THIRD_ERROR"), "ERROR", t0 + 20ms); + engine.process_fault(rec("FOURTH_ERROR"), "ERROR", t0 + 30ms); EXPECT_EQ(1u, engine.get_cluster_count()); auto clusters = engine.get_clusters(); @@ -645,16 +702,16 @@ TEST_F(CorrelationEngineTest, ClearAllFaultsRemovesPendingCluster) { auto t0 = std::chrono::steady_clock::now(); // Add 2 faults - engine.process_fault("MOTOR_COMM_FL", "ERROR", t0); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", t0 + 10ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", t0 + 10ms); // Clear both - engine.process_clear("MOTOR_COMM_FL"); - engine.process_clear("SENSOR_TIMEOUT"); + engine.process_clear(rec("MOTOR_COMM_FL")); + engine.process_clear(rec("SENSOR_TIMEOUT")); // Adding 2 new faults should start a fresh pending cluster, not join old one - engine.process_fault("DRIVE_COMM_NEW", "ERROR", t0 + 100ms); - engine.process_fault("MOTOR_COMM_NEW", "ERROR", t0 + 110ms); + engine.process_fault(rec("DRIVE_COMM_NEW"), "ERROR", t0 + 100ms); + engine.process_fault(rec("MOTOR_COMM_NEW"), "ERROR", t0 + 110ms); EXPECT_EQ(0u, engine.get_cluster_count()); // Still 2 faults, below min_count=3 } @@ -684,9 +741,9 @@ TEST_F(CorrelationEngineTest, ClearMostRecentRepresentativeReassignsActiveCluste auto t0 = std::chrono::steady_clock::now(); - engine.process_fault("FIRST_ERROR", "WARNING", t0); - engine.process_fault("SECOND_ERROR", "ERROR", t0 + 10ms); - engine.process_fault("THIRD_ERROR", "CRITICAL", t0 + 20ms); + engine.process_fault(rec("FIRST_ERROR"), "WARNING", t0); + engine.process_fault(rec("SECOND_ERROR"), "ERROR", t0 + 10ms); + engine.process_fault(rec("THIRD_ERROR"), "CRITICAL", t0 + 20ms); EXPECT_EQ(1u, engine.get_cluster_count()); auto clusters = engine.get_clusters(); @@ -694,7 +751,7 @@ TEST_F(CorrelationEngineTest, ClearMostRecentRepresentativeReassignsActiveCluste EXPECT_EQ("THIRD_ERROR", clusters[0].representative_code); EXPECT_EQ("CRITICAL", clusters[0].representative_severity); - engine.process_clear("THIRD_ERROR"); + engine.process_clear(rec("THIRD_ERROR")); clusters = engine.get_clusters(); ASSERT_EQ(1u, clusters.size()); @@ -713,9 +770,9 @@ TEST_F(CorrelationEngineTest, ClearHighestSeverityRepresentativeReassigns) { auto t0 = std::chrono::steady_clock::now(); // WARNING, CRITICAL (rep), ERROR - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "CRITICAL", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", t0 + 20ms); // SENSOR_TIMEOUT is representative (highest severity) auto clusters = engine.get_clusters(); @@ -723,7 +780,7 @@ TEST_F(CorrelationEngineTest, ClearHighestSeverityRepresentativeReassigns) { EXPECT_EQ("SENSOR_TIMEOUT", clusters[0].representative_code); // Clear the representative - engine.process_clear("SENSOR_TIMEOUT"); + engine.process_clear(rec("SENSOR_TIMEOUT")); // Remaining: WARNING, ERROR -> ERROR should become new representative clusters = engine.get_clusters(); @@ -742,8 +799,8 @@ TEST_F(CorrelationEngineTest, CleanupExpiredRemovesFaultToClusterEntries) { // Create pending cluster with old timestamp auto past = std::chrono::steady_clock::now() - std::chrono::milliseconds(1000); - engine.process_fault("MOTOR_COMM_FL", "ERROR", past); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", past + 10ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", past); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", past + 10ms); EXPECT_EQ(0u, engine.get_cluster_count()); // Cleanup removes expired pending cluster @@ -753,9 +810,9 @@ TEST_F(CorrelationEngineTest, CleanupExpiredRemovesFaultToClusterEntries) { // If fault_to_cluster_ was NOT cleaned, these would try to join // a non-existent cluster instead of creating a new one. auto t1 = std::chrono::steady_clock::now(); - engine.process_fault("MOTOR_COMM_FL", "ERROR", t1); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", t1 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", t1 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "ERROR", t1); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", t1 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", t1 + 20ms); // Should form a NEW cluster with all 3 faults EXPECT_EQ(1u, engine.get_cluster_count()); @@ -771,18 +828,18 @@ TEST_F(CorrelationEngineTest, ClearRepresentativeAfterNewWindowReplacesPendingRe auto t0 = std::chrono::steady_clock::now(); // Activate first cluster: SENSOR_TIMEOUT (CRITICAL) is representative - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "CRITICAL", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", t0 + 20ms); ASSERT_EQ(1u, engine.get_cluster_count()); // After window_ms (500ms), a new fault starts a fresh pending cluster for the same rule, // replacing pending_clusters_["comm_storm"] - engine.process_fault("OTHER_COMM_ERROR", "WARNING", t0 + 600ms); + engine.process_fault(rec("OTHER_COMM_ERROR"), "WARNING", t0 + 600ms); // Clearing the representative of the first active cluster must still reassign // its representative to the next-highest remaining fault (DRIVE_COMM_ERROR) - engine.process_clear("SENSOR_TIMEOUT"); + engine.process_clear(rec("SENSOR_TIMEOUT")); auto clusters = engine.get_clusters(); ASSERT_EQ(1u, clusters.size()); @@ -798,20 +855,20 @@ TEST_F(CorrelationEngineTest, ReReportRepresentativeAfterWindowAndClearReassigns auto t0 = std::chrono::steady_clock::now(); // Activate cluster: SENSOR_TIMEOUT (CRITICAL) is representative - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "CRITICAL", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", t0 + 20ms); ASSERT_EQ(1u, engine.get_cluster_count()); const auto active_id = engine.get_clusters()[0].cluster_id; // Re-report the representative after window_ms (500ms); it must stay in the active cluster // rather than starting a new pending cluster and overwriting fault_to_cluster_ - auto rereport = engine.process_fault("SENSOR_TIMEOUT", "CRITICAL", t0 + 600ms); + auto rereport = engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL", t0 + 600ms); EXPECT_EQ(active_id, rereport.cluster_id); EXPECT_FALSE(rereport.should_mute); // Clearing the representative must still update the active cluster - engine.process_clear("SENSOR_TIMEOUT"); + engine.process_clear(rec("SENSOR_TIMEOUT")); auto clusters = engine.get_clusters(); ASSERT_EQ(1u, clusters.size()); @@ -827,18 +884,18 @@ TEST_F(CorrelationEngineTest, ClearBelowMinCountInWindowThenNewFaultJoinsUpdates auto t0 = std::chrono::steady_clock::now(); // Activate cluster with 3 faults: SENSOR_TIMEOUT (CRITICAL) is representative - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "CRITICAL", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "WARNING", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "WARNING", t0 + 20ms); ASSERT_EQ(1u, engine.get_cluster_count()); // Clear 2 members inside the window so the cluster holds only 1 code (< min_count) - engine.process_clear("MOTOR_COMM_FL"); - engine.process_clear("DRIVE_COMM_ERROR"); + engine.process_clear(rec("MOTOR_COMM_FL")); + engine.process_clear(rec("DRIVE_COMM_ERROR")); ASSERT_EQ(1u, engine.get_cluster_count()); // A new code joins inside the window while cluster size (2) is still below min_count (3) - auto join_res = engine.process_fault("NEW_COMM_ERROR", "ERROR", t0 + 50ms); + auto join_res = engine.process_fault(rec("NEW_COMM_ERROR"), "ERROR", t0 + 50ms); EXPECT_TRUE(join_res.should_mute); auto clusters = engine.get_clusters(); @@ -846,7 +903,7 @@ TEST_F(CorrelationEngineTest, ClearBelowMinCountInWindowThenNewFaultJoinsUpdates EXPECT_EQ(2u, clusters[0].fault_codes.size()); // Clearing the original representative must reassign to the newly joined code - engine.process_clear("SENSOR_TIMEOUT"); + engine.process_clear(rec("SENSOR_TIMEOUT")); clusters = engine.get_clusters(); ASSERT_EQ(1u, clusters.size()); @@ -860,12 +917,12 @@ TEST_F(CorrelationEngineTest, ReReportActiveMemberWithHigherSeverityInWindowBeco CorrelationEngine engine(config); auto t0 = std::chrono::steady_clock::now(); - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "WARNING", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "WARNING", t0 + 20ms); ASSERT_EQ(1u, engine.get_cluster_count()); - auto res = engine.process_fault("MOTOR_COMM_FL", "CRITICAL", t0 + 30ms); + auto res = engine.process_fault(rec("MOTOR_COMM_FL"), "CRITICAL", t0 + 30ms); EXPECT_FALSE(res.should_mute); auto clusters = engine.get_clusters(); @@ -879,12 +936,12 @@ TEST_F(CorrelationEngineTest, ReReportActiveMemberWithHigherSeverityAfterWindowB CorrelationEngine engine(config); auto t0 = std::chrono::steady_clock::now(); - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "ERROR", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "WARNING", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "ERROR", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "WARNING", t0 + 20ms); ASSERT_EQ(1u, engine.get_cluster_count()); - auto res = engine.process_fault("MOTOR_COMM_FL", "CRITICAL", t0 + 600ms); + auto res = engine.process_fault(rec("MOTOR_COMM_FL"), "CRITICAL", t0 + 600ms); EXPECT_FALSE(res.should_mute); auto clusters = engine.get_clusters(); @@ -898,12 +955,12 @@ TEST_F(CorrelationEngineTest, ReReportRepresentativeWithLowerSeverityHandsOverRe CorrelationEngine engine(config); auto t0 = std::chrono::steady_clock::now(); - engine.process_fault("MOTOR_COMM_FL", "WARNING", t0); - engine.process_fault("SENSOR_TIMEOUT", "CRITICAL", t0 + 10ms); - engine.process_fault("DRIVE_COMM_ERROR", "ERROR", t0 + 20ms); + engine.process_fault(rec("MOTOR_COMM_FL"), "WARNING", t0); + engine.process_fault(rec("SENSOR_TIMEOUT"), "CRITICAL", t0 + 10ms); + engine.process_fault(rec("DRIVE_COMM_ERROR"), "ERROR", t0 + 20ms); ASSERT_EQ(1u, engine.get_cluster_count()); - auto res = engine.process_fault("SENSOR_TIMEOUT", "WARNING", t0 + 600ms); + auto res = engine.process_fault(rec("SENSOR_TIMEOUT"), "WARNING", t0 + 600ms); EXPECT_TRUE(res.should_mute); auto clusters = engine.get_clusters(); diff --git a/src/ros2_medkit_fault_manager/test/test_entity_thresholds.cpp b/src/ros2_medkit_fault_manager/test/test_entity_thresholds.cpp index 36fe66355..ba1e1ccde 100644 --- a/src/ros2_medkit_fault_manager/test/test_entity_thresholds.cpp +++ b/src/ros2_medkit_fault_manager/test/test_entity_thresholds.cpp @@ -30,6 +30,7 @@ namespace fs = std::filesystem; using ros2_medkit_fault_manager::DebounceConfig; using ros2_medkit_fault_manager::EntityDebounceOverride; using ros2_medkit_fault_manager::EntityThresholdResolver; +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::InMemoryFaultStorage; using ros2_medkit_msgs::msg::Fault; using ros2_medkit_msgs::srv::ReportFault; @@ -286,10 +287,10 @@ class PerEntityStorageTest : public ::testing::Test { // @verifies REQ_INTEROP_095 TEST_F(PerEntityStorageTest, LidarFaultConfirmsImmediately) { auto ts = clock_.now(); - storage_.report_fault_event("LIDAR_FAIL", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Lidar failure", - "/sensors/lidar", ts.nanoseconds(), lidar_config_); + storage_.report_fault_event(FaultId{"LIDAR_FAIL", "/sensors/lidar"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Lidar failure", "/sensors/lidar", ts.nanoseconds(), lidar_config_); - auto fault = storage_.get_fault("LIDAR_FAIL"); + auto fault = storage_.get_fault({"LIDAR_FAIL", "/sensors/lidar"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -299,35 +300,38 @@ TEST_F(PerEntityStorageTest, MotorFaultNeedsDebouncing) { auto ts = clock_.now(); // First event: counter = -1, threshold = -5, so PREFAILED - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor hot", - "/powertrain/motor_left", ts.nanoseconds(), motor_config_); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor_left"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Motor hot", "/powertrain/motor_left", ts.nanoseconds(), + motor_config_); - auto fault = storage_.get_fault("MOTOR_OVERHEAT"); + auto fault = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor_left"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Events 2-4: still PREFAILED (counter = -2, -3, -4) for (int i = 0; i < 3; ++i) { - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", - "/powertrain/motor_left", clock_.now().nanoseconds(), motor_config_); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor_left"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "", "/powertrain/motor_left", clock_.now().nanoseconds(), + motor_config_); } - fault = storage_.get_fault("MOTOR_OVERHEAT"); + fault = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor_left"}); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Event 5: counter = -5, meets threshold -> CONFIRMED - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", - "/powertrain/motor_left", clock_.now().nanoseconds(), motor_config_); - fault = storage_.get_fault("MOTOR_OVERHEAT"); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor_left"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "", "/powertrain/motor_left", clock_.now().nanoseconds(), + motor_config_); + fault = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor_left"}); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } // @verifies REQ_INTEROP_095 TEST_F(PerEntityStorageTest, UnknownEntityUsesGlobalConfig) { auto ts = clock_.now(); - storage_.report_fault_event("UNKNOWN_FAULT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Unknown", - "/some/unknown/entity", ts.nanoseconds(), global_); + storage_.report_fault_event(FaultId{"UNKNOWN_FAULT", "/some/unknown/entity"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Unknown", "/some/unknown/entity", ts.nanoseconds(), global_); - auto fault = storage_.get_fault("UNKNOWN_FAULT"); + auto fault = storage_.get_fault({"UNKNOWN_FAULT", "/some/unknown/entity"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Global threshold=-1 -> immediate } @@ -335,19 +339,50 @@ TEST_F(PerEntityStorageTest, UnknownEntityUsesGlobalConfig) { // @verifies REQ_INTEROP_095 TEST_F(PerEntityStorageTest, DifferentEntitiesSameFaultCode) { auto ts = clock_.now(); + // One record shared by every source of the code. + const FaultId shared{"OVERHEAT", ros2_medkit_fault_manager::kSharedOwner}; // Motor reports with motor config (debounced) - storage_.report_fault_event("OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheat", + storage_.report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheat", "/powertrain/motor_left", ts.nanoseconds(), motor_config_); - auto fault = storage_.get_fault("OVERHEAT"); + auto fault = storage_.get_fault(shared); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Motor needs 5 events // Lidar reports same fault_code with lidar config (threshold=-1) // Counter is now -2, lidar threshold is -1, -2 <= -1 -> CONFIRMED - storage_.report_fault_event("OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Lidar overheat", + storage_.report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Lidar overheat", "/sensors/lidar", clock_.now().nanoseconds(), lidar_config_); - fault = storage_.get_fault("OVERHEAT"); + fault = storage_.get_fault(shared); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Lidar config confirms at -1, counter is -2 + EXPECT_EQ(storage_.size(), 1u); +} + +// Under per-source identity two entities reporting one code are two records, each evaluated against +// its own entity's band. +// @verifies REQ_INTEROP_095 +TEST_F(PerEntityStorageTest, DifferentEntitiesSameFaultCodeDebounceIndependentlyWhenPerSource) { + auto ts = clock_.now(); + + // Motor reports with motor config (debounced, threshold -5) + storage_.report_fault_event(FaultId{"OVERHEAT", "/powertrain/motor_left"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Motor overheat", "/powertrain/motor_left", ts.nanoseconds(), + motor_config_); + + // Lidar reports the same fault_code with lidar config (threshold -1) + storage_.report_fault_event(FaultId{"OVERHEAT", "/sensors/lidar"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Lidar overheat", "/sensors/lidar", clock_.now().nanoseconds(), lidar_config_); + + auto motor = storage_.get_fault({"OVERHEAT", "/powertrain/motor_left"}); + ASSERT_TRUE(motor.has_value()); + EXPECT_EQ(motor->status, Fault::STATUS_PREFAILED) << "the motor's own record is one report into a band of five"; + EXPECT_EQ(motor->description, "Motor overheat"); + + auto lidar = storage_.get_fault({"OVERHEAT", "/sensors/lidar"}); + ASSERT_TRUE(lidar.has_value()); + EXPECT_EQ(lidar->status, Fault::STATUS_CONFIRMED) << "the lidar's own record confirms on its first report"; + EXPECT_EQ(lidar->description, "Lidar overheat"); + + EXPECT_EQ(storage_.size(), 2u); } // @verifies REQ_INTEROP_095 @@ -355,18 +390,18 @@ TEST_F(PerEntityStorageTest, LidarHealingWithThreshold1) { auto ts = clock_.now(); // Confirm fault - storage_.report_fault_event("LIDAR_FAIL", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Lidar failure", - "/sensors/lidar", ts.nanoseconds(), lidar_config_); - auto fault = storage_.get_fault("LIDAR_FAIL"); + storage_.report_fault_event(FaultId{"LIDAR_FAIL", "/sensors/lidar"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Lidar failure", "/sensors/lidar", ts.nanoseconds(), lidar_config_); + auto fault = storage_.get_fault({"LIDAR_FAIL", "/sensors/lidar"}); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // One PASSED event should heal (healing_threshold=1 for lidar) // Counter goes from -1 to 0 (no status change at 0) then +1 -> healed - storage_.report_fault_event("LIDAR_FAIL", fd::ReportKind::kPassed, 0, "", "/sensors/lidar", + storage_.report_fault_event(FaultId{"LIDAR_FAIL", "/sensors/lidar"}, fd::ReportKind::kPassed, 0, "", "/sensors/lidar", clock_.now().nanoseconds(), lidar_config_); - storage_.report_fault_event("LIDAR_FAIL", fd::ReportKind::kPassed, 0, "", "/sensors/lidar", + storage_.report_fault_event(FaultId{"LIDAR_FAIL", "/sensors/lidar"}, fd::ReportKind::kPassed, 0, "", "/sensors/lidar", clock_.now().nanoseconds(), lidar_config_); - fault = storage_.get_fault("LIDAR_FAIL"); + fault = storage_.get_fault({"LIDAR_FAIL", "/sensors/lidar"}); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); } @@ -374,10 +409,10 @@ TEST_F(PerEntityStorageTest, LidarHealingWithThreshold1) { TEST_F(PerEntityStorageTest, BackwardCompatibleWithoutResolver) { // When no per-entity config is used, passing global config gives same behavior as before auto ts = clock_.now(); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - ts.nanoseconds(), global_); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", ts.nanoseconds(), global_); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Global threshold=-1 } diff --git a/src/ros2_medkit_fault_manager/test/test_fault_code_thresholds.cpp b/src/ros2_medkit_fault_manager/test/test_fault_code_thresholds.cpp index a32295959..7bb34d60e 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_code_thresholds.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_code_thresholds.cpp @@ -33,6 +33,7 @@ using ros2_medkit_fault_manager::EntityDebounceOverride; using ros2_medkit_fault_manager::EntityThresholdResolver; using ros2_medkit_fault_manager::FaultCodeDebounceOverride; using ros2_medkit_fault_manager::FaultCodeThresholdResolver; +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::InMemoryFaultStorage; using ros2_medkit_msgs::msg::Fault; using ros2_medkit_msgs::srv::ReportFault; @@ -350,6 +351,26 @@ TEST_F(FaultCodeYamlLoadTest, BadValueReturnsEmpty) { EXPECT_TRUE(entries.empty()); } +// A per_source that is not the YAML boolean true or false would decide how a code is identified by +// accident, so it rejects the whole file like any other malformed value. yaml-cpp itself reads "yes" as true. +// @verifies REQ_INTEROP_107 +TEST_F(FaultCodeYamlLoadTest, PerSourceNonBoolRejectsTheFile) { + for (const char * bad : {"\"yes\"", "3"}) { + auto path = tmpdir_ / "bad_per_source.yaml"; + { + std::ofstream f(path); + f << "GOOD.CODE:\n" + << " per_source: true\n" + << "BAD.CODE:\n" + << " per_source: " << bad << "\n"; + } + + std::vector entries; + ASSERT_NO_THROW(entries = FaultCodeThresholdResolver::load_from_yaml(path.string(), test_log_sink())) << bad; + EXPECT_TRUE(entries.empty()) << "per_source: " << bad; + } +} + // yaml-cpp accepts a key twice. The first entry wins, the same as the resolver. // @verifies REQ_INTEROP_107 TEST_F(FaultCodeYamlLoadTest, RepeatedCodeKeepsTheFirstEntry) { @@ -448,9 +469,11 @@ class FaultCodeStorageTest : public ::testing::Test { codes_ = FaultCodeThresholdResolver({overheat}); } - void report(const std::string & fault_code, const std::string & source_id) { - storage_.report_fault_event(fault_code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", source_id, - clock_.now().nanoseconds(), + /// Reports into the shared record of the code unless @p owner names another record. + void report(const std::string & fault_code, const std::string & source_id, + const std::string & owner = ros2_medkit_fault_manager::kSharedOwner) { + storage_.report_fault_event(FaultId{fault_code, owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", + source_id, clock_.now().nanoseconds(), resolve_layered(entities_, codes_, source_id, fault_code, global_)); } }; @@ -460,23 +483,44 @@ TEST_F(FaultCodeStorageTest, LidarNoLongerConfirmsTheMotorsCodeEarly) { // Without the fault-code override, lidar's threshold of -1 would confirm this // on the second event (the counter is shared). With it, both sources debounce // at -3. + const FaultId shared{"OVERHEAT", ros2_medkit_fault_manager::kSharedOwner}; report("OVERHEAT", "/powertrain/motor/left"); - EXPECT_EQ(storage_.get_fault("OVERHEAT")->status, Fault::STATUS_PREFAILED); + EXPECT_EQ(storage_.get_fault(shared)->status, Fault::STATUS_PREFAILED); report("OVERHEAT", "/sensors/lidar/front"); - EXPECT_EQ(storage_.get_fault("OVERHEAT")->status, Fault::STATUS_PREFAILED); + EXPECT_EQ(storage_.get_fault(shared)->status, Fault::STATUS_PREFAILED); report("OVERHEAT", "/sensors/lidar/front"); - EXPECT_EQ(storage_.get_fault("OVERHEAT")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_.get_fault(shared)->status, Fault::STATUS_CONFIRMED); +} + +// @verifies REQ_INTEROP_107 +TEST_F(FaultCodeStorageTest, TheCodeOverrideGovernsEachOwnersRecordWhenPerSource) { + // Without the fault-code override, lidar's threshold of -1 would confirm its record on + // its first event. With it, the record of each owner debounces at -3, on its own counter. + const FaultId motor{"OVERHEAT", "/powertrain/motor/left"}; + const FaultId lidar{"OVERHEAT", "/sensors/lidar/front"}; + + report(motor.fault_code, motor.owner, motor.owner); + EXPECT_EQ(storage_.get_fault(motor)->status, Fault::STATUS_PREFAILED); + + report(lidar.fault_code, lidar.owner, lidar.owner); + EXPECT_EQ(storage_.get_fault(lidar)->status, Fault::STATUS_PREFAILED); + report(lidar.fault_code, lidar.owner, lidar.owner); + EXPECT_EQ(storage_.get_fault(lidar)->status, Fault::STATUS_PREFAILED); + report(lidar.fault_code, lidar.owner, lidar.owner); + EXPECT_EQ(storage_.get_fault(lidar)->status, Fault::STATUS_CONFIRMED); + + EXPECT_EQ(storage_.get_fault(motor)->status, Fault::STATUS_PREFAILED) << "the motor's own counter is at 1 of 3"; } // @verifies REQ_INTEROP_107 TEST_F(FaultCodeStorageTest, ACodeWithoutAnOverrideKeepsEntityBehaviour) { report("MOTOR_STALL", "/powertrain/motor/left"); - EXPECT_EQ(storage_.get_fault("MOTOR_STALL")->status, Fault::STATUS_PREFAILED); + EXPECT_EQ(storage_.get_fault({"MOTOR_STALL", "*"})->status, Fault::STATUS_PREFAILED); report("LIDAR_BLOCKED", "/sensors/lidar/front"); - EXPECT_EQ(storage_.get_fault("LIDAR_BLOCKED")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_.get_fault({"LIDAR_BLOCKED", "*"})->status, Fault::STATUS_CONFIRMED); } // The startup pass flips HEALED to CLEARED only for codes whose healing is off. @@ -486,12 +530,13 @@ TEST_F(FaultCodeStorageTest, ReclassifyKeepsCodesThatStillHeal) { healing.healing_enabled = true; healing.healing_threshold = 1; for (const std::string code : {"KEEPS.HEALING", "STOPPED.HEALING"}) { - storage_.report_fault_event(code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", "/src", + storage_.report_fault_event(FaultId{code, "/src"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", "/src", clock_.now().nanoseconds(), healing); - for (int i = 0; i < 3 && storage_.get_fault(code)->status != Fault::STATUS_HEALED; ++i) { - storage_.report_fault_event(code, fd::ReportKind::kPassed, 0, "", "/src", clock_.now().nanoseconds(), healing); + for (int i = 0; i < 3 && storage_.get_fault({code, "/src"})->status != Fault::STATUS_HEALED; ++i) { + storage_.report_fault_event(FaultId{code, "/src"}, fd::ReportKind::kPassed, 0, "", "/src", + clock_.now().nanoseconds(), healing); } - ASSERT_EQ(storage_.get_fault(code)->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_.get_fault({code, "/src"})->status, Fault::STATUS_HEALED); } auto reclassified = storage_.reclassify_healed_as_cleared([](const std::string & code) { @@ -499,7 +544,7 @@ TEST_F(FaultCodeStorageTest, ReclassifyKeepsCodesThatStillHeal) { }); ASSERT_EQ(reclassified.size(), 1u); - EXPECT_EQ(reclassified[0], "STOPPED.HEALING"); - EXPECT_EQ(storage_.get_fault("KEEPS.HEALING")->status, Fault::STATUS_HEALED); - EXPECT_EQ(storage_.get_fault("STOPPED.HEALING")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(reclassified[0], (FaultId{"STOPPED.HEALING", "/src"})); + EXPECT_EQ(storage_.get_fault({"KEEPS.HEALING", "/src"})->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_.get_fault({"STOPPED.HEALING", "/src"})->status, Fault::STATUS_CLEARED); } diff --git a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp index 7050e1d16..510da55db 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_manager.cpp @@ -15,15 +15,21 @@ #include #include +#include +#include #include #include +#include +#include #include #include #include +#include #include #include #include #include +#include #include #include @@ -32,6 +38,7 @@ #include #include "rclcpp/rclcpp.hpp" +#include "rcutils/logging.h" #include "ros2_medkit_fault_manager/fault_audit_log.hpp" #include "ros2_medkit_fault_manager/fault_manager_node.hpp" #include "ros2_medkit_fault_manager/fault_storage.hpp" @@ -44,6 +51,7 @@ #include "ros2_medkit_msgs/srv/get_fault.hpp" #include "ros2_medkit_msgs/srv/get_rosbag.hpp" #include "ros2_medkit_msgs/srv/get_snapshots.hpp" +#include "ros2_medkit_msgs/srv/list_faults.hpp" #include "ros2_medkit_msgs/srv/list_faults_for_entity.hpp" #include "ros2_medkit_msgs/srv/list_rosbags.hpp" #include "ros2_medkit_msgs/srv/report_fault.hpp" @@ -52,6 +60,7 @@ using ros2_medkit_fault_manager::clamp_debounce_counter; using ros2_medkit_fault_manager::compute_debounce_status; using ros2_medkit_fault_manager::DebounceConfig; +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::FaultManagerNode; using ros2_medkit_fault_manager::InMemoryFaultStorage; using ros2_medkit_fault_manager::sanitize_debounce_config; @@ -61,6 +70,7 @@ using ros2_medkit_msgs::srv::ClearFault; using ros2_medkit_msgs::srv::GetFault; using ros2_medkit_msgs::srv::GetRosbag; using ros2_medkit_msgs::srv::GetSnapshots; +using ros2_medkit_msgs::srv::ListFaults; using ros2_medkit_msgs::srv::ListFaultsForEntity; using ros2_medkit_msgs::srv::ListRosbags; using ros2_medkit_msgs::srv::ReportFault; @@ -91,21 +101,22 @@ TEST_F(FaultStorageTest, ReportNewFaultEvent) { rclcpp::Clock clock; auto timestamp = clock.now(); - bool is_new = storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "Motor temperature exceeded threshold", "/powertrain/motor", - timestamp.nanoseconds(), default_config()); + bool is_new = storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Motor temperature exceeded threshold", + "/powertrain/motor", timestamp.nanoseconds(), default_config()); EXPECT_TRUE(is_new); EXPECT_EQ(storage_.size(), 1u); - EXPECT_TRUE(storage_.contains("MOTOR_OVERHEAT")); + EXPECT_TRUE(storage_.contains({"MOTOR_OVERHEAT", "/powertrain/motor"})); } TEST_F(FaultStorageTest, PassedEventForNonExistentFaultIgnored) { rclcpp::Clock clock; auto timestamp = clock.now(); - bool is_new = storage_.report_fault_event("NON_EXISTENT", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "Test", - "/node1", timestamp.nanoseconds(), default_config()); + bool is_new = + storage_.report_fault_event(FaultId{"NON_EXISTENT", "/node1"}, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, + "Test", "/node1", timestamp.nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_.size(), 0u); @@ -116,23 +127,94 @@ TEST_F(FaultStorageTest, ReportExistingFaultEventUpdates) { auto timestamp1 = clock.now(); auto timestamp2 = clock.now(); - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Initial report", - "/powertrain/motor1", timestamp1.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "Initial report", "/powertrain/motor1", timestamp1.nanoseconds(), + default_config()); - bool is_new = - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Second report", - "/powertrain/motor2", timestamp2.nanoseconds(), default_config()); + bool is_new = storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Second report", "/powertrain/motor1", + timestamp2.nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_.size(), 1u); - auto fault = storage_.get_fault("MOTOR_OVERHEAT"); + auto fault = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor1"}); ASSERT_TRUE(fault.has_value()); // Still the same continuous occurrence (not CLEARED in between): occurrence_count - // does not bump on every report, only severity/sources/description update. + // does not bump on every report, only severity/description update. EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->severity, Fault::SEVERITY_ERROR); // Updated to higher severity - EXPECT_EQ(fault->reporting_sources.size(), 2u); + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources.front(), "/powertrain/motor1"); +} + +// Two sources reporting one code are two records. This used to be one row whose +// counter, status, occurrence count, severity and timestamps were all shared, so one +// source's report moved another source's fault. +TEST_F(FaultStorageTest, TwoSourcesReportingOneCodeAreTwoRecords) { + DebounceConfig config; + config.confirmation_threshold = -2; + + const rclcpp::Time a_time(1000, 0, RCL_SYSTEM_TIME); + const rclcpp::Time b_time(2000, 0, RCL_SYSTEM_TIME); + + // A reports twice and reaches its threshold. B reports once and does not. + EXPECT_TRUE(storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "from a", "/powertrain/motor1", a_time.nanoseconds(), + config)); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "from a", "/powertrain/motor1", a_time.nanoseconds(), config); + EXPECT_TRUE(storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor2"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "from b", "/powertrain/motor2", b_time.nanoseconds(), + config)) + << "a source that owns no record yet gets a new one, not an update"; + + EXPECT_EQ(storage_.size(), 2u); + + auto a = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor1"}); + auto b = storage_.get_fault({"MOTOR_OVERHEAT", "/powertrain/motor2"}); + ASSERT_TRUE(a.has_value()); + ASSERT_TRUE(b.has_value()); + + EXPECT_EQ(a->status, Fault::STATUS_CONFIRMED) << "A reached its own threshold"; + EXPECT_EQ(b->status, Fault::STATUS_PREFAILED) << "B is one report into its own band"; + EXPECT_EQ(a->severity, Fault::SEVERITY_WARN) << "B's ERROR must not escalate A's record"; + EXPECT_EQ(b->severity, Fault::SEVERITY_ERROR); + EXPECT_EQ(a->description, "from a"); + EXPECT_EQ(b->description, "from b"); + EXPECT_EQ(a->occurrence_count, 1u); + EXPECT_EQ(b->occurrence_count, 1u); + EXPECT_EQ(a->first_occurred_ns, a_time.nanoseconds()); + EXPECT_EQ(b->first_occurred_ns, b_time.nanoseconds()); + ASSERT_EQ(a->reporting_sources.size(), 1u); + EXPECT_EQ(a->reporting_sources.front(), "/powertrain/motor1"); + ASSERT_EQ(b->reporting_sources.size(), 1u); + EXPECT_EQ(b->reporting_sources.front(), "/powertrain/motor2"); +} + +// The resolution step behind an unscoped service call. +TEST_F(FaultStorageTest, GetIdsByCodeReturnsEveryOwnerAndGetFaultExactlyOne) { + rclcpp::Clock clock; + storage_.report_fault_event(FaultId{"SHARED", "/owner_b"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_b", clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"SHARED", "/owner_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_a", clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"OTHER", "/owner_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_a", clock.now().nanoseconds(), default_config()); + + const auto shared = storage_.get_ids_by_code("SHARED"); + ASSERT_EQ(shared.size(), 2u); + // Ordered by owner, so an ambiguity message names them the same way every time. + EXPECT_EQ(shared[0].owner, "/owner_a"); + EXPECT_EQ(shared[1].owner, "/owner_b"); + + EXPECT_EQ(storage_.get_ids_by_code("OTHER").size(), 1u); + EXPECT_TRUE(storage_.get_ids_by_code("NEVER_REPORTED").empty()); + + auto one = storage_.get_fault({"SHARED", "/owner_a"}); + ASSERT_TRUE(one.has_value()); + EXPECT_EQ(one->reporting_sources.front(), "/owner_a"); + EXPECT_FALSE(storage_.get_fault({"SHARED", "/nobody"}).has_value()); } TEST_F(FaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCount) { @@ -141,11 +223,11 @@ TEST_F(FaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCount) { // Simulate a level-triggered poller re-reporting the same still-true condition // every cycle (issue #11): occurrence_count must stay at 1, not grow per poll. for (int i = 0; i < 25; ++i) { - storage_.report_fault_event("TANK_OVERFILL", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "level = 95 > 80", - "/tank", clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"TANK_OVERFILL", "/tank"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "level = 95 > 80", "/tank", clock.now().nanoseconds(), default_config()); } - auto fault = storage_.get_fault("TANK_OVERFILL"); + auto fault = storage_.get_fault({"TANK_OVERFILL", "/tank"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -156,8 +238,8 @@ TEST_F(FaultStorageTest, ListFaultsDefaultReturnsConfirmedOnly) { auto timestamp = clock.now(); // With default threshold=-1, single report confirms immediately - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); // Default query should return the CONFIRMED fault auto faults = storage_.list_faults(false, 0, {}); @@ -174,8 +256,8 @@ TEST_F(FaultStorageTest, ListFaultsWithPrefailedStatus) { config.confirmation_threshold = -3; storage_.set_debounce_config(config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), config); // Query with PREFAILED status auto faults = storage_.list_faults(false, 0, {Fault::STATUS_PREFAILED}); @@ -189,10 +271,10 @@ TEST_F(FaultStorageTest, ListFaultsFilterBySeverity) { auto timestamp = clock.now(); // With default threshold=-1, faults are immediately CONFIRMED - storage_.report_fault_event("FAULT_INFO", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Info", "/node1", - timestamp.nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_ERROR", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Error", "/node1", - timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_INFO", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Info", + "/node1", timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_ERROR", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Error", + "/node1", timestamp.nanoseconds(), default_config()); // Filter by ERROR severity (query CONFIRMED since that's the default status now) auto faults = storage_.list_faults(true, Fault::SEVERITY_ERROR, {Fault::STATUS_CONFIRMED}); @@ -204,19 +286,19 @@ TEST_F(FaultStorageTest, ClearFault) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Test", "/node1", timestamp.nanoseconds(), default_config()); - bool cleared = storage_.clear_fault("MOTOR_OVERHEAT"); + bool cleared = storage_.clear_fault({"MOTOR_OVERHEAT", "/node1"}); EXPECT_TRUE(cleared); - auto fault = storage_.get_fault("MOTOR_OVERHEAT"); + auto fault = storage_.get_fault({"MOTOR_OVERHEAT", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); } TEST_F(FaultStorageTest, ClearNonExistentFault) { - bool cleared = storage_.clear_fault("NON_EXISTENT"); + bool cleared = storage_.clear_fault({"NON_EXISTENT", "/node1"}); EXPECT_FALSE(cleared); } @@ -224,9 +306,9 @@ TEST_F(FaultStorageTest, GetClearedFaults) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); - storage_.clear_fault("FAULT_1"); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); + storage_.clear_fault({"FAULT_1", "/node1"}); // Query cleared faults auto faults = storage_.list_faults(false, 0, {Fault::STATUS_CLEARED}); @@ -239,8 +321,8 @@ TEST_F(FaultStorageTest, InvalidStatusDefaultsToConfirmed) { auto timestamp = clock.now(); // With default threshold=-1, fault is immediately CONFIRMED - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); // Query with invalid status - defaults to CONFIRMED, which now matches our fault auto faults = storage_.list_faults(false, 0, {"INVALID_STATUS"}); @@ -283,12 +365,12 @@ TEST_F(FaultStorageTest, FaultStaysPrefailedAboveThreshold) { config.confirmation_threshold = -3; storage_.set_debounce_config(config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node2", clock.now().nanoseconds(), config); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Still debouncing towards confirmation, same occurrence EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); @@ -302,14 +384,14 @@ TEST_F(FaultStorageTest, FaultConfirmsAtThreshold) { config.confirmation_threshold = -3; storage_.set_debounce_config(config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node2", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node3", clock.now().nanoseconds(), config); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Debounce build-up is still one occurrence EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -318,13 +400,13 @@ TEST_F(FaultStorageTest, FaultConfirmsAtThreshold) { TEST_F(FaultStorageTest, ConfirmedFaultStaysConfirmed) { rclcpp::Clock clock; - // Report fault 4 times + // One source reports 4 times: occurrence_count counts outages, not reports. for (int i = 0; i < 4; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", - "/node" + std::to_string(i), clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Same occurrence throughout, not one per report EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -334,14 +416,17 @@ TEST_F(FaultStorageTest, MultiSourceConfirmsFault) { rclcpp::Clock clock; // Report same fault from 3 different sources - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/sensor1", + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", ros2_medkit_fault_manager::kSharedOwner}, + fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/sensor1", clock.now().nanoseconds(), default_config()); - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/sensor2", + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", ros2_medkit_fault_manager::kSharedOwner}, + fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/sensor2", clock.now().nanoseconds(), default_config()); - storage_.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/sensor3", + storage_.report_fault_event(FaultId{"MOTOR_OVERHEAT", ros2_medkit_fault_manager::kSharedOwner}, + fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/sensor3", clock.now().nanoseconds(), default_config()); - auto fault = storage_.get_fault("MOTOR_OVERHEAT"); + auto fault = storage_.get_fault({"MOTOR_OVERHEAT", ros2_medkit_fault_manager::kSharedOwner}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(fault->reporting_sources.size(), 3u); @@ -351,14 +436,14 @@ TEST_F(FaultStorageTest, SameSourceMultipleReportsConfirms) { rclcpp::Clock clock; // Same source reports 3 times - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); - - auto fault = storage_.get_fault("FAULT_1"); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); + + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Repeated reports from one source, one occurrence EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -372,10 +457,10 @@ TEST_F(FaultStorageTest, ImmediateConfirmationWithThresholdZero) { storage_.set_debounce_config(config); // Single report should confirm immediately - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -385,10 +470,10 @@ TEST_F(FaultStorageTest, CriticalSeverityBypassesDebounce) { rclcpp::Clock clock; // CRITICAL severity should confirm immediately - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "Critical test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, + "Critical test", "/node1", clock.now().nanoseconds(), default_config()); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -402,10 +487,10 @@ TEST_F(FaultStorageTest, CriticalBypassCanBeDisabled) { storage_.set_debounce_config(config); // CRITICAL should NOT confirm immediately when disabled - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "Critical test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, + "Critical test", "/node1", clock.now().nanoseconds(), config); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); } @@ -415,54 +500,69 @@ TEST_F(FaultStorageTest, ClearedFaultCanBeReactivated) { // Report to confirm (with default threshold=-1, single report confirms) auto first_ts = clock.now(); - bool is_new = storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Initial", - "/node1", first_ts.nanoseconds(), default_config()); + bool is_new = + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Initial", "/node1", first_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->first_occurred_ns, first_ts.nanoseconds()); - // Clear the fault - storage_.clear_fault("FAULT_1"); - fault = storage_.get_fault("FAULT_1"); + // Clear the record + storage_.clear_fault({"FAULT_1", "/node1"}); + fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); - // Report again after a gap - should reactivate as a new cycle + // The same owner reports again after a gap - reactivates its own record as a new cycle auto second_ts = rclcpp::Time(first_ts.nanoseconds() + 1'000'000'000LL); // +1s - is_new = storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Reactivated", - "/node2", second_ts.nanoseconds(), default_config()); + is_new = storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Reactivated", "/node1", second_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); // Should return true like a new fault - fault = storage_.get_fault("FAULT_1"); + fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Should be reconfirmed EXPECT_EQ(fault->occurrence_count, 2u); // Should increment - EXPECT_EQ(fault->reporting_sources.size(), 2u); // Both sources - EXPECT_EQ(fault->description, "Reactivated"); // Updated description + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources.front(), "/node1"); + EXPECT_EQ(fault->description, "Reactivated"); // Updated description // #25: first_occurred must reflect the new cycle, not the outage that already cleared. EXPECT_EQ(fault->first_occurred_ns, second_ts.nanoseconds()); + + // A different source reporting the same code after the clear opens its OWN record + // rather than reactivating another owner's. It used to reactivate the shared row, + // which is what made occurrence_count and reporting_sources grow across owners. + const rclcpp::Time third_ts(first_ts.nanoseconds() + 2'000'000'000LL, RCL_SYSTEM_TIME); + EXPECT_TRUE(storage_.report_fault_event(FaultId{"FAULT_1", "/node2"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "From the other source", "/node2", third_ts.nanoseconds(), default_config())); + auto other = storage_.get_fault({"FAULT_1", "/node2"}); + ASSERT_TRUE(other.has_value()); + EXPECT_EQ(other->occurrence_count, 1u) << "a first report is one occurrence, whatever the code has seen before"; + ASSERT_EQ(other->reporting_sources.size(), 1u); + EXPECT_EQ(other->reporting_sources.front(), "/node2"); + EXPECT_EQ(storage_.size(), 2u); } TEST_F(FaultStorageTest, PassedEventForClearedFaultIgnored) { rclcpp::Clock clock; // Report and confirm - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); // Clear the fault - storage_.clear_fault("FAULT_1"); + storage_.clear_fault({"FAULT_1", "/node1"}); // PASSED event should be ignored for CLEARED fault - bool result = storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", + bool result = storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), default_config()); EXPECT_FALSE(result); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); // Should stay cleared } @@ -475,36 +575,34 @@ TEST_F(FaultStorageTest, ClearedFaultReactivationRestartsDebounce) { config.confirmation_threshold = -3; storage_.set_debounce_config(config); - // Report 3 times to confirm - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), config); + // One source reports 3 times to confirm its own record + for (int i = 0; i < 3; ++i) { + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); - // Clear the fault - storage_.clear_fault("FAULT_1"); + // Clear the record + storage_.clear_fault({"FAULT_1", "/node1"}); // Reactivate - should start in PREFAILED with counter=-1 - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node4", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); - fault = storage_.get_fault("FAULT_1"); + fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Not yet confirmed, needs 2 more FAILED // Report 2 more times to re-confirm - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node5", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node6", - clock.now().nanoseconds(), config); + for (int i = 0; i < 2; ++i) { + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + } - fault = storage_.get_fault("FAULT_1"); + fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Now confirmed } @@ -518,21 +616,85 @@ TEST_F(FaultStorageTest, PassedEventIncrementsCounter) { config.healing_threshold = 3; // explicit upper clamp (don't depend on the default) // Report 2 FAILED events (counter = -2, PREFAILED) - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node2", clock.now().nanoseconds(), config); // Report 3 PASSED events (counter = -2 + 3 = +1) for (int i = 0; i < 3; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kPassed, 0, + "", "/node1", clock.now().nanoseconds(), config); } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREPASSED); // Counter > 0 } +// A PASSED moves only its own source's counter. It used to move the one counter the +// code had, so one reporter recovering walked another reporter's fault back from +// confirmation, and with healing on it healed it outright. +TEST_F(FaultStorageTest, PassedFromOneSourceMovesOnlyThatSourcesRecord) { + rclcpp::Clock clock; + DebounceConfig config; + config.confirmation_threshold = -1; + config.healing_enabled = true; + config.healing_threshold = 1; + + storage_.report_fault_event(FaultId{"FAULT_1", "/node_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node_a", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node_b"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node_b", clock.now().nanoseconds(), config); + + // Two PASSED from A only: -1 -> 0 -> +1, which is A's healing threshold. + for (int i = 0; i < 2; ++i) { + storage_.report_fault_event(FaultId{"FAULT_1", "/node_a"}, fd::ReportKind::kPassed, 0, "", "/node_a", + clock.now().nanoseconds(), config); + } + + auto a = storage_.get_fault({"FAULT_1", "/node_a"}); + auto b = storage_.get_fault({"FAULT_1", "/node_b"}); + ASSERT_TRUE(a.has_value()); + ASSERT_TRUE(b.has_value()); + EXPECT_EQ(a->status, Fault::STATUS_HEALED); + EXPECT_EQ(b->status, Fault::STATUS_CONFIRMED) << "B never reported PASSED, so B's fault is still active"; + EXPECT_EQ(b->last_passed_ns, 0) << "A's recovery must not stamp B's record"; +} + +// A PASSED from a source that owns no record for this code has nothing to heal. +TEST_F(FaultStorageTest, PassedFromASourceWithNoRecordIsIgnored) { + rclcpp::Clock clock; + storage_.report_fault_event(FaultId{"FAULT_1", "/node_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node_a", clock.now().nanoseconds(), default_config()); + + const bool is_new = storage_.report_fault_event(FaultId{"FAULT_1", "/node_b"}, fd::ReportKind::kPassed, 0, "", + "/node_b", clock.now().nanoseconds(), default_config()); + + EXPECT_FALSE(is_new); + EXPECT_EQ(storage_.size(), 1u) << "a PASSED must never create a record"; + EXPECT_FALSE(storage_.get_fault({"FAULT_1", "/node_b"}).has_value()); + auto owner_a = storage_.get_fault({"FAULT_1", "/node_a"}); + ASSERT_TRUE(owner_a.has_value()); + EXPECT_EQ(owner_a->status, Fault::STATUS_CONFIRMED); +} + +// Severity is per record: a CRITICAL from one source does not escalate another's. +TEST_F(FaultStorageTest, CriticalFromOneSourceDoesNotEscalateAnothersRecord) { + rclcpp::Clock clock; + storage_.report_fault_event(FaultId{"FAULT_1", "/node_b"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "warn", + "/node_b", clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, + "critical", "/node_a", clock.now().nanoseconds(), default_config()); + + auto a = storage_.get_fault({"FAULT_1", "/node_a"}); + auto b = storage_.get_fault({"FAULT_1", "/node_b"}); + ASSERT_TRUE(a.has_value()); + ASSERT_TRUE(b.has_value()); + EXPECT_EQ(a->severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(b->severity, Fault::SEVERITY_WARN) << "severity is the record's own maximum, not the code's"; +} + TEST_F(FaultStorageTest, PassedEventDoesNotAdvanceLastOccurred) { // Observed on a live box: a plugin sent one PASSED on reconnect, healing was // disabled so the fault stayed CONFIRMED (by design), but last_occurred jumped @@ -541,12 +703,12 @@ TEST_F(FaultStorageTest, PassedEventDoesNotAdvanceLastOccurred) { const rclcpp::Time failed_at(1000, 0, RCL_SYSTEM_TIME); const rclcpp::Time passed_at(9000, 0, RCL_SYSTEM_TIME); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - failed_at.nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", passed_at.nanoseconds(), - default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", failed_at.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + passed_at.nanoseconds(), default_config()); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // healing disabled: latched, by design EXPECT_EQ(fault->last_occurred_ns, failed_at.nanoseconds()); @@ -558,17 +720,17 @@ TEST_F(FaultStorageTest, HealingDisabledByDefault) { rclcpp::Clock clock; // Report 1 FAILED event -> confirmed (threshold -1) - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); // With healing disabled, a heal heartbeat must not downgrade a confirmed fault. // The counter clamps at healing_threshold and the status latches CONFIRMED. for (int i = 0; i < 10; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), default_config()); } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // stays confirmed, not silently downgraded } @@ -581,15 +743,16 @@ TEST_F(FaultStorageTest, HealingWhenEnabled) { storage_.set_debounce_config(config); // Report 1 FAILED event (counter = -1) - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); // Report 4 PASSED events (counter = -1 + 4 = +3, reaches healing threshold) for (int i = 0; i < 4; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); } @@ -602,8 +765,8 @@ TEST_F(FaultStorageTest, TimeBasedConfirmationDisabledByDefault) { config.confirmation_threshold = -3; storage_.set_debounce_config(config); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); // Advance time and check - should not auto-confirm (auto_confirm_after_sec = 0) auto future_time = rclcpp::Time(clock.now().nanoseconds() + static_cast(20e9)); @@ -611,7 +774,7 @@ TEST_F(FaultStorageTest, TimeBasedConfirmationDisabledByDefault) { EXPECT_TRUE(confirmed.empty()); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); } @@ -624,8 +787,8 @@ TEST_F(FaultStorageTest, TimeBasedConfirmationWhenEnabled) { storage_.set_debounce_config(config); auto now = clock.now(); - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - now.nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", now.nanoseconds(), config); // Check before timeout - should not confirm auto before_timeout = rclcpp::Time(now.nanoseconds() + static_cast(5e9)); @@ -636,9 +799,9 @@ TEST_F(FaultStorageTest, TimeBasedConfirmationWhenEnabled) { auto after_timeout = rclcpp::Time(now.nanoseconds() + static_cast(15e9)); auto confirmed = storage_.check_time_based_confirmation(after_timeout.nanoseconds()); ASSERT_EQ(confirmed.size(), 1u); - EXPECT_EQ(confirmed[0], "FAULT_1"); + EXPECT_EQ(confirmed[0], (ros2_medkit_fault_manager::FaultId{"FAULT_1", "/node1"})); - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -652,20 +815,22 @@ TEST_F(FaultStorageTest, ConfirmedFaultCanHealWithPassedEvents) { // Report 3 FAILED events to confirm (counter = -3) for (int i = 0; i < 3; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", - "/node" + std::to_string(i), clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node" + std::to_string(i)}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node" + std::to_string(i), clock.now().nanoseconds(), + config); } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Report 6 PASSED events (counter = -3 + 6 = +3, reaches healing threshold) for (int i = 0; i < 6; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } - fault = storage_.get_fault("FAULT_1"); + fault = storage_.get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); } @@ -678,25 +843,26 @@ TEST_F(FaultStorageTest, HealedFaultCanRecurWithFailedEvents) { storage_.set_debounce_config(config); // Report 1 FAILED event (counter = -1) - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test", "/node1", clock.now().nanoseconds(), config); // Report 4 PASSED events to heal (counter = -1 + 4 = +3) for (int i = 0; i < 4; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kPassed, 0, + "", "/node1", clock.now().nanoseconds(), config); } - auto fault = storage_.get_fault("FAULT_1"); + auto fault = storage_.get_fault({"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); // Report 3 FAILED events - fault should recur and confirm (counter = +3 - 3 = 0, then -3) for (int i = 0; i < 6; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Recurrence", "/node2", - clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Recurrence", "/node2", clock.now().nanoseconds(), config); } - fault = storage_.get_fault("FAULT_1"); + fault = storage_.get_fault({"FAULT_1", ros2_medkit_fault_manager::kSharedOwner}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -711,26 +877,28 @@ TEST_F(FaultStorageTest, HeartbeatHealClampedAndStatusLatched) { config.healing_enabled = true; config.healing_threshold = 3; - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - ASSERT_EQ(storage_.get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + ASSERT_EQ(storage_.get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); // Long heal heartbeat: counter clamps at healing_threshold, then heals. for (int i = 0; i < 100; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } - EXPECT_EQ(storage_.get_fault("FAULT_1")->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_.get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_HEALED); // Re-confirmation is bounded and the healed status latches until the counter // walks all the way back down to the confirmation threshold. for (int i = 0; i < 3; ++i) { - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("FAULT_1")->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + EXPECT_EQ(storage_.get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_HEALED) + << "latch broke after " << (i + 1); } - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + EXPECT_EQ(storage_.get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); } // Latch holds in both directions, same as the SQLite backend: CONFIRMED with a positive counter @@ -742,16 +910,17 @@ TEST_F(FaultStorageTest, ConfirmedLatchSurvivesFailedAtPositiveCounter) { config.healing_threshold = 3; config.critical_immediate_confirm = true; - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", - clock.now().nanoseconds(), config); - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CONFIRMED); + storage_.report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); + ASSERT_EQ(storage_.get_fault({"F", ros2_medkit_fault_manager::kSharedOwner})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kPassed, 0, "", + "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CONFIRMED); - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), - config); - EXPECT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_.get_fault({"F", ros2_medkit_fault_manager::kSharedOwner})->status, Fault::STATUS_CONFIRMED); + storage_.report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); + EXPECT_EQ(storage_.get_fault({"F", ros2_medkit_fault_manager::kSharedOwner})->status, Fault::STATUS_CONFIRMED); } // HEALED with a negative counter stays HEALED on PASSED. @@ -762,19 +931,21 @@ TEST_F(FaultStorageTest, HealedLatchSurvivesPassedAtNegativeCounter) { config.healing_enabled = true; config.healing_threshold = 3; - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), - config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); for (int i = 0; i < 5; ++i) { // +3 -> -2, latched HEALED - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); } - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); + EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } TEST_F(FaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePassed) { @@ -785,24 +956,27 @@ TEST_F(FaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePassed) { config.healing_threshold = 3; config.critical_immediate_confirm = true; - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { // -1 -> +3 -> HEALED - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); // CRITICAL confirms immediately from HEALED; counter must not stay at +2 where one PASSED heals it. - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { // -1 -> +2, still CONFIRMED - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CONFIRMED); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); + EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); } - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); + EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } TEST_F(FaultStorageTest, ReclassifyHealedAsCleared) { @@ -811,17 +985,18 @@ TEST_F(FaultStorageTest, ReclassifyHealedAsCleared) { config.healing_enabled = true; config.healing_threshold = 3; - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), - config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); const auto reclassified = storage_.reclassify_healed_as_cleared(); ASSERT_EQ(reclassified.size(), 1u); - EXPECT_EQ(reclassified[0], "F"); - EXPECT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(reclassified[0], (ros2_medkit_fault_manager::FaultId{"F", "/n"})); + EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CLEARED); EXPECT_TRUE(storage_.reclassify_healed_as_cleared().empty()); } @@ -862,11 +1037,12 @@ TEST_F(FaultStorageTest, HealingThresholdZeroHealsOnSinglePassed) { config.healing_enabled = true; config.healing_threshold = 0; - storage_.report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + clock.now().nanoseconds(), config); + ASSERT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); + storage_.report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_.get_fault("F")->status, Fault::STATUS_CONFIRMED); - storage_.report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_.get_fault("F")->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_.get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } TEST(DebounceHelpers, ComputeDebounceStatusLatches) { @@ -885,12 +1061,12 @@ TEST_F(FaultStorageTest, GetAllFaultsReturnsAllFaults) { auto timestamp = clock.now(); // Add faults with different statuses - storage_.report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test 1", "/node1", - timestamp.nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_2", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Test 2", "/node2", - timestamp.nanoseconds(), default_config()); - storage_.report_fault_event("FAULT_3", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Test 3", "/node3", - timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test 1", + "/node1", timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_2", "/node2"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Test 2", + "/node2", timestamp.nanoseconds(), default_config()); + storage_.report_fault_event(FaultId{"FAULT_3", "/node3"}, fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Test 3", + "/node3", timestamp.nanoseconds(), default_config()); auto all_faults = storage_.get_all_faults(); EXPECT_EQ(all_faults.size(), 3u); @@ -1007,8 +1183,9 @@ static void drive_near_misses(ros2_medkit_fault_manager::FaultStorage & storage, constexpr int64_t kBaseNs = 1700000000000000000LL; for (int i = 0; i < count; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", kBaseNs + static_cast(i) * 1000000LL, config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + kBaseNs + static_cast(i) * 1000000LL, config); } } @@ -1022,7 +1199,7 @@ TEST(FaultManagerNodeParameterTest, AppliesNearMissRetentionBound) { drive_near_misses(node->get_storage_for_test(), 10); - EXPECT_EQ(node->get_storage().get_near_misses("PUMP_PRESSURE_LOW").size(), 4u); + EXPECT_EQ(node->get_storage().get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 4u); } TEST(FaultManagerNodeParameterTest, NearMissRetentionDefaultsToBounded) { @@ -1032,8 +1209,8 @@ TEST(FaultManagerNodeParameterTest, NearMissRetentionDefaultsToBounded) { drive_near_misses(node->get_storage_for_test(), 205); - // The documented default is 200 per fault code, and it must be in force without configuration. - EXPECT_EQ(node->get_storage().get_near_misses("PUMP_PRESSURE_LOW").size(), 200u); + // The documented default is 200 per fault record, and it must be in force without configuration. + EXPECT_EQ(node->get_storage().get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 200u); } TEST(FaultManagerNodeParameterTest, NegativeNearMissBoundFallsBackToDefault) { @@ -1047,7 +1224,7 @@ TEST(FaultManagerNodeParameterTest, NegativeNearMissBoundFallsBackToDefault) { drive_near_misses(node->get_storage_for_test(), 205); - EXPECT_EQ(node->get_storage().get_near_misses("PUMP_PRESSURE_LOW").size(), 200u); + EXPECT_EQ(node->get_storage().get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 200u); } TEST(FaultManagerNodeParameterTest, ZeroNearMissBoundIsUnlimited) { @@ -1060,7 +1237,7 @@ TEST(FaultManagerNodeParameterTest, ZeroNearMissBoundIsUnlimited) { drive_near_misses(node->get_storage_for_test(), 205); - EXPECT_EQ(node->get_storage().get_near_misses("PUMP_PRESSURE_LOW").size(), 205u); + EXPECT_EQ(node->get_storage().get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 205u); } TEST(FaultManagerNodeParameterTest, ParsesDropOldestPolicy) { @@ -1092,8 +1269,10 @@ class FaultEventPublishingTest : public ::testing::Test { } void SetUp() override { - // Unique namespace per test iteration avoids DDS topic collisions - std::string ns = "/test_events_" + std::to_string(test_counter_.fetch_add(1)); + // Unique namespace per test iteration avoids DDS topic collisions. Kept as a + // member so a subclass can add a client of its own on the same namespace. + ns_ = "/test_events_" + std::to_string(test_counter_.fetch_add(1)); + const std::string & ns = ns_; // Create fault manager node with immediate confirmation rclcpp::NodeOptions fm_options; @@ -1137,6 +1316,25 @@ class FaultEventPublishingTest : public ::testing::Test { test_node_.reset(); fault_manager_.reset(); received_events_.clear(); + if (!thresholds_path_.empty()) { + std::error_code ec; + std::filesystem::remove(thresholds_path_, ec); + } + } + + /// The override that makes every code in @p codes per_source. The file is written now, because the node + /// reads it at construction, and removed in TearDown. + rclcpp::Parameter per_source_codes(const std::vector & codes) { + std::random_device rd; + std::mt19937_64 gen(rd()); + std::uniform_int_distribution dist; + thresholds_path_ = + std::filesystem::temp_directory_path() / ("test_per_source_" + std::to_string(dist(gen)) + ".yaml"); + std::ofstream out(thresholds_path_); + for (const auto & code : codes) { + out << code << ":\n per_source: true\n"; + } + return rclcpp::Parameter("fault_thresholds.config_file", thresholds_path_.string()); } void spin_for(std::chrono::milliseconds duration) { @@ -1207,20 +1405,31 @@ class FaultEventPublishingTest : public ::testing::Test { return future.get()->accepted; } - bool call_clear_fault(const std::string & fault_code) { + /// @param source_id owner to scope the call to. Empty leaves the call unscoped. + bool call_clear_fault(const std::string & fault_code, const std::string & source_id = "") { + const auto response = clear_fault_response(fault_code, source_id); + return response.has_value() && response->success; + } + + /// The whole response, for the tests that check the refusal message rather than the flag. + std::optional clear_fault_response(const std::string & fault_code, + const std::string & source_id = "") { auto request = std::make_shared(); request->fault_code = fault_code; + request->source_id = source_id; auto future = clear_fault_client_->async_send_request(request); if (!spin_until_future_ready(future)) { - return false; + return std::nullopt; } - return future.get()->success; + return *future.get(); } - std::optional call_get_fault(const std::string & fault_code) { + /// @param source_id owner to scope the call to. Empty leaves the call unscoped. + std::optional call_get_fault(const std::string & fault_code, const std::string & source_id = "") { auto request = std::make_shared(); request->fault_code = fault_code; + request->source_id = source_id; auto future = get_fault_client_->async_send_request(request); if (!spin_until_future_ready(future)) { @@ -1240,6 +1449,8 @@ class FaultEventPublishingTest : public ::testing::Test { return *future.get(); } + std::string ns_; + std::filesystem::path thresholds_path_; std::shared_ptr fault_manager_; std::shared_ptr test_node_; rclcpp::Subscription::SharedPtr event_subscription_; @@ -1277,7 +1488,7 @@ TEST_F(FaultEventPublishingTest, UpdateExistingFaultPublishesUpdatedEvent) { // Clear received events received_events_.clear(); - // Report same fault again - should trigger EVENT_UPDATED + // Report same fault again from another source - should trigger EVENT_UPDATED on the shared record ASSERT_TRUE(call_report_fault("TEST_FAULT_2", Fault::SEVERITY_ERROR, "/test_node2")); ASSERT_TRUE(spin_until([this]() { return !received_events_.empty(); @@ -1288,6 +1499,44 @@ TEST_F(FaultEventPublishingTest, UpdateExistingFaultPublishesUpdatedEvent) { EXPECT_EQ(received_events_[0].event_type, FaultEvent::EVENT_UPDATED); EXPECT_EQ(received_events_[0].fault.fault_code, "TEST_FAULT_2"); EXPECT_EQ(received_events_[0].fault.occurrence_count, 1u); + EXPECT_EQ(received_events_[0].fault.reporting_sources, (std::vector{"/test_node1", "/test_node2"})); +} + +/// A node where the codes the tests below report are configured per_source, so each source gets its own record. +class PerSourceNodeTest : public FaultEventPublishingTest { + protected: + std::vector fault_manager_overrides() override { + auto overrides = FaultEventPublishingTest::fault_manager_overrides(); + overrides.push_back(per_source_codes({"TWO_OWNERS", "SHARED_CODE", "ONLY_ONE"})); + return overrides; + } +}; + +// With a per_source code, a second source reporting a code someone else already reported opens its OWN +// record, so the event is a confirmation of a new record rather than an update of a shared one. +TEST_F(PerSourceNodeTest, ASecondSourceConfirmsItsOwnRecord) { + ASSERT_TRUE(call_report_fault("TWO_OWNERS", Fault::SEVERITY_WARN, "/test_node1")); + ASSERT_TRUE(spin_until([this]() { + return !received_events_.empty(); + })); + received_events_.clear(); + + ASSERT_TRUE(call_report_fault("TWO_OWNERS", Fault::SEVERITY_ERROR, "/test_node2")); + ASSERT_TRUE(spin_until([this]() { + return !received_events_.empty(); + })); + + ASSERT_EQ(received_events_.size(), 1u); + EXPECT_EQ(received_events_[0].event_type, FaultEvent::EVENT_CONFIRMED); + ASSERT_EQ(received_events_[0].fault.reporting_sources.size(), 1u); + EXPECT_EQ(received_events_[0].fault.reporting_sources.front(), "/test_node2"); + EXPECT_EQ(received_events_[0].fault.severity, Fault::SEVERITY_ERROR); + + // The first owner's record is untouched, severity included. + auto first = call_get_fault("TWO_OWNERS", "/test_node1"); + ASSERT_TRUE(first.has_value()); + ASSERT_TRUE(first->success); + EXPECT_EQ(first->fault.severity, Fault::SEVERITY_WARN); } TEST_F(FaultEventPublishingTest, ClearFaultPublishesClearedEvent) { @@ -1358,6 +1607,348 @@ TEST_F(HealingFaultEventPublishingTest, HealPublishesClearedEventSoStreamConsume EXPECT_EQ(event->fault.status, Fault::STATUS_HEALED); } +/// Records every message logged through rcutils while an instance exists, then forwards it. +class LogCapture { + public: + struct Line { + int severity; + std::string text; + }; + + LogCapture() : previous_(rcutils_logging_get_output_handler()) { + std::lock_guard lock(mutex()); + active() = this; + rcutils_logging_set_output_handler(&LogCapture::handle); + } + + ~LogCapture() { + std::lock_guard lock(mutex()); + rcutils_logging_set_output_handler(previous_); + active() = nullptr; + } + + LogCapture(const LogCapture &) = delete; + LogCapture & operator=(const LogCapture &) = delete; + LogCapture(LogCapture &&) = delete; + LogCapture & operator=(LogCapture &&) = delete; + + /// WARN lines whose text contains @p needle. + std::vector warnings_with(const std::string & needle) const { + std::lock_guard lock(mutex()); + std::vector out; + for (const auto & line : lines_) { + if (line.severity == RCUTILS_LOG_SEVERITY_WARN && line.text.find(needle) != std::string::npos) { + out.push_back(line.text); + } + } + return out; + } + + private: + static std::mutex & mutex() { + static std::mutex m; + return m; + } + + static LogCapture *& active() { + static LogCapture * instance = nullptr; + return instance; + } + + static void handle(const rcutils_log_location_t * location, int severity, const char * name, + rcutils_time_point_value_t timestamp, const char * format, va_list * args) { + rcutils_logging_output_handler_t forward = nullptr; + { + std::lock_guard lock(mutex()); + LogCapture * self = active(); + if (self != nullptr) { + va_list copy; + va_copy(copy, *args); + std::array buffer{}; + // The format comes from the logging call site. There is no literal to pass. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" + std::vsnprintf(buffer.data(), buffer.size(), format, copy); +#pragma GCC diagnostic pop + va_end(copy); + self->lines_.push_back(Line{severity, buffer.data()}); + forward = self->previous_; + } + } + if (forward != nullptr) { + forward(location, severity, name, timestamp, format, args); + } + } + + rcutils_logging_output_handler_t previous_; + std::vector lines_; +}; + +/// Severity consistency of a record through the node, on both storage backends: events and audit +/// rows carry the stored severity, near-miss rows the reported one, and a differing severity is logged. +class SeverityConsistencyNodeTest : public FaultEventPublishingTest, public ::testing::WithParamInterface { + protected: + SeverityConsistencyNodeTest() { + std::random_device rd; + std::mt19937_64 gen(rd()); + std::uniform_int_distribution dist; + dir_ = std::filesystem::temp_directory_path() / ("test_severity_node_" + std::to_string(dist(gen))); + std::filesystem::create_directories(dir_); + // Every code these tests report is per_source: the stored severity of a record decides immediate + // confirmation there, and the owner-scoped assertions below are about one record per source. + std::ofstream thresholds(dir_ / "fault_thresholds.yaml"); + for (const char * code : {"EVT_SEV", "AUDIT_SEV", "AUDIT_REACT", "MISMATCH", "MISMATCH_A", "MISMATCH_B", + "ENTITY_WARN", "ENTITY_CRIT", "ENTITY_MIXED"}) { + thresholds << code << ":\n per_source: true\n"; + } + } + + ~SeverityConsistencyNodeTest() override { + std::error_code ec; + std::filesystem::remove_all(dir_, ec); + } + + SeverityConsistencyNodeTest(const SeverityConsistencyNodeTest &) = delete; + SeverityConsistencyNodeTest & operator=(const SeverityConsistencyNodeTest &) = delete; + SeverityConsistencyNodeTest(SeverityConsistencyNodeTest &&) = delete; + SeverityConsistencyNodeTest & operator=(SeverityConsistencyNodeTest &&) = delete; + + std::vector fault_manager_overrides() override { + return { + rclcpp::Parameter("storage_type", GetParam()), + rclcpp::Parameter("database_path", (dir_ / "faults.db").string()), + rclcpp::Parameter("confirmation_threshold", -3), + rclcpp::Parameter("audit_log.enabled", true), + rclcpp::Parameter("audit_log.database_path", audit_path()), + rclcpp::Parameter("fault_thresholds.config_file", (dir_ / "fault_thresholds.yaml").string()), + }; + } + + std::string audit_path() const { + return (dir_ / "audit.db").string(); + } + + bool report(const std::string & code, uint8_t severity, const std::string & source) { + return call_report_fault(code, severity, source); + } + + /// The event of @p type for @p code in occurrence @p occurrence, once it arrives. + std::optional wait_for_event(const std::string & code, const std::string & type, uint32_t occurrence) { + std::optional found; + spin_until([&]() { + for (const auto & e : received_events_) { + if (e.fault.fault_code == code && e.event_type == type && e.fault.occurrence_count == occurrence) { + found = e; + return true; + } + } + return false; + }); + return found; + } + + /// @param source_id owner of the record. Empty resolves the code's only record. + Fault get(const std::string & code, const std::string & source_id = "") { + auto response = call_get_fault(code, source_id); + EXPECT_TRUE(response.has_value() && response->success) << code << " " << source_id; + return response ? response->fault : Fault{}; + } + + /// (transition, severity) of every audit row for @p code, oldest first. + std::vector> audit_rows(const std::string & code) const { + ros2_medkit_fault_manager::FaultAuditLog audit(audit_path()); + std::vector> rows; + for (const auto & record : audit.read()) { + if (record.event.fault_code == code) { + rows.emplace_back(record.event.transition, record.event.severity); + } + } + return rows; + } + + std::filesystem::path dir_; +}; + +INSTANTIATE_TEST_SUITE_P(Backends, SeverityConsistencyNodeTest, ::testing::Values("memory", "sqlite")); + +TEST_P(SeverityConsistencyNodeTest, EventsCarryTheSeverityGetShows) { + ASSERT_TRUE(report("EVT_SEV", Fault::SEVERITY_CRITICAL, "/a")); + auto confirmed = wait_for_event("EVT_SEV", FaultEvent::EVENT_CONFIRMED, 1); + ASSERT_TRUE(confirmed.has_value()); + EXPECT_EQ(confirmed->fault.severity, get("EVT_SEV").severity); + + ASSERT_TRUE(call_clear_fault("EVT_SEV")); + auto cleared = wait_for_event("EVT_SEV", FaultEvent::EVENT_CLEARED, 1); + ASSERT_TRUE(cleared.has_value()); + EXPECT_EQ(cleared->fault.severity, get("EVT_SEV").severity); + + // A WARN report on a CRITICAL code starts occurrence 2 and confirms it at once. + ASSERT_TRUE(report("EVT_SEV", Fault::SEVERITY_WARN, "/a")); + auto reconfirmed = wait_for_event("EVT_SEV", FaultEvent::EVENT_CONFIRMED, 2); + ASSERT_TRUE(reconfirmed.has_value()) << "no fault_confirmed event for occurrence 2"; + auto shown = get("EVT_SEV"); + EXPECT_EQ(shown.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(shown.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(reconfirmed->fault.severity, shown.severity); + EXPECT_EQ(reconfirmed->fault.status, shown.status); + + // A lower report inside the occurrence updates the fault and keeps the code's severity. + received_events_.clear(); + ASSERT_TRUE(report("EVT_SEV", Fault::SEVERITY_ERROR, "/a")); + auto updated = wait_for_event("EVT_SEV", FaultEvent::EVENT_UPDATED, 2); + ASSERT_TRUE(updated.has_value()); + EXPECT_EQ(updated->fault.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(updated->fault.severity, get("EVT_SEV").severity); +} + +TEST_P(SeverityConsistencyNodeTest, AuditRowsCarryTheStoredSeverity) { + using Rows = std::vector>; + const int warn = Fault::SEVERITY_WARN; + const int error = Fault::SEVERITY_ERROR; + + // Debounced: WARN opens the occurrence, ERROR escalates it, WARN confirms it. + ASSERT_TRUE(report("AUDIT_SEV", Fault::SEVERITY_WARN, "/a")); + ASSERT_TRUE(report("AUDIT_SEV", Fault::SEVERITY_ERROR, "/a")); + ASSERT_TRUE(report("AUDIT_SEV", Fault::SEVERITY_WARN, "/a")); + auto confirmed = wait_for_event("AUDIT_SEV", FaultEvent::EVENT_CONFIRMED, 1); + ASSERT_TRUE(confirmed.has_value()); + EXPECT_EQ(confirmed->fault.severity, error); + EXPECT_EQ(get("AUDIT_SEV").severity, error); + EXPECT_EQ(audit_rows("AUDIT_SEV"), (Rows{{"occurred", warn}, {"confirmed", error}})) + << "each row records the stored severity at its transition"; + + // ERROR confirmed and cleared, then WARN reports open and confirm occurrence 2. + for (int i = 0; i < 3; ++i) { + ASSERT_TRUE(report("AUDIT_REACT", Fault::SEVERITY_ERROR, "/a")); + } + ASSERT_TRUE(call_clear_fault("AUDIT_REACT")); + for (int i = 0; i < 3; ++i) { + ASSERT_TRUE(report("AUDIT_REACT", Fault::SEVERITY_WARN, "/a")); + } + ASSERT_TRUE(wait_for_event("AUDIT_REACT", FaultEvent::EVENT_CONFIRMED, 2).has_value()); + EXPECT_EQ(get("AUDIT_REACT").severity, error); + EXPECT_EQ( + audit_rows("AUDIT_REACT"), + (Rows{{"occurred", error}, {"confirmed", error}, {"cleared", error}, {"occurred", error}, {"confirmed", error}})); + + // The near-miss rows of the same reports keep what was reported. + std::vector near_miss_severities; + for (const auto & record : fault_manager_->get_storage().get_near_misses({"AUDIT_REACT", "/a"})) { + near_miss_severities.push_back(record.severity); + } + EXPECT_EQ(near_miss_severities, (std::vector{error, error, warn, warn})); +} + +/// Global confirmation threshold -1. Sources under /robot/slow have their own threshold of -3. +class SeverityEntityThresholdNodeTest : public SeverityConsistencyNodeTest { + protected: + SeverityEntityThresholdNodeTest() { + std::ofstream(dir_ / "thresholds.yaml") << "/robot/slow:\n confirmation_threshold: -3\n"; + } + + std::vector fault_manager_overrides() override { + return { + rclcpp::Parameter("storage_type", GetParam()), + rclcpp::Parameter("database_path", (dir_ / "faults.db").string()), + rclcpp::Parameter("confirmation_threshold", -1), + rclcpp::Parameter("entity_thresholds.config_file", (dir_ / "thresholds.yaml").string()), + rclcpp::Parameter("fault_thresholds.config_file", (dir_ / "fault_thresholds.yaml").string()), + }; + } +}; + +INSTANTIATE_TEST_SUITE_P(Backends, SeverityEntityThresholdNodeTest, ::testing::Values("memory", "sqlite")); + +TEST_P(SeverityEntityThresholdNodeTest, CriticalCodeConfirmsUnderAnEntityThreshold) { + const std::string slow = "/robot/slow/sensor"; + + // The override is in effect: one WARN report from the entity leaves a new code pending. + ASSERT_TRUE(report("ENTITY_WARN", Fault::SEVERITY_WARN, slow)); + EXPECT_EQ(get("ENTITY_WARN").status, Fault::STATUS_PREFAILED); + + // CRITICAL, clear, WARN, all from the entity. + ASSERT_TRUE(report("ENTITY_CRIT", Fault::SEVERITY_CRITICAL, slow)); + ASSERT_TRUE(wait_for_event("ENTITY_CRIT", FaultEvent::EVENT_CONFIRMED, 1).has_value()); + ASSERT_TRUE(call_clear_fault("ENTITY_CRIT")); + ASSERT_TRUE(report("ENTITY_CRIT", Fault::SEVERITY_WARN, slow)); + auto shown = get("ENTITY_CRIT"); + EXPECT_EQ(shown.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(shown.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(shown.occurrence_count, 2u); + auto event = wait_for_event("ENTITY_CRIT", FaultEvent::EVENT_CONFIRMED, 2); + ASSERT_TRUE(event.has_value()) << "no fault_confirmed event for occurrence 2"; + EXPECT_EQ(event->fault.severity, Fault::SEVERITY_CRITICAL); + + // A CRITICAL record of a source on the global threshold does not confirm the entity's record. + ASSERT_TRUE(report("ENTITY_MIXED", Fault::SEVERITY_CRITICAL, "/robot/fast/sensor")); + ASSERT_TRUE(report("ENTITY_MIXED", Fault::SEVERITY_WARN, slow)); + shown = get("ENTITY_MIXED", slow); + EXPECT_EQ(shown.status, Fault::STATUS_PREFAILED); + EXPECT_EQ(shown.severity, Fault::SEVERITY_WARN); + EXPECT_EQ(get("ENTITY_MIXED", "/robot/fast/sensor").status, Fault::STATUS_CONFIRMED); +} + +TEST_P(SeverityConsistencyNodeTest, SeverityMismatchIsLoggedOncePerCodeOwnerAndSeverity) { + LogCapture capture; + auto logged = [&capture]() { + return capture.warnings_with("'MISMATCH'"); + }; + + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_ERROR, "/a")); + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_ERROR, "/a")); + EXPECT_TRUE(logged().empty()) << "a new code and a matching report are not mismatches"; + + for (int i = 0; i < 50; ++i) { + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_WARN, "/a")); + } + auto lines = logged(); + ASSERT_EQ(lines.size(), 1u) << "fifty identical mismatches log one line"; + EXPECT_NE(lines[0].find("WARN"), std::string::npos) << lines[0]; + EXPECT_NE(lines[0].find("ERROR"), std::string::npos) << lines[0]; + EXPECT_NE(lines[0].find("'/a'"), std::string::npos) << lines[0]; + + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_WARN, "/b")); + EXPECT_EQ(logged().size(), 1u) << "a report that creates /b's record has nothing to differ from"; + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_ERROR, "/b")); + lines = logged(); + ASSERT_EQ(lines.size(), 2u) << "another owner is named on its own"; + EXPECT_NE(lines[1].find("'/b'"), std::string::npos) << lines[1]; + EXPECT_NE(lines[1].find("stored as WARN"), std::string::npos) << "compared with /b's own record: " << lines[1]; + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_INFO, "/a")); + EXPECT_EQ(logged().size(), 3u) << "another reported severity is named on its own"; + ASSERT_TRUE(call_report_passed("MISMATCH", "/a")); + EXPECT_EQ(logged().size(), 3u) << "a PASSED report carries no severity"; + + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_CRITICAL, "/a")); + EXPECT_EQ(logged().size(), 4u) << "escalation is a mismatch too"; + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_WARN, "/a")); + ASSERT_TRUE(call_clear_fault("MISMATCH", "/a")); + for (int i = 0; i < 20; ++i) { + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_WARN, "/a")); + } + EXPECT_EQ(logged().size(), 4u) << "the bound holds across the stored severity and across occurrences"; + + ASSERT_TRUE(report("MISMATCH", Fault::SEVERITY_ERROR, "/a")); + lines = logged(); + ASSERT_EQ(lines.size(), 5u); + EXPECT_NE(lines[4].find("CRITICAL"), std::string::npos) << lines[4]; +} + +TEST_P(SeverityConsistencyNodeTest, SeverityMismatchIsLoggedPerCode) { + LogCapture capture; + ASSERT_TRUE(report("MISMATCH_A", Fault::SEVERITY_ERROR, "/a")); + ASSERT_TRUE(report("MISMATCH_B", Fault::SEVERITY_ERROR, "/a")); + + ASSERT_TRUE(report("MISMATCH_A", Fault::SEVERITY_WARN, "/a")); + ASSERT_EQ(capture.warnings_with("'MISMATCH_A'").size(), 1u); + + ASSERT_TRUE(report("MISMATCH_B", Fault::SEVERITY_WARN, "/a")); + const auto lines = capture.warnings_with("'MISMATCH_B'"); + ASSERT_EQ(lines.size(), 1u) << "the same owner and severity on another code is a mismatch of its own"; + EXPECT_NE(lines[0].find("'/a'"), std::string::npos) << lines[0]; + EXPECT_NE(lines[0].find("stored as ERROR"), std::string::npos) << lines[0]; +} + TEST_F(FaultEventPublishingTest, ClearNonExistentFaultNoEvent) { // Clear non-existent fault - should not publish event ASSERT_FALSE(call_clear_fault("NON_EXISTENT_FAULT")); @@ -1491,6 +2082,7 @@ TEST_F(FaultEventPublishingTest, GetFaultReturnsExtendedDataRecords) { return received_events_.size() >= 2; })); + // One shared record carries the code, so the unscoped call resolves to it. auto response = call_get_fault("EDR_TEST"); ASSERT_TRUE(response.has_value()); EXPECT_TRUE(response->success); @@ -1501,6 +2093,415 @@ TEST_F(FaultEventPublishingTest, GetFaultReturnsExtendedDataRecords) { EXPECT_NE(edr.last_occurrence_ns, 0); } +// --- Record resolution on the single-record services --- + +TEST_F(PerSourceNodeTest, ScopedClearClearsOneRecordAndPublishesItsOwner) { + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_a")); + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_b")); + ASSERT_TRUE(spin_until([this]() { + return received_events_.size() >= 2; + })); + received_events_.clear(); + + ASSERT_TRUE(call_clear_fault("SHARED_CODE", "/owner_a")); + ASSERT_TRUE(spin_until([this]() { + return !received_events_.empty(); + })); + + ASSERT_EQ(received_events_.size(), 1u) << "one record was cleared, so one event"; + EXPECT_EQ(received_events_[0].event_type, FaultEvent::EVENT_CLEARED); + EXPECT_EQ(received_events_[0].fault.status, Fault::STATUS_CLEARED); + ASSERT_EQ(received_events_[0].fault.reporting_sources.size(), 1u); + EXPECT_EQ(received_events_[0].fault.reporting_sources.front(), "/owner_a"); + + auto other = call_get_fault("SHARED_CODE", "/owner_b"); + ASSERT_TRUE(other.has_value()); + ASSERT_TRUE(other->success); + EXPECT_EQ(other->fault.status, Fault::STATUS_CONFIRMED) << "clearing one owner must not clear another"; +} + +TEST_F(PerSourceNodeTest, UnscopedClearOfTwoRecordsIsRefusedAndClearsNothing) { + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_a")); + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_b")); + ASSERT_TRUE(spin_until([this]() { + return received_events_.size() >= 2; + })); + received_events_.clear(); + + const auto response = clear_fault_response("SHARED_CODE"); + ASSERT_TRUE(response.has_value()); + EXPECT_FALSE(response->success); + EXPECT_EQ(response->message.rfind("ambiguous:", 0), 0u) << "got: " << response->message; + EXPECT_NE(response->message.find("/owner_a"), std::string::npos) << "the owners must be in the message"; + EXPECT_NE(response->message.find("/owner_b"), std::string::npos); + // The message is prose an operator reads, so it is two sentences, not a semicolon + // splice. The instruction has to survive whatever a client does to the rest of it. + EXPECT_NE(response->message.find("Set source_id to pick one."), std::string::npos) << "got: " << response->message; + EXPECT_EQ(response->message.find(';'), std::string::npos) << "got: " << response->message; + + // Nothing was cleared, and nothing was published: events arrive in publish order, so the + // refused clear published nothing if a later report's event comes first. + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_ERROR, "/owner_c")); + ASSERT_TRUE(spin_until([this]() { + return !received_events_.empty(); + })); + ASSERT_EQ(received_events_.size(), 1u); + EXPECT_EQ(received_events_[0].event_type, FaultEvent::EVENT_CONFIRMED); + ASSERT_EQ(received_events_[0].fault.reporting_sources.size(), 1u); + EXPECT_EQ(received_events_[0].fault.reporting_sources.front(), "/owner_c"); + for (const char * owner : {"/owner_a", "/owner_b"}) { + auto still = call_get_fault("SHARED_CODE", owner); + ASSERT_TRUE(still.has_value()); + ASSERT_TRUE(still->success) << owner; + EXPECT_EQ(still->fault.status, Fault::STATUS_CONFIRMED) << owner; + } +} + +TEST_F(PerSourceNodeTest, UnscopedClearOfOneRecordSucceeds) { + ASSERT_TRUE(call_report_fault("ONLY_ONE", Fault::SEVERITY_ERROR, "/owner_a")); + ASSERT_TRUE(spin_until([this]() { + return !received_events_.empty(); + })); + + EXPECT_TRUE(call_clear_fault("ONLY_ONE")); + + auto response = call_get_fault("ONLY_ONE", "/owner_a"); + ASSERT_TRUE(response.has_value()); + ASSERT_TRUE(response->success); + EXPECT_EQ(response->fault.status, Fault::STATUS_CLEARED); +} + +TEST_F(PerSourceNodeTest, GetFaultResolvesTheSameWayAsClear) { + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_WARN, "/owner_a")); + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_CRITICAL, "/owner_b")); + ASSERT_TRUE(call_report_fault("ONLY_ONE", Fault::SEVERITY_ERROR, "/owner_a")); + ASSERT_TRUE(spin_until([this]() { + return received_events_.size() >= 3; + })); + + // Scoped: exactly that owner's record. + auto scoped = call_get_fault("SHARED_CODE", "/owner_a"); + ASSERT_TRUE(scoped.has_value()); + ASSERT_TRUE(scoped->success); + EXPECT_EQ(scoped->fault.severity, Fault::SEVERITY_WARN); + ASSERT_EQ(scoped->fault.reporting_sources.size(), 1u); + EXPECT_EQ(scoped->fault.reporting_sources.front(), "/owner_a"); + + // Unscoped with two records: refused, not an arbitrary pick. + auto ambiguous = call_get_fault("SHARED_CODE"); + ASSERT_TRUE(ambiguous.has_value()); + EXPECT_FALSE(ambiguous->success); + EXPECT_EQ(ambiguous->error_message.rfind("ambiguous:", 0), 0u) << "got: " << ambiguous->error_message; + + // Unscoped with one record: that record. + auto single = call_get_fault("ONLY_ONE"); + ASSERT_TRUE(single.has_value()); + ASSERT_TRUE(single->success); + EXPECT_EQ(single->fault.severity, Fault::SEVERITY_ERROR); +} + +/// Gives the node a hierarchical rule that mutes symptoms, which is the only way two +/// owners can hold the same symptom code with exactly one of them muted. +class MutedFaultListingTest : public FaultEventPublishingTest { + protected: + std::vector fault_manager_overrides() override { + // Written here rather than in SetUp() because the base SetUp() asks for the + // overrides before it builds the node, and the node reads the file at construction. + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dist; + config_path_ = + std::filesystem::temp_directory_path() / ("test_muted_listing_" + std::to_string(dist(gen)) + ".yaml"); + std::ofstream out(config_path_); + out << R"( +correlation: + enabled: true + default_window_ms: 30000 + patterns: + symptom_codes: + codes: ["SYMPTOM_*"] + rules: + - id: root_cascade + name: "Root Cascade" + mode: hierarchical + root_cause: + codes: ["ROOT_CAUSE"] + symptoms: + - pattern: symptom_codes + window_ms: 30000 + mute_symptoms: true + auto_clear_with_root: true +)"; + out.close(); + + auto overrides = FaultEventPublishingTest::fault_manager_overrides(); + overrides.emplace_back("correlation.config_file", config_path_.string()); + overrides.push_back(per_source_codes({"ROOT_CAUSE", "SYMPTOM_SPEED"})); + return overrides; + } + + void SetUp() override { + FaultEventPublishingTest::SetUp(); + list_faults_client_ = test_node_->create_client(ns_ + "/fault_manager/list_faults"); + ASSERT_TRUE(list_faults_client_->wait_for_service(std::chrono::seconds(5))); + } + + void TearDown() override { + list_faults_client_.reset(); + FaultEventPublishingTest::TearDown(); + std::filesystem::remove(config_path_); + } + + std::optional call_list_faults(bool include_muted) { + auto request = std::make_shared(); + request->include_muted = include_muted; + request->statuses = {Fault::STATUS_CONFIRMED}; + + auto future = list_faults_client_->async_send_request(request); + if (!spin_until_future_ready(future)) { + return std::nullopt; + } + return *future.get(); + } + + /// Owners of every listed record carrying @p fault_code, sorted so the order the + /// backend happens to return does not decide the assertion. + static std::vector owners_of(const ListFaults::Response & response, const std::string & fault_code) { + std::vector owners; + for (const auto & fault : response.faults) { + if (fault.fault_code == fault_code && !fault.reporting_sources.empty()) { + owners.push_back(fault.reporting_sources.front()); + } + } + std::sort(owners.begin(), owners.end()); + return owners; + } + + std::filesystem::path config_path_; + rclcpp::Client::SharedPtr list_faults_client_; +}; + +// Muting belongs to a RECORD, so the default listing hides the muted record alone. A root cause +// of owner A mutes the symptom record of owner B too. Owner C reported the same code before any +// root cause was active, so C's record was never muted and a listing that drops it would hide a +// live CRITICAL fault from the default view. +TEST_F(MutedFaultListingTest, DefaultListingHidesTheMutedRecordNotTheCode) { + ASSERT_TRUE(call_report_fault("SYMPTOM_SPEED", Fault::SEVERITY_CRITICAL, "/owner_c")); + ASSERT_TRUE(call_report_fault("ROOT_CAUSE", Fault::SEVERITY_CRITICAL, "/owner_a")); + ASSERT_TRUE(call_report_fault("SYMPTOM_SPEED", Fault::SEVERITY_CRITICAL, "/owner_a")); + ASSERT_TRUE(call_report_fault("SYMPTOM_SPEED", Fault::SEVERITY_CRITICAL, "/owner_b")); + + auto visible = call_list_faults(false); + ASSERT_TRUE(visible.has_value()); + EXPECT_EQ(visible->muted_count, 2u) << "the symptom records of A and B are muted"; + EXPECT_EQ(owners_of(*visible, "SYMPTOM_SPEED"), std::vector{"/owner_c"}); + // The root cause itself is never muted, so the listing is not simply empty. + EXPECT_EQ(owners_of(*visible, "ROOT_CAUSE"), std::vector{"/owner_a"}); + + auto everything = call_list_faults(true); + ASSERT_TRUE(everything.has_value()); + EXPECT_EQ(owners_of(*everything, "SYMPTOM_SPEED"), (std::vector{"/owner_a", "/owner_b", "/owner_c"})); + // One entry per muted RECORD, naming both halves of its identity. + ASSERT_EQ(everything->muted_faults.size(), 2u); + std::vector muted_owners; + for (const auto & muted : everything->muted_faults) { + EXPECT_EQ(muted.fault_code, "SYMPTOM_SPEED"); + EXPECT_EQ(muted.root_cause_code, "ROOT_CAUSE"); + muted_owners.push_back(muted.source_id); + } + std::sort(muted_owners.begin(), muted_owners.end()); + EXPECT_EQ(muted_owners, (std::vector{"/owner_a", "/owner_b"})); +} + +// Two owners muted on one code are two entries of that code. Without the owner on the +// wire they are indistinguishable, and a reader cannot tell one muted record from the +// other, nor address either of them. +TEST_F(MutedFaultListingTest, MutedEntriesOfOneCodeAreToldApartByTheirOwner) { + for (const char * owner : {"/owner_a", "/owner_b"}) { + ASSERT_TRUE(call_report_fault("ROOT_CAUSE", Fault::SEVERITY_CRITICAL, owner)); + ASSERT_TRUE(call_report_fault("SYMPTOM_SPEED", Fault::SEVERITY_CRITICAL, owner)); + } + + auto everything = call_list_faults(true); + ASSERT_TRUE(everything.has_value()); + EXPECT_EQ(everything->muted_count, 2u); + + std::vector muted_owners; + for (const auto & muted : everything->muted_faults) { + EXPECT_EQ(muted.fault_code, "SYMPTOM_SPEED"); + EXPECT_EQ(muted.root_cause_code, "ROOT_CAUSE"); + muted_owners.push_back(muted.source_id); + } + std::sort(muted_owners.begin(), muted_owners.end()); + EXPECT_EQ(muted_owners, (std::vector{"/owner_a", "/owner_b"})); + + // With both muted the default listing shows neither, which is what makes the owner + // on the entry the only way to tell the two suppressed records apart. + auto visible = call_list_faults(false); + ASSERT_TRUE(visible.has_value()); + EXPECT_TRUE(owners_of(*visible, "SYMPTOM_SPEED").empty()); +} + +/// A node backed by a database written before the owner column existed, so the record +/// the migration produced can be exercised over the wire the way a client reaches it. +class LegacyOwnedRecordTest : public FaultEventPublishingTest { + protected: + std::vector fault_manager_overrides() override { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dist; + db_path_ = std::filesystem::temp_directory_path() / ("test_legacy_owner_" + std::to_string(dist(gen)) + ".db"); + + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path_.string().c_str(), &raw), SQLITE_OK); + EXPECT_EQ(sqlite3_exec(raw, + "CREATE TABLE faults (fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, " + "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " + "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " + "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " + "debounce_counter INTEGER NOT NULL DEFAULT 0, " + "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " + "confirmed_at_ns INTEGER NOT NULL DEFAULT 0);" + "INSERT INTO faults VALUES ('SHARED_CODE', 2, 'from an older release', 1, 1, 1, " + "'CONFIRMED', '[\"/old_source\"]', -1, 1, 0, 0);", + nullptr, nullptr, nullptr), + SQLITE_OK); + sqlite3_close(raw); + + return { + rclcpp::Parameter("storage_type", "sqlite"), + rclcpp::Parameter("database_path", db_path_.string()), + rclcpp::Parameter("confirmation_threshold", -1), + }; + } + + void TearDown() override { + FaultEventPublishingTest::TearDown(); + std::filesystem::remove(db_path_); + std::filesystem::remove(db_path_.string() + "-wal"); + std::filesystem::remove(db_path_.string() + "-shm"); + } + + std::filesystem::path db_path_; +}; + +// The migrated record is the shared record of its code, so it behaves as a record always did: +// a live source reporting the code joins it, any reporting source and the unscoped call reach it, +// and a clear ends it. +TEST_F(LegacyOwnedRecordTest, AMigratedRecordIsTheSharedRecordOfItsCode) { + ASSERT_TRUE(call_report_fault("SHARED_CODE", Fault::SEVERITY_CRITICAL, "/live_source")); + + const std::vector both = {"/live_source", "/old_source"}; + for (const std::string & source_id : + {std::string("*"), std::string("/old_source"), std::string("/live_source"), std::string()}) { + auto record = call_get_fault("SHARED_CODE", source_id); + ASSERT_TRUE(record.has_value()); + ASSERT_TRUE(record->success) << "source_id '" << source_id << "': " << record->error_message; + EXPECT_EQ(record->fault.reporting_sources, both) << "source_id '" << source_id << "'"; + EXPECT_EQ(record->fault.occurrence_count, 1u) << "the live source joined the record, it did not open a new one"; + } + + ASSERT_TRUE(call_clear_fault("SHARED_CODE", "")); + auto cleared = call_get_fault("SHARED_CODE", ""); + ASSERT_TRUE(cleared.has_value()); + ASSERT_TRUE(cleared->success) << cleared->error_message; + EXPECT_EQ(cleared->fault.status, Fault::STATUS_CLEARED); +} + +/// GetRosbag over the wire, with the links written straight into the node's store: the +/// recording itself comes from a background capture this suite does not run, and what +/// is under test is how the handler reports the links, not how they got there. +class GetRosbagAttachedCodesTest : public FaultEventPublishingTest { + protected: + std::vector fault_manager_overrides() override { + auto overrides = FaultEventPublishingTest::fault_manager_overrides(); + overrides.push_back(per_source_codes({"BURST_CODE", "BURST_FIRST", "BURST_SECOND"})); + return overrides; + } + + void SetUp() override { + FaultEventPublishingTest::SetUp(); + get_rosbag_client_ = test_node_->create_client(ns_ + "/fault_manager/get_rosbag"); + ASSERT_TRUE(get_rosbag_client_->wait_for_service(std::chrono::seconds(5))); + + // The handler serves a link only while its bag is on disk, so the directory has to + // exist for the response to be an answer rather than a reap. + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dist; + bag_path_ = std::filesystem::temp_directory_path() / ("test_get_rosbag_" + std::to_string(dist(gen))); + std::filesystem::create_directories(bag_path_); + } + + void TearDown() override { + get_rosbag_client_.reset(); + FaultEventPublishingTest::TearDown(); + std::error_code ec; + std::filesystem::remove_all(bag_path_, ec); + } + + /// Link one record to the single shared recording, the way a burst does. + void link_recording(const std::string & fault_code, const std::string & owner) { + ros2_medkit_fault_manager::RosbagFileInfo info; + info.fault_code = fault_code; + info.owner = owner; + info.file_path = bag_path_.string(); + info.format = "mcap"; + info.duration_sec = 5.0; + info.size_bytes = 100; + info.created_at_ns = 1000; + fault_manager_->get_storage_for_test().store_rosbag_file(info); + } + + std::optional call_get_rosbag(const std::string & fault_code, const std::string & source_id) { + auto request = std::make_shared(); + request->fault_code = fault_code; + request->source_id = source_id; + + auto future = get_rosbag_client_->async_send_request(request); + if (!spin_until_future_ready(future)) { + return std::nullopt; + } + return *future.get(); + } + + std::filesystem::path bag_path_; + rclcpp::Client::SharedPtr get_rosbag_client_; +}; + +// fault_codes and owners are parallel, one entry per record the recording is attached to. Two +// owners of one code are two links to the one bag, so the code is listed once per owner and the +// caller can tell whose record each entry is. +TEST_F(GetRosbagAttachedCodesTest, ACodeHeldByTwoOwnersIsListedOncePerOwner) { + ASSERT_TRUE(call_report_fault("BURST_CODE", Fault::SEVERITY_CRITICAL, "/owner_a")); + ASSERT_TRUE(call_report_fault("BURST_CODE", Fault::SEVERITY_CRITICAL, "/owner_b")); + link_recording("BURST_CODE", "/owner_a"); + link_recording("BURST_CODE", "/owner_b"); + + auto response = call_get_rosbag("BURST_CODE", "/owner_a"); + ASSERT_TRUE(response.has_value()); + ASSERT_TRUE(response->success) << response->error_message; + EXPECT_EQ(response->fault_codes, (std::vector{"BURST_CODE", "BURST_CODE"})); + EXPECT_EQ(response->owners.size(), response->fault_codes.size()); + std::vector owners(response->owners.begin(), response->owners.end()); + std::sort(owners.begin(), owners.end()); + EXPECT_EQ(owners, (std::vector{"/owner_a", "/owner_b"})); +} + +// A burst of two different codes on one bag reports both, in first-seen order, with their owners. +TEST_F(GetRosbagAttachedCodesTest, EveryDistinctCodeOfTheBurstIsReported) { + ASSERT_TRUE(call_report_fault("BURST_FIRST", Fault::SEVERITY_CRITICAL, "/owner_a")); + ASSERT_TRUE(call_report_fault("BURST_SECOND", Fault::SEVERITY_CRITICAL, "/owner_a")); + link_recording("BURST_FIRST", "/owner_a"); + link_recording("BURST_SECOND", "/owner_a"); + + auto response = call_get_rosbag("BURST_FIRST", "/owner_a"); + ASSERT_TRUE(response.has_value()); + ASSERT_TRUE(response->success) << response->error_message; + EXPECT_EQ(response->fault_codes, (std::vector{"BURST_FIRST", "BURST_SECOND"})); + EXPECT_EQ(response->owners, (std::vector{"/owner_a", "/owner_a"})); +} + // @verifies REQ_INTEROP_012 TEST_F(FaultEventPublishingTest, ListFaultsForEntitySuccess) { // Report faults from different sources @@ -1580,7 +2581,7 @@ TEST_F(FreezeFrameRetentionTest, GetFaultServesRetainedFreezeFrameAfterClear) { // Capture runs asynchronously on the pool; wait for a non-empty freeze-frame. ASSERT_TRUE(spin_until( [this]() { - auto frame = fault_manager_->get_storage().get_freeze_frame("FF_FAULT"); + auto frame = fault_manager_->get_storage().get_freeze_frame({"FF_FAULT", "*"}); return frame.has_value() && frame->data != "{}"; }, std::chrono::milliseconds(10000))); @@ -1647,6 +2648,7 @@ struct ReportedBag { ros2_medkit_fault_manager::RosbagFileInfo row_for(const std::string & fault_code) const { ros2_medkit_fault_manager::RosbagFileInfo info; info.fault_code = fault_code; + info.owner = "*"; info.file_path = dir.string(); info.recording_id = ros2_medkit_fault_manager::rosbag_recording_id(info.file_path); info.format = "sqlite3"; @@ -1694,7 +2696,7 @@ TEST_F(FaultEventPublishingTest, GetFaultReportsARecordingsServedBytesNotItsFoot EXPECT_NE(rosbag_snapshot->size_bytes, bag.footprint) << "the directory total is the quota's figure, not the API's"; // The row itself is untouched: the quota still sees the whole recording. - auto row = fault_manager_->get_storage().get_rosbag_file("SERVED_BYTES_FAULT"); + auto row = fault_manager_->get_storage().get_rosbag_file({"SERVED_BYTES_FAULT", "*"}); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->size_bytes, bag.footprint) << "reporting must not have rewritten what the quota counts"; } @@ -1831,13 +2833,13 @@ TEST_F(UnlimitedSnapshotRetentionTest, RetentionAppliesWithNoPerFaultCap) { ASSERT_TRUE(spin_until( [this]() { - return !fault_manager_->get_storage().get_snapshots("UNCAPPED_FAULT").empty(); + return !fault_manager_->get_storage().get_snapshots({"UNCAPPED_FAULT", "*"}).empty(); }, std::chrono::milliseconds(10000))); ASSERT_TRUE(call_clear_fault("UNCAPPED_FAULT")); - EXPECT_FALSE(fault_manager_->get_storage().get_snapshots("UNCAPPED_FAULT").empty()) + EXPECT_FALSE(fault_manager_->get_storage().get_snapshots({"UNCAPPED_FAULT", "*"}).empty()) << "acknowledgement deleted readings the operator asked to keep"; } @@ -1880,12 +2882,14 @@ TEST(MatchesEntityTest, EmptySources) { TEST(InMemorySnapshotRetentionTest, ClearKeepsSnapshotsWhenEvidenceIsRetained) { InMemoryFaultStorage storage; rclcpp::Clock clock; - storage.report_fault_event("KEEP", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "keep", - "/n", clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage.report_fault_event(FaultId{"KEEP", "/n"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "keep", "/n", clock.now().nanoseconds(), + ros2_medkit_fault_manager::DebounceConfig{}); storage.set_retain_snapshots_on_clear(true); ros2_medkit_fault_manager::SnapshotData row; row.fault_code = "KEEP"; + row.owner = "/n"; row.topic = "/t"; row.message_type = "std_msgs/msg/String"; row.data = "{}"; @@ -1893,29 +2897,31 @@ TEST(InMemorySnapshotRetentionTest, ClearKeepsSnapshotsWhenEvidenceIsRetained) { row.capture_id = 1; storage.store_snapshots({row}); - ASSERT_TRUE(storage.clear_fault("KEEP")); + ASSERT_TRUE(storage.clear_fault({"KEEP", "/n"})); // Same rule as the SQLite side: with a history configured, acknowledging must // not leave recordings whose matching readings were deleted. - EXPECT_EQ(storage.get_snapshots("KEEP").size(), 1u); + EXPECT_EQ(storage.get_snapshots({"KEEP", "/n"}).size(), 1u); } TEST(InMemorySnapshotRetentionTest, ClearStillDropsSnapshotsByDefault) { InMemoryFaultStorage storage; rclcpp::Clock clock; - storage.report_fault_event("DROP", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "drop", - "/n", clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage.report_fault_event(FaultId{"DROP", "/n"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "drop", "/n", clock.now().nanoseconds(), + ros2_medkit_fault_manager::DebounceConfig{}); ros2_medkit_fault_manager::SnapshotData row; row.fault_code = "DROP"; + row.owner = "/n"; row.topic = "/t"; row.message_type = "std_msgs/msg/String"; row.data = "{}"; row.capture_id = 1; storage.store_snapshots({row}); - ASSERT_TRUE(storage.clear_fault("DROP")); - EXPECT_TRUE(storage.get_snapshots("DROP").empty()) << "the default must stay what it always was"; + ASSERT_TRUE(storage.clear_fault({"DROP", "/n"})); + EXPECT_TRUE(storage.get_snapshots({"DROP", "/n"}).empty()) << "the default must stay what it always was"; } TEST(InMemorySnapshotLimitTest, KeepsTheNewestCaptureWholeAndDropsTheOldest) { @@ -1927,6 +2933,7 @@ TEST(InMemorySnapshotLimitTest, KeepsTheNewestCaptureWholeAndDropsTheOldest) { for (const char * topic : {"/a", "/b"}) { ros2_medkit_fault_manager::SnapshotData row; row.fault_code = "TEST"; + row.owner = "/n"; row.topic = topic; row.message_type = "std_msgs/msg/String"; row.data = "{}"; @@ -1943,7 +2950,7 @@ TEST(InMemorySnapshotLimitTest, KeepsTheNewestCaptureWholeAndDropsTheOldest) { // Whole captures in, whole capture out. Counting rows and rejecting the new one // stored capture 3 in part: one topic present, the rest silently absent. - auto result = storage.get_snapshots("TEST"); + auto result = storage.get_snapshots({"TEST", "/n"}); ASSERT_EQ(result.size(), 4u); for (const auto & s : result) { EXPECT_NE(s.capture_id, 1) << "the oldest capture should have gone whole"; @@ -1956,6 +2963,7 @@ TEST(InMemorySnapshotLimitTest, UnlimitedWhenZero) { ros2_medkit_fault_manager::SnapshotData snap; snap.fault_code = "TEST"; + snap.owner = "/n"; snap.topic = "/test"; snap.message_type = "std_msgs/msg/String"; snap.data = "{}"; @@ -1965,7 +2973,7 @@ TEST(InMemorySnapshotLimitTest, UnlimitedWhenZero) { storage.store_snapshot(snap); } - auto result = storage.get_snapshots("TEST"); + auto result = storage.get_snapshots({"TEST", "/n"}); EXPECT_EQ(result.size(), 20u); } @@ -1976,11 +2984,12 @@ TEST(InMemoryFreezeFrameTest, StoreAndRetrieve) { ros2_medkit_fault_manager::FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc"; frame.data = R"({"/plc/pressure":{"data":8.4}})"; frame.captured_at_ns = 1000; storage.store_freeze_frame(frame); - auto retrieved = storage.get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage.get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, frame.data); EXPECT_EQ(retrieved->captured_at_ns, 1000); @@ -1988,7 +2997,7 @@ TEST(InMemoryFreezeFrameTest, StoreAndRetrieve) { TEST(InMemoryFreezeFrameTest, AbsentForUnknownFault) { InMemoryFaultStorage storage; - EXPECT_FALSE(storage.get_freeze_frame("NEVER_CAPTURED").has_value()); + EXPECT_FALSE(storage.get_freeze_frame({"NEVER_CAPTURED", "/plc"}).has_value()); } TEST(InMemoryFreezeFrameTest, ReplacedOnRecapture) { @@ -1996,6 +3005,7 @@ TEST(InMemoryFreezeFrameTest, ReplacedOnRecapture) { ros2_medkit_fault_manager::FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc"; frame.data = R"({"/plc/pressure":{"data":8.4}})"; frame.captured_at_ns = 1000; storage.store_freeze_frame(frame); @@ -2004,7 +3014,7 @@ TEST(InMemoryFreezeFrameTest, ReplacedOnRecapture) { frame.captured_at_ns = 2000; storage.store_freeze_frame(frame); - auto retrieved = storage.get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage.get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, R"({"/plc/pressure":{"data":9.9}})"); EXPECT_EQ(retrieved->captured_at_ns, 2000); @@ -2014,11 +3024,12 @@ TEST(InMemoryFreezeFrameTest, SurvivesClearFault) { InMemoryFaultStorage storage; rclcpp::Clock clock; - storage.report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", - "/plc_node", clock.now().nanoseconds(), DebounceConfig{}); + storage.report_fault_event(FaultId{"PLC_PRESSURE_HIGH", "/plc_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Pressure high", "/plc_node", clock.now().nanoseconds(), DebounceConfig{}); ros2_medkit_fault_manager::SnapshotData snap; snap.fault_code = "PLC_PRESSURE_HIGH"; + snap.owner = "/plc_node"; snap.topic = "/plc/pressure"; snap.message_type = "std_msgs/msg/Float64"; snap.data = R"({"data":8.4})"; @@ -2027,14 +3038,15 @@ TEST(InMemoryFreezeFrameTest, SurvivesClearFault) { ros2_medkit_fault_manager::FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc_node"; frame.data = R"({"/plc/pressure":{"data":8.4}})"; frame.captured_at_ns = 1000; storage.store_freeze_frame(frame); - ASSERT_TRUE(storage.clear_fault("PLC_PRESSURE_HIGH")); + ASSERT_TRUE(storage.clear_fault({"PLC_PRESSURE_HIGH", "/plc_node"})); - EXPECT_TRUE(storage.get_snapshots("PLC_PRESSURE_HIGH").empty()); - auto retrieved = storage.get_freeze_frame("PLC_PRESSURE_HIGH"); + EXPECT_TRUE(storage.get_snapshots({"PLC_PRESSURE_HIGH", "/plc_node"}).empty()); + auto retrieved = storage.get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc_node"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, frame.data); } @@ -2051,6 +3063,7 @@ TEST(InMemoryRosbagRestoreTest, RestoreWithNewPathUnlinksTheOldExclusiveBag) { ros2_medkit_fault_manager::RosbagFileInfo info; info.fault_code = "X"; + info.owner = "/n"; info.file_path = old_path; info.format = "mcap"; info.duration_sec = 5.0; @@ -2062,7 +3075,7 @@ TEST(InMemoryRosbagRestoreTest, RestoreWithNewPathUnlinksTheOldExclusiveBag) { storage.store_rosbag_file(info); EXPECT_FALSE(std::filesystem::exists(old_path)) << "nobody references the old bag, it must be unlinked"; - auto row = storage.get_rosbag_file("X"); + auto row = storage.get_rosbag_file({"X", "/n"}); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->file_path, old_path + "_new"); } @@ -2080,16 +3093,19 @@ TEST(InMemoryRosbagRestoreTest, RestoreWithNewPathKeepsTheBagASiblingStillRefere info.size_bytes = 100; info.created_at_ns = 1000; info.fault_code = "X"; + info.owner = "/n"; storage.store_rosbag_file(info); info.fault_code = "Y"; + info.owner = "/n"; storage.store_rosbag_file(info); info.fault_code = "X"; + info.owner = "/n"; info.file_path = shared_path + "_new"; storage.store_rosbag_file(info); EXPECT_TRUE(std::filesystem::exists(shared_path)) << "the sibling fault still owns the shared bag"; - auto sibling = storage.get_rosbag_file("Y"); + auto sibling = storage.get_rosbag_file({"Y", "/n"}); ASSERT_TRUE(sibling.has_value()); EXPECT_EQ(sibling->file_path, shared_path); @@ -2332,17 +3348,23 @@ TEST_F(FaultAuditIntegrationTest, TransitionsAppendVerifiableChain) { EXPECT_EQ(records[2].event.transition, ros2_medkit_fault_manager::kTransitionConfirmed); EXPECT_EQ(records[3].event.transition, ros2_medkit_fault_manager::kTransitionCleared); EXPECT_EQ(records[1].event.fault_code, "AUDIT_FAULT"); + // The rows a report drove name the reporting source, the clear row its mechanism. A shared record has no owner. + EXPECT_EQ(records[1].event.source_id, "/plc/pump"); EXPECT_EQ(records[2].event.source_id, "/plc/pump"); + EXPECT_EQ(records[3].event.source_id, ros2_medkit_fault_manager::kAuditSourceClearService); + for (size_t i = 1; i < records.size(); ++i) { + EXPECT_TRUE(records[i].event.owner.empty()) << "row " << i; + } auto result = audit.verify(); EXPECT_TRUE(result.ok) << result.error; EXPECT_EQ(result.checked, 4); } -// Completeness: an auto-healed fault must record its END. One FAILED confirms the -// fault (occurred + confirmed); two PASSED drive the debounce counter to the -// healing threshold, which must append a distinct "healed" row (source auto_heal), -// and the full occurred -> confirmed -> healed chain must verify. +// Completeness: an auto-healed record must record its END. One FAILED confirms the +// record (occurred + confirmed), two PASSED drive the debounce counter to the +// healing threshold, which must append a distinct "healed" row. That row names its mechanism. +// The full occurred -> confirmed -> healed chain must verify. TEST_F(FaultAuditIntegrationTest, AutoHealAppendsHealedRow) { auto send_report = [&](uint8_t event_type) { auto req = std::make_shared(); @@ -2381,7 +3403,7 @@ TEST_F(FaultAuditIntegrationTest, AutoHealAppendsHealedRow) { EXPECT_EQ(transitions[0], ros2_medkit_fault_manager::kTransitionOccurred); EXPECT_EQ(transitions[1], ros2_medkit_fault_manager::kTransitionConfirmed); EXPECT_EQ(transitions[2], ros2_medkit_fault_manager::kTransitionHealed); - EXPECT_EQ(heal_source, "auto_heal"); + EXPECT_EQ(heal_source, ros2_medkit_fault_manager::kAuditSourceAutoHeal); auto result = audit.verify(); EXPECT_TRUE(result.ok) << result.error; @@ -2500,9 +3522,10 @@ TEST(FaultAuditTimerTest, TimerConfirmationAppendsConfirmedAuditRow) { config.confirmation_threshold = -3; config.auto_confirm_after_sec = 0.2; rclcpp::Clock clock(RCL_SYSTEM_TIME); - node->get_storage_for_test().report_fault_event("AUTO_CONF_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "stuck", "/robot/src", clock.now().nanoseconds(), config); - ASSERT_EQ(node->get_storage().get_fault("AUTO_CONF_1")->status, Fault::STATUS_PREFAILED); + node->get_storage_for_test().report_fault_event(FaultId{"AUTO_CONF_1", "/robot/src"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "stuck", "/robot/src", + clock.now().nanoseconds(), config); + ASSERT_EQ(node->get_storage().get_fault({"AUTO_CONF_1", "/robot/src"})->status, Fault::STATUS_PREFAILED); // Spin until a confirmed audit row appears or the budget expires (the wall // timer fires once per second). @@ -2524,7 +3547,7 @@ TEST(FaultAuditTimerTest, TimerConfirmationAppendsConfirmedAuditRow) { } EXPECT_TRUE(saw_confirmed) << "timer-driven confirmation was not audited"; - EXPECT_EQ(node->get_storage().get_fault("AUTO_CONF_1")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(node->get_storage().get_fault({"AUTO_CONF_1", "/robot/src"})->status, Fault::STATUS_CONFIRMED); EXPECT_TRUE(audit->verify().ok); } @@ -2537,13 +3560,13 @@ void seed_healed_fault(const std::string & db_path, const std::string & fault_co config.healing_enabled = true; config.healing_threshold = 3; rclcpp::Clock clock; - seed.report_fault_event(fault_code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "stale", "/robot/src", - clock.now().nanoseconds(), config); + seed.report_fault_event(FaultId{fault_code, "/robot/src"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "stale", + "/robot/src", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - seed.report_fault_event(fault_code, fd::ReportKind::kPassed, 0, "", "/robot/src", clock.now().nanoseconds(), - config); + seed.report_fault_event(FaultId{fault_code, "/robot/src"}, fd::ReportKind::kPassed, 0, "", "/robot/src", + clock.now().nanoseconds(), config); } - ASSERT_EQ(seed.get_fault(fault_code)->status, Fault::STATUS_HEALED); + ASSERT_EQ(seed.get_fault({fault_code, "/robot/src"})->status, Fault::STATUS_HEALED); } std::filesystem::path make_temp_dir(const char * prefix) { @@ -2578,7 +3601,7 @@ TEST(FaultAuditStartupReclassifyTest, StartupReclassifyAppendsClearedRow) { }); auto node = std::make_shared(options); - EXPECT_EQ(node->get_storage().get_fault("STALE_HEALED")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(node->get_storage().get_fault({"STALE_HEALED", "/robot/src"})->status, Fault::STATUS_CLEARED); const auto * audit = node->get_audit_log_for_test(); ASSERT_NE(audit, nullptr); @@ -2586,7 +3609,8 @@ TEST(FaultAuditStartupReclassifyTest, StartupReclassifyAppendsClearedRow) { for (const auto & rec : audit->read()) { if (rec.event.fault_code == "STALE_HEALED") { EXPECT_EQ(rec.event.transition, ros2_medkit_fault_manager::kTransitionCleared); - EXPECT_EQ(rec.event.source_id, "startup_reclassify"); + EXPECT_EQ(rec.event.source_id, ros2_medkit_fault_manager::kAuditSourceStartupReclassify); + EXPECT_EQ(rec.event.owner, "/robot/src") << "a per-source record's row names its owner"; EXPECT_EQ(rec.event.status, Fault::STATUS_CLEARED); ++cleared_rows; } @@ -2615,7 +3639,7 @@ TEST(FaultAuditStartupReclassifyTest, DisabledAuditStillReclassifies) { }); auto node = std::make_shared(options); - EXPECT_EQ(node->get_storage().get_fault("STALE_HEALED")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(node->get_storage().get_fault({"STALE_HEALED", "/robot/src"})->status, Fault::STATUS_CLEARED); EXPECT_EQ(node->get_audit_log_for_test(), nullptr); EXPECT_FALSE(std::filesystem::exists(dir / "fault_audit.db")); } @@ -2722,10 +3746,12 @@ TEST(FaultAuditFailClosedTest, FailClosedAbortsAndFlags) { // --- InMemoryFaultStorage snapshot retention tests --- -static void store_two_snapshots(InMemoryFaultStorage & storage, const std::string & fault_code) { +static void store_two_snapshots(InMemoryFaultStorage & storage, const std::string & fault_code, + const std::string & owner = "/test_node") { for (int i = 0; i < 2; ++i) { ros2_medkit_fault_manager::SnapshotData snapshot; snapshot.fault_code = fault_code; + snapshot.owner = owner; snapshot.topic = "/test/topic" + std::to_string(i); snapshot.message_type = "std_msgs/msg/String"; snapshot.data = R"({"data": "value"})"; @@ -2738,26 +3764,26 @@ TEST(InMemorySnapshotRetentionTest, RetainedOnClearWhenConfigured) { InMemoryFaultStorage storage; storage.set_retain_snapshots_on_clear(true); - storage.report_fault_event("SNAPSHOT_RETAIN_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "fault with evidence", "/test_node", 1000, default_config()); + storage.report_fault_event(FaultId{"SNAPSHOT_RETAIN_TEST", "/test_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "fault with evidence", "/test_node", 1000, default_config()); store_two_snapshots(storage, "SNAPSHOT_RETAIN_TEST"); - ASSERT_EQ(storage.get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u); + ASSERT_EQ(storage.get_snapshots({"SNAPSHOT_RETAIN_TEST", "/test_node"}).size(), 2u); - ASSERT_TRUE(storage.clear_fault("SNAPSHOT_RETAIN_TEST")); + ASSERT_TRUE(storage.clear_fault({"SNAPSHOT_RETAIN_TEST", "/test_node"})); - EXPECT_EQ(storage.get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u); + EXPECT_EQ(storage.get_snapshots({"SNAPSHOT_RETAIN_TEST", "/test_node"}).size(), 2u); } TEST(InMemorySnapshotRetentionTest, DeletedOnClearByDefault) { InMemoryFaultStorage storage; - storage.report_fault_event("SNAPSHOT_DROP_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "fault with evidence", "/test_node", 1000, default_config()); + storage.report_fault_event(FaultId{"SNAPSHOT_DROP_TEST", "/test_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "fault with evidence", "/test_node", 1000, default_config()); store_two_snapshots(storage, "SNAPSHOT_DROP_TEST"); - ASSERT_TRUE(storage.clear_fault("SNAPSHOT_DROP_TEST")); + ASSERT_TRUE(storage.clear_fault({"SNAPSHOT_DROP_TEST", "/test_node"})); - EXPECT_TRUE(storage.get_snapshots("SNAPSHOT_DROP_TEST").empty()); + EXPECT_TRUE(storage.get_snapshots({"SNAPSHOT_DROP_TEST", "/test_node"}).empty()); } TEST(InMemorySnapshotRetentionTest, HealedReclassificationDropsSnapshotsByDefault) { @@ -2768,20 +3794,20 @@ TEST(InMemorySnapshotRetentionTest, HealedReclassificationDropsSnapshotsByDefaul config.healing_enabled = true; config.healing_threshold = 1; - storage.report_fault_event("SNAPSHOT_DROP_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "fault with evidence", "/test_node", 1000, config); + storage.report_fault_event(FaultId{"SNAPSHOT_DROP_TEST", "/test_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "fault with evidence", "/test_node", 1000, config); store_two_snapshots(storage, "SNAPSHOT_DROP_TEST"); for (int i = 1; i <= 2; ++i) { - storage.report_fault_event("SNAPSHOT_DROP_TEST", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "/test_node", - 1000 + i, config); + storage.report_fault_event(FaultId{"SNAPSHOT_DROP_TEST", "/test_node"}, fd::ReportKind::kPassed, + Fault::SEVERITY_ERROR, "", "/test_node", 1000 + i, config); } - auto healed = storage.get_fault("SNAPSHOT_DROP_TEST"); + auto healed = storage.get_fault({"SNAPSHOT_DROP_TEST", "/test_node"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; ASSERT_EQ(storage.reclassify_healed_as_cleared().size(), 1u); - EXPECT_TRUE(storage.get_snapshots("SNAPSHOT_DROP_TEST").empty()); + EXPECT_TRUE(storage.get_snapshots({"SNAPSHOT_DROP_TEST", "/test_node"}).empty()); } TEST(InMemorySnapshotRetentionTest, HealedReclassificationKeepsSnapshotsWhenConfigured) { @@ -2791,16 +3817,16 @@ TEST(InMemorySnapshotRetentionTest, HealedReclassificationKeepsSnapshotsWhenConf config.healing_enabled = true; config.healing_threshold = 1; - storage.report_fault_event("SNAPSHOT_RETAIN_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "fault with evidence", "/test_node", 1000, config); + storage.report_fault_event(FaultId{"SNAPSHOT_RETAIN_TEST", "/test_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "fault with evidence", "/test_node", 1000, config); store_two_snapshots(storage, "SNAPSHOT_RETAIN_TEST"); for (int i = 1; i <= 2; ++i) { - storage.report_fault_event("SNAPSHOT_RETAIN_TEST", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "/test_node", - 1000 + i, config); + storage.report_fault_event(FaultId{"SNAPSHOT_RETAIN_TEST", "/test_node"}, fd::ReportKind::kPassed, + Fault::SEVERITY_ERROR, "", "/test_node", 1000 + i, config); } ASSERT_EQ(storage.reclassify_healed_as_cleared().size(), 1u); - EXPECT_EQ(storage.get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u); + EXPECT_EQ(storage.get_snapshots({"SNAPSHOT_RETAIN_TEST", "/test_node"}).size(), 2u); } TEST(InMemorySnapshotRetentionTest, HealedReclassificationLeavesOtherFaultsSnapshotsAlone) { @@ -2809,22 +3835,23 @@ TEST(InMemorySnapshotRetentionTest, HealedReclassificationLeavesOtherFaultsSnaps healing.healing_enabled = true; healing.healing_threshold = 1; - storage.report_fault_event("HEALED_FAULT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "/test_node", - 1000, healing); + storage.report_fault_event(FaultId{"HEALED_FAULT", "/test_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "fault", "/test_node", 1000, healing); store_two_snapshots(storage, "HEALED_FAULT"); for (int i = 1; i <= 2; ++i) { - storage.report_fault_event("HEALED_FAULT", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "/test_node", - 1000 + i, healing); + storage.report_fault_event(FaultId{"HEALED_FAULT", "/test_node"}, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, + "", "/test_node", 1000 + i, healing); } - storage.report_fault_event("ACTIVE_FAULT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "/test_node", - 1000, default_config()); + storage.report_fault_event(FaultId{"ACTIVE_FAULT", "/test_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "fault", "/test_node", 1000, default_config()); store_two_snapshots(storage, "ACTIVE_FAULT"); ASSERT_EQ(storage.reclassify_healed_as_cleared().size(), 1u); - EXPECT_TRUE(storage.get_snapshots("HEALED_FAULT").empty()); - EXPECT_EQ(storage.get_snapshots("ACTIVE_FAULT").size(), 2u) << "reclassification touched an unrelated fault"; + EXPECT_TRUE(storage.get_snapshots({"HEALED_FAULT", "/test_node"}).empty()); + EXPECT_EQ(storage.get_snapshots({"ACTIVE_FAULT", "/test_node"}).size(), 2u) + << "reclassification touched an unrelated fault"; } TEST(InMemorySnapshotRetentionTest, RetentionIsPerFaultCodeNotGlobal) { @@ -2832,15 +3859,15 @@ TEST(InMemorySnapshotRetentionTest, RetentionIsPerFaultCodeNotGlobal) { storage.set_retain_snapshots_on_clear(true); for (const auto & code : {"FAULT_A", "FAULT_B"}) { - storage.report_fault_event(code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "/test_node", 1000, - default_config()); + storage.report_fault_event(FaultId{code, "/test_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", + "/test_node", 1000, default_config()); store_two_snapshots(storage, code); } - ASSERT_TRUE(storage.clear_fault("FAULT_A")); + ASSERT_TRUE(storage.clear_fault({"FAULT_A", "/test_node"})); - EXPECT_EQ(storage.get_snapshots("FAULT_A").size(), 2u); - EXPECT_EQ(storage.get_snapshots("FAULT_B").size(), 2u); + EXPECT_EQ(storage.get_snapshots({"FAULT_A", "/test_node"}).size(), 2u); + EXPECT_EQ(storage.get_snapshots({"FAULT_B", "/test_node"}).size(), 2u); } // --- InMemoryFaultStorage near-miss series tests --- @@ -2866,18 +3893,19 @@ TEST(InMemoryNearMissTest, SeriesIsAppendedAndSurvivesClear) { const auto config = near_miss_config(); for (int i = 0; i < 3; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); } - auto series = storage.get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u); EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[2].debounce_counter, -3); EXPECT_EQ(series[0].confirmation_threshold, -4); - ASSERT_TRUE(storage.clear_fault("PUMP_PRESSURE_LOW")); - EXPECT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + ASSERT_TRUE(storage.clear_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); + EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); } TEST(InMemoryNearMissTest, ConfirmingReportIsNotANearMiss) { @@ -2885,14 +3913,15 @@ TEST(InMemoryNearMissTest, ConfirmingReportIsNotANearMiss) { const auto config = near_miss_config(); for (int i = 0; i < 4; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); } - auto fault = storage.get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage.get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); - EXPECT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); } TEST(InMemoryNearMissTest, SeriesBoundedKeepingNewest) { @@ -2902,11 +3931,12 @@ TEST(InMemoryNearMissTest, SeriesBoundedKeepingNewest) { storage.set_max_near_misses_per_fault(3); for (int i = 0; i < 5; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); } - auto series = storage.get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u); EXPECT_EQ(series[0].debounce_counter, -3); EXPECT_EQ(series[2].debounce_counter, -5); @@ -2917,13 +3947,14 @@ TEST(InMemoryNearMissTest, PassedReportIsNotANearMiss) { const auto config = near_miss_config(); for (int i = 0; i < 2; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); } - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/hydraulics/pump", - near_miss_time(2).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", near_miss_time(2).nanoseconds(), config); - EXPECT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); + EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); } TEST(InMemoryNearMissTest, SeriesSurvivesHealedReclassification) { @@ -2932,20 +3963,21 @@ TEST(InMemoryNearMissTest, SeriesSurvivesHealedReclassification) { config.healing_enabled = true; config.healing_threshold = 1; - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(0).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(0).nanoseconds(), config); for (int i = 1; i <= 2; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); } - auto healed = storage.get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage.get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; - ASSERT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + ASSERT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); ASSERT_EQ(storage.reclassify_healed_as_cleared().size(), 1u); - EXPECT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); } TEST(InMemoryNearMissTest, BoundIsPerFaultCode) { @@ -2955,14 +3987,16 @@ TEST(InMemoryNearMissTest, BoundIsPerFaultCode) { storage.set_max_near_misses_per_fault(2); for (int i = 0; i < 3; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); - storage.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + near_miss_time(i).nanoseconds(), config); } - EXPECT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); - EXPECT_EQ(storage.get_near_misses("MOTOR_OVERHEAT").size(), 2u); + EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); + EXPECT_EQ(storage.get_near_misses({"MOTOR_OVERHEAT", "/powertrain/motor"}).size(), 2u); } TEST(InMemoryNearMissTest, BoundZeroIsUnlimited) { @@ -2972,11 +4006,12 @@ TEST(InMemoryNearMissTest, BoundZeroIsUnlimited) { storage.set_max_near_misses_per_fault(0); for (int i = 0; i < 150; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); } - EXPECT_EQ(storage.get_near_misses("PUMP_PRESSURE_LOW").size(), 150u); + EXPECT_EQ(storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 150u); } TEST(InMemoryNearMissTest, ApplyingSmallerBoundTrimsExistingSeries) { @@ -2986,13 +4021,14 @@ TEST(InMemoryNearMissTest, ApplyingSmallerBoundTrimsExistingSeries) { storage.set_max_near_misses_per_fault(0); for (int i = 0; i < 5; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); } storage.set_max_near_misses_per_fault(2); - auto series = storage.get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[0].debounce_counter, -4); EXPECT_EQ(series[1].debounce_counter, -5); @@ -3005,10 +4041,12 @@ TEST(InMemoryNearMissTest, ApplyingBoundReportsHowManyEntriesItDropped) { storage.set_max_near_misses_per_fault(0); for (int i = 0; i < 5; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); - storage.report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + near_miss_time(i).nanoseconds(), config); } EXPECT_EQ(storage.set_max_near_misses_per_fault(2), 6u); @@ -3021,20 +4059,22 @@ TEST(InMemoryNearMissTest, EntriesCarryTheResultingStatus) { config.healing_enabled = true; config.healing_threshold = 2; - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(0).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(0).nanoseconds(), config); for (int i = 1; i <= 3; ++i) { - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", near_miss_time(i).nanoseconds(), config); } - auto healed = storage.get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage.get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; - storage.report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", near_miss_time(4).nanoseconds(), config); + storage.report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + near_miss_time(4).nanoseconds(), config); - auto series = storage.get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage.get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[0].resulting_status, Fault::STATUS_PREFAILED); EXPECT_EQ(series[1].resulting_status, Fault::STATUS_HEALED) @@ -3042,10 +4082,11 @@ TEST(InMemoryNearMissTest, EntriesCarryTheResultingStatus) { } /// Run one identical report sequence against both backends and return their near-miss series. -/// Per-entity threshold overrides mean two reports for the SAME fault code can carry different -/// DebounceConfig values, which is the only way a stored counter ends up outside the band the next -/// report is evaluated against. -TEST(StorageBackendParityTest, MixedEntityThresholdsProduceTheSameSeries) { +/// A record's counter belongs to one source, but that source's resolved band can change under +/// it - an entity-threshold file reloaded with different values, or a global config that moved +/// between runs - which is how a stored counter ends up outside the band the next report is +/// evaluated against. Both backends clamp on read, and they have to agree on the result. +TEST(StorageBackendParityTest, AChangedBandProducesTheSameSeriesInBothBackends) { std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution dist; @@ -3057,34 +4098,36 @@ TEST(StorageBackendParityTest, MixedEntityThresholdsProduceTheSameSeries) { wide.healing_threshold = 6; wide.critical_immediate_confirm = false; - DebounceConfig narrow; // a second source, with a lower ceiling + DebounceConfig narrow; // the same source, re-resolved to a lower ceiling narrow.confirmation_threshold = -4; narrow.healing_threshold = 3; narrow.critical_immediate_confirm = false; + const ros2_medkit_fault_manager::FaultId id{"SHARED_CODE", "/one_source"}; + auto drive = [&](ros2_medkit_fault_manager::FaultStorage & storage) { - storage.report_fault_event("SHARED_CODE", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "dip", "/wide_source", - 1000, wide); + storage.report_fault_event(FaultId{"SHARED_CODE", "/one_source"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, + "dip", "/one_source", 1000, wide); for (int i = 1; i <= 7; ++i) { - storage.report_fault_event("SHARED_CODE", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", "/wide_source", - 1000 + i, wide); + storage.report_fault_event(FaultId{"SHARED_CODE", "/one_source"}, fd::ReportKind::kPassed, Fault::SEVERITY_WARN, + "", "/one_source", 1000 + i, wide); } - storage.report_fault_event("SHARED_CODE", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "dip", "/narrow_source", - 2000, narrow); + storage.report_fault_event(FaultId{"SHARED_CODE", "/one_source"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, + "dip", "/one_source", 2000, narrow); }; InMemoryFaultStorage memory; drive(memory); - const auto memory_series = memory.get_near_misses("SHARED_CODE"); - const auto memory_fault = memory.get_fault("SHARED_CODE"); + const auto memory_series = memory.get_near_misses(id); + const auto memory_fault = memory.get_fault(id); std::vector sqlite_series; std::optional sqlite_fault; { ros2_medkit_fault_manager::SqliteFaultStorage sqlite(db_path.string(), test_log_sink()); drive(sqlite); - sqlite_series = sqlite.get_near_misses("SHARED_CODE"); - sqlite_fault = sqlite.get_fault("SHARED_CODE"); + sqlite_series = sqlite.get_near_misses(id); + sqlite_fault = sqlite.get_fault(id); } std::filesystem::remove(db_path); std::filesystem::remove(db_path.string() + "-wal"); @@ -3105,7 +4148,7 @@ TEST(StorageBackendParityTest, MixedEntityThresholdsProduceTheSameSeries) { TEST(InMemoryNearMissTest, EmptyForUnknownFault) { InMemoryFaultStorage storage; - EXPECT_TRUE(storage.get_near_misses("NEVER_REPORTED").empty()); + EXPECT_TRUE(storage.get_near_misses({"NEVER_REPORTED", "/node1"}).empty()); } // --- Snapshot read path with retention enabled --- @@ -3170,9 +4213,10 @@ class SnapshotReadPathTest : public ::testing::Test { } void store_snapshot(const std::string & fault_code, const std::string & topic, const std::string & data, - int64_t captured_at_ns) { + int64_t captured_at_ns, const std::string & owner = "/plc") { ros2_medkit_fault_manager::SnapshotData snapshot; snapshot.fault_code = fault_code; + snapshot.owner = owner; snapshot.topic = topic; snapshot.message_type = "std_msgs/msg/String"; snapshot.data = data; @@ -3189,8 +4233,8 @@ class SnapshotReadPathTest : public ::testing::Test { TEST_F(SnapshotReadPathTest, NewestSnapshotOfATopicWins) { auto & storage = fault_manager_->get_storage_for_test(); - storage.report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure high", - "/plc", 1000, DebounceConfig{}); + storage.report_fault_event(FaultId{"PLC_PRESSURE_HIGH", "/plc"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "pressure high", "/plc", 1000, DebounceConfig{}); store_snapshot("PLC_PRESSURE_HIGH", "/plc/pressure", R"({"data":"old"})", 1000); store_snapshot("PLC_PRESSURE_HIGH", "/plc/pressure", R"({"data":"new"})", 5000); @@ -3210,12 +4254,13 @@ TEST_F(SnapshotReadPathTest, NewestSnapshotOfATopicWins) { TEST_F(SnapshotReadPathTest, FreezeFrameStaysVisibleBehindRetainedSnapshots) { auto & storage = fault_manager_->get_storage_for_test(); - storage.report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure high", - "/plc", 1000, DebounceConfig{}); + storage.report_fault_event(FaultId{"PLC_PRESSURE_HIGH", "/plc"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "pressure high", "/plc", 1000, DebounceConfig{}); store_snapshot("PLC_PRESSURE_HIGH", "/plc/pressure", R"({"data":"from an earlier occurrence"})", 1000); ros2_medkit_fault_manager::FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc"; frame.data = R"({"/plc/pressure":{"data":"latest confirmation"}})"; frame.captured_at_ns = 9000; storage.store_freeze_frame(frame); diff --git a/src/ros2_medkit_fault_manager/test/test_fault_manager_core_concurrency.cpp b/src/ros2_medkit_fault_manager/test/test_fault_manager_core_concurrency.cpp index 530adff39..708c97318 100644 --- a/src/ros2_medkit_fault_manager/test/test_fault_manager_core_concurrency.cpp +++ b/src/ros2_medkit_fault_manager/test/test_fault_manager_core_concurrency.cpp @@ -39,6 +39,9 @@ constexpr int kReportsPerThread = 500; constexpr int kCodes = 4; constexpr int kClearEvery = 50; const char * const kClearedCode = "CLEARED_CODE"; +/// Every load thread reports as one source, so the threads contend on the same records. +const char * const kLoadSource = "/load"; +const char * const kSeedSource = "/seed"; class NullLogSink : public core::LogSink { public: @@ -64,12 +67,12 @@ class CountingEventSink : public core::EventSink { class NoCaptureHooks : public core::CaptureHooks { public: - core::CaptureResult capture_now(const std::string & /*code*/) override { + core::CaptureResult capture_now(const ros2_medkit_fault_manager::FaultId & /*id*/) override { return {}; } - void on_prefailed(const std::string & /*code*/) override { + void on_prefailed(const ros2_medkit_fault_manager::FaultId & /*id*/) override { } - void on_cleared(const std::string & /*code*/) override { + void on_cleared(const ros2_medkit_fault_manager::FaultId & /*id*/) override { } }; @@ -85,15 +88,14 @@ class FixedClock : public core::WallClock { /// overlaps with others on every first report of a code. class YieldingStorage : public ros2_medkit_fault_manager::InMemoryFaultStorage { public: - bool report_fault_event(const std::string & fault_code, fd::ReportKind kind, uint8_t severity, + bool report_fault_event(const ros2_medkit_fault_manager::FaultId & id, fd::ReportKind kind, uint8_t severity, const std::string & description, const std::string & source_id, int64_t timestamp_ns, const ros2_medkit_fault_manager::DebounceConfig & config) override { pause(); - return InMemoryFaultStorage::report_fault_event(fault_code, kind, severity, description, source_id, timestamp_ns, - config); + return InMemoryFaultStorage::report_fault_event(id, kind, severity, description, source_id, timestamp_ns, config); } - std::optional get_fault(const std::string & fault_code) const override { - auto result = InMemoryFaultStorage::get_fault(fault_code); + std::optional get_fault(const ros2_medkit_fault_manager::FaultId & id) const override { + auto result = InMemoryFaultStorage::get_fault(id); pause(); return result; } @@ -130,7 +132,7 @@ struct Fixture { fd::FaultReport seed; seed.fault_code = kClearedCode; seed.severity = fd::severity::kError; - seed.source_id = "/seed"; + seed.source_id = kSeedSource; EXPECT_TRUE(core.report(seed).accepted); } }; @@ -147,10 +149,10 @@ void run_thread_sequence(core::FaultManagerCore & core, int thread_index) { report.kind = fd::ReportKind::kFailed; report.severity = fd::severity::kError; report.description = "load"; - report.source_id = "/thread_" + std::to_string(thread_index); + report.source_id = kLoadSource; EXPECT_TRUE(core.report(report).accepted); if (i % kClearEvery == 0) { - EXPECT_TRUE(core.clear(kClearedCode, true).success); + EXPECT_TRUE(core.clear(kClearedCode, kSeedSource, true).success); } } } @@ -199,8 +201,8 @@ TEST(FaultManagerCoreConcurrency, ParallelReportsKeepCounts) { // Per code occurrence_count and status equal the replay's. for (int c = 0; c < kCodes; ++c) { const auto code = code_name(c); - const auto got = parallel.core.get(code); - const auto want = replay.core.get(code); + const auto got = parallel.core.get(code, kLoadSource); + const auto want = replay.core.get(code, kLoadSource); ASSERT_TRUE(got.success) << code; ASSERT_TRUE(want.success) << code; EXPECT_EQ(got.fault.occurrence_count, want.fault.occurrence_count) << code; diff --git a/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp b/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp new file mode 100644 index 000000000..24db91440 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/test_fault_severity.cpp @@ -0,0 +1,916 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Severity contract of a fault record, run against both storage backends. +/// +/// A record (fault_code, owner) has one severity: the highest any FAILED report from its owner has +/// carried, within an occurrence and across occurrences. With critical_immediate_confirm, a report +/// on a record whose stored severity is CRITICAL confirms it, so no stored record is CRITICAL in +/// PREFAILED or PREPASSED. Another owner's record of the same code is untouched. + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rclcpp/rclcpp.hpp" +#include "ros2_medkit_fault_manager/fault_storage.hpp" +#include "ros2_medkit_fault_manager/sqlite_fault_storage.hpp" +#include "ros2_medkit_msgs/msg/fault.hpp" +#include "test_log_sink.hpp" + +namespace { + +using ros2_medkit_fault_manager::DebounceConfig; +using ros2_medkit_fault_manager::FaultId; +using ros2_medkit_fault_manager::FaultStorage; +using ros2_medkit_fault_manager::InMemoryFaultStorage; +using ros2_medkit_fault_manager::SqliteFaultStorage; +using ros2_medkit_msgs::msg::Fault; + +constexpr const char * kOwner = "/reporter"; + +constexpr int64_t kBaseNs = 1'700'000'000'000'000'000LL; + +int64_t at(int64_t second) { + return kBaseNs + second * 1'000'000'000LL; +} + +DebounceConfig confirm_at(int32_t confirmation_threshold) { + DebounceConfig config; + config.confirmation_threshold = confirmation_threshold; + return config; +} + +bool pending(const std::string & status) { + return status == Fault::STATUS_PREFAILED || status == Fault::STATUS_PREPASSED; +} + +std::filesystem::path unique_temp_dir() { + std::random_device rd; + std::mt19937_64 gen(rd()); + std::uniform_int_distribution dist; + return std::filesystem::temp_directory_path() / ("medkit_severity_" + std::to_string(dist(gen))); +} + +template +struct BackendFactory; + +template <> +struct BackendFactory { + static std::unique_ptr make(const std::filesystem::path & /*dir*/) { + return std::make_unique(); + } +}; + +template <> +struct BackendFactory { + static std::unique_ptr make(const std::filesystem::path & dir) { + return std::make_unique((dir / "faults.db").string(), test_log_sink()); + } +}; + +template +class FaultSeverityTest : public ::testing::Test { + protected: + void SetUp() override { + dir_ = unique_temp_dir(); + std::filesystem::create_directories(dir_); + storage_ = BackendFactory::make(dir_); + } + + void TearDown() override { + storage_.reset(); + std::error_code ec; + std::filesystem::remove_all(dir_, ec); + } + + bool failed(const std::string & code, uint8_t severity, int64_t second, const DebounceConfig & config, + const std::string & owner = kOwner) { + return storage_->report_fault_event(FaultId{code, owner}, fd::ReportKind::kFailed, severity, "report", owner, + at(second), config); + } + + void passed(const std::string & code, int64_t second, const DebounceConfig & config, + const std::string & owner = kOwner) { + storage_->report_fault_event(FaultId{code, owner}, fd::ReportKind::kPassed, 0, "", owner, at(second), config); + } + + bool clear(const std::string & code, const std::string & owner = kOwner) { + return storage_->clear_fault(FaultId{code, owner}); + } + + fd::FaultRecord fault(const std::string & code, const std::string & owner = kOwner) { + auto f = storage_->get_fault(FaultId{code, owner}); + EXPECT_TRUE(f.has_value()) << code << " " << owner; + return f.value_or(fd::FaultRecord{}); + } + + /// Every stored CRITICAL record in a pending status. + std::vector pending_critical() { + return storage_->list_faults(true, Fault::SEVERITY_CRITICAL, {Fault::STATUS_PREFAILED, Fault::STATUS_PREPASSED}); + } + + std::filesystem::path dir_; + std::unique_ptr storage_; +}; + +using Backends = ::testing::Types; +TYPED_TEST_SUITE(FaultSeverityTest, Backends); + +TYPED_TEST(FaultSeverityTest, CriticalThenClearedWarnConfirmsAtEveryThreshold) { + for (const int32_t threshold : {-1, -2, -3}) { + const auto config = confirm_at(threshold); + ASSERT_TRUE(config.critical_immediate_confirm); + const std::string code = "CRIT_CLEAR_WARN_T" + std::to_string(-threshold); + SCOPED_TRACE(code); + + this->failed(code, Fault::SEVERITY_CRITICAL, 0, config); + ASSERT_EQ(this->fault(code).status, Fault::STATUS_CONFIRMED); + ASSERT_TRUE(this->clear(code)); + + EXPECT_TRUE(this->failed(code, Fault::SEVERITY_WARN, 1, config)) << "a report on a CLEARED record is an occurrence"; + const auto f = this->fault(code); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL) << "the record keeps the highest severity ever reported"; + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED) << "a CRITICAL record confirms on its first report"; + EXPECT_EQ(f.occurrence_count, 2u); + EXPECT_TRUE(this->pending_critical().empty()); + } +} + +TYPED_TEST(FaultSeverityTest, TimeBasedConfirmationFindsACriticalRecordConfirmedAtTheReport) { + auto config = confirm_at(-3); + config.auto_confirm_after_sec = 5.0; + this->storage_->set_debounce_config(config); + + this->failed("TIMED_CRIT", Fault::SEVERITY_CRITICAL, 0, config); + ASSERT_TRUE(this->clear("TIMED_CRIT")); + this->failed("TIMED_CRIT", Fault::SEVERITY_WARN, 1, config); + this->failed("TIMED_WARN", Fault::SEVERITY_WARN, 1, config); + EXPECT_EQ(this->fault("TIMED_CRIT").status, Fault::STATUS_CONFIRMED) << "confirmed by the report, not the sweep"; + EXPECT_EQ(this->fault("TIMED_WARN").status, Fault::STATUS_PREFAILED); + + EXPECT_TRUE(this->storage_->check_time_based_confirmation(at(3)).empty()) << "before the window ends"; + const auto swept = this->storage_->check_time_based_confirmation(at(10)); + ASSERT_EQ(swept.size(), 1u) << "the sweep confirms the pending record only"; + EXPECT_EQ(swept[0].fault_code, "TIMED_WARN"); + EXPECT_EQ(this->fault("TIMED_WARN").status, Fault::STATUS_CONFIRMED); + const auto f = this->fault("TIMED_CRIT"); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(f.occurrence_count, 2u); +} + +TYPED_TEST(FaultSeverityTest, AnotherOwnersCriticalDoesNotConfirmThisRecord) { + const auto config = confirm_at(-3); + + this->failed("SHARED_CRIT", Fault::SEVERITY_CRITICAL, 0, config, "/a"); + ASSERT_TRUE(this->clear("SHARED_CRIT", "/a")); + + EXPECT_TRUE(this->failed("SHARED_CRIT", Fault::SEVERITY_WARN, 1, config, "/b")) << "/b owns no record yet"; + auto b = this->fault("SHARED_CRIT", "/b"); + EXPECT_EQ(b.severity, Fault::SEVERITY_WARN) << "severity is per record"; + EXPECT_EQ(b.status, Fault::STATUS_PREFAILED); + + this->failed("SHARED_CRIT", Fault::SEVERITY_WARN, 2, config, "/a"); + const auto a = this->fault("SHARED_CRIT", "/a"); + EXPECT_EQ(a.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(a.status, Fault::STATUS_CONFIRMED); + b = this->fault("SHARED_CRIT", "/b"); + EXPECT_EQ(b.status, Fault::STATUS_PREFAILED) << "confirming /a leaves /b's record as it was"; + EXPECT_TRUE(this->pending_critical().empty()); +} + +TYPED_TEST(FaultSeverityTest, WarnCriticalWarnWithinAnOccurrence) { + const auto config = confirm_at(-3); + + this->failed("W_C_W", Fault::SEVERITY_WARN, 0, config); + EXPECT_EQ(this->fault("W_C_W").status, Fault::STATUS_PREFAILED); + EXPECT_EQ(this->fault("W_C_W").severity, Fault::SEVERITY_WARN); + + this->failed("W_C_W", Fault::SEVERITY_CRITICAL, 1, config); + EXPECT_EQ(this->fault("W_C_W").status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(this->fault("W_C_W").severity, Fault::SEVERITY_CRITICAL) << "escalation within an occurrence"; + + this->failed("W_C_W", Fault::SEVERITY_WARN, 2, config); + const auto f = this->fault("W_C_W"); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL) << "a lower report does not lower the record"; + EXPECT_EQ(f.occurrence_count, 1u); +} + +TYPED_TEST(FaultSeverityTest, WarnCriticalClearWarnConfirmsTheNewOccurrence) { + const auto config = confirm_at(-3); + + this->failed("W_C_CLR_W", Fault::SEVERITY_WARN, 0, config); + this->failed("W_C_CLR_W", Fault::SEVERITY_CRITICAL, 1, config); + ASSERT_TRUE(this->clear("W_C_CLR_W")); + + this->failed("W_C_CLR_W", Fault::SEVERITY_WARN, 2, config); + const auto f = this->fault("W_C_CLR_W"); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(f.occurrence_count, 2u); + EXPECT_TRUE(this->pending_critical().empty()); +} + +TYPED_TEST(FaultSeverityTest, HealedCriticalRecordConfirmsOnTheNextFailed) { + // Healing at 1: one FAILED confirms at -1, two PASSED reach 1 and heal. + DebounceConfig config = confirm_at(-3); + config.healing_enabled = true; + config.healing_threshold = 1; + + this->failed("HEAL_CRIT", Fault::SEVERITY_CRITICAL, 0, config); + this->passed("HEAL_CRIT", 1, config); + this->passed("HEAL_CRIT", 2, config); + ASSERT_EQ(this->fault("HEAL_CRIT").status, Fault::STATUS_HEALED); + + EXPECT_FALSE(this->failed("HEAL_CRIT", Fault::SEVERITY_WARN, 3, config)) << "HEALED is not an occurrence boundary"; + const auto f = this->fault("HEAL_CRIT"); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED) << "the latch does not hold a CRITICAL record back"; + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(f.occurrence_count, 1u); + EXPECT_EQ(f.first_occurred_ns, at(0)); +} + +TYPED_TEST(FaultSeverityTest, HealedWarnRecordStillWalksBackThroughTheLatch) { + // Control for the case above: a non-CRITICAL record keeps the latch. + DebounceConfig config = confirm_at(-3); + config.healing_enabled = true; + config.healing_threshold = 1; + + this->failed("HEAL_WARN", Fault::SEVERITY_WARN, 0, config); + this->failed("HEAL_WARN", Fault::SEVERITY_WARN, 1, config); + this->failed("HEAL_WARN", Fault::SEVERITY_WARN, 2, config); + ASSERT_EQ(this->fault("HEAL_WARN").status, Fault::STATUS_CONFIRMED); + for (int64_t s = 3; s < 7; ++s) { + this->passed("HEAL_WARN", s, config); + } + ASSERT_EQ(this->fault("HEAL_WARN").status, Fault::STATUS_HEALED); + + this->failed("HEAL_WARN", Fault::SEVERITY_WARN, 7, config); + EXPECT_EQ(this->fault("HEAL_WARN").status, Fault::STATUS_HEALED); +} + +TYPED_TEST(FaultSeverityTest, CriticalBypassOffLeavesTheDebounceInCharge) { + DebounceConfig config = confirm_at(-3); + config.critical_immediate_confirm = false; + + this->failed("NO_BYPASS", Fault::SEVERITY_CRITICAL, 0, config); + EXPECT_EQ(this->fault("NO_BYPASS").status, Fault::STATUS_PREFAILED); + this->failed("NO_BYPASS", Fault::SEVERITY_WARN, 1, config); + EXPECT_EQ(this->fault("NO_BYPASS").status, Fault::STATUS_PREFAILED); +} + +TYPED_TEST(FaultSeverityTest, CriticalBypassOffLeavesTheReactivationToTheDebounce) { + DebounceConfig config = confirm_at(-3); + config.critical_immediate_confirm = false; + + for (const int64_t second : {0, 1, 2}) { + this->failed("NO_BYPASS_REACT", Fault::SEVERITY_CRITICAL, second, config); + } + ASSERT_EQ(this->fault("NO_BYPASS_REACT").status, Fault::STATUS_CONFIRMED) << "confirmed by the debounce"; + ASSERT_TRUE(this->clear("NO_BYPASS_REACT")); + + this->failed("NO_BYPASS_REACT", Fault::SEVERITY_WARN, 3, config); + auto f = this->fault("NO_BYPASS_REACT"); + EXPECT_EQ(f.status, Fault::STATUS_PREFAILED) << "without the bypass a CRITICAL record reactivates like any other"; + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(f.occurrence_count, 2u); + + this->failed("NO_BYPASS_REACT", Fault::SEVERITY_WARN, 4, config); + EXPECT_EQ(this->fault("NO_BYPASS_REACT").status, Fault::STATUS_PREFAILED); + this->failed("NO_BYPASS_REACT", Fault::SEVERITY_WARN, 5, config); + EXPECT_EQ(this->fault("NO_BYPASS_REACT").status, Fault::STATUS_CONFIRMED); +} + +TYPED_TEST(FaultSeverityTest, SeverityNeverDropsWithinOrAcrossOccurrences) { + for (const int32_t threshold : {-1, -3}) { + const auto config = confirm_at(threshold); + const std::string code = "NO_DROP_T" + std::to_string(-threshold); + SCOPED_TRACE(code); + + this->failed(code, Fault::SEVERITY_ERROR, 0, config); + this->failed(code, Fault::SEVERITY_WARN, 1, config); + this->failed(code, Fault::SEVERITY_INFO, 2, config); + EXPECT_EQ(this->fault(code).severity, Fault::SEVERITY_ERROR); + + ASSERT_TRUE(this->clear(code)); + this->failed(code, Fault::SEVERITY_WARN, 3, config); + const auto f = this->fault(code); + EXPECT_EQ(f.severity, Fault::SEVERITY_ERROR); + EXPECT_EQ(f.occurrence_count, 2u); + EXPECT_TRUE( + this->storage_->list_faults(true, Fault::SEVERITY_WARN, {Fault::STATUS_PREFAILED, Fault::STATUS_CONFIRMED}) + .empty()) + << "the severity filter matches the stored severity"; + } +} + +TYPED_TEST(FaultSeverityTest, NearMissRowsKeepTheReportedSeverity) { + const auto config = confirm_at(-3); + + this->failed("NEAR", Fault::SEVERITY_ERROR, 0, config); + this->failed("NEAR", Fault::SEVERITY_WARN, 1, config); + ASSERT_EQ(this->fault("NEAR").severity, Fault::SEVERITY_ERROR); + + const auto series = this->storage_->get_near_misses(FaultId{"NEAR", kOwner}); + ASSERT_EQ(series.size(), 2u); + EXPECT_EQ(series[0].severity, Fault::SEVERITY_ERROR); + EXPECT_EQ(series[1].severity, Fault::SEVERITY_WARN) << "a near-miss row records its report"; +} + +// --------------------------------------------------------------------------- +// SQLite only: restart, the migration of a database without owners, and the confirmation time +// the in-memory backend does not store. +// --------------------------------------------------------------------------- + +/// The faults table as releases before record identity created it: one row per code. +constexpr const char * kLegacyFaultsDdl = + "CREATE TABLE faults (fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, " + "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " + "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " + "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " + "debounce_counter INTEGER NOT NULL DEFAULT 0, " + "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " + "confirmed_at_ns INTEGER NOT NULL DEFAULT 0);"; + +class SqliteSeverityTest : public ::testing::Test { + protected: + void SetUp() override { + dir_ = unique_temp_dir(); + std::filesystem::create_directories(dir_); + reopen(); + } + + void TearDown() override { + storage_.reset(); + std::error_code ec; + std::filesystem::remove_all(dir_, ec); + } + + /// Close the database and open it again, as a fault manager restart does. + void reopen() { + storage_.reset(); + storage_ = std::make_unique(db_path(), test_log_sink()); + } + + std::string db_path() const { + return (dir_ / "faults.db").string(); + } + + struct LegacyRow { + std::string code; + int severity; + std::string status; + int64_t confirmed_at_ns; + }; + + /// Replace the database with a legacy one holding these rows, then open it, which migrates it. + void open_legacy_database(const std::vector & rows) { + storage_.reset(); + std::error_code ec; + std::filesystem::remove(db_path(), ec); + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path().c_str(), &raw), SQLITE_OK); + const int64_t t1 = at(1); + std::ostringstream sql; + sql << kLegacyFaultsDdl; + for (const auto & row : rows) { + sql << "INSERT INTO faults VALUES ('" << row.code << "', " << row.severity << ", 'legacy', " << t1 << ", " << t1 + << ", 2, '" << row.status << "', '[\"" << kOwner << "\"]', -1, " << t1 << ", 0, " << row.confirmed_at_ns + << ");"; + } + EXPECT_EQ(sqlite3_exec(raw, sql.str().c_str(), nullptr, nullptr, nullptr), SQLITE_OK) << sqlite3_errmsg(raw); + sqlite3_close(raw); + record_owner_ = ros2_medkit_fault_manager::kSharedOwner; // a migrated row is the shared record of its code + reopen(); + } + + void open_legacy_database(const std::string & code, int severity, const std::string & status, + int64_t confirmed_at_ns) { + open_legacy_database({LegacyRow{code, severity, status, confirmed_at_ns}}); + } + + /// open_legacy_database, then hand the migrated row to the test's reporter, so it is a per-source record + /// and the stored severity decides immediate confirmation. + void open_legacy_per_source_database(const std::string & code, int severity, const std::string & status, + int64_t confirmed_at_ns) { + open_legacy_database(code, severity, status, confirmed_at_ns); + storage_.reset(); + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path().c_str(), &raw), SQLITE_OK); + const std::string sql = std::string("UPDATE faults SET owner = '") + kOwner + "'"; + EXPECT_EQ(sqlite3_exec(raw, sql.c_str(), nullptr, nullptr, nullptr), SQLITE_OK) << sqlite3_errmsg(raw); + sqlite3_close(raw); + record_owner_ = kOwner; + reopen(); + } + + void failed(const std::string & code, uint8_t severity, int64_t second, const DebounceConfig & config) { + storage_->report_fault_event(FaultId{code, record_owner_}, fd::ReportKind::kFailed, severity, "report", kOwner, + at(second), config); + } + + void passed(const std::string & code, int64_t second, const DebounceConfig & config) { + storage_->report_fault_event(FaultId{code, record_owner_}, fd::ReportKind::kPassed, 0, "", kOwner, at(second), + config); + } + + bool clear(const std::string & code) { + return storage_->clear_fault(FaultId{code, record_owner_}); + } + + fd::FaultRecord fault(const std::string & code) { + auto f = storage_->get_fault(FaultId{code, record_owner_}); + EXPECT_TRUE(f.has_value()) << code; + return f.value_or(fd::FaultRecord{}); + } + + int64_t confirmed_at(const std::string & code) const { + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path().c_str(), &raw), SQLITE_OK); + sqlite3_stmt * stmt = nullptr; + EXPECT_EQ(sqlite3_prepare_v2(raw, "SELECT confirmed_at_ns FROM faults WHERE fault_code = ? AND owner = ?", -1, + &stmt, nullptr), + SQLITE_OK); + sqlite3_bind_text(stmt, 1, code.c_str(), -1, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, record_owner_.c_str(), -1, SQLITE_TRANSIENT); + int64_t value = -1; + if (sqlite3_step(stmt) == SQLITE_ROW) { + value = sqlite3_column_int64(stmt, 0); + } + sqlite3_finalize(stmt); + sqlite3_close(raw); + return value; + } + + std::filesystem::path dir_; + std::unique_ptr storage_; + /// Owner of the records the helpers address: the test's reporter, or the shared owner after a legacy open. + std::string record_owner_{kOwner}; +}; + +TEST_F(SqliteSeverityTest, CriticalClearRestartWarnConfirms) { + const auto config = confirm_at(-3); + failed("CRIT_RESTART", Fault::SEVERITY_CRITICAL, 0, config); + ASSERT_TRUE(clear("CRIT_RESTART")); + + reopen(); + failed("CRIT_RESTART", Fault::SEVERITY_WARN, 1, config); + + const auto f = fault("CRIT_RESTART"); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(f.occurrence_count, 2u); + EXPECT_EQ(confirmed_at("CRIT_RESTART"), at(1)); +} + +TEST_F(SqliteSeverityTest, ReactivationHasNoConfirmationTimeUntilItConfirms) { + const auto config = confirm_at(-3); + failed("CONF_AT", Fault::SEVERITY_ERROR, 0, config); + failed("CONF_AT", Fault::SEVERITY_ERROR, 1, config); + failed("CONF_AT", Fault::SEVERITY_ERROR, 2, config); + ASSERT_EQ(fault("CONF_AT").status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(confirmed_at("CONF_AT"), at(2)); + ASSERT_TRUE(clear("CONF_AT")); + EXPECT_EQ(confirmed_at("CONF_AT"), at(2)) << "a CLEARED record keeps the time it confirmed"; + + failed("CONF_AT", Fault::SEVERITY_ERROR, 3, config); + ASSERT_EQ(fault("CONF_AT").status, Fault::STATUS_PREFAILED); + EXPECT_EQ(confirmed_at("CONF_AT"), 0) << "this occurrence has not confirmed"; + + failed("CONF_AT", Fault::SEVERITY_ERROR, 4, config); + EXPECT_EQ(confirmed_at("CONF_AT"), 0); + + failed("CONF_AT", Fault::SEVERITY_ERROR, 5, config); + ASSERT_EQ(fault("CONF_AT").status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(confirmed_at("CONF_AT"), at(5)); +} + +TEST_F(SqliteSeverityTest, ReactivationConfirmationTimeSurvivesRestarts) { + const auto config = confirm_at(-2); + failed("CONF_AT_RESTART", Fault::SEVERITY_ERROR, 0, config); + failed("CONF_AT_RESTART", Fault::SEVERITY_ERROR, 1, config); + ASSERT_EQ(confirmed_at("CONF_AT_RESTART"), at(1)); + ASSERT_TRUE(clear("CONF_AT_RESTART")); + + reopen(); + failed("CONF_AT_RESTART", Fault::SEVERITY_ERROR, 2, config); + ASSERT_EQ(fault("CONF_AT_RESTART").status, Fault::STATUS_PREFAILED); + EXPECT_EQ(confirmed_at("CONF_AT_RESTART"), 0); + + reopen(); + EXPECT_EQ(confirmed_at("CONF_AT_RESTART"), 0); + failed("CONF_AT_RESTART", Fault::SEVERITY_ERROR, 3, config); + ASSERT_EQ(fault("CONF_AT_RESTART").status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(confirmed_at("CONF_AT_RESTART"), at(3)); +} + +TEST_F(SqliteSeverityTest, ReactivationThatConfirmsAtOnceRecordsItsOwnTime) { + const auto config = confirm_at(-1); + failed("CONF_AT_NOW", Fault::SEVERITY_WARN, 0, config); + ASSERT_EQ(confirmed_at("CONF_AT_NOW"), at(0)); + ASSERT_TRUE(clear("CONF_AT_NOW")); + + failed("CONF_AT_NOW", Fault::SEVERITY_WARN, 1, config); + EXPECT_EQ(confirmed_at("CONF_AT_NOW"), at(1)); + + // A CRITICAL record confirms a debounced reactivation at once, and records that instant. + const auto debounced = confirm_at(-3); + failed("CONF_AT_CRIT", Fault::SEVERITY_CRITICAL, 2, debounced); + ASSERT_TRUE(clear("CONF_AT_CRIT")); + failed("CONF_AT_CRIT", Fault::SEVERITY_INFO, 3, debounced); + EXPECT_EQ(confirmed_at("CONF_AT_CRIT"), at(3)); +} + +TEST_F(SqliteSeverityTest, MigratedClearedRecordHasNoConfirmationTimeUntilItConfirms) { + const auto config = confirm_at(-3); + open_legacy_database("MIGRATED", Fault::SEVERITY_ERROR, Fault::STATUS_CLEARED, at(0)); + ASSERT_EQ(fault("MIGRATED").status, Fault::STATUS_CLEARED); + EXPECT_EQ(confirmed_at("MIGRATED"), at(0)) << "the migration keeps the cleared occurrence's time"; + + failed("MIGRATED", Fault::SEVERITY_ERROR, 2, config); + ASSERT_EQ(fault("MIGRATED").status, Fault::STATUS_PREFAILED); + EXPECT_EQ(fault("MIGRATED").occurrence_count, 3u); + EXPECT_EQ(confirmed_at("MIGRATED"), 0) << "this occurrence has not confirmed"; + + reopen(); + EXPECT_EQ(confirmed_at("MIGRATED"), 0); + failed("MIGRATED", Fault::SEVERITY_ERROR, 3, config); + failed("MIGRATED", Fault::SEVERITY_ERROR, 4, config); + ASSERT_EQ(fault("MIGRATED").status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(confirmed_at("MIGRATED"), at(4)); +} + +TEST_F(SqliteSeverityTest, MigratedClearedCriticalRecordConfirmsOnAWarn) { + const auto config = confirm_at(-3); + open_legacy_per_source_database("MIGRATED_CRIT", Fault::SEVERITY_CRITICAL, Fault::STATUS_CLEARED, at(0)); + + failed("MIGRATED_CRIT", Fault::SEVERITY_WARN, 2, config); + const auto f = fault("MIGRATED_CRIT"); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(confirmed_at("MIGRATED_CRIT"), at(2)); +} + +TEST_F(SqliteSeverityTest, PendingCriticalRowFromAnEarlierReleaseStaysUntilAReport) { + // The row an earlier release left after CRITICAL, clear, WARN at threshold -3: CRITICAL in + // PREFAILED, occurrence 2, still holding the first occurrence's confirmation time. + open_legacy_per_source_database("LEGACY", Fault::SEVERITY_CRITICAL, Fault::STATUS_PREFAILED, at(0)); + auto f = fault("LEGACY"); + EXPECT_EQ(f.status, Fault::STATUS_PREFAILED) << "the migration does not rewrite the status"; + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(confirmed_at("LEGACY"), 0) << "the migration drops the earlier occurrence's time"; + + reopen(); + f = fault("LEGACY"); + EXPECT_EQ(f.status, Fault::STATUS_PREFAILED); + EXPECT_EQ(f.occurrence_count, 2u); + EXPECT_EQ(confirmed_at("LEGACY"), 0); +} + +TEST_F(SqliteSeverityTest, MigrationKeepsAConfirmationTimeOnlyOnRowsThatConfirmed) { + // Under the latch a row that confirmed stays CONFIRMED or HEALED until a clear, so a pending row + // with a time carries it from the occurrence before a reactivation. + const int64_t earlier = at(0); + open_legacy_database({ + {"PENDING_FAILED", Fault::SEVERITY_ERROR, Fault::STATUS_PREFAILED, earlier}, + {"PENDING_PASSED", Fault::SEVERITY_WARN, Fault::STATUS_PREPASSED, earlier}, + {"PENDING_NEVER", Fault::SEVERITY_ERROR, Fault::STATUS_PREFAILED, 0}, + {"CONFIRMED", Fault::SEVERITY_ERROR, Fault::STATUS_CONFIRMED, earlier}, + {"HEALED", Fault::SEVERITY_ERROR, Fault::STATUS_HEALED, earlier}, + {"CLEARED", Fault::SEVERITY_ERROR, Fault::STATUS_CLEARED, earlier}, + }); + + const std::vector> expected{ + {"PENDING_FAILED", 0}, {"PENDING_PASSED", 0}, {"PENDING_NEVER", 0}, + {"CONFIRMED", earlier}, {"HEALED", earlier}, {"CLEARED", earlier}, + }; + for (int open = 0; open < 2; ++open) { + for (const auto & [code, confirmed] : expected) { + SCOPED_TRACE(code + " open " + std::to_string(open)); + EXPECT_EQ(confirmed_at(code), confirmed); + } + reopen(); + } + EXPECT_EQ(fault("PENDING_FAILED").status, Fault::STATUS_PREFAILED); + EXPECT_EQ(fault("PENDING_PASSED").status, Fault::STATUS_PREPASSED); + EXPECT_EQ(fault("CONFIRMED").status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(fault("HEALED").status, Fault::STATUS_HEALED); + EXPECT_EQ(fault("CLEARED").status, Fault::STATUS_CLEARED); +} + +TEST_F(SqliteSeverityTest, SweepRecordsTheConfirmationTimeOfAReactivatedOccurrence) { + auto config = confirm_at(-3); + config.auto_confirm_after_sec = 5.0; + storage_->set_debounce_config(config); + + for (const int64_t second : {0, 1, 2}) { + failed("SWEPT", Fault::SEVERITY_ERROR, second, config); + } + ASSERT_EQ(fault("SWEPT").status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(confirmed_at("SWEPT"), at(2)); + ASSERT_TRUE(clear("SWEPT")); + failed("SWEPT", Fault::SEVERITY_ERROR, 3, config); + ASSERT_EQ(fault("SWEPT").status, Fault::STATUS_PREFAILED); + EXPECT_EQ(confirmed_at("SWEPT"), 0) << "this occurrence has not confirmed"; + + reopen(); + storage_->set_debounce_config(config); + EXPECT_EQ(confirmed_at("SWEPT"), 0); + EXPECT_TRUE(storage_->check_time_based_confirmation(at(5)).empty()) << "before the window ends"; + ASSERT_EQ(storage_->check_time_based_confirmation(at(10)).size(), 1u); + EXPECT_EQ(fault("SWEPT").status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(confirmed_at("SWEPT"), at(10)) << "the sweep confirmed this occurrence"; +} + +TEST_F(SqliteSeverityTest, TimeBasedConfirmationKeepsTheConfirmationTimeOfTheReport) { + auto config = confirm_at(-3); + config.auto_confirm_after_sec = 5.0; + storage_->set_debounce_config(config); + + failed("TIMED", Fault::SEVERITY_CRITICAL, 0, config); + ASSERT_TRUE(clear("TIMED")); + failed("TIMED", Fault::SEVERITY_WARN, 1, config); + ASSERT_EQ(fault("TIMED").status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(confirmed_at("TIMED"), at(1)) << "confirmed by the report"; + + EXPECT_TRUE(storage_->check_time_based_confirmation(at(10)).empty()); + EXPECT_EQ(confirmed_at("TIMED"), at(1)) << "the sweep does not rewrite it"; + + reopen(); + storage_->set_debounce_config(config); + EXPECT_TRUE(storage_->check_time_based_confirmation(at(20)).empty()); + EXPECT_EQ(confirmed_at("TIMED"), at(1)); +} + +TEST_F(SqliteSeverityTest, PendingCriticalRowConfirmsOnTheNextFailedOfAnySeverity) { + const auto config = confirm_at(-3); + for (const uint8_t severity : + {Fault::SEVERITY_INFO, Fault::SEVERITY_WARN, Fault::SEVERITY_ERROR, Fault::SEVERITY_CRITICAL}) { + const std::string code = "LEGACY_FAILED_" + std::to_string(severity); + SCOPED_TRACE(code); + open_legacy_per_source_database(code, Fault::SEVERITY_CRITICAL, Fault::STATUS_PREFAILED, at(0)); + + failed(code, severity, 2, config); + const auto f = fault(code); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(f.occurrence_count, 2u) << "the row is not CLEARED, so this is the same occurrence"; + EXPECT_EQ(confirmed_at(code), at(2)); + } +} + +TEST_F(SqliteSeverityTest, PendingCriticalRowCanMoveToPrepassedOnPassed) { + const auto config = confirm_at(-3); + open_legacy_per_source_database("LEGACY_PASSED", Fault::SEVERITY_CRITICAL, Fault::STATUS_PREFAILED, at(0)); + + passed("LEGACY_PASSED", 2, config); + EXPECT_EQ(fault("LEGACY_PASSED").status, Fault::STATUS_PREFAILED) << "counter 0 keeps the status"; + passed("LEGACY_PASSED", 3, config); + auto f = fault("LEGACY_PASSED"); + EXPECT_EQ(f.status, Fault::STATUS_PREPASSED) << "a PASSED report does not apply the CRITICAL bypass"; + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); + + failed("LEGACY_PASSED", Fault::SEVERITY_WARN, 4, config); + f = fault("LEGACY_PASSED"); + EXPECT_EQ(f.status, Fault::STATUS_CONFIRMED) << "the next FAILED confirms it"; + EXPECT_EQ(f.severity, Fault::SEVERITY_CRITICAL); +} + +// --------------------------------------------------------------------------- +// Exhaustive sweep: every event sequence of a given length over a small alphabet, applied to both +// backends and checked after every step, for both records of one code. +// --------------------------------------------------------------------------- + +enum class Kind { kFailed, kPassed, kClear }; + +struct Event { + Kind kind; + uint8_t severity; + const char * owner; +}; + +constexpr std::array kOwners = {"/a", "/b"}; + +constexpr std::array kAlphabet = {{ + {Kind::kFailed, Fault::SEVERITY_INFO, "/a"}, + {Kind::kFailed, Fault::SEVERITY_WARN, "/a"}, + {Kind::kFailed, Fault::SEVERITY_CRITICAL, "/a"}, + {Kind::kFailed, Fault::SEVERITY_WARN, "/b"}, + {Kind::kFailed, Fault::SEVERITY_CRITICAL, "/b"}, + {Kind::kPassed, 0, "/a"}, + {Kind::kClear, 0, "/a"}, + {Kind::kClear, 0, "/b"}, +}}; + +/// The fields this contract fixes for one record, independent of the debounce state machine. +struct Model { + bool exists{false}; + bool cleared{false}; + uint8_t severity{0}; + uint32_t occurrence_count{0}; + + void apply(const Event & e) { + switch (e.kind) { + case Kind::kFailed: + if (!exists || cleared) { + ++occurrence_count; + } + severity = exists ? std::max(severity, e.severity) : e.severity; + exists = true; + cleared = false; + break; + case Kind::kPassed: + break; + case Kind::kClear: + cleared = exists; + break; + } + } +}; + +std::string describe(const std::vector & seq, int upto) { + std::ostringstream out; + for (int i = 0; i <= upto; ++i) { + const auto & e = kAlphabet[seq[static_cast(i)]]; + if (i > 0) { + out << ", "; + } + switch (e.kind) { + case Kind::kFailed: + out << "FAILED(" << static_cast(e.severity) << "," << e.owner << ")"; + break; + case Kind::kPassed: + out << "PASSED(" << e.owner << ")"; + break; + case Kind::kClear: + out << "CLEAR(" << e.owner << ")"; + break; + } + } + return out.str(); +} + +/// Empty when the record holds the contract after @p e; otherwise what broke. +/// @param addressed Whether @p e was addressed to this record +std::string check(const fd::FaultRecord & f, const Model & m, const Event & e, bool addressed, + const DebounceConfig & config, const std::string & where) { + std::ostringstream out; + if (f.severity != m.severity) { + out << where << " severity " << static_cast(f.severity) << " != " << static_cast(m.severity) << "; "; + } + if (f.occurrence_count != m.occurrence_count) { + out << where << " occurrence_count " << f.occurrence_count << " != " << m.occurrence_count << "; "; + } + if (m.cleared != (f.status == Fault::STATUS_CLEARED)) { + out << where << " status " << f.status << "; "; + } + if (config.critical_immediate_confirm && f.severity == Fault::SEVERITY_CRITICAL) { + if (pending(f.status)) { + out << where << " CRITICAL in " << f.status << "; "; + } + if (addressed && e.kind == Kind::kFailed && f.status != Fault::STATUS_CONFIRMED) { + out << where << " FAILED on a CRITICAL record left " << f.status << "; "; + } + } + return out.str(); +} + +std::string check_parity(const fd::FaultRecord & mem, const fd::FaultRecord & sql, const std::string & owner) { + std::ostringstream out; + if (mem.severity != sql.severity || mem.status != sql.status || mem.description != sql.description || + mem.occurrence_count != sql.occurrence_count || mem.reporting_sources != sql.reporting_sources || + mem.first_occurred_ns != sql.first_occurred_ns || mem.last_occurred_ns != sql.last_occurred_ns || + mem.last_passed_ns != sql.last_passed_ns) { + out << owner << " backends disagree (memory: sev=" << static_cast(mem.severity) << " status=" << mem.status + << " occ=" << mem.occurrence_count << "; sqlite: sev=" << static_cast(sql.severity) + << " status=" << sql.status << " occ=" << sql.occurrence_count << "); "; + } + return out.str(); +} + +/// @param length Events per sequence; long enough to reach HEALED and report FAILED on it. +void run_sweep(const DebounceConfig & config, const std::string & tag, int length) { + InMemoryFaultStorage memory; + SqliteFaultStorage sqlite(":memory:", test_log_sink()); + + size_t total = 1; + for (int i = 0; i < length; ++i) { + total *= kAlphabet.size(); + } + + size_t failures = 0; + std::string first_failure; + std::vector seq(static_cast(length), 0); + for (size_t index = 0; index < total; ++index) { + size_t rest = index; + for (int i = 0; i < length; ++i) { + seq[static_cast(i)] = rest % kAlphabet.size(); + rest /= kAlphabet.size(); + } + const std::string code = tag + "_" + std::to_string(index); + std::array models{}; + for (int step = 0; step < length; ++step) { + const auto & e = kAlphabet[seq[static_cast(step)]]; + const auto t = at(step); + if (e.kind == Kind::kClear) { + memory.clear_fault(FaultId{code, e.owner}); + sqlite.clear_fault(FaultId{code, e.owner}); + } else { + const fd::ReportKind type = e.kind == Kind::kFailed ? fd::ReportKind::kFailed : fd::ReportKind::kPassed; + memory.report_fault_event(FaultId{code, e.owner}, type, e.severity, "step " + std::to_string(step), e.owner, t, + config); + sqlite.report_fault_event(FaultId{code, e.owner}, type, e.severity, "step " + std::to_string(step), e.owner, t, + config); + } + + std::string diff; + for (size_t o = 0; o < kOwners.size(); ++o) { + const std::string owner = kOwners[o]; + const bool addressed = owner == e.owner; + if (addressed) { + models[o].apply(e); + } + const auto mem = memory.get_fault(FaultId{code, owner}); + const auto sql = sqlite.get_fault(FaultId{code, owner}); + if (mem.has_value() != models[o].exists || sql.has_value() != models[o].exists) { + diff += owner + " existence differs; "; + } else if (models[o].exists) { + diff += check(*mem, models[o], e, addressed, config, "memory " + owner) + + check(*sql, models[o], e, addressed, config, "sqlite " + owner) + check_parity(*mem, *sql, owner); + } + } + if (!diff.empty()) { + if (failures == 0) { + first_failure = "[" + describe(seq, step) + "] " + diff; + } + ++failures; + break; + } + } + } + + EXPECT_EQ(failures, 0u) << failures << " of " << total << " sequences diverged; first: " << first_failure; +} + +TEST(FaultSeveritySweep, ImmediateConfirmation) { + run_sweep(DebounceConfig{}, "IC", 4); +} + +TEST(FaultSeveritySweep, ImmediateConfirmationHealOnSinglePassed) { + DebounceConfig config; + config.healing_enabled = true; + config.healing_threshold = 0; + run_sweep(config, "IH", 4); +} + +TEST(FaultSeveritySweep, DebouncedWithShortHealing) { + // HEALED within three events, so a FAILED on a HEALED record fits in four. + DebounceConfig config; + config.confirmation_threshold = -2; + config.healing_enabled = true; + config.healing_threshold = 1; + run_sweep(config, "DS", 4); +} + +TEST(FaultSeveritySweep, DeepDebounceWithHealing) { + // CRITICAL confirms at -1 and heals after three PASSED, so a FAILED on it needs five events. + DebounceConfig config; + config.confirmation_threshold = -3; + config.healing_enabled = true; + config.healing_threshold = 2; + run_sweep(config, "DH", 5); +} + +TEST(FaultSeveritySweep, CriticalBypassOff) { + // Without the bypass nothing forces a CRITICAL record to confirm; the other checks still hold. + DebounceConfig config; + config.confirmation_threshold = -3; + config.critical_immediate_confirm = false; + run_sweep(config, "NB", 4); +} + +} // namespace diff --git a/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp b/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp index d3bdb419e..1df9f61d1 100644 --- a/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp +++ b/src/ros2_medkit_fault_manager/test/test_postgres_storage.cpp @@ -53,6 +53,7 @@ #include "test_log_sink.hpp" using ros2_medkit_fault_manager::DebounceConfig; +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::PgFaultStorage; using ros2_medkit_fault_manager::RosbagFileInfo; using ros2_medkit_msgs::msg::Fault; @@ -150,52 +151,87 @@ class PgFaultStorageTest : public ::testing::Test { std::unique_ptr storage_; }; +// The per-record trims and reads of snapshots and near misses filter on (fault_code, owner): the +// indexes behind them exist, and a store that lacks them gets them when it is opened. +TEST_F(PgFaultStorageTest, RecordIndexesExist) { + const auto index_definition = [this](const std::string & name) { + auto conn = raw_connection(); + pqxx::nontransaction tx(*conn); + const auto rows = tx.exec( + "SELECT indexdef FROM pg_indexes WHERE schemaname = current_schema() AND " + "indexname = $1", + pqxx::params{name}); + return rows.empty() ? std::string() : rows[0][0].as(); + }; + + EXPECT_NE(index_definition("idx_snapshots_record").find("(fault_code, owner, capture_id)"), std::string::npos); + EXPECT_NE(index_definition("idx_near_misses_record").find("(fault_code, owner, id)"), std::string::npos); + + storage_.reset(); + exec_raw("DROP INDEX idx_snapshots_record; DROP INDEX idx_near_misses_record"); + EXPECT_TRUE(index_definition("idx_snapshots_record").empty()) << "the fixture must start without them"; + storage_ = std::make_unique(conn_info_, test_log_sink()); + EXPECT_NE(index_definition("idx_snapshots_record").find("(fault_code, owner, capture_id)"), std::string::npos); + EXPECT_NE(index_definition("idx_near_misses_record").find("(fault_code, owner, id)"), std::string::npos); +} + TEST_F(PgFaultStorageTest, ReportNewFaultEvent) { rclcpp::Clock clock; auto timestamp = clock.now(); - bool is_new = storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "Motor temperature exceeded threshold", "/powertrain/motor", - timestamp.nanoseconds(), default_config()); + bool is_new = storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Motor temperature exceeded threshold", + "/powertrain/motor", timestamp.nanoseconds(), default_config()); EXPECT_TRUE(is_new); EXPECT_EQ(storage_->size(), 1u); - EXPECT_TRUE(storage_->contains("MOTOR_OVERHEAT")); + EXPECT_TRUE(storage_->contains(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"})); } TEST_F(PgFaultStorageTest, PassedEventForNonExistentFaultIgnored) { rclcpp::Clock clock; auto timestamp = clock.now(); - bool is_new = storage_->report_fault_event("NON_EXISTENT", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "Test", - "/node1", timestamp.nanoseconds(), default_config()); + bool is_new = + storage_->report_fault_event(FaultId{"NON_EXISTENT", "/node1"}, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, + "Test", "/node1", timestamp.nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_->size(), 0u); } -TEST_F(PgFaultStorageTest, ReportExistingFaultEventUpdates) { +TEST_F(PgFaultStorageTest, SameCodeDifferentSourcesAreSeparateRecords) { rclcpp::Clock clock; auto timestamp1 = clock.now(); auto timestamp2 = clock.now(); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Initial report", - "/powertrain/motor1", timestamp1.nanoseconds(), default_config()); + bool first_new = storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, + fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Initial report", + "/powertrain/motor1", timestamp1.nanoseconds(), default_config()); - bool is_new = - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Second report", - "/powertrain/motor2", timestamp2.nanoseconds(), default_config()); + // A second source reporting the same code owns a NEW record, not an update of the first. + bool second_new = storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor2"}, + fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Second report", + "/powertrain/motor2", timestamp2.nanoseconds(), default_config()); - EXPECT_FALSE(is_new); - EXPECT_EQ(storage_->size(), 1u); + EXPECT_TRUE(first_new); + EXPECT_TRUE(second_new); + EXPECT_EQ(storage_->size(), 2u); - auto fault = storage_->get_fault("MOTOR_OVERHEAT"); - ASSERT_TRUE(fault.has_value()); - // Still the same continuous occurrence (not CLEARED in between): occurrence_count - // does not bump on every report, only severity/sources/description update. - EXPECT_EQ(fault->occurrence_count, 1u); - EXPECT_EQ(fault->severity, Fault::SEVERITY_ERROR); // Updated to higher severity - EXPECT_EQ(fault->reporting_sources.size(), 2u); + auto by_code = storage_->get_ids_by_code("MOTOR_OVERHEAT"); + ASSERT_EQ(by_code.size(), 2u); + + auto first = storage_->get_fault(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}); + auto second = storage_->get_fault(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor2"}); + ASSERT_TRUE(first.has_value()); + ASSERT_TRUE(second.has_value()); + // No merge: each record carries only its own owner and its own severity. + EXPECT_EQ(first->occurrence_count, 1u); + EXPECT_EQ(second->occurrence_count, 1u); + ASSERT_EQ(first->reporting_sources.size(), 1u); + EXPECT_EQ(first->reporting_sources[0], "/powertrain/motor1"); + EXPECT_EQ(first->severity, Fault::SEVERITY_WARN); + EXPECT_EQ(second->severity, Fault::SEVERITY_ERROR); } TEST_F(PgFaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCount) { @@ -204,23 +240,137 @@ TEST_F(PgFaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCount) // Simulate a level-triggered poller re-reporting the same still-true condition // every cycle (issue #11): occurrence_count must stay at 1, not grow per poll. for (int i = 0; i < 25; ++i) { - storage_->report_fault_event("TANK_OVERFILL", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "level = 95 > 80", - "/tank", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"TANK_OVERFILL", "/tank"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "level = 95 > 80", "/tank", clock.now().nanoseconds(), default_config()); } - auto fault = storage_->get_fault("TANK_OVERFILL"); + auto fault = storage_->get_fault(FaultId{"TANK_OVERFILL", "/tank"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } +// --- Record identity is (fault_code, owner) --- +// Two sources reporting one code own independent records. A single-record service driven by +// one owner never reaches another owner's record of the same code. + +TEST_F(PgFaultStorageTest, ClearingOneOwnerLeavesTheOtherOwnersRecord) { + rclcpp::Clock clock; + const FaultId a{"SHARED_CODE", "/node_a"}; + const FaultId b{"SHARED_CODE", "/node_b"}; + storage_->report_fault_event(FaultId{"SHARED_CODE", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + a.owner, clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"SHARED_CODE", b.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", + b.owner, clock.now().nanoseconds(), default_config()); + ASSERT_EQ(storage_->size(), 2u); + + EXPECT_TRUE(storage_->clear_fault(a)); + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_CLEARED); + EXPECT_TRUE(storage_->contains(b)); + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->size(), 2u); +} + +TEST_F(PgFaultStorageTest, RecordsAreDistinctPerOwnerIncludingUnscoped) { + rclcpp::Clock clock; + const FaultId a{"DUP", "/owner_a"}; + const FaultId b{"DUP", "/owner_b"}; + const FaultId unscoped{"DUP", ""}; + storage_->report_fault_event(FaultId{"DUP", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "a", a.owner, + clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"DUP", b.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, + clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"DUP", unscoped.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "u", + unscoped.owner, clock.now().nanoseconds(), default_config()); + + EXPECT_EQ(storage_->size(), 3u); + EXPECT_EQ(storage_->get_ids_by_code("DUP").size(), 3u); + EXPECT_EQ(storage_->get_fault(a)->severity, Fault::SEVERITY_WARN); + EXPECT_EQ(storage_->get_fault(b)->severity, Fault::SEVERITY_ERROR); + EXPECT_EQ(storage_->get_fault(unscoped)->severity, Fault::SEVERITY_CRITICAL); +} + +TEST_F(PgFaultStorageTest, DebounceCountersAreIndependentPerOwner) { + rclcpp::Clock clock; + DebounceConfig config; + config.confirmation_threshold = -3; // three of one owner's own events confirm + storage_->set_debounce_config(config); + const FaultId a{"GRIND", "/node_a"}; + const FaultId b{"GRIND", "/node_b"}; + + storage_->report_fault_event(FaultId{"GRIND", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"GRIND", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_PREFAILED); + + storage_->report_fault_event(FaultId{"GRIND", b.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, + clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_PREFAILED); + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_PREFAILED); + + storage_->report_fault_event(FaultId{"GRIND", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_PREFAILED); +} + +TEST_F(PgFaultStorageTest, EvidenceAttachesToOneOwnerAndIsInvisibleUnderAnother) { + using ros2_medkit_fault_manager::FreezeFrameData; + using ros2_medkit_fault_manager::SnapshotData; + rclcpp::Clock clock; + const FaultId a{"EVID", "/node_a"}; + const FaultId b{"EVID", "/node_b"}; + storage_->report_fault_event(FaultId{"EVID", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", a.owner, + clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"EVID", b.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", b.owner, + clock.now().nanoseconds(), default_config()); + + SnapshotData snap; + snap.fault_code = a.fault_code; + snap.owner = a.owner; + snap.topic = "/t"; + snap.message_type = "std_msgs/msg/Float64"; + snap.data = R"({"data": 1.0})"; + snap.captured_at_ns = 1000; + snap.capture_id = 1; + storage_->store_snapshots({snap}); + + FreezeFrameData frame; + frame.fault_code = a.fault_code; + frame.owner = a.owner; + frame.data = R"({"/t": 1.0})"; + frame.captured_at_ns = 1000; + storage_->store_freeze_frame(frame); + + RosbagFileInfo bag; + bag.fault_code = a.fault_code; + bag.owner = a.owner; + bag.recording_id = "fault_EVID_1"; + bag.file_path = "/tmp/evid.mcap"; + bag.format = "mcap"; + bag.created_at_ns = 1000; + storage_->store_rosbag_files({bag}); + + EXPECT_EQ(storage_->get_snapshots(a).size(), 1u); + EXPECT_TRUE(storage_->get_snapshots(b).empty()); + EXPECT_TRUE(storage_->get_freeze_frame(a).has_value()); + EXPECT_FALSE(storage_->get_freeze_frame(b).has_value()); + EXPECT_TRUE(storage_->get_rosbag_file(a).has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(b).has_value()); + + EXPECT_TRUE(storage_->clear_fault(a)); + EXPECT_TRUE(storage_->get_snapshots(a).empty()); + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_CONFIRMED); +} + TEST_F(PgFaultStorageTest, ListFaultsDefaultReturnsConfirmedOnly) { rclcpp::Clock clock; auto timestamp = clock.now(); // With default threshold=-1, single report confirms immediately - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); // Default query should return the CONFIRMED fault auto faults = storage_->list_faults(false, 0, {}); @@ -237,8 +387,8 @@ TEST_F(PgFaultStorageTest, ListFaultsWithPrefailedStatus) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), config); // Query with PREFAILED status auto faults = storage_->list_faults(false, 0, {Fault::STATUS_PREFAILED}); @@ -252,10 +402,10 @@ TEST_F(PgFaultStorageTest, ListFaultsFilterBySeverity) { auto timestamp = clock.now(); // With default threshold=-1, faults are immediately CONFIRMED - storage_->report_fault_event("FAULT_INFO", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Info", "/node1", - timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_ERROR", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Error", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_INFO", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Info", + "/node1", timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_ERROR", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Error", "/node1", timestamp.nanoseconds(), default_config()); // Filter by ERROR severity (query CONFIRMED since that's the default status now) auto faults = storage_->list_faults(true, Fault::SEVERITY_ERROR, {Fault::STATUS_CONFIRMED}); @@ -267,19 +417,19 @@ TEST_F(PgFaultStorageTest, ClearFault) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Test", "/node1", timestamp.nanoseconds(), default_config()); - bool cleared = storage_->clear_fault("MOTOR_OVERHEAT"); + bool cleared = storage_->clear_fault(FaultId{"MOTOR_OVERHEAT", "/node1"}); EXPECT_TRUE(cleared); - auto fault = storage_->get_fault("MOTOR_OVERHEAT"); + auto fault = storage_->get_fault(FaultId{"MOTOR_OVERHEAT", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); } TEST_F(PgFaultStorageTest, ClearNonExistentFault) { - bool cleared = storage_->clear_fault("NON_EXISTENT"); + bool cleared = storage_->clear_fault(FaultId{"NON_EXISTENT", ""}); EXPECT_FALSE(cleared); } @@ -290,12 +440,12 @@ TEST_F(PgFaultStorageTest, PassedEventDoesNotAdvanceLastOccurred) { const rclcpp::Time failed_at(1000, 0, RCL_SYSTEM_TIME); const rclcpp::Time passed_at(9000, 0, RCL_SYSTEM_TIME); - storage_->report_fault_event("FAULT_LO", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - failed_at.nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_LO", fd::ReportKind::kPassed, 0, "", "/node1", passed_at.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"FAULT_LO", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", failed_at.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_LO", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + passed_at.nanoseconds(), default_config()); - auto fault = storage_->get_fault("FAULT_LO"); + auto fault = storage_->get_fault(FaultId{"FAULT_LO", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // healing disabled: latched, by design EXPECT_EQ(fault->last_occurred_ns, failed_at.nanoseconds()); @@ -307,9 +457,9 @@ TEST_F(PgFaultStorageTest, GetClearedFaults) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); - storage_->clear_fault("FAULT_1"); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); + storage_->clear_fault(FaultId{"FAULT_1", "/node1"}); // Query cleared faults auto faults = storage_->list_faults(false, 0, {Fault::STATUS_CLEARED}); @@ -322,8 +472,8 @@ TEST_F(PgFaultStorageTest, InvalidStatusDefaultsToConfirmed) { auto timestamp = clock.now(); // With default threshold=-1, fault is immediately CONFIRMED - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); // Query with invalid status - defaults to CONFIRMED, which now matches our fault auto faults = storage_->list_faults(false, 0, {"INVALID_STATUS"}); @@ -338,11 +488,11 @@ TEST_F(PgFaultStorageTest, PersistenceAcrossRestarts) { // With default threshold=-1, faults are immediately CONFIRMED // Report some faults - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Persistent fault 1", - "/node1", timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_2", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Persistent fault 2", "/node2", - timestamp.nanoseconds(), default_config()); - storage_->clear_fault("FAULT_2"); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Persistent fault 1", "/node1", timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_2", "/node2"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, + "Persistent fault 2", "/node2", timestamp.nanoseconds(), default_config()); + storage_->clear_fault(FaultId{"FAULT_2", "/node2"}); // Verify initial state EXPECT_EQ(storage_->size(), 2u); @@ -355,16 +505,16 @@ TEST_F(PgFaultStorageTest, PersistenceAcrossRestarts) { // Verify faults persisted EXPECT_EQ(storage_->size(), 2u); - EXPECT_TRUE(storage_->contains("FAULT_1")); - EXPECT_TRUE(storage_->contains("FAULT_2")); + EXPECT_TRUE(storage_->contains(FaultId{"FAULT_1", "/node1"})); + EXPECT_TRUE(storage_->contains(FaultId{"FAULT_2", "/node2"})); - auto fault1 = storage_->get_fault("FAULT_1"); + auto fault1 = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault1.has_value()); EXPECT_EQ(fault1->severity, Fault::SEVERITY_ERROR); EXPECT_EQ(fault1->status, Fault::STATUS_CONFIRMED); // Immediately confirmed with threshold=-1 EXPECT_EQ(fault1->description, "Persistent fault 1"); - auto fault2 = storage_->get_fault("FAULT_2"); + auto fault2 = storage_->get_fault(FaultId{"FAULT_2", "/node2"}); ASSERT_TRUE(fault2.has_value()); EXPECT_EQ(fault2->status, Fault::STATUS_CLEARED); } @@ -375,10 +525,10 @@ TEST_F(PgFaultStorageTest, TimestampPrecision) { int64_t test_ns = 1735312456123456789LL; // Specific nanosecond timestamp rclcpp::Time timestamp(test_ns, RCL_SYSTEM_TIME); - storage_->report_fault_event("FAULT_TS", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Timestamp test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_TS", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_INFO, + "Timestamp test", "/node1", timestamp.nanoseconds(), default_config()); - auto fault = storage_->get_fault("FAULT_TS"); + auto fault = storage_->get_fault(FaultId{"FAULT_TS", "/node1"}); ASSERT_TRUE(fault.has_value()); // Wrap the stored nanoseconds in rclcpp::Time for comparison @@ -458,8 +608,8 @@ TEST(PgFaultStorageConnectionTest, CreatesTheSchemaWhenTheDatabaseAppears) { const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(15); while (!stored && std::chrono::steady_clock::now() < deadline) { try { - storage.report_fault_event("LATE", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "late", "/node", - clock.now().nanoseconds(), default_config()); + storage.report_fault_event(FaultId{"LATE", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "late", + "/node", clock.now().nanoseconds(), default_config()); stored = true; } catch (const ros2_medkit_fault_manager::FaultStorage::IgnorableConnectionException &) { std::this_thread::sleep_for(std::chrono::milliseconds(250)); @@ -1041,7 +1191,7 @@ TEST(PgFaultStorageConnectionTest, ALostConnectionIsOpenedAgainWithTheSchema) { " application_name=" + application + " options='-csearch_path=" + schema + "'"), test_log_sink()); ASSERT_TRUE(storage.connected()); - storage.report_fault_event("LOST_A", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/node", + storage.report_fault_event(FaultId{"LOST_A", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/node", clock.now().nanoseconds(), default_config()); EXPECT_EQ(storage.size(), 1u); @@ -1054,7 +1204,7 @@ TEST(PgFaultStorageConnectionTest, ALostConnectionIsOpenedAgainWithTheSchema) { ntx.quote(application)); } EXPECT_EQ(storage.size(), 0u); - storage.report_fault_event("LOST_B", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", "/node", + storage.report_fault_event(FaultId{"LOST_B", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", "/node", clock.now().nanoseconds(), default_config()); EXPECT_EQ(storage.size(), 1u); } @@ -1067,23 +1217,22 @@ TEST_F(PgFaultStorageTest, ReportingSourcesJsonHandling) { rclcpp::Clock clock; auto timestamp = clock.now(); - // Add multiple sources for the same fault - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", - "/node/path/with/slashes", timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", - "/another/node", timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", - "/special\"chars", timestamp.nanoseconds(), default_config()); - - auto fault = storage_->get_fault("MULTI_SRC"); - ASSERT_TRUE(fault.has_value()); - EXPECT_EQ(fault->reporting_sources.size(), 3u); + // Each source owns its own record of the same code. The owner round-trips through the JSON + // reporting_sources column, including slashes and an embedded quote. + const std::vector owners = {"/node/path/with/slashes", "/another/node", "/special\"chars"}; + for (const auto & owner : owners) { + storage_->report_fault_event(FaultId{"MULTI_SRC", owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Multi-source", owner, timestamp.nanoseconds(), default_config()); + } - // Verify all sources are present (order may vary due to set) - std::set sources(fault->reporting_sources.begin(), fault->reporting_sources.end()); - EXPECT_TRUE(sources.count("/node/path/with/slashes") > 0); - EXPECT_TRUE(sources.count("/another/node") > 0); - EXPECT_TRUE(sources.count("/special\"chars") > 0); + EXPECT_EQ(storage_->size(), owners.size()); + EXPECT_EQ(storage_->get_ids_by_code("MULTI_SRC").size(), owners.size()); + for (const auto & owner : owners) { + auto fault = storage_->get_fault(FaultId{"MULTI_SRC", owner}); + ASSERT_TRUE(fault.has_value()) << owner; + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources[0], owner); + } } // Test connection string accessor (equivalent of sqlite's DbPathAccessor). @@ -1124,12 +1273,12 @@ TEST_F(PgFaultStorageTest, FaultStaysPrefailedAboveThreshold) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Still debouncing towards confirmation, same occurrence EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); @@ -1143,14 +1292,14 @@ TEST_F(PgFaultStorageTest, FaultConfirmsAtThreshold) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Debounce build-up is still one occurrence EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -1163,10 +1312,10 @@ TEST_F(PgFaultStorageTest, ImmediateConfirmationWithThresholdZero) { storage_->set_debounce_config(config); // Single report should confirm immediately - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -1176,10 +1325,10 @@ TEST_F(PgFaultStorageTest, CriticalSeverityBypassesDebounce) { rclcpp::Clock clock; // CRITICAL severity should confirm immediately - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "Critical test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, + "Critical test", "/node1", clock.now().nanoseconds(), default_config()); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -1189,35 +1338,38 @@ TEST_F(PgFaultStorageTest, ClearedFaultCanBeReactivated) { rclcpp::Clock clock; // Report to confirm (with default threshold=-1, single report confirms) + const FaultId id{"FAULT_1", "/node1"}; auto first_ts = clock.now(); - bool is_new = storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Initial", - "/node1", first_ts.nanoseconds(), default_config()); + bool is_new = + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Initial", "/node1", first_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(id); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->first_occurred_ns, first_ts.nanoseconds()); // Clear the fault - storage_->clear_fault("FAULT_1"); - fault = storage_->get_fault("FAULT_1"); + storage_->clear_fault(id); + fault = storage_->get_fault(id); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); - // Report again after a gap - should reactivate as a new cycle + // The SAME owner re-raising after a clear reopens its record as a new cycle. rclcpp::Time second_ts(first_ts.nanoseconds() + 1'000'000'000LL); // +1s - is_new = storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Reactivated", - "/node2", second_ts.nanoseconds(), default_config()); - EXPECT_TRUE(is_new); // Should return true like a new fault + is_new = storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Reactivated", "/node1", second_ts.nanoseconds(), default_config()); + EXPECT_TRUE(is_new); // Reactivation returns true like a new fault - fault = storage_->get_fault("FAULT_1"); + fault = storage_->get_fault(id); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Should be reconfirmed EXPECT_EQ(fault->occurrence_count, 2u); // Should increment - EXPECT_EQ(fault->reporting_sources.size(), 2u); // Both sources - EXPECT_EQ(fault->description, "Reactivated"); // Updated description + ASSERT_EQ(fault->reporting_sources.size(), 1u); // One record, one owner + EXPECT_EQ(fault->reporting_sources[0], "/node1"); + EXPECT_EQ(fault->description, "Reactivated"); // Updated description // #25: first_occurred must reflect the new cycle, not the outage that already cleared. EXPECT_EQ(fault->first_occurred_ns, second_ts.nanoseconds()); } @@ -1226,18 +1378,18 @@ TEST_F(PgFaultStorageTest, PassedEventForClearedFaultIgnored) { rclcpp::Clock clock; // Report and confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); // Clear the fault - storage_->clear_fault("FAULT_1"); + storage_->clear_fault(FaultId{"FAULT_1", "/node1"}); // PASSED event should be ignored for CLEARED fault - bool result = storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", + bool result = storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), default_config()); EXPECT_FALSE(result); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); // Should stay cleared } @@ -1251,35 +1403,35 @@ TEST_F(PgFaultStorageTest, ClearedFaultReactivationRestartsDebounce) { storage_->set_debounce_config(config); // Report 3 times to confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), config); - - auto fault = storage_->get_fault("FAULT_1"); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Clear the fault - storage_->clear_fault("FAULT_1"); + storage_->clear_fault(FaultId{"FAULT_1", "/node1"}); // Reactivate - should start in PREFAILED with counter=-1 - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node4", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); - fault = storage_->get_fault("FAULT_1"); + fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Not yet confirmed, needs 2 more FAILED // Report 2 more times to re-confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node5", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node6", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); - fault = storage_->get_fault("FAULT_1"); + fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Now confirmed } @@ -1288,19 +1440,19 @@ TEST_F(PgFaultStorageTest, ConfirmationPersistsAfterReopen) { rclcpp::Clock clock; // Report 3 times to confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); // Close and reopen storage storage_.reset(); storage_ = std::make_unique(conn_info_, test_log_sink()); // Verify status persisted - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -1313,18 +1465,18 @@ TEST_F(PgFaultStorageTest, PassedEventIncrementsCounter) { config.confirmation_threshold = -3; // Report 2 FAILED events (counter -2, PREFAILED) - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); // Report 3 PASSED events (counter -2 -> +1) for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREPASSED); // Counter > 0, never confirmed so not latched } @@ -1340,29 +1492,30 @@ TEST_F(PgFaultStorageTest, HeartbeatHealClampedAndStatusLatched) { config.healing_enabled = true; config.healing_threshold = 3; - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + ASSERT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); // Long heal heartbeat: counter clamps at healing_threshold instead of running off. for (int i = 0; i < 100; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } - auto healed = storage_->get_fault("FAULT_1"); + auto healed = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(healed.has_value()); EXPECT_EQ(healed->status, Fault::STATUS_HEALED); // Re-confirmation is now bounded to (healing_threshold - confirmation_threshold) // reports, and the healed status latches until the counter walks all the way down. for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); - } - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - auto reconfirmed = storage_->get_fault("FAULT_1"); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_HEALED) + << "latch broke after " << (i + 1); + } + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + auto reconfirmed = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(reconfirmed.has_value()); EXPECT_EQ(reconfirmed->status, Fault::STATUS_CONFIRMED); } @@ -1376,21 +1529,22 @@ TEST_F(PgFaultStorageTest, ConfirmedFaultSurvivesHealHeartbeat) { config.healing_enabled = false; config.healing_threshold = 3; // upper clamp on the counter, even with healing disabled - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + ASSERT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 20; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); - EXPECT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED) << "un-confirmed after " << (i + 1); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED) + << "un-confirmed after " << (i + 1); } // The counter is now positive (clamped at +3). One FAILED must keep it CONFIRMED, not flip it to // PREPASSED via the `counter > 0` branch - that was the asymmetric-latch bug. - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(FaultId{"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); } // Latch must hold in BOTH directions and identically on all backends (regression for the @@ -1403,18 +1557,22 @@ TEST_F(PgFaultStorageTest, ConfirmedLatchSurvivesFailedAtPositiveCounter) { config.critical_immediate_confirm = true; // CRITICAL confirms immediately at counter -1. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", - clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); + ASSERT_EQ(storage_->get_fault(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner})->status, + Fault::STATUS_CONFIRMED); // PASSED events raise the counter into positive territory; latch keeps it CONFIRMED. for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kPassed, 0, "", + "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner})->status, + Fault::STATUS_CONFIRMED); // One normal FAILED: counter still positive, must NOT become PREPASSED. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", - clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner})->status, + Fault::STATUS_CONFIRMED); } // A HEALED fault with a negative counter stays HEALED on PASSED (opposite direction of the bug above). @@ -1425,21 +1583,23 @@ TEST_F(PgFaultStorageTest, HealedLatchSurvivesPassedAtNegativeCounter) { config.healing_enabled = true; config.healing_threshold = 3; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { // counter -1 -> +3 -> HEALED - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); // FAILED events drive the counter negative while HEALED is latched (not yet at confirmation -3). for (int i = 0; i < 5; ++i) { // +3 -> -2 - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); + ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); } // One PASSED: counter goes -2 -> -1, still negative; must stay HEALED, not flip to PREFAILED. - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); } TEST_F(PgFaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePassed) { @@ -1450,24 +1610,27 @@ TEST_F(PgFaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePassed) { config.healing_threshold = 3; config.critical_immediate_confirm = true; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { // -1 -> +3 -> HEALED - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); // CRITICAL confirms immediately from HEALED; counter must not stay at +2 where one PASSED heals it. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { // -1 -> +2, still CONFIRMED - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); } - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); } // A database written by an older build can hold a runaway counter. It must be clamped back on first @@ -1478,7 +1641,7 @@ TEST_F(PgFaultStorageTest, RunawayCounterFromOldRowRecovers) { config.confirmation_threshold = -1; config.healing_threshold = 3; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); // Simulate the old bug: poke a huge counter directly into the table behind the storage's back. exec_raw("UPDATE faults SET debounce_counter = 100000, status = 'PREPASSED'"); @@ -1486,10 +1649,10 @@ TEST_F(PgFaultStorageTest, RunawayCounterFromOldRowRecovers) { // The counter is clamped back into range on first touch, so a bounded number of FAILED events // re-confirms (healing_threshold - confirmation_threshold = 4), not the ~100000 the runaway implied. for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); } - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); } TEST_F(PgFaultStorageTest, ReclassifyHealedAsCleared) { @@ -1498,17 +1661,18 @@ TEST_F(PgFaultStorageTest, ReclassifyHealedAsCleared) { config.healing_enabled = true; config.healing_threshold = 3; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_HEALED); const auto reclassified = storage_->reclassify_healed_as_cleared(); ASSERT_EQ(reclassified.size(), 1u); - EXPECT_EQ(reclassified[0], "F"); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(reclassified[0].fault_code, "F"); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CLEARED); EXPECT_TRUE(storage_->reclassify_healed_as_cleared().empty()); } @@ -1519,21 +1683,22 @@ TEST_F(PgFaultStorageTest, ReclassifyKeepsCodesTheCallerKeepsHealed) { config.healing_threshold = 3; for (const std::string code : {"KEEP", "FLIP"}) { - storage_->report_fault_event(code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{code, "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event(code, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{code, "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } - ASSERT_EQ(storage_->get_fault(code)->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault(FaultId{code, "/n"})->status, Fault::STATUS_HEALED); } const auto reclassified = storage_->reclassify_healed_as_cleared([](const std::string & code) { return code == "KEEP"; }); ASSERT_EQ(reclassified.size(), 1u); - EXPECT_EQ(reclassified[0], "FLIP"); - EXPECT_EQ(storage_->get_fault("KEEP")->status, Fault::STATUS_HEALED); - EXPECT_EQ(storage_->get_fault("FLIP")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(reclassified[0].fault_code, "FLIP"); + EXPECT_EQ(storage_->get_fault(FaultId{"KEEP", "/n"})->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_->get_fault(FaultId{"FLIP", "/n"})->status, Fault::STATUS_CLEARED); } TEST_F(PgFaultStorageTest, HealingWhenEnabled) { @@ -1544,16 +1709,16 @@ TEST_F(PgFaultStorageTest, HealingWhenEnabled) { storage_->set_debounce_config(config); // Report 1 FAILED event - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); // Report 4 PASSED events (counter = -1 + 4 = +3, reaches healing threshold) for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); } @@ -1566,8 +1731,8 @@ TEST_F(PgFaultStorageTest, TimeBasedConfirmationWhenEnabled) { storage_->set_debounce_config(config); auto now = clock.now(); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - now.nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", now.nanoseconds(), config); // Check before timeout - should not confirm auto before_timeout = rclcpp::Time(now.nanoseconds() + static_cast(5e9)); @@ -1578,9 +1743,9 @@ TEST_F(PgFaultStorageTest, TimeBasedConfirmationWhenEnabled) { auto after_timeout = rclcpp::Time(now.nanoseconds() + static_cast(15e9)); auto confirmed = storage_->check_time_based_confirmation(after_timeout.nanoseconds()); ASSERT_EQ(confirmed.size(), 1u); - EXPECT_EQ(confirmed[0], "FAULT_1"); + EXPECT_EQ(confirmed[0].fault_code, "FAULT_1"); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault(FaultId{"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -1594,19 +1759,19 @@ TEST_F(PgFaultStorageTest, TimeBasedConfirmationConfirmsOnlyReturnedFaults) { storage_->set_debounce_config(config); auto now = clock.now(); - storage_->report_fault_event("TIMED", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - now.nanoseconds(), config); - storage_->report_fault_event("NO_FAILURE_TIME", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - now.nanoseconds(), config); + storage_->report_fault_event(FaultId{"TIMED", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", now.nanoseconds(), config); + storage_->report_fault_event(FaultId{"NO_FAILURE_TIME", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Test", "/node1", now.nanoseconds(), config); // A PREFAILED row without a failure time is not eligible. exec_raw("UPDATE faults SET last_failed_ns = 0 WHERE fault_code = 'NO_FAILURE_TIME'"); auto after_timeout = rclcpp::Time(now.nanoseconds() + static_cast(15e9)); auto confirmed = storage_->check_time_based_confirmation(after_timeout.nanoseconds()); ASSERT_EQ(confirmed.size(), 1u); - EXPECT_EQ(confirmed[0], "TIMED"); + EXPECT_EQ(confirmed[0].fault_code, "TIMED"); - auto skipped = storage_->get_fault("NO_FAILURE_TIME"); + auto skipped = storage_->get_fault(FaultId{"NO_FAILURE_TIME", "/node1"}); ASSERT_TRUE(skipped.has_value()); EXPECT_EQ(skipped->status, Fault::STATUS_PREFAILED); } @@ -1615,15 +1780,15 @@ TEST_F(PgFaultStorageTest, ConfirmedAtRecordedOnImmediateConfirmation) { rclcpp::Clock clock; auto t = clock.now(); // Default config confirms on the first FAILED event. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t.nanoseconds(), - default_config()); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t.nanoseconds(), default_config()); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(read_confirmed_at("F"), t.nanoseconds()); // A later FAILED on an already-confirmed fault must NOT move the timestamp. auto t2 = rclcpp::Time(t.nanoseconds() + static_cast(5e9)); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t2.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t2.nanoseconds(), default_config()); EXPECT_EQ(read_confirmed_at("F"), t.nanoseconds()); } @@ -1633,15 +1798,15 @@ TEST_F(PgFaultStorageTest, ConfirmedAtRecordedOnDebouncedConfirmation) { config.confirmation_threshold = -2; // second FAILED event confirms auto t1 = clock.now(); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t1.nanoseconds(), - config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_PREFAILED); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t1.nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_PREFAILED); EXPECT_EQ(read_confirmed_at("F"), 0) << "not confirmed yet: no confirmation timestamp"; auto t2 = rclcpp::Time(t1.nanoseconds() + static_cast(1e9)); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t2.nanoseconds(), - config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t2.nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(FaultId{"F", "/n"})->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(read_confirmed_at("F"), t2.nanoseconds()); } @@ -1653,8 +1818,8 @@ TEST_F(PgFaultStorageTest, ConfirmedAtRecordedOnTimeBasedConfirmation) { storage_->set_debounce_config(config); auto now = clock.now(); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", now.nanoseconds(), - config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + now.nanoseconds(), config); auto after_timeout = rclcpp::Time(now.nanoseconds() + static_cast(15e9)); auto confirmed = storage_->check_time_based_confirmation(after_timeout.nanoseconds()); @@ -1669,13 +1834,13 @@ TEST_F(PgFaultStorageTest, ConfirmedAtDefaultsToZeroForExternallyInsertedRow) { "occurrence_count, status, reporting_sources) " "VALUES ('OLD', 2, 'd', 1, 1, 1, 'CONFIRMED', '[\"/n\"]')"); - EXPECT_TRUE(storage_->contains("OLD")); + EXPECT_TRUE(storage_->contains(FaultId{"OLD", ""})); EXPECT_EQ(read_confirmed_at("OLD"), 0) << "rows carrying no confirmation time default to 0"; rclcpp::Clock clock; auto t = clock.now(); - storage_->report_fault_event("NEW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"NEW", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t.nanoseconds(), default_config()); EXPECT_EQ(read_confirmed_at("NEW"), t.nanoseconds()); } @@ -1701,13 +1866,13 @@ TEST_F(PgFaultStorageTest, OneFaultKeepsSeveralRecordingsWhenTheCapAllows) { storage_->store_rosbag_file(make_rosbag("FLAP", "/bags/fault_FLAP_100", 100)); storage_->store_rosbag_file(make_rosbag("FLAP", "/bags/fault_FLAP_200", 200)); - const auto rows = storage_->get_rosbag_files("FLAP"); + const auto rows = storage_->get_rosbag_files(FaultId{"FLAP", ""}); ASSERT_EQ(rows.size(), 2u) << "the second recording must not replace the first"; EXPECT_EQ(rows[0].recording_id, "fault_FLAP_200") << "newest first"; EXPECT_EQ(rows[1].recording_id, "fault_FLAP_100"); // get_rosbag_file is "the newest", deterministically. - const auto newest = storage_->get_rosbag_file("FLAP"); + const auto newest = storage_->get_rosbag_file(FaultId{"FLAP", ""}); ASSERT_TRUE(newest.has_value()); EXPECT_EQ(newest->recording_id, "fault_FLAP_200"); } @@ -1722,7 +1887,7 @@ TEST_F(PgFaultStorageTest, CapKeepsTheNewestAndUnlinksTheEvictedBag) { storage_->store_rosbag_file(make_rosbag("CAP", dir_a.string(), 100)); storage_->store_rosbag_file(make_rosbag("CAP", dir_b.string(), 200)); - const auto rows = storage_->get_rosbag_files("CAP"); + const auto rows = storage_->get_rosbag_files(FaultId{"CAP", ""}); ASSERT_EQ(rows.size(), 1u) << "cap 1 is the historical behaviour: one recording per fault"; EXPECT_EQ(rows[0].file_path, dir_b.string()); EXPECT_FALSE(std::filesystem::exists(dir_a)) << "the evicted bag must be unlinked"; @@ -1743,7 +1908,7 @@ TEST_F(PgFaultStorageTest, EvictingOneFaultsLinkKeepsABagASiblingStillReferences storage_->store_rosbag_file(make_rosbag("A", later.string(), 200)); EXPECT_TRUE(std::filesystem::exists(shared)) << "B still references it"; - const auto b_rows = storage_->get_rosbag_files("B"); + const auto b_rows = storage_->get_rosbag_files(FaultId{"B", ""}); ASSERT_EQ(b_rows.size(), 1u); EXPECT_EQ(b_rows[0].file_path, shared.string()); } @@ -1765,8 +1930,8 @@ TEST_F(PgFaultStorageTest, DeleteRecordingRemovesEveryLinkAndTheBag) { storage_->store_rosbag_files({make_rosbag("A", dir.string(), 100), make_rosbag("B", dir.string(), 100)}); EXPECT_EQ(storage_->delete_rosbag_recording("fault_A_100"), 2u); - EXPECT_TRUE(storage_->get_rosbag_files("A").empty()); - EXPECT_TRUE(storage_->get_rosbag_files("B").empty()); + EXPECT_TRUE(storage_->get_rosbag_files(FaultId{"A", ""}).empty()); + EXPECT_TRUE(storage_->get_rosbag_files(FaultId{"B", ""}).empty()); EXPECT_FALSE(std::filesystem::exists(dir)); } @@ -1775,8 +1940,8 @@ TEST_F(PgFaultStorageTest, DeletingAFaultDropsAllItsRecordings) { storage_->store_rosbag_file(make_rosbag("A", "/bags/fault_A_100", 100)); storage_->store_rosbag_file(make_rosbag("A", "/bags/fault_A_200", 200)); - EXPECT_TRUE(storage_->delete_rosbag_file("A")); - EXPECT_TRUE(storage_->get_rosbag_files("A").empty()) << "auto_cleanup drops the fault's whole history"; + EXPECT_TRUE(storage_->delete_rosbag_file(FaultId{"A", ""})); + EXPECT_TRUE(storage_->get_rosbag_files(FaultId{"A", ""}).empty()) << "auto_cleanup drops the fault's whole history"; } TEST_F(PgFaultStorageTest, SharedRecordingStillCountsOnceTowardsStorageWithSeveralRecordings) { @@ -1795,8 +1960,8 @@ TEST_F(PgFaultStorageTest, StoreAndRetrieveSnapshot) { // First, create a fault to associate the snapshot with rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); // Store a snapshot SnapshotData snapshot; @@ -1809,7 +1974,7 @@ TEST_F(PgFaultStorageTest, StoreAndRetrieveSnapshot) { storage_->store_snapshot(snapshot); // Retrieve snapshots - auto snapshots = storage_->get_snapshots("MOTOR_OVERHEAT"); + auto snapshots = storage_->get_snapshots(FaultId{"MOTOR_OVERHEAT", ""}); ASSERT_EQ(snapshots.size(), 1u); EXPECT_EQ(snapshots[0].fault_code, "MOTOR_OVERHEAT"); @@ -1824,8 +1989,8 @@ TEST_F(PgFaultStorageTest, MultipleSnapshotsForSameFault) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); // Store multiple snapshots for the same fault SnapshotData snapshot1; @@ -1845,7 +2010,7 @@ TEST_F(PgFaultStorageTest, MultipleSnapshotsForSameFault) { storage_->store_snapshot(snapshot1); storage_->store_snapshot(snapshot2); - auto snapshots = storage_->get_snapshots("MOTOR_OVERHEAT"); + auto snapshots = storage_->get_snapshots(FaultId{"MOTOR_OVERHEAT", ""}); EXPECT_EQ(snapshots.size(), 2u); } @@ -1854,8 +2019,8 @@ TEST_F(PgFaultStorageTest, FilterSnapshotsByTopic) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); SnapshotData snapshot1; snapshot1.fault_code = "MOTOR_OVERHEAT"; @@ -1875,14 +2040,14 @@ TEST_F(PgFaultStorageTest, FilterSnapshotsByTopic) { storage_->store_snapshot(snapshot2); // Filter by topic - auto filtered = storage_->get_snapshots("MOTOR_OVERHEAT", "/motor/temperature"); + auto filtered = storage_->get_snapshots(FaultId{"MOTOR_OVERHEAT", ""}, "/motor/temperature"); ASSERT_EQ(filtered.size(), 1u); EXPECT_EQ(filtered[0].topic, "/motor/temperature"); } // @verifies REQ_INTEROP_088 TEST_F(PgFaultStorageTest, NoSnapshotsForUnknownFault) { - auto snapshots = storage_->get_snapshots("UNKNOWN_FAULT"); + auto snapshots = storage_->get_snapshots(FaultId{"UNKNOWN_FAULT", ""}); EXPECT_TRUE(snapshots.empty()); } @@ -1892,9 +2057,8 @@ TEST_F(PgFaultStorageTest, ClearFaultDeletesAssociatedSnapshots) { rclcpp::Clock clock; // Create a fault using report_fault_event - storage_->report_fault_event("SNAPSHOT_CLEAR_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "Test fault for snapshot cleanup", "/test_node", clock.now().nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"SNAPSHOT_CLEAR_TEST", ""}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Test fault for snapshot cleanup", "", clock.now().nanoseconds(), default_config()); // Store snapshots for this fault SnapshotData snapshot1; @@ -1914,15 +2078,15 @@ TEST_F(PgFaultStorageTest, ClearFaultDeletesAssociatedSnapshots) { storage_->store_snapshot(snapshot2); // Verify snapshots exist - auto snapshots_before = storage_->get_snapshots("SNAPSHOT_CLEAR_TEST"); + auto snapshots_before = storage_->get_snapshots(FaultId{"SNAPSHOT_CLEAR_TEST", ""}); ASSERT_EQ(snapshots_before.size(), 2u); // Clear the fault - bool cleared = storage_->clear_fault("SNAPSHOT_CLEAR_TEST"); + bool cleared = storage_->clear_fault(FaultId{"SNAPSHOT_CLEAR_TEST", ""}); EXPECT_TRUE(cleared); // Verify snapshots are deleted - auto snapshots_after = storage_->get_snapshots("SNAPSHOT_CLEAR_TEST"); + auto snapshots_after = storage_->get_snapshots(FaultId{"SNAPSHOT_CLEAR_TEST", ""}); EXPECT_TRUE(snapshots_after.empty()); } @@ -1932,7 +2096,7 @@ TEST_F(PgFaultStorageTest, ClearFaultKeepsSnapshotsWhenEvidenceIsRetained) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("KEEP", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "keep", "/n", + storage_->report_fault_event(FaultId{"KEEP", ""}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "keep", "", clock.now().nanoseconds(), default_config()); storage_->set_retain_snapshots_on_clear(true); EXPECT_TRUE(storage_->retains_snapshots_on_clear()); @@ -1946,20 +2110,20 @@ TEST_F(PgFaultStorageTest, ClearFaultKeepsSnapshotsWhenEvidenceIsRetained) { row.capture_id = 1; storage_->store_snapshots({row}); - ASSERT_TRUE(storage_->clear_fault("KEEP")); + ASSERT_TRUE(storage_->clear_fault(FaultId{"KEEP", ""})); // Recordings survive an acknowledgement once a history is configured, so the // readings captured beside them have to as well - otherwise the fault is left // holding bags whose values are gone, which is worse than losing both. - EXPECT_EQ(storage_->get_snapshots("KEEP").size(), 1u); + EXPECT_EQ(storage_->get_snapshots(FaultId{"KEEP", ""}).size(), 1u); } TEST_F(PgFaultStorageTest, StoreAndRetrieveFreezeFrame) { using ros2_medkit_fault_manager::FreezeFrameData; rclcpp::Clock clock; - storage_->report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", - "/plc_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"PLC_PRESSURE_HIGH", ""}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Pressure high", "", clock.now().nanoseconds(), default_config()); FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; @@ -1967,7 +2131,7 @@ TEST_F(PgFaultStorageTest, StoreAndRetrieveFreezeFrame) { frame.captured_at_ns = clock.now().nanoseconds(); storage_->store_freeze_frame(frame); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage_->get_freeze_frame(FaultId{"PLC_PRESSURE_HIGH", ""}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->fault_code, "PLC_PRESSURE_HIGH"); EXPECT_EQ(retrieved->data, frame.data); @@ -1976,7 +2140,7 @@ TEST_F(PgFaultStorageTest, StoreAndRetrieveFreezeFrame) { // @verifies REQ_INTEROP_088 TEST_F(PgFaultStorageTest, NoFreezeFrameForUnknownFault) { - auto retrieved = storage_->get_freeze_frame("NEVER_CAPTURED"); + auto retrieved = storage_->get_freeze_frame(FaultId{"NEVER_CAPTURED", ""}); EXPECT_FALSE(retrieved.has_value()); } @@ -1996,7 +2160,7 @@ TEST_F(PgFaultStorageTest, FreezeFrameReplacedOnRecapture) { second.captured_at_ns = 2000; storage_->store_freeze_frame(second); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage_->get_freeze_frame(FaultId{"PLC_PRESSURE_HIGH", ""}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, second.data); EXPECT_EQ(retrieved->captured_at_ns, 2000); @@ -2008,8 +2172,8 @@ TEST_F(PgFaultStorageTest, FreezeFrameSurvivesClearFault) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", - "/plc_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"PLC_PRESSURE_HIGH", ""}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Pressure high", "", clock.now().nanoseconds(), default_config()); // A per-topic snapshot (removed on clear) plus a freeze-frame (retained on clear). SnapshotData snapshot; @@ -2026,11 +2190,11 @@ TEST_F(PgFaultStorageTest, FreezeFrameSurvivesClearFault) { frame.captured_at_ns = clock.now().nanoseconds(); storage_->store_freeze_frame(frame); - ASSERT_TRUE(storage_->clear_fault("PLC_PRESSURE_HIGH")); + ASSERT_TRUE(storage_->clear_fault(FaultId{"PLC_PRESSURE_HIGH", ""})); // Snapshots are wiped on clear, the freeze-frame is retained and still retrievable. - EXPECT_TRUE(storage_->get_snapshots("PLC_PRESSURE_HIGH").empty()); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + EXPECT_TRUE(storage_->get_snapshots(FaultId{"PLC_PRESSURE_HIGH", ""}).empty()); + auto retrieved = storage_->get_freeze_frame(FaultId{"PLC_PRESSURE_HIGH", ""}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, frame.data); } @@ -2049,7 +2213,7 @@ TEST_F(PgFaultStorageTest, FreezeFramePersistsAcrossReopen) { storage_.reset(); storage_ = std::make_unique(conn_info_, test_log_sink()); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage_->get_freeze_frame(FaultId{"PLC_PRESSURE_HIGH", ""}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, frame.data); EXPECT_EQ(retrieved->captured_at_ns, 4242); @@ -2063,17 +2227,19 @@ TEST_F(PgFaultStorageTest, ListRosbagsForEntityFiltersCorrectly) { rclcpp::Clock clock; // Create fault with reporting source for entity - storage_->report_fault_event("ENTITY_FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Fault from entity", - "/powertrain/motor", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"ENTITY_FAULT_1", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Fault from entity", "/powertrain/motor", + clock.now().nanoseconds(), default_config()); // Create another fault with different reporting source - storage_->report_fault_event("ENTITY_FAULT_2", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, - "Fault from other entity", "/chassis/brake", clock.now().nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"ENTITY_FAULT_2", "/chassis/brake"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "Fault from other entity", "/chassis/brake", + clock.now().nanoseconds(), default_config()); // Store rosbags for both faults RosbagFileInfo info1; info1.fault_code = "ENTITY_FAULT_1"; + info1.owner = "/powertrain/motor"; info1.file_path = "/tmp/entity1.mcap"; info1.format = "mcap"; info1.duration_sec = 5.0; @@ -2083,6 +2249,7 @@ TEST_F(PgFaultStorageTest, ListRosbagsForEntityFiltersCorrectly) { RosbagFileInfo info2; info2.fault_code = "ENTITY_FAULT_2"; + info2.owner = "/chassis/brake"; info2.file_path = "/tmp/entity2.mcap"; info2.format = "mcap"; info2.duration_sec = 3.0; @@ -2128,11 +2295,12 @@ TEST_F(PgFaultStorageTest, AFailingRowDeleteKeepsTheBagOnDisk) { "CREATE TRIGGER block_rosbag_delete BEFORE DELETE ON rosbag_files " "FOR EACH ROW EXECUTE PROCEDURE block_rosbag_delete()"); - EXPECT_THROW(storage_->delete_rosbag_file("DELETE_FAILS"), std::runtime_error); + EXPECT_THROW(storage_->delete_rosbag_file(FaultId{"DELETE_FAILS", ""}), std::runtime_error); exec_raw("DROP TRIGGER block_rosbag_delete ON rosbag_files"); - EXPECT_TRUE(storage_->get_rosbag_file("DELETE_FAILS").has_value()) << "the DELETE failed, so its row must remain"; + EXPECT_TRUE(storage_->get_rosbag_file(FaultId{"DELETE_FAILS", ""}).has_value()) + << "the DELETE failed, so its row must remain"; EXPECT_TRUE(std::filesystem::exists(bag_dir)) << "the bag was unlinked before its row was deleted, so the surviving row now points at nothing"; } @@ -2189,11 +2357,11 @@ TEST_F(PgFaultStorageTest, SharedRosbagSurvivesUntilTheLastFaultIsDeleted) { info.fault_code = "CORRELATED"; storage_->store_rosbag_file(info); - EXPECT_TRUE(storage_->delete_rosbag_file("CORRELATED")); + EXPECT_TRUE(storage_->delete_rosbag_file(FaultId{"CORRELATED", ""})); EXPECT_TRUE(std::filesystem::exists(bag_path)); - EXPECT_TRUE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); + EXPECT_TRUE(storage_->get_rosbag_file(FaultId{"ROOT_CAUSE", ""}).has_value()); - EXPECT_TRUE(storage_->delete_rosbag_file("ROOT_CAUSE")); + EXPECT_TRUE(storage_->delete_rosbag_file(FaultId{"ROOT_CAUSE", ""})); EXPECT_FALSE(std::filesystem::exists(bag_path)); } @@ -2246,7 +2414,7 @@ TEST_F(PgFaultStorageTest, RestoreWithNewPathUnlinksTheOldExclusiveBag) { storage_->store_rosbag_file(info); EXPECT_FALSE(std::filesystem::exists(old_path)) << "nobody references the old bag, it must be unlinked"; - auto row = storage_->get_rosbag_file("X"); + auto row = storage_->get_rosbag_file(FaultId{"X", ""}); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->file_path, old_path + "_new"); } @@ -2273,7 +2441,7 @@ TEST_F(PgFaultStorageTest, RestoreWithNewPathKeepsTheBagASiblingStillReferences) storage_->store_rosbag_file(info); EXPECT_TRUE(std::filesystem::exists(shared_path)) << "the sibling fault still owns the shared bag"; - auto sibling = storage_->get_rosbag_file("Y"); + auto sibling = storage_->get_rosbag_file(FaultId{"Y", ""}); ASSERT_TRUE(sibling.has_value()); EXPECT_EQ(sibling->file_path, shared_path); } @@ -2298,7 +2466,7 @@ TEST_F(PgFaultStorageTest, BulkStoreRegistersEveryFaultOfTheBurst) { storage_->store_rosbag_files(rows); for (const char * code : {"ROOT_CAUSE", "CORRELATED_A", "CORRELATED_B"}) { - auto row = storage_->get_rosbag_file(code); + auto row = storage_->get_rosbag_file(FaultId{code, ""}); ASSERT_TRUE(row.has_value()) << code; EXPECT_EQ(row->file_path, "/tmp/burst_bag"); } @@ -2325,13 +2493,15 @@ TEST_F(PgFaultStorageTest, BulkDeleteRemovesTheBurstAndUnlinksTheBagOnce) { storage_->store_rosbag_files(rows); // A partial delete leaves the bag on disk for the remaining fault. - EXPECT_EQ(storage_->delete_rosbag_files({"CORRELATED_A", "CORRELATED_B", "NEVER_STORED"}), 2u); + EXPECT_EQ(storage_->delete_rosbag_files( + {FaultId{"CORRELATED_A", ""}, FaultId{"CORRELATED_B", ""}, FaultId{"NEVER_STORED", ""}}), + 2u); EXPECT_TRUE(std::filesystem::exists(bag_path)); - EXPECT_TRUE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); - EXPECT_FALSE(storage_->get_rosbag_file("CORRELATED_A").has_value()); + EXPECT_TRUE(storage_->get_rosbag_file(FaultId{"ROOT_CAUSE", ""}).has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(FaultId{"CORRELATED_A", ""}).has_value()); // The last reference going away unlinks the directory. - EXPECT_EQ(storage_->delete_rosbag_files({"ROOT_CAUSE"}), 1u); + EXPECT_EQ(storage_->delete_rosbag_files({FaultId{"ROOT_CAUSE", ""}}), 1u); EXPECT_FALSE(std::filesystem::exists(bag_path)); EXPECT_EQ(storage_->get_total_rosbag_storage_bytes(), 0u); } @@ -2344,8 +2514,8 @@ TEST_F(PgFaultStorageTest, SnapshotCapKeepsTheNewestCaptureWholeAndDropsTheOldes using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); // Two topics per capture, room for two captures. storage_->set_max_snapshots_per_fault(4); @@ -2369,7 +2539,7 @@ TEST_F(PgFaultStorageTest, SnapshotCapKeepsTheNewestCaptureWholeAndDropsTheOldes storage_->store_snapshots(capture(2, 2000)); storage_->store_snapshots(capture(3, 3000)); - auto snapshots = storage_->get_snapshots("MOTOR_OVERHEAT"); + auto snapshots = storage_->get_snapshots(FaultId{"MOTOR_OVERHEAT", ""}); // The third capture is stored WHOLE and the first goes whole. The old rule // counted rows and rejected the new one once full, so capture 3 landed with one @@ -2395,8 +2565,8 @@ TEST_F(PgFaultStorageTest, ACaptureLargerThanTheCapIsKeptWholeRatherThanTorn) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("WIDE", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "many topics", "/n", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"WIDE", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "many topics", + "/n", clock.now().nanoseconds(), default_config()); storage_->set_max_snapshots_per_fault(2); std::vector rows; @@ -2415,15 +2585,15 @@ TEST_F(PgFaultStorageTest, ACaptureLargerThanTheCapIsKeptWholeRatherThanTorn) { // The cap is smaller than this fault's topic count. Trimming to it would mean // storing the reading with holes, which is the failure being fixed; the capture // stays whole and the operator can see the cap is too small. - EXPECT_EQ(storage_->get_snapshots("WIDE").size(), 4u); + EXPECT_EQ(storage_->get_snapshots(FaultId{"WIDE", ""}).size(), 4u); } TEST_F(PgFaultStorageTest, SnapshotLimitZeroMeansUnlimited) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("FAULT_A", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", "/node", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_A", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", + "/node", clock.now().nanoseconds(), default_config()); // Default (0) = unlimited storage_->set_max_snapshots_per_fault(0); @@ -2438,7 +2608,7 @@ TEST_F(PgFaultStorageTest, SnapshotLimitZeroMeansUnlimited) { storage_->store_snapshot(snap); } - auto snapshots = storage_->get_snapshots("FAULT_A"); + auto snapshots = storage_->get_snapshots(FaultId{"FAULT_A", ""}); EXPECT_EQ(snapshots.size(), 20u) << "Unlimited mode should store all snapshots"; } @@ -2446,10 +2616,10 @@ TEST_F(PgFaultStorageTest, SnapshotLimitPerFaultNotGlobal) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("FAULT_A", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", "/node", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_B", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", "/node", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_A", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", + "/node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_B", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", + "/node", clock.now().nanoseconds(), default_config()); storage_->set_max_snapshots_per_fault(1); @@ -2467,8 +2637,8 @@ TEST_F(PgFaultStorageTest, SnapshotLimitPerFaultNotGlobal) { storage_->store_snapshot(snap_b); // Both faults should have 1 snapshot each (limit is per-fault) - EXPECT_EQ(storage_->get_snapshots("FAULT_A").size(), 1u); - EXPECT_EQ(storage_->get_snapshots("FAULT_B").size(), 1u); + EXPECT_EQ(storage_->get_snapshots(FaultId{"FAULT_A", ""}).size(), 1u); + EXPECT_EQ(storage_->get_snapshots(FaultId{"FAULT_B", ""}).size(), 1u); } // --- Near-miss series --- @@ -2495,15 +2665,16 @@ TEST_F(PgFaultStorageTest, NearMissSeriesIsAppendedNotOverwritten) { const auto config = four_strike_config(); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - auto fault = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_NE(fault->status, Fault::STATUS_CONFIRMED) << "test setup: these reports must not confirm"; - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "each near miss must append an entry, not overwrite the last"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[1].debounce_counter, -2); @@ -2520,30 +2691,32 @@ TEST_F(PgFaultStorageTest, ConfirmingReportIsNotANearMiss) { const auto config = four_strike_config(); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - auto fault = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); // The fourth report is the fault happening, not nearly happening. - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); } TEST_F(PgFaultStorageTest, NearMissSeriesSurvivesClearFault) { const auto config = four_strike_config(); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); - ASSERT_TRUE(storage_->clear_fault("PUMP_PRESSURE_LOW")); + ASSERT_TRUE(storage_->clear_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "acknowledging the fault destroyed the near-miss record"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[2].debounce_counter, -3); @@ -2553,16 +2726,18 @@ TEST_F(PgFaultStorageTest, NearMissSeriesContinuesAcrossReactivation) { const auto config = four_strike_config(); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - ASSERT_TRUE(storage_->clear_fault("PUMP_PRESSURE_LOW")); + ASSERT_TRUE(storage_->clear_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); // A new outage cycle starts: the reactivating report resets the counter to -1 without confirming. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, - "pressure dipping again", "/hydraulics/pump", nth_report_time(10).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping again", "/hydraulics/pump", + nth_report_time(10).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 4u) << "the series must span fault cycles, one entry per occurrence"; EXPECT_EQ(series[3].debounce_counter, -1); EXPECT_EQ(series[3].occurred_at_ns, nth_report_time(10).nanoseconds()); @@ -2572,15 +2747,16 @@ TEST_F(PgFaultStorageTest, NearMissSeriesSurvivesReopen) { const auto config = four_strike_config(); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - ASSERT_TRUE(storage_->clear_fault("PUMP_PRESSURE_LOW")); + ASSERT_TRUE(storage_->clear_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); storage_.reset(); storage_ = std::make_unique(conn_info_, test_log_sink()); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "the series must outlive the process, not just the fault cycle"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[2].debounce_counter, -3); @@ -2590,29 +2766,31 @@ TEST_F(PgFaultStorageTest, PassedReportIsNotANearMiss) { const auto config = four_strike_config(); for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); + ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); // A PASSED report moves the counter in the healing direction: the fault receding, not nearing. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); } TEST_F(PgFaultStorageTest, CriticalImmediateConfirmIsNotANearMiss) { DebounceConfig config = four_strike_config(); config.critical_immediate_confirm = true; - storage_->report_fault_event("BATTERY_THERMAL_RUNAWAY", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, - "cell over temperature", "/power/bms", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"BATTERY_THERMAL_RUNAWAY", "/power/bms"}, fd::ReportKind::kFailed, + Fault::SEVERITY_CRITICAL, "cell over temperature", "/power/bms", + nth_report_time(0).nanoseconds(), config); - auto fault = storage_->get_fault("BATTERY_THERMAL_RUNAWAY"); + auto fault = storage_->get_fault(FaultId{"BATTERY_THERMAL_RUNAWAY", "/power/bms"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); - EXPECT_TRUE(storage_->get_near_misses("BATTERY_THERMAL_RUNAWAY").empty()); + EXPECT_TRUE(storage_->get_near_misses(FaultId{"BATTERY_THERMAL_RUNAWAY", "/power/bms"}).empty()); } TEST_F(PgFaultStorageTest, ImmediateConfirmThresholdRecordsNoNearMiss) { @@ -2620,13 +2798,14 @@ TEST_F(PgFaultStorageTest, ImmediateConfirmThresholdRecordsNoNearMiss) { // so a fault under this config never has a near miss to record. const auto config = default_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure lost", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "pressure lost", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); - auto fault = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); - EXPECT_TRUE(storage_->get_near_misses("PUMP_PRESSURE_LOW").empty()); + EXPECT_TRUE(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).empty()); } TEST_F(PgFaultStorageTest, FailedReportUnderHealedLatchIsNearMiss) { @@ -2634,23 +2813,26 @@ TEST_F(PgFaultStorageTest, FailedReportUnderHealedLatchIsNearMiss) { config.healing_enabled = true; config.healing_threshold = 1; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); + ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); for (int i = 1; i <= 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), + config); } - auto healed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; // The latch keeps the status at HEALED, but the counter moved toward confirmation: a near miss. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(3).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(3).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[1].debounce_counter, 0); } @@ -2661,11 +2843,12 @@ TEST_F(PgFaultStorageTest, NearMissSeriesBoundedKeepingNewest) { storage_->set_max_near_misses_per_fault(3); for (int i = 0; i < 5; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u); // Oldest-first eviction: a series frozen at boot would say nothing about a trend. EXPECT_EQ(series[0].debounce_counter, -3); @@ -2679,11 +2862,12 @@ TEST_F(PgFaultStorageTest, NearMissBoundOfOneKeepsLatestOnly) { storage_->set_max_near_misses_per_fault(1); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 1u); EXPECT_EQ(series[0].debounce_counter, -4); } @@ -2694,14 +2878,16 @@ TEST_F(PgFaultStorageTest, NearMissBoundIsPerFaultCode) { storage_->set_max_near_misses_per_fault(2); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + nth_report_time(i).nanoseconds(), config); } - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); - EXPECT_EQ(storage_->get_near_misses("MOTOR_OVERHEAT").size(), 2u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}).size(), 2u); } TEST_F(PgFaultStorageTest, NearMissBoundZeroIsUnlimited) { @@ -2710,11 +2896,12 @@ TEST_F(PgFaultStorageTest, NearMissBoundZeroIsUnlimited) { storage_->set_max_near_misses_per_fault(0); for (int i = 0; i < 150; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 150u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 150u); } TEST_F(PgFaultStorageTest, NearMissSeriesSurvivesHealedReclassification) { @@ -2724,20 +2911,22 @@ TEST_F(PgFaultStorageTest, NearMissSeriesSurvivesHealedReclassification) { config.healing_enabled = true; config.healing_threshold = 1; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); for (int i = 1; i <= 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), + config); } - auto healed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); ASSERT_EQ(storage_->reclassify_healed_as_cleared().size(), 1u); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); } TEST_F(PgFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { @@ -2748,14 +2937,16 @@ TEST_F(PgFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { storage_->set_max_near_misses_per_fault(2); for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(10 + i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(10 + i).nanoseconds(), config); } // Arrives third, but carries the earliest timestamp of the three. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[0].occurred_at_ns, nth_report_time(11).nanoseconds()); EXPECT_EQ(series[1].occurred_at_ns, nth_report_time(0).nanoseconds()) @@ -2766,40 +2957,51 @@ TEST_F(PgFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { TEST_F(PgFaultStorageTest, NearMissEntriesDescribeTheirOwnReport) { // Each entry must describe the report that produced it, not the fault's current state, or a // series spanning a threshold change or a new reporting source reads as if nothing changed. + const FaultId pump{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}; + const FaultId backup{"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}; + DebounceConfig first = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), first); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", pump.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, + "pressure dipping", pump.owner, nth_report_time(0).nanoseconds(), first); DebounceConfig second = four_strike_config(); second.confirmation_threshold = -8; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure dipping", - "/hydraulics/backup_pump", nth_report_time(1).nanoseconds(), second); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", backup.owner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "pressure dipping", backup.owner, + nth_report_time(1).nanoseconds(), second); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); - ASSERT_EQ(series.size(), 2u); - EXPECT_EQ(series[0].confirmation_threshold, -4); - EXPECT_EQ(series[0].severity, Fault::SEVERITY_WARN); - EXPECT_EQ(series[0].source_id, "/hydraulics/pump"); - EXPECT_EQ(series[1].confirmation_threshold, -8); - EXPECT_EQ(series[1].severity, Fault::SEVERITY_ERROR); - EXPECT_EQ(series[1].source_id, "/hydraulics/backup_pump"); + // Two sources of one code own separate records, so each near-miss series holds only its + // owner's reports and is not visible under the other owner. + auto pump_series = storage_->get_near_misses(pump); + ASSERT_EQ(pump_series.size(), 1u); + EXPECT_EQ(pump_series[0].confirmation_threshold, -4); + EXPECT_EQ(pump_series[0].severity, Fault::SEVERITY_WARN); + EXPECT_EQ(pump_series[0].source_id, "/hydraulics/pump"); + + auto backup_series = storage_->get_near_misses(backup); + ASSERT_EQ(backup_series.size(), 1u); + EXPECT_EQ(backup_series[0].confirmation_threshold, -8); + EXPECT_EQ(backup_series[0].severity, Fault::SEVERITY_ERROR); + EXPECT_EQ(backup_series[0].source_id, "/hydraulics/backup_pump"); } TEST_F(PgFaultStorageTest, NearMissSeriesContinuesAfterReopen) { const auto config = four_strike_config(); for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } storage_.reset(); storage_ = std::make_unique(conn_info_, test_log_sink()); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(2).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "a restart must extend the series, not restart or reorder it"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[1].debounce_counter, -2); @@ -2816,20 +3018,23 @@ TEST_F(PgFaultStorageTest, ApplyingSmallerBoundTrimsExistingSeries) { storage_->set_max_near_misses_per_fault(0); for (int i = 0; i < 5; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + nth_report_time(i).nanoseconds(), config); } - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 5u); + ASSERT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 5u); storage_->set_max_near_misses_per_fault(2); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u) << "applying the bound left the stored series over it"; EXPECT_EQ(series[0].debounce_counter, -4); EXPECT_EQ(series[1].debounce_counter, -5); - EXPECT_EQ(storage_->get_near_misses("MOTOR_OVERHEAT").size(), 2u) << "the bound is applied per fault code"; + EXPECT_EQ(storage_->get_near_misses(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}).size(), 2u) + << "the bound is applied per fault code"; } TEST_F(PgFaultStorageTest, ApplyingUnlimitedBoundKeepsExistingSeries) { @@ -2838,13 +3043,14 @@ TEST_F(PgFaultStorageTest, ApplyingUnlimitedBoundKeepsExistingSeries) { storage_->set_max_near_misses_per_fault(4); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } storage_->set_max_near_misses_per_fault(0); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 4u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 4u); } TEST_F(PgFaultStorageTest, UnlimitedBoundSpeltAsSizeMaxKeepsTheSeries) { @@ -2854,16 +3060,18 @@ TEST_F(PgFaultStorageTest, UnlimitedBoundSpeltAsSizeMaxKeepsTheSeries) { config.confirmation_threshold = -20; for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } EXPECT_EQ(storage_->set_max_near_misses_per_fault(std::numeric_limits::max()), 0u); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(3).nanoseconds(), config); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 4u); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(3).nanoseconds(), config); + EXPECT_EQ(storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 4u); } TEST_F(PgFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { @@ -2872,10 +3080,12 @@ TEST_F(PgFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { storage_->set_max_near_misses_per_fault(0); for (int i = 0; i < 5; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + nth_report_time(i).nanoseconds(), config); } // Two codes, five entries each, bound of 2: three dropped per code. @@ -2886,10 +3096,11 @@ TEST_F(PgFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { TEST_F(PgFaultStorageTest, PassedReportOnUnknownFaultWritesNothing) { // A heal heartbeat for a fault that does not exist must stay a read: it writes no row. - EXPECT_FALSE(storage_->report_fault_event("NEVER_REPORTED", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/test_node", nth_report_time(0).nanoseconds(), default_config())); + EXPECT_FALSE(storage_->report_fault_event(FaultId{"NEVER_REPORTED", "/test_node"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/test_node", nth_report_time(0).nanoseconds(), + default_config())); EXPECT_EQ(storage_->size(), 0u); - EXPECT_TRUE(storage_->get_near_misses("NEVER_REPORTED").empty()); + EXPECT_TRUE(storage_->get_near_misses(FaultId{"NEVER_REPORTED", "/test_node"}).empty()); } TEST_F(PgFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) { @@ -2901,13 +3112,15 @@ TEST_F(PgFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) { config.healing_threshold = 2; // An approach that recedes: counter moves, nothing is latched. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); for (int i = 1; i <= 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), + config); } - auto healed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; @@ -2915,14 +3128,15 @@ TEST_F(PgFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) { // From the healing threshold (+2) it takes six FAILED reports to reach the confirmation // threshold (-4). for (int i = 4; i <= 9; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - auto confirmed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto confirmed = storage_->get_fault(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(confirmed.has_value()); ASSERT_EQ(confirmed->status, Fault::STATUS_CONFIRMED) << "test setup: the ramp must end in a real fault"; - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_FALSE(series.empty()); EXPECT_EQ(series[0].resulting_status, Fault::STATUS_PREFAILED) << "the first report was a genuine approach"; @@ -2938,19 +3152,20 @@ TEST_F(PgFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) { TEST_F(PgFaultStorageTest, NearMissResultingStatusSurvivesReopen) { const auto config = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); storage_.reset(); storage_ = std::make_unique(conn_info_, test_log_sink()); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 1u); EXPECT_EQ(series[0].resulting_status, Fault::STATUS_PREFAILED); } TEST_F(PgFaultStorageTest, NearMissSeriesEmptyForUnknownFault) { - EXPECT_TRUE(storage_->get_near_misses("NEVER_REPORTED").empty()); + EXPECT_TRUE(storage_->get_near_misses(FaultId{"NEVER_REPORTED", ""}).empty()); } // --- Snapshot retention through the startup reclassification --- @@ -2978,12 +3193,12 @@ static void drive_to_healed(ros2_medkit_fault_manager::FaultStorage & storage, c config.healing_enabled = true; config.healing_threshold = 1; - storage.report_fault_event(fault_code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "/test_node", + storage.report_fault_event(FaultId{fault_code, ""}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "", nth_report_time(0).nanoseconds(), config); store_snapshots_for(storage, fault_code, 2); // Two PASSED reports: the first only lifts the counter to 0, which the CONFIRMED latch holds. for (int i = 1; i <= 2; ++i) { - storage.report_fault_event(fault_code, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "/test_node", + storage.report_fault_event(FaultId{fault_code, ""}, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "", nth_report_time(i).nanoseconds(), config); } } @@ -2992,14 +3207,14 @@ TEST_F(PgFaultStorageTest, SnapshotsRetainedThroughHealedReclassification) { storage_->set_retain_snapshots_on_clear(true); drive_to_healed(*storage_, "SNAPSHOT_RETAIN_TEST"); - auto healed = storage_->get_fault("SNAPSHOT_RETAIN_TEST"); + auto healed = storage_->get_fault(FaultId{"SNAPSHOT_RETAIN_TEST", ""}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; - ASSERT_EQ(storage_->get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u); + ASSERT_EQ(storage_->get_snapshots(FaultId{"SNAPSHOT_RETAIN_TEST", ""}).size(), 2u); ASSERT_EQ(storage_->reclassify_healed_as_cleared().size(), 1u); - EXPECT_EQ(storage_->get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u) + EXPECT_EQ(storage_->get_snapshots(FaultId{"SNAPSHOT_RETAIN_TEST", ""}).size(), 2u) << "the restart reclassification deleted snapshots the configuration asked to keep"; } @@ -3007,7 +3222,284 @@ TEST_F(PgFaultStorageTest, SnapshotsDroppedByHealedReclassificationByDefault) { drive_to_healed(*storage_, "SNAPSHOT_DROP_TEST"); ASSERT_EQ(storage_->reclassify_healed_as_cleared().size(), 1u); - EXPECT_TRUE(storage_->get_snapshots("SNAPSHOT_DROP_TEST").empty()); + EXPECT_TRUE(storage_->get_snapshots(FaultId{"SNAPSHOT_DROP_TEST", ""}).empty()); +} + +// --- Record identity: the same per-record contract as the SQLite backend --- + +TEST_F(PgFaultStorageTest, GetIdsByCodeReturnsEveryOwnerAndGetFaultExactlyOne) { + rclcpp::Clock clock; + storage_->report_fault_event(FaultId{"SHARED", "/owner_b"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_b", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"SHARED", "/owner_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_a", clock.now().nanoseconds(), default_config()); + + const auto shared = storage_->get_ids_by_code("SHARED"); + ASSERT_EQ(shared.size(), 2u); + EXPECT_EQ(shared[0].owner, "/owner_a") << "ordered by owner, like the other backends"; + EXPECT_EQ(shared[1].owner, "/owner_b"); + + EXPECT_TRUE(storage_->get_ids_by_code("NEVER_REPORTED").empty()); + ASSERT_TRUE(storage_->get_fault(FaultId{"SHARED", "/owner_a"}).has_value()); + EXPECT_FALSE(storage_->get_fault(FaultId{"SHARED", "/nobody"}).has_value()); +} + +TEST_F(PgFaultStorageTest, NearMissSeriesIsPerRecord) { + const auto config = four_strike_config(); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "dip", "/hydraulics/pump", nth_report_time(0).nanoseconds(), + config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "dip", "/hydraulics/backup_pump", + nth_report_time(1).nanoseconds(), config); + + const auto pump = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); + const auto backup = storage_->get_near_misses(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}); + ASSERT_EQ(pump.size(), 1u); + ASSERT_EQ(backup.size(), 1u); + EXPECT_EQ(pump.front().source_id, "/hydraulics/pump"); + EXPECT_EQ(pump.front().severity, Fault::SEVERITY_WARN); + EXPECT_EQ(backup.front().source_id, "/hydraulics/backup_pump"); + EXPECT_EQ(backup.front().severity, Fault::SEVERITY_ERROR); +} + +TEST_F(PgFaultStorageTest, ASharedRecordConfirmsCriticalOnTheReportsSeverity) { + rclcpp::Clock clock; + DebounceConfig config; + config.confirmation_threshold = -3; + ASSERT_TRUE(config.critical_immediate_confirm); + const FaultId shared{"CRIT_SHARED", ros2_medkit_fault_manager::kSharedOwner}; + const FaultId per_source{"CRIT_PER", "/a"}; + + storage_->report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "", "/a", + clock.now().nanoseconds(), config); + ASSERT_EQ(storage_->get_fault(shared)->status, Fault::STATUS_CONFIRMED); + ASSERT_TRUE(storage_->clear_fault(shared)); + EXPECT_TRUE(storage_->report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "", "/b", + clock.now().nanoseconds(), config)); + EXPECT_EQ(storage_->get_fault(shared)->severity, Fault::SEVERITY_CRITICAL); + EXPECT_EQ(storage_->get_fault(shared)->status, Fault::STATUS_PREFAILED) << "the report's severity decides"; + + storage_->report_fault_event(per_source, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "", "/a", + clock.now().nanoseconds(), config); + ASSERT_TRUE(storage_->clear_fault(per_source)); + storage_->report_fault_event(per_source, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "", "/a", + clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(per_source)->status, Fault::STATUS_CONFIRMED) << "the stored severity decides"; +} + +TEST_F(PgFaultStorageTest, ACriticalRecordConfirmsOnAnyFailedReportAndAnotherOwnersIsUntouched) { + rclcpp::Clock clock; + DebounceConfig config; + config.confirmation_threshold = -3; + ASSERT_TRUE(config.critical_immediate_confirm); + const FaultId a{"CRIT", "/a"}; + const FaultId b{"CRIT", "/b"}; + + storage_->report_fault_event(FaultId{"CRIT", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "", a.owner, + clock.now().nanoseconds(), config); + ASSERT_TRUE(storage_->clear_fault(a)); + + EXPECT_TRUE(storage_->report_fault_event(FaultId{"CRIT", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "", + a.owner, clock.now().nanoseconds(), config)) + << "a report on a CLEARED record is an occurrence"; + EXPECT_EQ(storage_->get_fault(a)->severity, Fault::SEVERITY_CRITICAL) << "the record keeps its highest severity"; + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_CONFIRMED) << "the stored severity decides, not the report's"; + + storage_->report_fault_event(FaultId{"CRIT", b.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "", b.owner, + clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault(b)->severity, Fault::SEVERITY_WARN) << "severity is per record"; + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_PREFAILED) << "another owner's CRITICAL confirms nothing"; +} + +TEST_F(PgFaultStorageTest, HealingAndTimeBasedConfirmationActOnOneRecord) { + rclcpp::Clock clock; + DebounceConfig config; + config.confirmation_threshold = -3; + config.healing_enabled = true; + config.healing_threshold = 1; + config.auto_confirm_after_sec = 5.0; + storage_->set_debounce_config(config); + const FaultId a{"TWO", "/a"}; + const FaultId b{"TWO", "/b"}; + + const int64_t t0 = 1'000'000'000'000LL; + storage_->report_fault_event(FaultId{"TWO", a.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", a.owner, t0, + config); + storage_->report_fault_event(FaultId{"TWO", b.owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "", b.owner, + t0 + 1, config); + + const auto swept = storage_->check_time_based_confirmation(t0 + 10'000'000'000LL); + ASSERT_EQ(swept.size(), 2u); + EXPECT_EQ(swept[0].fault_code, "TWO"); + EXPECT_NE(swept[0].owner, swept[1].owner) << "one entry per record, not per code"; + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_CONFIRMED); + + // Heal a only: its counter moves, b's does not. + for (int i = 0; i < 4; ++i) { + storage_->report_fault_event(FaultId{"TWO", a.owner}, fd::ReportKind::kPassed, 0, "", a.owner, t0 + 20 + i, config); + } + EXPECT_EQ(storage_->get_fault(a)->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_CONFIRMED); + + // The startup pass reclassifies the HEALED record and names it by identity. + const auto reclassified = storage_->reclassify_healed_as_cleared(); + ASSERT_EQ(reclassified.size(), 1u); + EXPECT_EQ(reclassified.front(), a); + EXPECT_EQ(storage_->get_fault(b)->status, Fault::STATUS_CONFIRMED); +} + +namespace { + +/// Tables as the previous layout wrote them: the code alone is the key. +constexpr const char * kLegacyPgSchema = R"( + CREATE TABLE faults ( + fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, description TEXT NOT NULL, + first_occurred_ns BIGINT NOT NULL, last_occurred_ns BIGINT NOT NULL, occurrence_count BIGINT NOT NULL, + status TEXT NOT NULL, reporting_sources TEXT NOT NULL, debounce_counter INTEGER NOT NULL DEFAULT 0, + last_failed_ns BIGINT NOT NULL DEFAULT 0, last_passed_ns BIGINT NOT NULL DEFAULT 0, + confirmed_at_ns BIGINT NOT NULL DEFAULT 0); + CREATE TABLE snapshots ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, topic TEXT NOT NULL, + message_type TEXT NOT NULL, data TEXT NOT NULL, captured_at_ns BIGINT NOT NULL, + capture_id BIGINT NOT NULL DEFAULT 0); + CREATE INDEX idx_snapshots_fault_code ON snapshots(fault_code); + CREATE TABLE freeze_frames (fault_code TEXT PRIMARY KEY, data TEXT NOT NULL, captured_at_ns BIGINT NOT NULL); + CREATE TABLE rosbag_files ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + recording_id TEXT NOT NULL DEFAULT '', file_path TEXT NOT NULL, format TEXT NOT NULL, + duration_sec DOUBLE PRECISION NOT NULL, size_bytes BIGINT NOT NULL, created_at_ns BIGINT NOT NULL); + CREATE UNIQUE INDEX idx_rosbag_files_fault_path ON rosbag_files(fault_code, file_path); + CREATE INDEX idx_rosbag_files_fault_created ON rosbag_files(fault_code, created_at_ns, id); + )"; + +} // namespace + +// Every record of a table written when a fault code was one record, fed by every source, becomes the +// shared record of its code, and every child row attaches to it. +TEST_F(PgFaultStorageTest, ATableWrittenByThePreviousLayoutIsMigratedInPlace) { + storage_.reset(); + exec_raw("DROP TABLE faults, snapshots, freeze_frames, near_misses, rosbag_files"); + exec_raw(kLegacyPgSchema); + exec_raw(R"( + CREATE TABLE near_misses ( + id BIGINT PRIMARY KEY GENERATED ALWAYS AS IDENTITY, fault_code TEXT NOT NULL, + occurred_at_ns BIGINT NOT NULL, debounce_counter INTEGER NOT NULL, confirmation_threshold INTEGER NOT NULL, + severity INTEGER NOT NULL, source_id TEXT NOT NULL, resulting_status TEXT NOT NULL DEFAULT ''); + CREATE INDEX idx_near_misses_fault_code ON near_misses(fault_code, id); + INSERT INTO faults VALUES ('SHARED', 2, 'shared', 1, 1, 1, 'CONFIRMED', '["/owner_a","/owner_b"]', -1, 1, 0, 5); + INSERT INTO faults VALUES ('SOLO', 1, 'solo', 2, 2, 1, 'PREFAILED', '["/owner_c"]', -1, 2, 0, 99); + INSERT INTO freeze_frames VALUES ('SHARED', '{"/t":1}', 7); + INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) + VALUES ('SHARED', '/t', 'std_msgs/msg/Float64', '{}', 9); + INSERT INTO near_misses (fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, severity, + source_id, resulting_status) + VALUES ('SHARED', 5, -1, -3, 2, '/owner_a', 'PREFAILED'), ('SHARED', 6, -2, -3, 2, '/owner_b', 'PREFAILED'); + INSERT INTO rosbag_files (fault_code, file_path, format, duration_sec, size_bytes, created_at_ns) + VALUES ('SHARED', '/bags/fault_SHARED_1', 'mcap', 5.0, 1024, 11); + )"); + + ASSERT_NO_THROW(storage_ = std::make_unique(conn_info_, test_log_sink())); + + const auto scalar = [this](const std::string & sql) { + auto conn = raw_connection(); + pqxx::nontransaction tx(*conn); + return tx.exec(sql)[0][0].as(); + }; + + // The identity is unique over (fault_code, owner), and the old PRIMARY KEY on the code is gone. + EXPECT_EQ(scalar("SELECT COUNT(*) FROM pg_constraint WHERE conrelid = to_regclass('faults') AND contype = 'p'"), "0"); + EXPECT_EQ(scalar("SELECT COUNT(*) FROM pg_constraint WHERE conrelid = to_regclass('freeze_frames') AND " + "contype = 'p'"), + "0"); + EXPECT_NE(scalar("SELECT indexdef FROM pg_indexes WHERE schemaname = current_schema() AND " + "indexname = 'idx_rosbag_files_fault_path'") + .find("(fault_code, owner, file_path)"), + std::string::npos) + << "the rosbag unique index must widen with the identity"; + + // Every row is the shared record of its code and keeps its reporting_sources as written. + EXPECT_EQ(scalar("SELECT COUNT(*) FROM faults WHERE owner = '*'"), "2"); + EXPECT_EQ(scalar("SELECT reporting_sources FROM faults WHERE fault_code = 'SHARED'"), "[\"/owner_a\",\"/owner_b\"]"); + EXPECT_EQ(scalar("SELECT confirmed_at_ns FROM faults WHERE fault_code = 'SOLO'"), "0") + << "a pending record has no confirmation time"; + EXPECT_EQ(scalar("SELECT confirmed_at_ns FROM faults WHERE fault_code = 'SHARED'"), "5"); + + const FaultId shared{"SHARED", ros2_medkit_fault_manager::kSharedOwner}; + auto migrated = storage_->get_fault(shared); + ASSERT_TRUE(migrated.has_value()); + EXPECT_EQ(migrated->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(migrated->reporting_sources, (std::vector{"/owner_a", "/owner_b"})); + EXPECT_EQ(storage_->get_ids_by_code("SHARED").size(), 1u); + EXPECT_TRUE(storage_->get_fault(FaultId{"SOLO", "*"}).has_value()); + + // Every child row attached to the shared record, none lost. + ASSERT_TRUE(storage_->get_freeze_frame(shared).has_value()); + EXPECT_EQ(storage_->get_freeze_frame(shared)->captured_at_ns, 7); + ASSERT_EQ(storage_->get_snapshots(shared).size(), 1u); + EXPECT_EQ(storage_->get_snapshots(shared).front().owner, "*"); + ASSERT_TRUE(storage_->get_rosbag_file(shared).has_value()); + EXPECT_EQ(storage_->get_rosbag_file(shared)->owner, "*"); + const auto near_misses = storage_->get_near_misses(shared); + ASSERT_EQ(near_misses.size(), 2u) << "the near misses of both sources belong to the one shared record"; + EXPECT_EQ(near_misses[0].source_id, "/owner_a"); + EXPECT_EQ(near_misses[1].source_id, "/owner_b"); + EXPECT_EQ(scalar("SELECT COUNT(*) FROM near_misses n WHERE NOT EXISTS " + "(SELECT 1 FROM faults f WHERE f.fault_code = n.fault_code AND f.owner = n.owner)"), + "0"); + + // The migrated record behaves as any shared record: another source joins it. + rclcpp::Clock clock; + EXPECT_FALSE(storage_->report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "c", "/owner_c", + clock.now().nanoseconds(), default_config())); + EXPECT_EQ(storage_->get_fault(shared)->reporting_sources, + (std::vector{"/owner_a", "/owner_b", "/owner_c"})); + + // Re-opening a migrated database changes nothing. + storage_.reset(); + storage_ = std::make_unique(conn_info_, test_log_sink()); + EXPECT_EQ(storage_->size(), 2u); + EXPECT_EQ(scalar("SELECT COUNT(*) FROM faults WHERE owner = '*'"), "2"); + ASSERT_TRUE(storage_->get_freeze_frame(shared).has_value()); + EXPECT_EQ(storage_->get_near_misses(shared).size(), 2u); +} + +// One record shared by every source of a code: one debounce counter, every source listed. +TEST_F(PgFaultStorageTest, ASharedRecordHasOneCounterAndListsEverySource) { + rclcpp::Clock clock; + const FaultId shared{"ONE_COUNTER", ros2_medkit_fault_manager::kSharedOwner}; + auto config = default_config(); + config.confirmation_threshold = -3; + + EXPECT_TRUE(storage_->report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", "/src_a", + clock.now().nanoseconds(), config)); + EXPECT_FALSE(storage_->report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "b", "/src_b", + clock.now().nanoseconds(), config)); + EXPECT_EQ(storage_->get_fault(shared)->status, Fault::STATUS_PREFAILED); + EXPECT_FALSE(storage_->report_fault_event(shared, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "c", "/src_c", + clock.now().nanoseconds(), config)); + + EXPECT_EQ(storage_->size(), 1u); + auto record = storage_->get_fault(shared); + ASSERT_TRUE(record.has_value()); + EXPECT_EQ(record->status, Fault::STATUS_CONFIRMED) << "three sources, one counter"; + EXPECT_EQ(record->reporting_sources, (std::vector{"/src_a", "/src_b", "/src_c"})); + EXPECT_EQ(record->occurrence_count, 1u); + ASSERT_EQ(storage_->list_faults(false, 0, {}).size(), 1u); + EXPECT_EQ(storage_->list_fault_entries(false, 0, {}).front().id, shared); + + // The sources of the record are the entities that own its recordings. + RosbagFileInfo link; + link.fault_code = shared.fault_code; + link.owner = shared.owner; + link.file_path = "/bags/fault_ONE_COUNTER_1"; + link.format = "mcap"; + link.created_at_ns = 1; + storage_->store_rosbag_file(link); + for (const char * entity : {"/src_a", "/src_b", "/src_c"}) { + EXPECT_EQ(storage_->list_rosbags_for_entity(entity).size(), 1u) << entity; + } + EXPECT_TRUE(storage_->list_rosbags_for_entity("/src_d").empty()); } int main(int argc, char ** argv) { diff --git a/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp b/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp index 6f86b69bf..a6c1d6ffb 100644 --- a/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp +++ b/src/ros2_medkit_fault_manager/test/test_rosbag_capture.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -52,6 +53,7 @@ #include "ros2_medkit_msgs/srv/report_fault.hpp" #include "test_log_sink.hpp" +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::FaultStorage; using ros2_medkit_fault_manager::InMemoryFaultStorage; using ros2_medkit_fault_manager::RosbagCapture; @@ -68,6 +70,15 @@ void spin_some_once(const rclcpp::Node::SharedPtr & node) { executor.remove_node(node); } +/// Every record this suite drives belongs to one owner. These tests are about the +/// recording state machine and the bag lifecycle, which the record identity does not +/// change. The cross-owner cases live with the storage and node tests. +constexpr const char * kOwner = "/reporter"; + +FaultId rec(const std::string & code) { + return FaultId{code, kOwner}; +} + /// Captures rcutils log output while alive and restores the console handler on /// every exit path - leaving the process-global handler installed would swallow /// the output of every later case in this binary. @@ -186,8 +197,8 @@ class RosbagCaptureTest : public ::testing::Test { node_ = std::make_shared("test_rosbag_capture_node"); storage_ = std::make_unique(); - // Create temp directory for test bags - temp_dir_ = std::filesystem::temp_directory_path() / "rosbag_capture_test"; + // Per process, so concurrent runs of this suite do not delete each other's bags. + temp_dir_ = std::filesystem::temp_directory_path() / ("rosbag_capture_test_" + std::to_string(::getpid())); std::filesystem::create_directories(temp_dir_); } @@ -849,7 +860,7 @@ TEST_F(RosbagCaptureTest, OnFaultPrefailedWhileDisabled) { auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); // Should not throw when disabled - EXPECT_NO_THROW(capture.on_fault_prefailed("TEST_FAULT")); + EXPECT_NO_THROW(capture.on_fault_prefailed(rec("TEST_FAULT"))); } TEST_F(RosbagCaptureTest, OnFaultConfirmedWhileDisabled) { @@ -857,7 +868,7 @@ TEST_F(RosbagCaptureTest, OnFaultConfirmedWhileDisabled) { auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); // Should not throw when disabled - EXPECT_NO_THROW(capture.on_fault_confirmed("TEST_FAULT")); + EXPECT_NO_THROW(capture.on_fault_confirmed(rec("TEST_FAULT"))); } TEST_F(RosbagCaptureTest, OnFaultClearedWhileDisabled) { @@ -865,7 +876,7 @@ TEST_F(RosbagCaptureTest, OnFaultClearedWhileDisabled) { auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); // Should not throw when disabled - EXPECT_NO_THROW(capture.on_fault_cleared("TEST_FAULT")); + EXPECT_NO_THROW(capture.on_fault_cleared(rec("TEST_FAULT"))); } TEST_F(RosbagCaptureTest, OnFaultPrefailedStartsLazyCapture) { @@ -874,7 +885,7 @@ TEST_F(RosbagCaptureTest, OnFaultPrefailedStartsLazyCapture) { auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); EXPECT_FALSE(capture.is_running()); - capture.on_fault_prefailed("TEST_FAULT"); + capture.on_fault_prefailed(rec("TEST_FAULT")); EXPECT_TRUE(capture.is_running()); } @@ -953,6 +964,7 @@ TEST_F(RosbagCaptureTest, SharedBagTotalTakesTheLargestRowPerPath) { auto store = [this](const char * code, const char * path, size_t bytes) { ros2_medkit_fault_manager::RosbagFileInfo info; info.fault_code = code; + info.owner = kOwner; info.file_path = path; info.format = "mcap"; info.duration_sec = 5.0; @@ -986,11 +998,13 @@ TEST_F(RosbagCaptureTest, EvictionFreesASharedBagOnceAndKeepsTheQuotaHonest) { shared.created_at_ns = 1000; for (const char * code : {"ROOT_CAUSE", "CORRELATED_A", "CORRELATED_B"}) { shared.fault_code = code; + shared.owner = kOwner; storage_->store_rosbag_file(shared); } ros2_medkit_fault_manager::RosbagFileInfo solo; solo.fault_code = "UNRELATED"; + solo.owner = kOwner; solo.file_path = solo_path; solo.format = "mcap"; solo.duration_sec = 5.0; @@ -1007,12 +1021,12 @@ TEST_F(RosbagCaptureTest, EvictionFreesASharedBagOnceAndKeepsTheQuotaHonest) { EXPECT_EQ(evicted, std::vector{shared_path}); EXPECT_EQ(storage_->get_total_rosbag_storage_bytes(), 600u); EXPECT_FALSE(std::filesystem::exists(shared_path)); - EXPECT_FALSE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); - EXPECT_FALSE(storage_->get_rosbag_file("CORRELATED_A").has_value()); - EXPECT_FALSE(storage_->get_rosbag_file("CORRELATED_B").has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("ROOT_CAUSE")).has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("CORRELATED_A")).has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("CORRELATED_B")).has_value()); // The newer bag fits under the quota and must survive. - EXPECT_TRUE(storage_->get_rosbag_file("UNRELATED").has_value()); + EXPECT_TRUE(storage_->get_rosbag_file(rec("UNRELATED")).has_value()); EXPECT_TRUE(std::filesystem::exists(solo_path)); } @@ -1034,14 +1048,17 @@ TEST_F(RosbagCaptureTest, EvictionKeepsGoingWhenTheSharedBagIsNotEnough) { shared.duration_sec = 5.0; shared.created_at_ns = 1000; shared.fault_code = "ROOT_CAUSE"; + shared.owner = kOwner; shared.size_bytes = 500; storage_->store_rosbag_file(shared); shared.fault_code = "CORRELATED_A"; + shared.owner = kOwner; shared.size_bytes = 400; storage_->store_rosbag_file(shared); ros2_medkit_fault_manager::RosbagFileInfo solo; solo.fault_code = "UNRELATED"; + solo.owner = kOwner; solo.file_path = solo_path; solo.format = "mcap"; solo.duration_sec = 5.0; @@ -1070,6 +1087,7 @@ TEST_F(RosbagCaptureTest, EvictionNeverDeletesTheRecordingJustStored) { ros2_medkit_fault_manager::RosbagFileInfo older; older.fault_code = "OLDER"; + older.owner = kOwner; older.file_path = older_path; older.format = "mcap"; older.size_bytes = 300; @@ -1078,6 +1096,7 @@ TEST_F(RosbagCaptureTest, EvictionNeverDeletesTheRecordingJustStored) { ros2_medkit_fault_manager::RosbagFileInfo newest; newest.fault_code = "NEWEST"; + newest.owner = kOwner; newest.file_path = newest_path; newest.format = "mcap"; newest.size_bytes = 2000; @@ -1089,7 +1108,7 @@ TEST_F(RosbagCaptureTest, EvictionNeverDeletesTheRecordingJustStored) { EXPECT_EQ(evicted, std::vector{older_path}); EXPECT_FALSE(std::filesystem::exists(older_path)); EXPECT_TRUE(std::filesystem::exists(newest_path)); - ASSERT_TRUE(storage_->get_rosbag_file("NEWEST").has_value()); + ASSERT_TRUE(storage_->get_rosbag_file(rec("NEWEST")).has_value()); // Still over the quota: that is the state the caller warns about. EXPECT_EQ(storage_->get_total_rosbag_storage_bytes(), 2000u); } @@ -1104,6 +1123,7 @@ TEST_F(RosbagCaptureTest, EvictionSkipsTheKeptRecordingWhereverItSorts) { ros2_medkit_fault_manager::RosbagFileInfo kept; kept.fault_code = "KEPT"; + kept.owner = kOwner; kept.file_path = kept_path; kept.format = "mcap"; kept.size_bytes = 900; @@ -1112,6 +1132,7 @@ TEST_F(RosbagCaptureTest, EvictionSkipsTheKeptRecordingWhereverItSorts) { ros2_medkit_fault_manager::RosbagFileInfo later; later.fault_code = "LATER"; + later.owner = kOwner; later.file_path = later_path; later.format = "mcap"; later.size_bytes = 900; @@ -1253,8 +1274,10 @@ TEST_F(RosbagCaptureTest, ABatchStoreUnlinksASharedReplacedBagExactlyOnce) { shared.size_bytes = 10; shared.created_at_ns = 1000; shared.fault_code = "SHARED_A"; + shared.owner = kOwner; storage_->store_rosbag_file(shared); shared.fault_code = "SHARED_B"; + shared.owner = kOwner; storage_->store_rosbag_file(shared); ASSERT_TRUE(std::filesystem::exists(old_bag)); @@ -1264,15 +1287,17 @@ TEST_F(RosbagCaptureTest, ABatchStoreUnlinksASharedReplacedBagExactlyOnce) { std::vector rows; rows.reserve(2); moved.fault_code = "SHARED_A"; + moved.owner = kOwner; rows.push_back(moved); moved.fault_code = "SHARED_B"; + moved.owner = kOwner; rows.push_back(moved); storage_->store_rosbag_files(rows); EXPECT_FALSE(std::filesystem::exists(old_bag)) << "the replaced bag leaked: nothing references it any more"; EXPECT_TRUE(std::filesystem::exists(new_bag)) << "the bag the batch points at was removed"; - auto row_a = storage_->get_rosbag_file("SHARED_A"); - auto row_b = storage_->get_rosbag_file("SHARED_B"); + auto row_a = storage_->get_rosbag_file(rec("SHARED_A")); + auto row_b = storage_->get_rosbag_file(rec("SHARED_B")); ASSERT_TRUE(row_a.has_value()); ASSERT_TRUE(row_b.has_value()); EXPECT_EQ(row_a->file_path, new_bag.string()); @@ -1361,9 +1386,14 @@ class RosbagNonStandardThrowStorage : public InMemoryFaultStorage { /// Storage counting fault lookups, which is what resolving an entity scope costs. class RosbagFaultLookupCountingStorage : public InMemoryFaultStorage { public: - std::optional get_fault(const std::string & fault_code) const override { + std::optional get_fault(const FaultId & id) const override { ++lookups; - return InMemoryFaultStorage::get_fault(fault_code); + return InMemoryFaultStorage::get_fault(id); + } + + bool contains(const FaultId & id) const override { + ++lookups; + return InMemoryFaultStorage::contains(id); } mutable size_t lookups = 0; @@ -1372,7 +1402,7 @@ class RosbagFaultLookupCountingStorage : public InMemoryFaultStorage { /// Storage that cannot read faults, as the PostgreSQL backend does while its server is down. class RosbagFaultLookupFailingStorage : public InMemoryFaultStorage { public: - std::optional get_fault(const std::string & /*fault_code*/) const override { + std::optional get_fault(const FaultId & /*id*/) const override { ++lookups; throw FaultStorage::IgnorableConnectionException("database unreachable"); } @@ -1383,7 +1413,7 @@ class RosbagFaultLookupFailingStorage : public InMemoryFaultStorage { /// Storage whose bag cleanup fails, as the PostgreSQL backend does while its server is down. class RosbagCleanupFailingStorage : public InMemoryFaultStorage { public: - bool delete_rosbag_file(const std::string & /*fault_code*/) override { + bool delete_rosbag_file(const FaultId & /*id*/) override { ++delete_attempts; throw FaultStorage::IgnorableConnectionException("database unreachable"); } @@ -1466,11 +1496,11 @@ class RosbagCaptureIntegrationTest : public RosbagCaptureTest { } } - /// Spin until the post-roll finalises and @p fault_code has its metadata row. - bool wait_for_row(const std::string & fault_code, std::chrono::milliseconds timeout) { + /// Spin until the post-roll finalises and @p id has its metadata row. + bool wait_for_row(const FaultId & id, std::chrono::milliseconds timeout) { auto deadline = std::chrono::steady_clock::now() + timeout; while (std::chrono::steady_clock::now() < deadline) { - if (storage_->get_rosbag_file(fault_code).has_value()) { + if (storage_->get_rosbag_file(id).has_value()) { return true; } spin_for(std::chrono::milliseconds(100)); @@ -1557,18 +1587,18 @@ TEST_F(RosbagCaptureIntegrationTest, FullFaultLifecycleWithNoMessages) { EXPECT_TRUE(capture.is_running()); // Simulate fault lifecycle - capture.on_fault_prefailed("TEST_FAULT_001"); + capture.on_fault_prefailed(rec("TEST_FAULT_001")); // Let some time pass spin_for(std::chrono::milliseconds(100)); - capture.on_fault_confirmed("TEST_FAULT_001"); + capture.on_fault_confirmed(rec("TEST_FAULT_001")); // Wait for post-fault timer spin_for(std::chrono::milliseconds(300)); // Clear the fault - capture.on_fault_cleared("TEST_FAULT_001"); + capture.on_fault_cleared(rec("TEST_FAULT_001")); // Stop capture capture.stop(); @@ -1593,15 +1623,15 @@ TEST_F(RosbagCaptureIntegrationTest, AcknowledgingKeepsAHistoryTheOperatorConfig RosbagCapture capture(node_.get(), &storage, rosbag_config, snapshot_config); capture.start(); fill_buffer("/keep_history_probe"); - capture.on_fault_confirmed("KEEPS_HISTORY"); + capture.on_fault_confirmed(rec("KEEPS_HISTORY")); spin_for(std::chrono::milliseconds(300)); - ASSERT_FALSE(storage.get_rosbag_files("KEEPS_HISTORY").empty()) << "nothing was recorded to keep"; - const auto before = storage.get_rosbag_files("KEEPS_HISTORY").size(); + ASSERT_FALSE(storage.get_rosbag_files(rec("KEEPS_HISTORY")).empty()) << "nothing was recorded to keep"; + const auto before = storage.get_rosbag_files(rec("KEEPS_HISTORY")).size(); - capture.on_fault_cleared("KEEPS_HISTORY"); + capture.on_fault_cleared(rec("KEEPS_HISTORY")); - EXPECT_EQ(storage.get_rosbag_files("KEEPS_HISTORY").size(), before) + EXPECT_EQ(storage.get_rosbag_files(rec("KEEPS_HISTORY")).size(), before) << "acknowledging the fault destroyed the recordings the cap was raised to keep"; capture.stop(); } @@ -1621,25 +1651,25 @@ TEST_F(RosbagCaptureIntegrationTest, AcknowledgingDuringThePostRollKeepsAConfigu storage.set_max_rosbags_per_fault(rosbag_config.max_bags_per_fault); rclcpp::Clock clock; - storage.report_fault_event("ACK_MID_ROLL", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "mid roll", "/node", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage.report_fault_event(FaultId{"ACK_MID_ROLL", "/node"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "mid roll", "/node", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), &storage, rosbag_config, snapshot_config); capture.start(); fill_buffer("/ack_mid_roll_probe"); - capture.on_fault_confirmed("ACK_MID_ROLL"); + capture.on_fault_confirmed({"ACK_MID_ROLL", "/node"}); // The acknowledgement arrives while the post-fault window still runs, so the // store reports the fault as CLEARED by the time finalize asks. - ASSERT_TRUE(storage.clear_fault("ACK_MID_ROLL")); - capture.on_fault_cleared("ACK_MID_ROLL"); + ASSERT_TRUE(storage.clear_fault({"ACK_MID_ROLL", "/node"})); + capture.on_fault_cleared({"ACK_MID_ROLL", "/node"}); spin_for(std::chrono::milliseconds(900)); // past the post-roll and the finalize capture.wait_for_stored_recordings(); - EXPECT_FALSE(storage.get_rosbag_files("ACK_MID_ROLL").empty()) + EXPECT_FALSE(storage.get_rosbag_files({"ACK_MID_ROLL", "/node"}).empty()) << "the post-roll finalize deleted a history on_fault_cleared had decided to keep"; capture.stop(); } @@ -1658,14 +1688,14 @@ TEST_F(RosbagCaptureIntegrationTest, AcknowledgingStillCleansUpWhenNoHistoryIsCo RosbagCapture capture(node_.get(), &storage, rosbag_config, snapshot_config); capture.start(); fill_buffer("/cleanup_probe"); - capture.on_fault_confirmed("CLEANED_UP"); + capture.on_fault_confirmed(rec("CLEANED_UP")); spin_for(std::chrono::milliseconds(300)); - ASSERT_FALSE(storage.get_rosbag_files("CLEANED_UP").empty()); + ASSERT_FALSE(storage.get_rosbag_files(rec("CLEANED_UP")).empty()); - capture.on_fault_cleared("CLEANED_UP"); + capture.on_fault_cleared(rec("CLEANED_UP")); - EXPECT_TRUE(storage.get_rosbag_files("CLEANED_UP").empty()) << "auto_cleanup stopped working at the default cap"; + EXPECT_TRUE(storage.get_rosbag_files(rec("CLEANED_UP")).empty()) << "auto_cleanup stopped working at the default cap"; capture.stop(); } @@ -1679,16 +1709,16 @@ TEST_F(RosbagCaptureIntegrationTest, MultipleFaultsHandled) { capture.start(); // Multiple faults in sequence - capture.on_fault_prefailed("FAULT_A"); + capture.on_fault_prefailed(rec("FAULT_A")); spin_for(std::chrono::milliseconds(50)); - capture.on_fault_prefailed("FAULT_B"); + capture.on_fault_prefailed(rec("FAULT_B")); spin_for(std::chrono::milliseconds(50)); - capture.on_fault_confirmed("FAULT_A"); + capture.on_fault_confirmed(rec("FAULT_A")); spin_for(std::chrono::milliseconds(50)); - capture.on_fault_confirmed("FAULT_B"); + capture.on_fault_confirmed(rec("FAULT_B")); spin_for(std::chrono::milliseconds(200)); capture.stop(); @@ -1710,7 +1740,7 @@ TEST_F(RosbagCaptureIntegrationTest, StoringARecordingDoesNotHoldTheExecutor) { capture.start(); fill_buffer("/store_off_executor_probe"); - capture.on_fault_confirmed("STORE_OFF_EXECUTOR"); + capture.on_fault_confirmed(rec("STORE_OFF_EXECUTOR")); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); auto longest_spin = std::chrono::steady_clock::duration::zero(); @@ -1734,11 +1764,11 @@ TEST_F(RosbagCaptureIntegrationTest, StoringARecordingDoesNotHoldTheExecutor) { bag_path = gated.bag_path; } EXPECT_TRUE(read_manifest(bag_path).is_object()) << "the row was being stored before the manifest existed"; - EXPECT_FALSE(gated.get_rosbag_file("STORE_OFF_EXECUTOR").has_value()); + EXPECT_FALSE(gated.get_rosbag_file(rec("STORE_OFF_EXECUTOR")).has_value()); gated.release(); capture.wait_for_stored_recordings(); - auto row = gated.get_rosbag_file("STORE_OFF_EXECUTOR"); + auto row = gated.get_rosbag_file(rec("STORE_OFF_EXECUTOR")); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->file_path, bag_path); capture.stop(); @@ -1756,15 +1786,15 @@ TEST_F(RosbagCaptureIntegrationTest, AClearWhileTheRowIsStoredTakesTheRowWithIt) rosbag_config.auto_cleanup = true; rosbag_config.max_bags_per_fault = 1; rclcpp::Clock clock; - gated.report_fault_event("CLEARED_IN_STORE", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "store race", "/node", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + gated.report_fault_event(FaultId{"CLEARED_IN_STORE", kOwner}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "store race", kOwner, + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), &gated, rosbag_config, snapshot_config); capture.start(); fill_buffer("/clear_during_store_probe"); - capture.on_fault_confirmed("CLEARED_IN_STORE"); + capture.on_fault_confirmed(rec("CLEARED_IN_STORE")); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); while (!gated.wait_entered(std::chrono::milliseconds(0)) && std::chrono::steady_clock::now() < deadline) { spin_for(std::chrono::milliseconds(10)); @@ -1778,9 +1808,9 @@ TEST_F(RosbagCaptureIntegrationTest, AClearWhileTheRowIsStoredTakesTheRowWithIt) // The worker found the fault not cleared and is inside the insert. The clear marks // the fault in the store, then runs the hook, as FaultManagerCore does. - ASSERT_TRUE(gated.clear_fault("CLEARED_IN_STORE")); + ASSERT_TRUE(gated.clear_fault(rec("CLEARED_IN_STORE"))); auto cleared = std::async(std::launch::async, [&capture]() { - capture.on_fault_cleared("CLEARED_IN_STORE"); + capture.on_fault_cleared(rec("CLEARED_IN_STORE")); }); EXPECT_EQ(cleared.wait_for(std::chrono::milliseconds(500)), std::future_status::timeout) << "the clear's delete ran while the row was still being inserted"; @@ -1788,7 +1818,7 @@ TEST_F(RosbagCaptureIntegrationTest, AClearWhileTheRowIsStoredTakesTheRowWithIt) gated.release(); cleared.get(); capture.wait_for_stored_recordings(); - EXPECT_TRUE(gated.get_rosbag_files("CLEARED_IN_STORE").empty()) << "a cleared fault kept its row"; + EXPECT_TRUE(gated.get_rosbag_files(rec("CLEARED_IN_STORE")).empty()) << "a cleared fault kept its row"; EXPECT_FALSE(std::filesystem::exists(bag_path)) << "a cleared fault kept its bag"; capture.stop(); } @@ -1806,16 +1836,16 @@ TEST_F(RosbagCaptureIntegrationTest, FaultConfirmedDuringPostRollGetsTheSameBag) capture.start(); fill_buffer("/rosbag_burst_probe"); - capture.on_fault_confirmed("ROOT_CAUSE"); - capture.on_fault_confirmed("CORRELATED"); + capture.on_fault_confirmed(rec("ROOT_CAUSE")); + capture.on_fault_confirmed(rec("CORRELATED")); // Let the post-fault timer fire and finalise the recording. spin_for(std::chrono::milliseconds(1600)); capture.wait_for_stored_recordings(); - auto root = storage_->get_rosbag_file("ROOT_CAUSE"); + auto root = storage_->get_rosbag_file(rec("ROOT_CAUSE")); ASSERT_TRUE(root.has_value()) << "the first fault must produce a bag"; - auto correlated = storage_->get_rosbag_file("CORRELATED"); + auto correlated = storage_->get_rosbag_file(rec("CORRELATED")); ASSERT_TRUE(correlated.has_value()) << "the fault confirmed during the post-roll lost its recording"; EXPECT_EQ(correlated->file_path, root->file_path); EXPECT_TRUE(std::filesystem::exists(root->file_path)); @@ -1833,28 +1863,73 @@ TEST_F(RosbagCaptureIntegrationTest, ClearingOneFaultOfABurstKeepsTheSharedBag) capture.start(); fill_buffer("/rosbag_burst_cleanup_probe"); - capture.on_fault_confirmed("ROOT_CAUSE"); - capture.on_fault_confirmed("CORRELATED"); + capture.on_fault_confirmed(rec("ROOT_CAUSE")); + capture.on_fault_confirmed(rec("CORRELATED")); spin_for(std::chrono::milliseconds(1600)); capture.wait_for_stored_recordings(); - auto root = storage_->get_rosbag_file("ROOT_CAUSE"); + auto root = storage_->get_rosbag_file(rec("ROOT_CAUSE")); ASSERT_TRUE(root.has_value()); - ASSERT_TRUE(storage_->get_rosbag_file("CORRELATED").has_value()); + ASSERT_TRUE(storage_->get_rosbag_file(rec("CORRELATED")).has_value()); const std::string bag_path = root->file_path; // Auto-cleanup of one fault must not pull the bag out from under its sibling. - capture.on_fault_cleared("CORRELATED"); + capture.on_fault_cleared(rec("CORRELATED")); EXPECT_TRUE(std::filesystem::exists(bag_path)); - EXPECT_TRUE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); + EXPECT_TRUE(storage_->get_rosbag_file(rec("ROOT_CAUSE")).has_value()); // The last reference going away does delete it. - capture.on_fault_cleared("ROOT_CAUSE"); + capture.on_fault_cleared(rec("ROOT_CAUSE")); EXPECT_FALSE(std::filesystem::exists(bag_path)); capture.stop(); } +// The bag directory name and the recording id stay keyed by the CODE, so two +// owners confirming one code inside a burst attach to the one in-flight recording - +// and each of them still gets its own rosbag_files row, because the download and the +// acknowledgement are addressed to a record, not to a code. +TEST_F(RosbagCaptureIntegrationTest, TwoOwnersOfOneCodeShareTheBagAndGetARowEach) { + auto rosbag_config = create_rosbag_config(); + rosbag_config.duration_sec = 10.0; + rosbag_config.duration_after_sec = 1.0; + auto snapshot_config = create_snapshot_config(); + RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); + + capture.start(); + fill_buffer("/rosbag_two_owners_probe"); + + const FaultId owner_a{"BURST_CODE", "/owner_a"}; + const FaultId owner_b{"BURST_CODE", "/owner_b"}; + capture.on_fault_confirmed(owner_a); + capture.on_fault_confirmed(owner_b); // lands inside A's post-roll, attaches + + ASSERT_TRUE(wait_for_row(owner_b, std::chrono::milliseconds(8000))); + + auto row_a = storage_->get_rosbag_file(owner_a); + auto row_b = storage_->get_rosbag_file(owner_b); + ASSERT_TRUE(row_a.has_value()); + ASSERT_TRUE(row_b.has_value()); + EXPECT_EQ(row_a->owner, "/owner_a"); + EXPECT_EQ(row_b->owner, "/owner_b"); + EXPECT_EQ(row_b->file_path, row_a->file_path) << "one recording, one bag"; + EXPECT_EQ(row_b->recording_id, row_a->recording_id); + EXPECT_EQ(storage_->get_rosbag_files_by_recording(row_a->recording_id).size(), 2u); + + // Acknowledging one owner's record unlinks only that row. The bag survives while + // the other owner still holds it. + capture.on_fault_cleared(owner_a); + EXPECT_FALSE(storage_->get_rosbag_file(owner_a).has_value()); + ASSERT_TRUE(storage_->get_rosbag_file(owner_b).has_value()); + EXPECT_TRUE(std::filesystem::exists(row_a->file_path)) << "the other owner still references the bag"; + + capture.on_fault_cleared(owner_b); + EXPECT_FALSE(storage_->get_rosbag_file(owner_b).has_value()); + EXPECT_FALSE(std::filesystem::exists(row_a->file_path)); + + capture.stop(); +} + TEST_F(RosbagCaptureIntegrationTest, FaultClearedDuringPostRollNeverGetsARow) { auto rosbag_config = create_rosbag_config(); rosbag_config.duration_sec = 10.0; @@ -1865,23 +1940,23 @@ TEST_F(RosbagCaptureIntegrationTest, FaultClearedDuringPostRollNeverGetsARow) { capture.start(); fill_buffer("/rosbag_cleared_mid_roll_probe"); - capture.on_fault_confirmed("ROOT_CAUSE"); - capture.on_fault_confirmed("CORRELATED"); + capture.on_fault_confirmed(rec("ROOT_CAUSE")); + capture.on_fault_confirmed(rec("CORRELATED")); // The attached fault clears before the post-roll timer fires. Its row has not // been written yet - it must be forgotten, not resurrected at finalize, where // a leftover row would pin the shared bag forever. - capture.on_fault_cleared("CORRELATED"); + capture.on_fault_cleared(rec("CORRELATED")); - ASSERT_TRUE(wait_for_row("ROOT_CAUSE", std::chrono::milliseconds(8000))); - EXPECT_FALSE(storage_->get_rosbag_file("CORRELATED").has_value()) + ASSERT_TRUE(wait_for_row(rec("ROOT_CAUSE"), std::chrono::milliseconds(8000))); + EXPECT_FALSE(storage_->get_rosbag_file(rec("CORRELATED")).has_value()) << "a fault cleared during the post-roll must not get a row at finalize"; // With no leftover row the bag belongs to the primary alone: clearing it // must unlink the recording. - auto root = storage_->get_rosbag_file("ROOT_CAUSE"); + auto root = storage_->get_rosbag_file(rec("ROOT_CAUSE")); ASSERT_TRUE(root.has_value()); - capture.on_fault_cleared("ROOT_CAUSE"); - EXPECT_FALSE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); + capture.on_fault_cleared(rec("ROOT_CAUSE")); + EXPECT_FALSE(storage_->get_rosbag_file(rec("ROOT_CAUSE")).has_value()); EXPECT_FALSE(std::filesystem::exists(root->file_path)); capture.stop(); @@ -1898,10 +1973,10 @@ TEST_F(RosbagCaptureIntegrationTest, PostRollKeepsTheRowWhenTheFaultCannotBeRead capture.start(); fill_buffer("/rosbag_lookup_failure_probe"); - capture.on_fault_confirmed("UNREADABLE"); + capture.on_fault_confirmed(rec("UNREADABLE")); // The post-roll timer runs inside spin_some, so a throw from the finalize ends the test here. - ASSERT_TRUE(wait_for_row("UNREADABLE", std::chrono::milliseconds(8000))); + ASSERT_TRUE(wait_for_row(rec("UNREADABLE"), std::chrono::milliseconds(8000))); EXPECT_GT(failing_storage->lookups, 0u) << "the finalize never asked the store"; capture.stop(); @@ -1913,7 +1988,7 @@ TEST_F(RosbagCaptureIntegrationTest, ClearSurvivesAFailingBagCleanup) { auto snapshot_config = create_snapshot_config(); RosbagCapture capture(node_.get(), &storage, rosbag_config, snapshot_config); - EXPECT_NO_THROW(capture.on_fault_cleared("UNREACHABLE")); + EXPECT_NO_THROW(capture.on_fault_cleared(rec("UNREACHABLE"))); EXPECT_EQ(storage.delete_attempts, 1u) << "the cleanup never reached the store"; } @@ -1927,21 +2002,21 @@ TEST_F(RosbagCaptureIntegrationTest, PrimaryClearedDuringPostRollLeavesTheBagToI capture.start(); fill_buffer("/rosbag_primary_cleared_probe"); - capture.on_fault_confirmed("ROOT_CAUSE"); - capture.on_fault_confirmed("CORRELATED"); + capture.on_fault_confirmed(rec("ROOT_CAUSE")); + capture.on_fault_confirmed(rec("CORRELATED")); // The fault that opened the recording clears mid post-roll; the attached // sibling still needs the bag, so the recording finalises for it alone. - capture.on_fault_cleared("ROOT_CAUSE"); + capture.on_fault_cleared(rec("ROOT_CAUSE")); - ASSERT_TRUE(wait_for_row("CORRELATED", std::chrono::milliseconds(8000))); - EXPECT_FALSE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()) + ASSERT_TRUE(wait_for_row(rec("CORRELATED"), std::chrono::milliseconds(8000))); + EXPECT_FALSE(storage_->get_rosbag_file(rec("ROOT_CAUSE")).has_value()) << "the cleared primary must not get a row at finalize"; - auto correlated = storage_->get_rosbag_file("CORRELATED"); + auto correlated = storage_->get_rosbag_file(rec("CORRELATED")); ASSERT_TRUE(correlated.has_value()); EXPECT_TRUE(std::filesystem::exists(correlated->file_path)); - capture.on_fault_cleared("CORRELATED"); + capture.on_fault_cleared(rec("CORRELATED")); EXPECT_FALSE(std::filesystem::exists(correlated->file_path)); capture.stop(); @@ -1957,17 +2032,17 @@ TEST_F(RosbagCaptureIntegrationTest, BurstFullyClearedDuringPostRollDiscardsTheB capture.start(); fill_buffer("/rosbag_all_cleared_probe"); - capture.on_fault_confirmed("ROOT_CAUSE"); - capture.on_fault_confirmed("CORRELATED"); - capture.on_fault_cleared("CORRELATED"); - capture.on_fault_cleared("ROOT_CAUSE"); + capture.on_fault_confirmed(rec("ROOT_CAUSE")); + capture.on_fault_confirmed(rec("CORRELATED")); + capture.on_fault_cleared(rec("CORRELATED")); + capture.on_fault_cleared(rec("ROOT_CAUSE")); // Let the post-roll timer fire with nothing left to register. spin_for(std::chrono::milliseconds(1500)); capture.wait_for_stored_recordings(); - EXPECT_FALSE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); - EXPECT_FALSE(storage_->get_rosbag_file("CORRELATED").has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("ROOT_CAUSE")).has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("CORRELATED")).has_value()); EXPECT_EQ(count_bag_dirs(), 0u) << "a bag nobody references must not be left on disk"; capture.stop(); @@ -1992,19 +2067,19 @@ TEST_F(RosbagCaptureIntegrationTest, AttachmentCapDropsFaultsPastIt) { codes.push_back("BURST_" + std::string(i < 10 ? "0" : "") + std::to_string(i)); } for (const auto & code : codes) { - capture.on_fault_confirmed(code); + capture.on_fault_confirmed(rec(code)); } - ASSERT_TRUE(wait_for_row(codes[0], std::chrono::milliseconds(10000))); + ASSERT_TRUE(wait_for_row(rec(codes[0]), std::chrono::milliseconds(10000))); - auto primary = storage_->get_rosbag_file(codes[0]); + auto primary = storage_->get_rosbag_file(rec(codes[0])); ASSERT_TRUE(primary.has_value()); for (size_t i = 1; i < 33; ++i) { - auto row = storage_->get_rosbag_file(codes[i]); + auto row = storage_->get_rosbag_file(rec(codes[i])); ASSERT_TRUE(row.has_value()) << codes[i] << " is within the cap and must resolve to the recording"; EXPECT_EQ(row->file_path, primary->file_path); } - EXPECT_FALSE(storage_->get_rosbag_file(codes[33]).has_value()) + EXPECT_FALSE(storage_->get_rosbag_file(rec(codes[33])).has_value()) << "fault 34 of the burst is past the cap and is dropped with a WARN"; EXPECT_EQ(count_bag_dirs(), 1u) << "the dropped fault must not open a second bag"; @@ -2029,12 +2104,12 @@ TEST_F(RosbagCaptureIntegrationTest, AttachedFaultTopicsJoinTheEntityScopedRecor auto pub_b = node_b->create_publisher("/scope_topic_b", 10); rclcpp::Clock clock; - storage_->report_fault_event("FAULT_A", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "fault a", "/scope_source_a", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); - storage_->report_fault_event("FAULT_B", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "fault b", "/scope_source_b", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"FAULT_A", "/scope_source_a"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "fault a", "/scope_source_a", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"FAULT_B", "/scope_source_b"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "fault b", "/scope_source_b", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); @@ -2048,8 +2123,8 @@ TEST_F(RosbagCaptureIntegrationTest, AttachedFaultTopicsJoinTheEntityScopedRecor spin_for(std::chrono::milliseconds(50)); } - capture.on_fault_confirmed("FAULT_A"); - capture.on_fault_confirmed("FAULT_B"); // lands inside A's post-roll, attaches + capture.on_fault_confirmed({"FAULT_A", "/scope_source_a"}); + capture.on_fault_confirmed({"FAULT_B", "/scope_source_b"}); // lands inside A's post-roll, attaches // Keep both topics flowing during the post-roll so the widened filter has // B's data to write. @@ -2059,10 +2134,10 @@ TEST_F(RosbagCaptureIntegrationTest, AttachedFaultTopicsJoinTheEntityScopedRecor spin_for(std::chrono::milliseconds(50)); } - ASSERT_TRUE(wait_for_row("FAULT_B", std::chrono::milliseconds(12000))); + ASSERT_TRUE(wait_for_row({"FAULT_B", "/scope_source_b"}, std::chrono::milliseconds(12000))); - auto row_a = storage_->get_rosbag_file("FAULT_A"); - auto row_b = storage_->get_rosbag_file("FAULT_B"); + auto row_a = storage_->get_rosbag_file({"FAULT_A", "/scope_source_a"}); + auto row_b = storage_->get_rosbag_file({"FAULT_B", "/scope_source_b"}); ASSERT_TRUE(row_a.has_value()); ASSERT_TRUE(row_b.has_value()); EXPECT_EQ(row_b->file_path, row_a->file_path); @@ -2095,20 +2170,20 @@ TEST_F(RosbagCaptureIntegrationTest, StopDuringPostRollFinalisesTheRecording) { capture.start(); fill_buffer("/rosbag_stop_probe"); - capture.on_fault_confirmed("BEFORE_STOP"); + capture.on_fault_confirmed(rec("BEFORE_STOP")); capture.stop(); - auto before = storage_->get_rosbag_file("BEFORE_STOP"); + auto before = storage_->get_rosbag_file(rec("BEFORE_STOP")); ASSERT_TRUE(before.has_value()) << "a post-roll cut short by stop() must still leave a usable bag"; // Restart and confirm again: the new fault must get its own recording. capture.start(); fill_buffer("/rosbag_stop_probe_2"); - capture.on_fault_confirmed("AFTER_RESTART"); + capture.on_fault_confirmed(rec("AFTER_RESTART")); capture.stop(); - auto after = storage_->get_rosbag_file("AFTER_RESTART"); + auto after = storage_->get_rosbag_file(rec("AFTER_RESTART")); ASSERT_TRUE(after.has_value()) << "confirmation after restart was swallowed by stale recording state"; EXPECT_NE(after->file_path, before->file_path); } @@ -2133,18 +2208,18 @@ TEST_F(RosbagCaptureIntegrationTest, ConfirmRightAfterFinalizeGetsAPostFaultOnly // buffer after the flush - the boundary state is reached deterministically. fill_buffer("/rosbag_boundary_probe"); - capture.on_fault_confirmed("BOUNDARY_A"); - ASSERT_TRUE(wait_for_row("BOUNDARY_A", std::chrono::milliseconds(8000))); + capture.on_fault_confirmed(rec("BOUNDARY_A")); + ASSERT_TRUE(wait_for_row(rec("BOUNDARY_A"), std::chrono::milliseconds(8000))); // A's flush drained the deque, its post-roll diverted direct writes, and no // publisher exists any more: the buffer is empty. Confirming now used to be // a warn-and-return no-op that left the fault with no recording at all. - capture.on_fault_confirmed("BOUNDARY_B"); + capture.on_fault_confirmed(rec("BOUNDARY_B")); - ASSERT_TRUE(wait_for_row("BOUNDARY_B", std::chrono::milliseconds(8000))) + ASSERT_TRUE(wait_for_row(rec("BOUNDARY_B"), std::chrono::milliseconds(8000))) << "a fault confirmed right after the previous post-roll finalised must get a post-fault-only bag"; - auto row_a = storage_->get_rosbag_file("BOUNDARY_A"); - auto row_b = storage_->get_rosbag_file("BOUNDARY_B"); + auto row_a = storage_->get_rosbag_file(rec("BOUNDARY_A")); + auto row_b = storage_->get_rosbag_file(rec("BOUNDARY_B")); ASSERT_TRUE(row_a.has_value()); ASSERT_TRUE(row_b.has_value()); EXPECT_NE(row_b->file_path, row_a->file_path) << "the boundary fault opens its own recording, not the closed one"; @@ -2171,15 +2246,15 @@ TEST_F(RosbagCaptureIntegrationTest, BurstFaultAttachesToThePostFaultOnlyRecordi capture.start(); fill_buffer("/rosbag_boundary_attach_probe"); - capture.on_fault_confirmed("BOUNDARY_A"); - ASSERT_TRUE(wait_for_row("BOUNDARY_A", std::chrono::milliseconds(8000))); + capture.on_fault_confirmed(rec("BOUNDARY_A")); + ASSERT_TRUE(wait_for_row(rec("BOUNDARY_A"), std::chrono::milliseconds(8000))); - capture.on_fault_confirmed("BOUNDARY_B"); // empty buffer -> post-fault-only recording - capture.on_fault_confirmed("BOUNDARY_C"); // lands inside B's post-roll -> attaches + capture.on_fault_confirmed(rec("BOUNDARY_B")); // empty buffer -> post-fault-only recording + capture.on_fault_confirmed(rec("BOUNDARY_C")); // lands inside B's post-roll -> attaches - ASSERT_TRUE(wait_for_row("BOUNDARY_B", std::chrono::milliseconds(8000))); - auto row_b = storage_->get_rosbag_file("BOUNDARY_B"); - auto row_c = storage_->get_rosbag_file("BOUNDARY_C"); + ASSERT_TRUE(wait_for_row(rec("BOUNDARY_B"), std::chrono::milliseconds(8000))); + auto row_b = storage_->get_rosbag_file(rec("BOUNDARY_B")); + auto row_c = storage_->get_rosbag_file(rec("BOUNDARY_C")); ASSERT_TRUE(row_b.has_value()); ASSERT_TRUE(row_c.has_value()) << "a burst fault confirming inside the post-fault-only window lost its attachment"; EXPECT_EQ(row_c->file_path, row_b->file_path); @@ -2199,10 +2274,10 @@ TEST_F(RosbagCaptureIntegrationTest, EmptyBufferWithNoPostFaultWindowGetsNoBag) capture.start(); // No publisher at all: the buffer is empty at confirmation. - capture.on_fault_confirmed("NO_WINDOW"); + capture.on_fault_confirmed(rec("NO_WINDOW")); spin_for(std::chrono::milliseconds(300)); - EXPECT_FALSE(storage_->get_rosbag_file("NO_WINDOW").has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("NO_WINDOW")).has_value()); EXPECT_EQ(count_bag_dirs(), 0u); capture.stop(); @@ -2221,10 +2296,10 @@ TEST_F(RosbagCaptureIntegrationTest, FullBagDurationReflectsActualContentNotConf capture.start(); fill_buffer("/rosbag_duration_probe"); - capture.on_fault_confirmed("SHORT_CONTENT"); - ASSERT_TRUE(wait_for_row("SHORT_CONTENT", std::chrono::milliseconds(8000))); + capture.on_fault_confirmed(rec("SHORT_CONTENT")); + ASSERT_TRUE(wait_for_row(rec("SHORT_CONTENT"), std::chrono::milliseconds(8000))); - auto row = storage_->get_rosbag_file("SHORT_CONTENT"); + auto row = storage_->get_rosbag_file(rec("SHORT_CONTENT")); ASSERT_TRUE(row.has_value()); EXPECT_GT(row->duration_sec, 0.0); EXPECT_LE(row->duration_sec, 5.0) << "duration_sec must reflect the ~2.5s actually recorded, not config pre+post"; @@ -2244,9 +2319,9 @@ TEST_F(RosbagCaptureIntegrationTest, ImmediateFinalizeDurationReflectsActualCont capture.start(); fill_buffer("/rosbag_immediate_duration_probe"); - capture.on_fault_confirmed("IMMEDIATE_CONTENT"); + capture.on_fault_confirmed(rec("IMMEDIATE_CONTENT")); - auto row = storage_->get_rosbag_file("IMMEDIATE_CONTENT"); + auto row = storage_->get_rosbag_file(rec("IMMEDIATE_CONTENT")); ASSERT_TRUE(row.has_value()); EXPECT_GT(row->duration_sec, 0.0); EXPECT_LE(row->duration_sec, 5.0) << "duration_sec must reflect the ~2s actually buffered, not config duration_sec"; @@ -2276,26 +2351,26 @@ TEST_F(RosbagCaptureIntegrationTest, WriterOpenFailureNeverOpensAPostRollAndReco fill_buffer("/rosbag_io_error_probe"); // Buffered flush hits the I/O error. - capture.on_fault_confirmed("IO_FAIL_FULL"); + capture.on_fault_confirmed(rec("IO_FAIL_FULL")); // Wait past duration_after_sec: a wrongly-opened post-roll would finalise // and store a row for the fault. spin_for(std::chrono::milliseconds(900)); - EXPECT_FALSE(storage_->get_rosbag_file("IO_FAIL_FULL").has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("IO_FAIL_FULL")).has_value()); // Empty buffer at the boundary + broken path: the post-fault-only writer // open fails. Never open a post-roll after an I/O failure. - capture.on_fault_confirmed("IO_FAIL_EMPTY"); + capture.on_fault_confirmed(rec("IO_FAIL_EMPTY")); spin_for(std::chrono::milliseconds(900)); - EXPECT_FALSE(storage_->get_rosbag_file("IO_FAIL_EMPTY").has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("IO_FAIL_EMPTY")).has_value()); // Repair the path (now a directory) - later faults must capture normally. std::filesystem::remove(blocked); std::filesystem::create_directories(blocked); fill_buffer("/rosbag_io_error_probe_2"); - capture.on_fault_confirmed("IO_RECOVERED"); - ASSERT_TRUE(wait_for_row("IO_RECOVERED", std::chrono::milliseconds(8000))) + capture.on_fault_confirmed(rec("IO_RECOVERED")); + ASSERT_TRUE(wait_for_row(rec("IO_RECOVERED"), std::chrono::milliseconds(8000))) << "an earlier I/O failure must not corrupt the state machine for a fault with a working path"; - EXPECT_TRUE(std::filesystem::exists(storage_->get_rosbag_file("IO_RECOVERED")->file_path)); + EXPECT_TRUE(std::filesystem::exists(storage_->get_rosbag_file(rec("IO_RECOVERED"))->file_path)); capture.stop(); } @@ -2310,17 +2385,42 @@ TEST_F(RosbagCaptureIntegrationTest, ZeroMessagePostFaultOnlyBagFinalizesCleanly rosbag_config.duration_sec = 2.0; rosbag_config.duration_after_sec = 0.5; rosbag_config.format = format; + // A topic only this test knows, so traffic from the rest of the ROS domain cannot reach the bag. + rosbag_config.topics = "/zero_message_post_fault_probe"; auto snapshot_config = create_snapshot_config(); + // The probe topic never publishes. Traffic on another topic must stay out of the bag. Both + // publishers exist before the capture, which subscribes to the topics it finds on construction. + auto probe_pub = node_->create_publisher(rosbag_config.topics, 10); + const std::string unrelated_topic = "/zero_message_unrelated_traffic"; + auto unrelated_pub = node_->create_publisher(unrelated_topic, 10); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); - // No publisher at all: the buffer is empty and the post-roll records nothing. + const auto discovery_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (node_->count_subscribers(rosbag_config.topics) < 1 && + std::chrono::steady_clock::now() < discovery_deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + ASSERT_GE(node_->count_subscribers(rosbag_config.topics), 1u) << "the capture never subscribed to its topic"; + ASSERT_EQ(node_->count_subscribers(unrelated_topic), 0u) << "the capture subscribed to a topic it was not given"; + + std::atomic stop_traffic{false}; + std::thread traffic([&]() { + std_msgs::msg::String msg; + msg.data = "unrelated"; + while (!stop_traffic.load()) { + unrelated_pub->publish(msg); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + }); const std::string fault_code = "ZERO_MSG_" + format; - capture.on_fault_confirmed(fault_code); + capture.on_fault_confirmed(rec(fault_code)); - ASSERT_TRUE(wait_for_row(fault_code, std::chrono::milliseconds(8000))) - << "zero-message post-fault-only bag did not finalise on " << format; - auto row = storage_->get_rosbag_file(fault_code); + const bool finalised = wait_for_row(rec(fault_code), std::chrono::milliseconds(8000)); + stop_traffic.store(true); + traffic.join(); + ASSERT_TRUE(finalised) << "zero-message post-fault-only bag did not finalise on " << format; + auto row = storage_->get_rosbag_file(rec(fault_code)); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->format, format); ASSERT_TRUE(std::filesystem::is_directory(row->file_path)); @@ -2366,16 +2466,16 @@ TEST_F(RosbagCaptureIntegrationTest, PostFaultOnlyBagContainsThePostFaultWindow) // A's finalise and B's flush, or B is not the boundary case at all. publish_for(pub, std::chrono::milliseconds(2000)); - capture.on_fault_confirmed("CONTENT_A"); - ASSERT_TRUE(wait_for_row("CONTENT_A", std::chrono::milliseconds(10000))); + capture.on_fault_confirmed(rec("CONTENT_A")); + ASSERT_TRUE(wait_for_row(rec("CONTENT_A"), std::chrono::milliseconds(10000))); - capture.on_fault_confirmed("CONTENT_B"); + capture.on_fault_confirmed(rec("CONTENT_B")); // on_fault_confirmed armed B's window before returning, so everything published // from here lands inside it and has to reach B's bag. publish_for(pub, std::chrono::milliseconds(1000)); - ASSERT_TRUE(wait_for_row("CONTENT_B", std::chrono::milliseconds(10000))); - auto row_b = storage_->get_rosbag_file("CONTENT_B"); + ASSERT_TRUE(wait_for_row(rec("CONTENT_B"), std::chrono::milliseconds(10000))); + auto row_b = storage_->get_rosbag_file(rec("CONTENT_B")); ASSERT_TRUE(row_b.has_value()); EXPECT_TRUE(bag_has_topic(row_b->file_path, topic)) << "the post-fault-only bag never recorded the topic - an empty black box is the failure #574 is about"; @@ -2401,12 +2501,12 @@ TEST_F(RosbagCaptureIntegrationTest, EntityScopedPostFaultOnlyBagCarriesOnlyTheF auto pub_b = node_b->create_publisher("/boundary_scope_b", 10); rclcpp::Clock clock; - storage_->report_fault_event("SCOPED_A", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "scoped a", "/boundary_scope_source_a", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); - storage_->report_fault_event("SCOPED_B", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "scoped b", "/boundary_scope_source_b", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"SCOPED_A", "/boundary_scope_source_a"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "scoped a", "/boundary_scope_source_a", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"SCOPED_B", "/boundary_scope_source_b"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "scoped b", "/boundary_scope_source_b", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); @@ -2419,19 +2519,19 @@ TEST_F(RosbagCaptureIntegrationTest, EntityScopedPostFaultOnlyBagCarriesOnlyTheF spin_for(std::chrono::milliseconds(50)); } - capture.on_fault_confirmed("SCOPED_A"); - ASSERT_TRUE(wait_for_row("SCOPED_A", std::chrono::milliseconds(12000))); + capture.on_fault_confirmed({"SCOPED_A", "/boundary_scope_source_a"}); + ASSERT_TRUE(wait_for_row({"SCOPED_A", "/boundary_scope_source_a"}, std::chrono::milliseconds(12000))); // Nothing was published since A's flush drained the buffer: B is the boundary. - capture.on_fault_confirmed("SCOPED_B"); + capture.on_fault_confirmed({"SCOPED_B", "/boundary_scope_source_b"}); for (int i = 0; i < 20; ++i) { pub_a->publish(msg); pub_b->publish(msg); spin_for(std::chrono::milliseconds(50)); } - ASSERT_TRUE(wait_for_row("SCOPED_B", std::chrono::milliseconds(12000))); - auto row_b = storage_->get_rosbag_file("SCOPED_B"); + ASSERT_TRUE(wait_for_row({"SCOPED_B", "/boundary_scope_source_b"}, std::chrono::milliseconds(12000))); + auto row_b = storage_->get_rosbag_file({"SCOPED_B", "/boundary_scope_source_b"}); ASSERT_TRUE(row_b.has_value()); EXPECT_GT(bag_message_count(row_b->file_path), 0) << "the entity-scoped post-fault-only bag finalised empty"; EXPECT_TRUE(bag_has_topic(row_b->file_path, "/boundary_scope_b")) @@ -2467,12 +2567,12 @@ TEST_F(RosbagCaptureIntegrationTest, ABoundaryRecordingKeepsItsScopeWhileTheOldF auto pub_b = node_b->create_publisher("/scope_race_b", 10); rclcpp::Clock clock; - storage_->report_fault_event("SCOPE_RACE_A", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "scope race a", "/scope_race_source_a", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); - storage_->report_fault_event("SCOPE_RACE_B", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "scope race b", "/scope_race_source_b", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"SCOPE_RACE_A", "/scope_race_source_a"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "scope race a", "/scope_race_source_a", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"SCOPE_RACE_B", "/scope_race_source_b"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "scope race b", "/scope_race_source_b", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); @@ -2485,7 +2585,7 @@ TEST_F(RosbagCaptureIntegrationTest, ABoundaryRecordingKeepsItsScopeWhileTheOldF spin_for(std::chrono::milliseconds(50)); } - capture.on_fault_confirmed("SCOPE_RACE_A"); + capture.on_fault_confirmed({"SCOPE_RACE_A", "/scope_race_source_a"}); // The racer is the capture pool and this thread is the executor, the production // shape. Only the racer ever confirms B, so no two threads call on_fault_confirmed @@ -2504,7 +2604,7 @@ TEST_F(RosbagCaptureIntegrationTest, ABoundaryRecordingKeepsItsScopeWhileTheOldF }; std::thread racer([&capture, &stop_racer, &boundary_bag_opened]() { while (!stop_racer.load()) { - capture.on_fault_confirmed("SCOPE_RACE_B"); + capture.on_fault_confirmed({"SCOPE_RACE_B", "/scope_race_source_b"}); if (boundary_bag_opened()) { return; } @@ -2527,8 +2627,8 @@ TEST_F(RosbagCaptureIntegrationTest, ABoundaryRecordingKeepsItsScopeWhileTheOldF spin_for(std::chrono::milliseconds(50)); } - ASSERT_TRUE(wait_for_row("SCOPE_RACE_B", std::chrono::milliseconds(12000))); - auto row_b = storage_->get_rosbag_file("SCOPE_RACE_B"); + ASSERT_TRUE(wait_for_row({"SCOPE_RACE_B", "/scope_race_source_b"}, std::chrono::milliseconds(12000))); + auto row_b = storage_->get_rosbag_file({"SCOPE_RACE_B", "/scope_race_source_b"}); ASSERT_TRUE(row_b.has_value()); size_t b_recordings = 0; for (const auto & entry : std::filesystem::directory_iterator(temp_dir_)) { @@ -2580,7 +2680,7 @@ TEST_F(RosbagCaptureIntegrationTest, BoundaryConfirmationRacingAFinaliseKeepsIts capture.start(); publish_for(pub, std::chrono::milliseconds(1500)); - capture.on_fault_confirmed("RACE_A"); + capture.on_fault_confirmed(rec("RACE_A")); // Hammer the boundary for the whole of A's window: every call while A records // attaches and returns, and the first one after the guard drops takes the @@ -2588,7 +2688,7 @@ TEST_F(RosbagCaptureIntegrationTest, BoundaryConfirmationRacingAFinaliseKeepsIts std::atomic stop_racer{false}; std::thread racer([&capture, &stop_racer]() { while (!stop_racer.load()) { - capture.on_fault_confirmed("RACE_B"); + capture.on_fault_confirmed(rec("RACE_B")); std::this_thread::sleep_for(std::chrono::microseconds(200)); } }); @@ -2598,8 +2698,8 @@ TEST_F(RosbagCaptureIntegrationTest, BoundaryConfirmationRacingAFinaliseKeepsIts racer.join(); publish_for(pub, std::chrono::milliseconds(600)); - ASSERT_TRUE(wait_for_row("RACE_B", std::chrono::milliseconds(12000))); - auto row_b = storage_->get_rosbag_file("RACE_B"); + ASSERT_TRUE(wait_for_row(rec("RACE_B"), std::chrono::milliseconds(12000))); + auto row_b = storage_->get_rosbag_file(rec("RACE_B")); ASSERT_TRUE(row_b.has_value()); EXPECT_GT(bag_message_count(row_b->file_path), 0) << "the boundary recording finalised empty - its writer was taken by the finalise it raced"; @@ -2625,18 +2725,18 @@ TEST_F(RosbagCaptureIntegrationTest, AttachmentCapAppliesToAPostFaultOnlyRecordi codes.push_back("POSTONLY_BURST_" + std::string(i < 10 ? "0" : "") + std::to_string(i)); } for (const auto & code : codes) { - capture.on_fault_confirmed(code); + capture.on_fault_confirmed(rec(code)); } - ASSERT_TRUE(wait_for_row(codes[0], std::chrono::milliseconds(12000))); - auto primary = storage_->get_rosbag_file(codes[0]); + ASSERT_TRUE(wait_for_row(rec(codes[0]), std::chrono::milliseconds(12000))); + auto primary = storage_->get_rosbag_file(rec(codes[0])); ASSERT_TRUE(primary.has_value()); for (size_t i = 1; i < 33; ++i) { - auto row = storage_->get_rosbag_file(codes[i]); + auto row = storage_->get_rosbag_file(rec(codes[i])); ASSERT_TRUE(row.has_value()) << codes[i] << " is within the cap and must resolve to the recording"; EXPECT_EQ(row->file_path, primary->file_path); } - EXPECT_FALSE(storage_->get_rosbag_file(codes[33]).has_value()) + EXPECT_FALSE(storage_->get_rosbag_file(rec(codes[33])).has_value()) << "fault 34 of the burst is past the cap and is dropped with a WARN"; EXPECT_EQ(count_bag_dirs(), 1u) << "the dropped fault must not open a second bag"; @@ -2654,21 +2754,21 @@ TEST_F(RosbagCaptureIntegrationTest, BoundaryFaultClearedDuringItsOwnWindowDisca RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); - capture.on_fault_confirmed("CLEARED_BOUNDARY"); // empty buffer -> post-fault-only + capture.on_fault_confirmed(rec("CLEARED_BOUNDARY")); // empty buffer -> post-fault-only spin_for(std::chrono::milliseconds(200)); - capture.on_fault_cleared("CLEARED_BOUNDARY"); + capture.on_fault_cleared(rec("CLEARED_BOUNDARY")); spin_for(std::chrono::milliseconds(2000)); // past the window, finalise ran capture.wait_for_stored_recordings(); - EXPECT_FALSE(storage_->get_rosbag_file("CLEARED_BOUNDARY").has_value()); + EXPECT_FALSE(storage_->get_rosbag_file(rec("CLEARED_BOUNDARY")).has_value()); EXPECT_EQ(count_bag_dirs(), 0u) << "the bag of a fault cleared inside its own window must be discarded"; capture.stop(); } TEST_F(RosbagCaptureIntegrationTest, MetadataStoreFailureDiscardsTheBagInsteadOfOrphaningIt) { - // If the row cannot be written, nothing can ever reach the bag: retrieval is - // keyed by fault code and quota accounting enumerates rows, so a kept directory + // If the row cannot be written, nothing can ever reach the bag: retrieval goes + // through the rows and quota accounting enumerates rows, so a kept directory // would occupy disk that nothing can find and nothing can evict. RosbagMetadataFailingStorage failing_storage; auto rosbag_config = create_rosbag_config(); @@ -2678,13 +2778,13 @@ TEST_F(RosbagCaptureIntegrationTest, MetadataStoreFailureDiscardsTheBagInsteadOf RosbagCapture capture(node_.get(), &failing_storage, rosbag_config, snapshot_config); capture.start(); - capture.on_fault_confirmed("STORE_FAILS"); // empty buffer -> post-fault-only - spin_for(std::chrono::milliseconds(1200)); // past the window, finalise ran + capture.on_fault_confirmed(rec("STORE_FAILS")); // empty buffer -> post-fault-only + spin_for(std::chrono::milliseconds(1200)); // past the window, finalise ran capture.wait_for_stored_recordings(); ASSERT_GT(failing_storage.store_attempts, 0u) << "the recording never reached the metadata store, so no row and no directory prove nothing"; - EXPECT_FALSE(failing_storage.get_rosbag_file("STORE_FAILS").has_value()); + EXPECT_FALSE(failing_storage.get_rosbag_file(rec("STORE_FAILS")).has_value()); EXPECT_EQ(count_bag_dirs(), 0u) << "a bag that no row can reference must not be left on disk"; capture.stop(); @@ -2694,7 +2794,7 @@ TEST_F(RosbagCaptureIntegrationTest, AFailingQuotaSweepKeepsTheBagItJustStored) // The mirror image of the test above, and the case its double cannot reach. The // sweep runs AFTER store_rosbag_files() has committed, so a failure there says // nothing about this recording. Discarding the bag then strands the rows just - // written: unreadable for good, because retrieval is keyed by fault code, and + // written: unreadable for good, because retrieval goes through the rows, and // still charged against max_total_storage_mb, which sums rows - the very pressure // that made the sweep run. RosbagQuotaSweepFailingStorage sweep_failing_storage; @@ -2710,6 +2810,7 @@ TEST_F(RosbagCaptureIntegrationTest, AFailingQuotaSweepKeepsTheBagItJustStored) // reach its delete. ros2_medkit_fault_manager::RosbagFileInfo older; older.fault_code = "OLDER"; + older.owner = kOwner; older.file_path = (temp_dir_ / "fault_OLDER_1000").string(); older.format = "mcap"; older.size_bytes = 1; @@ -2719,11 +2820,11 @@ TEST_F(RosbagCaptureIntegrationTest, AFailingQuotaSweepKeepsTheBagItJustStored) capture.start(); fill_buffer("/sweep_failure_probe"); - capture.on_fault_confirmed("SWEEP_FAILS"); + capture.on_fault_confirmed(rec("SWEEP_FAILS")); spin_for(std::chrono::milliseconds(1200)); // past the window, finalise ran capture.wait_for_stored_recordings(); - auto row = sweep_failing_storage.get_rosbag_file("SWEEP_FAILS"); + auto row = sweep_failing_storage.get_rosbag_file(rec("SWEEP_FAILS")); ASSERT_TRUE(row.has_value()) << "the metadata store succeeded, so its row must be there"; // Without this the test would pass on a build where eviction never engaged, and // would be proving nothing about a failing sweep at all. @@ -2739,7 +2840,7 @@ TEST_F(RosbagCaptureIntegrationTest, AFailingStoreLeavesNoBagOnTheImmediatePath) // rather than in the post-roll finalise, and only the finalise was ever covered // for a failing store. Nothing catches it here: in production the capture pool // logs the exception and moves on, leaving a directory no row names - unreachable, - // because retrieval is keyed by fault code, and uncounted, because the quota + // because retrieval goes through the rows, and uncounted, because the quota // enumerates rows, so it is never evicted either and accumulates per failure. RosbagMetadataFailingStorage failing_storage; auto rosbag_config = create_rosbag_config(); @@ -2752,13 +2853,13 @@ TEST_F(RosbagCaptureIntegrationTest, AFailingStoreLeavesNoBagOnTheImmediatePath) fill_buffer("/immediate_store_probe"); // The buffer has to hold something, or the confirmation returns before it ever // opens a bag and there is nothing for the store to fail on. - EXPECT_NO_THROW(capture.on_fault_confirmed("IMMEDIATE_STORE_FAILS")); + EXPECT_NO_THROW(capture.on_fault_confirmed(rec("IMMEDIATE_STORE_FAILS"))); spin_for(std::chrono::milliseconds(200)); ASSERT_GT(failing_storage.store_attempts, 0u) << "fill_buffer() buffered nothing, so the confirmation returned before opening a bag and the " "assertions below hold for the wrong reason"; - EXPECT_FALSE(failing_storage.get_rosbag_file("IMMEDIATE_STORE_FAILS").has_value()); + EXPECT_FALSE(failing_storage.get_rosbag_file(rec("IMMEDIATE_STORE_FAILS")).has_value()); EXPECT_EQ(count_bag_dirs(), 0u) << "a bag that no row can reference must not be left on disk"; capture.stop(); @@ -2779,9 +2880,9 @@ TEST_F(RosbagCaptureIntegrationTest, TheStorageQuotaEngagesOnABoundaryRecording) capture.start(); // Nothing has published yet, so the buffer is empty and this takes the boundary // path - the startup case, not just the one after a window closes. - capture.on_fault_confirmed("QUOTA_ROOMY"); - ASSERT_TRUE(wait_for_row("QUOTA_ROOMY", std::chrono::milliseconds(8000))); - auto row = storage_->get_rosbag_file("QUOTA_ROOMY"); + capture.on_fault_confirmed(rec("QUOTA_ROOMY")); + ASSERT_TRUE(wait_for_row(rec("QUOTA_ROOMY"), std::chrono::milliseconds(8000))); + auto row = storage_->get_rosbag_file(rec("QUOTA_ROOMY")); ASSERT_TRUE(row.has_value()); EXPECT_TRUE(std::filesystem::exists(row->file_path)); capture.stop(); @@ -2795,14 +2896,14 @@ TEST_F(RosbagCaptureIntegrationTest, TheStorageQuotaEngagesOnABoundaryRecording) strict.max_total_storage_mb = 0; RosbagCapture capture(node_.get(), storage_.get(), strict, snapshot_config); capture.start(); - capture.on_fault_confirmed("QUOTA_TIGHT"); - ASSERT_TRUE(wait_for_row("QUOTA_TIGHT", std::chrono::milliseconds(8000))) + capture.on_fault_confirmed(rec("QUOTA_TIGHT")); + ASSERT_TRUE(wait_for_row(rec("QUOTA_TIGHT"), std::chrono::milliseconds(8000))) << "a recording larger than the quota was deleted by its own sweep"; // The sweep runs on the store worker right after the row is stored. capture.wait_for_stored_recordings(); - EXPECT_TRUE(std::filesystem::exists(storage_->get_rosbag_file("QUOTA_TIGHT")->file_path)); + EXPECT_TRUE(std::filesystem::exists(storage_->get_rosbag_file(rec("QUOTA_TIGHT"))->file_path)); - EXPECT_FALSE(storage_->get_rosbag_file("QUOTA_ROOMY").has_value()) + EXPECT_FALSE(storage_->get_rosbag_file(rec("QUOTA_ROOMY")).has_value()) << "the quota did not engage on a post-fault-only recording"; capture.stop(); } @@ -2819,9 +2920,9 @@ TEST_F(RosbagCaptureIntegrationTest, ARecordingWithoutAPostRollCarriesAManifestT capture.start(); fill_buffer("/manifest_immediate_probe"); - capture.on_fault_confirmed("MANIFEST_IMMEDIATE"); + capture.on_fault_confirmed(rec("MANIFEST_IMMEDIATE")); - auto row = storage_->get_rosbag_file("MANIFEST_IMMEDIATE"); + auto row = storage_->get_rosbag_file(rec("MANIFEST_IMMEDIATE")); ASSERT_TRUE(row.has_value()); const auto manifest = read_manifest(row->file_path); ASSERT_TRUE(manifest.is_object()) << "a recording closed without a post-roll carries no manifest.json"; @@ -2866,9 +2967,9 @@ TEST_F(RosbagCaptureIntegrationTest, ARecordingNeverTakesOverAnExistingBagDirect std::ofstream(taken.back() / "earlier_recording") << "kept"; } - capture.on_fault_confirmed("PATH_COLLISION"); + capture.on_fault_confirmed(rec("PATH_COLLISION")); - auto row = storage_->get_rosbag_file("PATH_COLLISION"); + auto row = storage_->get_rosbag_file(rec("PATH_COLLISION")); ASSERT_TRUE(row.has_value()) << "a taken bag name cost the fault its recording"; EXPECT_TRUE(std::filesystem::exists(std::filesystem::path(row->file_path) / "metadata.yaml")); for (const auto & earlier : taken) { @@ -2892,7 +2993,7 @@ TEST_F(RosbagCaptureIntegrationTest, ABoundaryRecordingSplitsAndReportsTheWholeB RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); - capture.on_fault_confirmed("SPLIT_BOUNDARY"); // empty buffer -> post-fault-only + capture.on_fault_confirmed(rec("SPLIT_BOUNDARY")); // empty buffer -> post-fault-only auto pub = node_->create_publisher("/split_probe", 10); std_msgs::msg::String msg; @@ -2902,8 +3003,8 @@ TEST_F(RosbagCaptureIntegrationTest, ABoundaryRecordingSplitsAndReportsTheWholeB spin_for(std::chrono::milliseconds(50)); } - ASSERT_TRUE(wait_for_row("SPLIT_BOUNDARY", std::chrono::milliseconds(12000))); - auto row = storage_->get_rosbag_file("SPLIT_BOUNDARY"); + ASSERT_TRUE(wait_for_row(rec("SPLIT_BOUNDARY"), std::chrono::milliseconds(12000))); + auto row = storage_->get_rosbag_file(rec("SPLIT_BOUNDARY")); ASSERT_TRUE(row.has_value()); size_t data_files = 0; @@ -2971,18 +3072,18 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultAttachingToABoundaryRecordingWidensIt auto pub_bystander = node_bystander->create_publisher("/widen_bystander", 10); rclcpp::Clock clock; - storage_->report_fault_event("WIDEN_C", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "widen c", "/widen_source_c", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); - storage_->report_fault_event("WIDEN_D", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "widen d", "/widen_source_d", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"WIDEN_C", "/widen_source_c"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "widen c", "/widen_source_c", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"WIDEN_D", "/widen_source_d"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "widen d", "/widen_source_d", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); // The fault that opens the boundary recording needs a reported source too, or its // scope resolves to nothing and the recording degrades to writing every topic - // which is what the bystander assertion below catches. - storage_->report_fault_event("WIDEN_C2", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "widen c2", "/widen_source_c", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{"WIDEN_C2", "/widen_source_c"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "widen c2", "/widen_source_c", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); @@ -2997,14 +3098,14 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultAttachingToABoundaryRecordingWidensIt spin_for(std::chrono::milliseconds(50)); } - capture.on_fault_confirmed("WIDEN_C"); - ASSERT_TRUE(wait_for_row("WIDEN_C", std::chrono::milliseconds(12000))); + capture.on_fault_confirmed({"WIDEN_C", "/widen_source_c"}); + ASSERT_TRUE(wait_for_row({"WIDEN_C", "/widen_source_c"}, std::chrono::milliseconds(12000))); // Buffer drained by C's flush: this one is the boundary, scoped to C alone. - capture.on_fault_confirmed("WIDEN_C2"); + capture.on_fault_confirmed({"WIDEN_C2", "/widen_source_c"}); spin_for(std::chrono::milliseconds(300)); // D confirms inside that window and attaches, so its topics join from here on. - capture.on_fault_confirmed("WIDEN_D"); + capture.on_fault_confirmed({"WIDEN_D", "/widen_source_d"}); for (int i = 0; i < 30; ++i) { pub_c->publish(msg); pub_d->publish(msg); @@ -3012,8 +3113,8 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultAttachingToABoundaryRecordingWidensIt spin_for(std::chrono::milliseconds(50)); } - ASSERT_TRUE(wait_for_row("WIDEN_D", std::chrono::milliseconds(12000))); - auto row_d = storage_->get_rosbag_file("WIDEN_D"); + ASSERT_TRUE(wait_for_row({"WIDEN_D", "/widen_source_d"}, std::chrono::milliseconds(12000))); + auto row_d = storage_->get_rosbag_file({"WIDEN_D", "/widen_source_d"}); ASSERT_TRUE(row_d.has_value()); EXPECT_TRUE(bag_has_topic(row_d->file_path, "/widen_d")) << "the attached fault's row serves a boundary recording holding none of its topics"; @@ -3032,9 +3133,9 @@ TEST_F(RosbagCaptureIntegrationTest, FaultClearedBeforeConfirmed) { capture.start(); // Prefail then clear (fault didn't confirm) - capture.on_fault_prefailed("CLEARED_EARLY"); + capture.on_fault_prefailed(rec("CLEARED_EARLY")); spin_for(std::chrono::milliseconds(50)); - capture.on_fault_cleared("CLEARED_EARLY"); + capture.on_fault_cleared(rec("CLEARED_EARLY")); // Should not crash spin_for(std::chrono::milliseconds(100)); @@ -3050,7 +3151,7 @@ TEST_F(RosbagCaptureIntegrationTest, ConfirmedWithoutPrefailed) { capture.start(); // Direct confirm (edge case) - capture.on_fault_confirmed("DIRECT_CONFIRM"); + capture.on_fault_confirmed(rec("DIRECT_CONFIRM")); spin_for(std::chrono::milliseconds(200)); capture.stop(); @@ -3130,18 +3231,18 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultClearedWhileItsBagWasOpeningGetsNoRow auto snapshot_config = create_snapshot_config(); rclcpp::Clock clock; - storage_->report_fault_event("CLEARED_WHILE_OPENING", fd::ReportKind::kFailed, + storage_->report_fault_event(FaultId{"CLEARED_WHILE_OPENING", "/test_source"}, fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "cleared while opening", "/test_source", clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); - capture.on_fault_confirmed("CLEARED_WHILE_OPENING"); - storage_->clear_fault("CLEARED_WHILE_OPENING"); + capture.on_fault_confirmed({"CLEARED_WHILE_OPENING", "/test_source"}); + storage_->clear_fault({"CLEARED_WHILE_OPENING", "/test_source"}); spin_for(std::chrono::milliseconds(1500)); capture.wait_for_stored_recordings(); - EXPECT_FALSE(storage_->get_rosbag_file("CLEARED_WHILE_OPENING").has_value()) + EXPECT_FALSE(storage_->get_rosbag_file({"CLEARED_WHILE_OPENING", "/test_source"}).has_value()) << "a fault cleared before its recording finalised kept a row nothing will ever remove"; EXPECT_EQ(count_bag_dirs(), 0u) << "the bag nobody references any more was left on disk"; @@ -3166,10 +3267,12 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultPastTheAttachmentCapStillGetsItsTopic rclcpp::Clock clock; const ros2_medkit_fault_manager::DebounceConfig debounce{}; - storage_->report_fault_event("CAP_PRIMARY", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "cap primary", "/cap_primary_source", clock.now().nanoseconds(), debounce); - storage_->report_fault_event("CAP_LATE", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "cap late", "/cap_late_source", clock.now().nanoseconds(), debounce); + storage_->report_fault_event(FaultId{"CAP_PRIMARY", "/cap_primary_source"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "cap primary", "/cap_primary_source", + clock.now().nanoseconds(), debounce); + storage_->report_fault_event(FaultId{"CAP_LATE", "/cap_late_source"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "cap late", "/cap_late_source", + clock.now().nanoseconds(), debounce); RosbagCapture capture(node_.get(), storage_.get(), rosbag_config, snapshot_config); capture.start(); @@ -3182,28 +3285,30 @@ TEST_F(RosbagCaptureIntegrationTest, AFaultPastTheAttachmentCapStillGetsItsTopic spin_for(std::chrono::milliseconds(50)); } - capture.on_fault_confirmed("CAP_PRIMARY"); + capture.on_fault_confirmed({"CAP_PRIMARY", "/cap_primary_source"}); // Fill the attachment set past its cap, then confirm the one whose topics matter. // The fillers report the PRIMARY node as their source on purpose: a fault whose // scope cannot be resolved widens the recording to every topic, which would let the // late fault's topic into the bag for a reason that has nothing to do with the cap. for (int i = 0; i < 40; ++i) { const std::string filler = "CAP_FILLER_" + std::string(i < 10 ? "0" : "") + std::to_string(i); - storage_->report_fault_event(filler, fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "cap filler", "/cap_primary_source", clock.now().nanoseconds(), debounce); - capture.on_fault_confirmed(filler); + storage_->report_fault_event(FaultId{filler, "/cap_primary_source"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "cap filler", "/cap_primary_source", + clock.now().nanoseconds(), debounce); + capture.on_fault_confirmed({filler, "/cap_primary_source"}); } - capture.on_fault_confirmed("CAP_LATE"); + capture.on_fault_confirmed({"CAP_LATE", "/cap_late_source"}); for (int i = 0; i < 30; ++i) { pub_primary->publish(msg); pub_late->publish(msg); spin_for(std::chrono::milliseconds(50)); } - ASSERT_TRUE(wait_for_row("CAP_PRIMARY", std::chrono::milliseconds(12000))); - auto row = storage_->get_rosbag_file("CAP_PRIMARY"); + ASSERT_TRUE(wait_for_row({"CAP_PRIMARY", "/cap_primary_source"}, std::chrono::milliseconds(12000))); + auto row = storage_->get_rosbag_file({"CAP_PRIMARY", "/cap_primary_source"}); ASSERT_TRUE(row.has_value()); - EXPECT_FALSE(storage_->get_rosbag_file("CAP_LATE").has_value()) << "the cap is supposed to withhold the lookup key"; + EXPECT_FALSE(storage_->get_rosbag_file({"CAP_LATE", "/cap_late_source"}).has_value()) + << "the cap is supposed to withhold the lookup key"; EXPECT_TRUE(bag_has_topic(row->file_path, "/cap_primary")) << "the recording lost the topics it was opened for"; EXPECT_TRUE(bag_has_topic(row->file_path, "/cap_late")) << "an over-cap fault's topics never reached the bag, so the burst's black box is missing its data"; @@ -3224,10 +3329,10 @@ TEST_F(RosbagCaptureIntegrationTest, AnUnusualExceptionFromTheStoreDoesNotEscape capture.start(); fill_buffer("/odd_throw_probe"); - EXPECT_NO_THROW(capture.on_fault_confirmed("ODD_THROW")); + EXPECT_NO_THROW(capture.on_fault_confirmed(rec("ODD_THROW"))); ASSERT_GT(odd_storage.store_attempts, 0u) << "the store was never reached, so nothing was caught"; - EXPECT_FALSE(odd_storage.get_rosbag_file("ODD_THROW").has_value()); + EXPECT_FALSE(odd_storage.get_rosbag_file(rec("ODD_THROW")).has_value()); EXPECT_EQ(count_bag_dirs(), 0u) << "the bag survived a failed store"; EXPECT_NO_THROW(capture.stop()); @@ -3250,9 +3355,9 @@ TEST_F(RosbagCaptureIntegrationTest, ReconfirmingTheRecordingsOwnFaultResolvesNo auto pub = source->create_publisher("/reconfirm_probe", 10); rclcpp::Clock clock; - storage_ptr->report_fault_event("RECONFIRMED", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "reconfirmed", "/reconfirm_source", clock.now().nanoseconds(), - ros2_medkit_fault_manager::DebounceConfig{}); + storage_ptr->report_fault_event(FaultId{"RECONFIRMED", "/reconfirm_source"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "reconfirmed", "/reconfirm_source", + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); RosbagCapture capture(node_.get(), storage_ptr, rosbag_config, snapshot_config); capture.start(); @@ -3264,12 +3369,12 @@ TEST_F(RosbagCaptureIntegrationTest, ReconfirmingTheRecordingsOwnFaultResolvesNo spin_for(std::chrono::milliseconds(50)); } - capture.on_fault_confirmed("RECONFIRMED"); + capture.on_fault_confirmed({"RECONFIRMED", "/reconfirm_source"}); const size_t after_first = storage_ptr->lookups; constexpr int kRepeats = 20; for (int i = 0; i < kRepeats; ++i) { - capture.on_fault_confirmed("RECONFIRMED"); + capture.on_fault_confirmed({"RECONFIRMED", "/reconfirm_source"}); } EXPECT_LT(storage_ptr->lookups - after_first, static_cast(kRepeats)) @@ -3352,7 +3457,7 @@ TEST_F(RosbagCaptureIntegrationTest, ConcurrentCapturesInOneProcessSurviveEachOt std::thread confirmer([&]() { int n = 0; while (!stop_racers.load()) { - capture.on_fault_confirmed("PLUGIN_RACE_" + std::to_string(n++)); + capture.on_fault_confirmed(rec("PLUGIN_RACE_" + std::to_string(n++))); confirms_done.fetch_add(1); std::this_thread::sleep_for(std::chrono::milliseconds(5)); } @@ -3381,7 +3486,7 @@ TEST_F(RosbagCaptureIntegrationTest, ConcurrentCapturesInOneProcessSurviveEachOt << "the confirming thread barely ran, so few bags were opened off-executor"; size_t rows = 0; for (int i = 0; i < confirms_done.load(); ++i) { - if (storage_->get_rosbag_file("PLUGIN_RACE_" + std::to_string(i)).has_value()) { + if (storage_->get_rosbag_file(rec("PLUGIN_RACE_" + std::to_string(i))).has_value()) { ++rows; } } diff --git a/src/ros2_medkit_fault_manager/test/test_rosbag_storage_parity.cpp b/src/ros2_medkit_fault_manager/test/test_rosbag_storage_parity.cpp index 7cb06c262..07b358370 100644 --- a/src/ros2_medkit_fault_manager/test/test_rosbag_storage_parity.cpp +++ b/src/ros2_medkit_fault_manager/test/test_rosbag_storage_parity.cpp @@ -44,6 +44,7 @@ namespace { +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::FaultStorage; using ros2_medkit_fault_manager::InMemoryFaultStorage; using ros2_medkit_fault_manager::rosbag_recording_id; @@ -96,9 +97,19 @@ class RosbagRetentionParityTest : public ::testing::Test { return path.string(); } + /// Every row in this suite belongs to one owner: these tests are about retention + /// and ordering, which the record identity does not change. Cross-owner separation + /// is covered where it belongs, in the storage identity tests. + static constexpr const char * kOwner = "/node"; + + static FaultId id(const std::string & code) { + return FaultId{code, kOwner}; + } + RosbagFileInfo row(const std::string & code, const std::string & bag_name, int64_t created_ns) { RosbagFileInfo info; info.fault_code = code; + info.owner = kOwner; info.file_path = make_bag(bag_name); info.recording_id = rosbag_recording_id(info.file_path); info.format = "mcap"; @@ -122,7 +133,7 @@ TYPED_TEST(RosbagRetentionParityTest, RecordingIdIsTheBasenameOfEveryStoredRow) this->storage_->store_rosbag_file(this->row("BASENAME", "fault_BASENAME_1", 1000)); this->storage_->store_rosbag_file(this->row("BASENAME", "fault_BASENAME_2", 2000)); - const auto rows = this->storage_->get_rosbag_files("BASENAME"); + const auto rows = this->storage_->get_rosbag_files(this->id("BASENAME")); ASSERT_EQ(rows.size(), 2u); for (const auto & r : rows) { EXPECT_EQ(r.recording_id, std::filesystem::path(r.file_path).filename().string()); @@ -137,7 +148,7 @@ TYPED_TEST(RosbagRetentionParityTest, RecordingIdIsDerivedWhenTheCallerLeftItEmp info.recording_id.clear(); this->storage_->store_rosbag_file(info); - const auto rows = this->storage_->get_rosbag_files("DERIVED"); + const auto rows = this->storage_->get_rosbag_files(this->id("DERIVED")); ASSERT_EQ(rows.size(), 1u); EXPECT_EQ(rows[0].recording_id, "fault_DERIVED_1"); } @@ -149,7 +160,7 @@ TYPED_TEST(RosbagRetentionParityTest, SeveralRecordingsForOneFaultSurviveUnderAC this->storage_->store_rosbag_file(this->row("FLAP", "fault_FLAP_2", 2000)); this->storage_->store_rosbag_file(this->row("FLAP", "fault_FLAP_3", 3000)); - const auto rows = this->storage_->get_rosbag_files("FLAP"); + const auto rows = this->storage_->get_rosbag_files(this->id("FLAP")); ASSERT_EQ(rows.size(), 3u); EXPECT_EQ(rows[0].recording_id, "fault_FLAP_3") << "newest first"; EXPECT_EQ(rows[2].recording_id, "fault_FLAP_1"); @@ -165,7 +176,7 @@ TYPED_TEST(RosbagRetentionParityTest, CapKeepsTheNewestAndUnlinksTheEvicted) { this->storage_->store_rosbag_file(this->row("CAP", "fault_CAP_2", 2000)); this->storage_->store_rosbag_file(this->row("CAP", "fault_CAP_3", 3000)); - const auto rows = this->storage_->get_rosbag_files("CAP"); + const auto rows = this->storage_->get_rosbag_files(this->id("CAP")); ASSERT_EQ(rows.size(), 2u); EXPECT_EQ(rows[0].recording_id, "fault_CAP_3"); EXPECT_EQ(rows[1].recording_id, "fault_CAP_2"); @@ -180,12 +191,12 @@ TYPED_TEST(RosbagRetentionParityTest, ACapOfOneReproducesThePreviousBehaviourExa this->storage_->store_rosbag_file(first); this->storage_->store_rosbag_file(this->row("ONE", "fault_ONE_2", 2000)); - const auto rows = this->storage_->get_rosbag_files("ONE"); + const auto rows = this->storage_->get_rosbag_files(this->id("ONE")); ASSERT_EQ(rows.size(), 1u); EXPECT_EQ(rows[0].recording_id, "fault_ONE_2"); EXPECT_FALSE(std::filesystem::exists(first.file_path)); - const auto newest = this->storage_->get_rosbag_file("ONE"); + const auto newest = this->storage_->get_rosbag_file(this->id("ONE")); ASSERT_TRUE(newest.has_value()); EXPECT_EQ(newest->recording_id, "fault_ONE_2"); } @@ -197,7 +208,7 @@ TYPED_TEST(RosbagRetentionParityTest, ACapOfOneIsTheDefaultWithoutConfiguringAny this->storage_->store_rosbag_file(first); this->storage_->store_rosbag_file(this->row("DEFAULT", "fault_DEFAULT_2", 2000)); - EXPECT_EQ(this->storage_->get_rosbag_files("DEFAULT").size(), 1u); + EXPECT_EQ(this->storage_->get_rosbag_files(this->id("DEFAULT")).size(), 1u); EXPECT_FALSE(std::filesystem::exists(first.file_path)); } @@ -206,7 +217,7 @@ TYPED_TEST(RosbagRetentionParityTest, ZeroMeansUnlimited) { for (int i = 1; i <= 6; ++i) { this->storage_->store_rosbag_file(this->row("UNBOUNDED", "fault_UNBOUNDED_" + std::to_string(i), i * 1000)); } - EXPECT_EQ(this->storage_->get_rosbag_files("UNBOUNDED").size(), 6u); + EXPECT_EQ(this->storage_->get_rosbag_files(this->id("UNBOUNDED")).size(), 6u); } TYPED_TEST(RosbagRetentionParityTest, ReStoringTheSamePathIsAnUpsertNotASecondRecording) { @@ -219,7 +230,7 @@ TYPED_TEST(RosbagRetentionParityTest, ReStoringTheSamePathIsAnUpsertNotASecondRe info.duration_sec = 9.0; this->storage_->store_rosbag_file(info); - const auto rows = this->storage_->get_rosbag_files("UPSERT"); + const auto rows = this->storage_->get_rosbag_files(this->id("UPSERT")); ASSERT_EQ(rows.size(), 1u); EXPECT_EQ(rows[0].size_bytes, 4096u); EXPECT_DOUBLE_EQ(rows[0].duration_sec, 9.0); @@ -239,7 +250,7 @@ TYPED_TEST(RosbagRetentionParityTest, RefreshingALinkDoesNotMoveItWithinATieGrou first.size_bytes = 8192; this->storage_->store_rosbag_file(first); - const auto rows = this->storage_->get_rosbag_files("REFRESH"); + const auto rows = this->storage_->get_rosbag_files(this->id("REFRESH")); ASSERT_EQ(rows.size(), 2u); EXPECT_EQ(rows[0].recording_id, "fault_REFRESH_2") << "the refresh must not reorder the tie group"; EXPECT_EQ(rows[1].recording_id, "fault_REFRESH_1"); @@ -256,7 +267,7 @@ TYPED_TEST(RosbagRetentionParityTest, NewestFirstIsStableWhenAWholeBurstSharesAT this->storage_->store_rosbag_file(this->row("TIE", "fault_TIE_" + std::to_string(i), 7000)); } - const auto rows = this->storage_->get_rosbag_files("TIE"); + const auto rows = this->storage_->get_rosbag_files(this->id("TIE")); ASSERT_EQ(rows.size(), 4u); EXPECT_EQ(rows[0].recording_id, "fault_TIE_4") << "insertion order breaks the tie, newest first"; EXPECT_EQ(rows[1].recording_id, "fault_TIE_3"); @@ -273,7 +284,7 @@ TYPED_TEST(RosbagRetentionParityTest, ACapTieIsBrokenByInsertionOrderNotArbitrar const auto second = this->row("TIECAP", "fault_TIECAP_2", 7000); this->storage_->store_rosbag_file(second); - const auto rows = this->storage_->get_rosbag_files("TIECAP"); + const auto rows = this->storage_->get_rosbag_files(this->id("TIECAP")); ASSERT_EQ(rows.size(), 1u); EXPECT_EQ(rows[0].recording_id, "fault_TIECAP_2"); EXPECT_TRUE(std::filesystem::exists(second.file_path)); @@ -292,7 +303,7 @@ TYPED_TEST(RosbagRetentionParityTest, ABurstRecordingSurvivesWhileASiblingRowRef this->storage_->store_rosbag_file(this->row("BURST_A", "fault_BURST_2", 2000)); EXPECT_TRUE(std::filesystem::exists(shared_path)) << "the sibling fault still owns the shared bag"; - const auto sibling = this->storage_->get_rosbag_files("BURST_B"); + const auto sibling = this->storage_->get_rosbag_files(this->id("BURST_B")); ASSERT_EQ(sibling.size(), 1u); EXPECT_EQ(sibling[0].file_path, shared_path); } @@ -323,8 +334,8 @@ TYPED_TEST(RosbagRetentionParityTest, AnUnknownRecordingLooksUpEmptyRatherThanTh // The gateway's compatibility path depends on this: empty means "not a recording, // try it as a fault code", so it must be a normal answer. EXPECT_TRUE(this->storage_->get_rosbag_files_by_recording("fault_NOPE_1").empty()); - EXPECT_TRUE(this->storage_->get_rosbag_files("NOPE").empty()); - EXPECT_FALSE(this->storage_->get_rosbag_file("NOPE").has_value()); + EXPECT_TRUE(this->storage_->get_rosbag_files(this->id("NOPE")).empty()); + EXPECT_FALSE(this->storage_->get_rosbag_file(this->id("NOPE")).has_value()); } TYPED_TEST(RosbagRetentionParityTest, DeletingARecordingRemovesEveryLinkAndTheBag) { @@ -340,9 +351,9 @@ TYPED_TEST(RosbagRetentionParityTest, DeletingARecordingRemovesEveryLinkAndTheBa EXPECT_EQ(this->storage_->delete_rosbag_recording("fault_DELREC_1"), 2u); EXPECT_FALSE(std::filesystem::exists(shared.file_path)); - EXPECT_TRUE(this->storage_->get_rosbag_files("DELREC_B").empty()); + EXPECT_TRUE(this->storage_->get_rosbag_files(this->id("DELREC_B")).empty()); - const auto survivors = this->storage_->get_rosbag_files("DELREC_A"); + const auto survivors = this->storage_->get_rosbag_files(this->id("DELREC_A")); ASSERT_EQ(survivors.size(), 1u) << "the other fault's other recording is untouched"; EXPECT_EQ(survivors[0].recording_id, "fault_DELREC_2"); EXPECT_TRUE(std::filesystem::exists(kept.file_path)); @@ -357,8 +368,8 @@ TYPED_TEST(RosbagRetentionParityTest, DeletingAFaultDropsAllItsRecordings) { this->storage_->store_rosbag_file(a); this->storage_->store_rosbag_file(b); - EXPECT_TRUE(this->storage_->delete_rosbag_file("DELALL")); - EXPECT_TRUE(this->storage_->get_rosbag_files("DELALL").empty()); + EXPECT_TRUE(this->storage_->delete_rosbag_file(this->id("DELALL"))); + EXPECT_TRUE(this->storage_->get_rosbag_files(this->id("DELALL")).empty()); EXPECT_FALSE(std::filesystem::exists(a.file_path)); EXPECT_FALSE(std::filesystem::exists(b.file_path)); } @@ -417,6 +428,7 @@ class SnapshotRetentionParityTest : public ::testing::Test { for (size_t i = 0; i < topics; ++i) { SnapshotData row; row.fault_code = code; + row.owner = kOwner; row.topic = "/topic_" + std::to_string(i); row.message_type = "std_msgs/msg/Float64"; row.data = R"({"data": )" + std::to_string(capture_id) + "}"; @@ -427,10 +439,19 @@ class SnapshotRetentionParityTest : public ::testing::Test { return rows; } + /// Every record in this suite belongs to one owner: these tests are about retention + /// and ordering, which the record identity does not change. + static constexpr const char * kOwner = "/node"; + + static FaultId id(const std::string & code) { + return FaultId{code, kOwner}; + } + void confirm(const std::string & code) { rclcpp::Clock clock; - storage_->report_fault_event(code, fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "parity", - "/node", clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage_->report_fault_event(FaultId{code, kOwner}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "parity", kOwner, + clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); } std::filesystem::path dir_; @@ -448,7 +469,7 @@ TYPED_TEST(SnapshotRetentionParityTest, ACaptureLargerThanTheCapIsKeptWholeRathe this->storage_->store_snapshots(this->capture("BIG", 1, 5)); - const auto rows = this->storage_->get_snapshots("BIG"); + const auto rows = this->storage_->get_snapshots(this->id("BIG")); EXPECT_EQ(rows.size(), 5u) << "a capture over the cap on its own is kept entire"; for (const auto & row : rows) { EXPECT_EQ(row.capture_id, 1); @@ -463,7 +484,7 @@ TYPED_TEST(SnapshotRetentionParityTest, TheOldestWholeCaptureGoesFirstPastTheCap this->storage_->store_snapshots(this->capture("EVICT", 2, 2)); this->storage_->store_snapshots(this->capture("EVICT", 3, 2)); - const auto rows = this->storage_->get_snapshots("EVICT"); + const auto rows = this->storage_->get_snapshots(this->id("EVICT")); ASSERT_EQ(rows.size(), 4u); for (const auto & row : rows) { EXPECT_NE(row.capture_id, 1) << "the oldest set goes whole, not row by row"; @@ -480,7 +501,7 @@ TYPED_TEST(SnapshotRetentionParityTest, GetSnapshotsReturnsTheNewestCaptureFirst this->storage_->store_snapshots(this->capture("ORDER", 1, 2)); this->storage_->store_snapshots(this->capture("ORDER", 2, 2)); - const auto rows = this->storage_->get_snapshots("ORDER"); + const auto rows = this->storage_->get_snapshots(this->id("ORDER")); ASSERT_EQ(rows.size(), 4u); EXPECT_EQ(rows.front().capture_id, 2) << "newest capture leads"; EXPECT_EQ(rows.back().capture_id, 1); @@ -507,9 +528,9 @@ TYPED_TEST(SnapshotRetentionParityTest, ClearFaultKeepsSnapshotsWhenEvidenceIsRe this->storage_->set_retain_snapshots_on_clear(true); this->storage_->store_snapshots(this->capture("KEEP", 1, 2)); - ASSERT_TRUE(this->storage_->clear_fault("KEEP")); + ASSERT_TRUE(this->storage_->clear_fault(this->id("KEEP"))); - EXPECT_EQ(this->storage_->get_snapshots("KEEP").size(), 2u); + EXPECT_EQ(this->storage_->get_snapshots(this->id("KEEP")).size(), 2u); EXPECT_TRUE(this->storage_->retains_snapshots_on_clear()); } @@ -518,9 +539,9 @@ TYPED_TEST(SnapshotRetentionParityTest, ClearFaultDropsSnapshotsByDefault) { this->confirm("DROP"); this->storage_->store_snapshots(this->capture("DROP", 1, 2)); - ASSERT_TRUE(this->storage_->clear_fault("DROP")); + ASSERT_TRUE(this->storage_->clear_fault(this->id("DROP"))); - EXPECT_TRUE(this->storage_->get_snapshots("DROP").empty()); + EXPECT_TRUE(this->storage_->get_snapshots(this->id("DROP")).empty()); EXPECT_FALSE(this->storage_->retains_snapshots_on_clear()); } diff --git a/src/ros2_medkit_fault_manager/test/test_snapshot_capture.cpp b/src/ros2_medkit_fault_manager/test/test_snapshot_capture.cpp index fa8f656d1..80c7283fa 100644 --- a/src/ros2_medkit_fault_manager/test/test_snapshot_capture.cpp +++ b/src/ros2_medkit_fault_manager/test/test_snapshot_capture.cpp @@ -62,6 +62,15 @@ void await_topic_type(const rclcpp::Node::SharedPtr & node, const std::string & FAIL() << "the graph never reported a type for " << topic; } +/// The owner every record in this suite belongs to. Snapshot capture resolves topics +/// by fault CODE and writes per RECORD, so one owner is enough except in the +/// entity-default suite, which resolves the scope from the owner itself. +constexpr const char * kOwner = "/node"; + +ros2_medkit_fault_manager::FaultId rec(const std::string & code) { + return ros2_medkit_fault_manager::FaultId{code, kOwner}; +} + } // namespace class SnapshotCaptureTest : public ::testing::Test { @@ -108,10 +117,10 @@ TEST_F(SnapshotCaptureTest, DisabledCaptureSkipsProcessing) { config.default_topics = {"/test_topic"}; SnapshotCapture capture(node_.get(), storage_.get(), config); - capture.capture("TEST_FAULT"); + capture.capture(rec("TEST_FAULT")); // No snapshots should be stored when disabled - auto snapshots = storage_->get_snapshots("TEST_FAULT"); + auto snapshots = storage_->get_snapshots(rec("TEST_FAULT")); EXPECT_TRUE(snapshots.empty()); } @@ -227,10 +236,10 @@ TEST_F(SnapshotCaptureTest, EmptyConfigurationHandledGracefully) { // No topics configured at all SnapshotCapture capture(node_.get(), storage_.get(), config); - capture.capture("ANY_FAULT"); + capture.capture(rec("ANY_FAULT")); // Should not crash, just log that no topics configured - auto snapshots = storage_->get_snapshots("ANY_FAULT"); + auto snapshots = storage_->get_snapshots(rec("ANY_FAULT")); EXPECT_TRUE(snapshots.empty()); } @@ -256,14 +265,14 @@ TEST_F(SnapshotCaptureTest, OnDemandCaptureHandlesNonExistentTopic) { config.default_topics = {"/nonexistent_topic"}; SnapshotCapture capture(node_.get(), storage_.get(), config); - capture.capture("TEST_FAULT"); + capture.capture(rec("TEST_FAULT")); // Should timeout gracefully, no snapshot stored - auto snapshots = storage_->get_snapshots("TEST_FAULT"); + auto snapshots = storage_->get_snapshots(rec("TEST_FAULT")); EXPECT_TRUE(snapshots.empty()); // A configured capture that sampled nothing still records an empty {} frame. - auto frame = storage_->get_freeze_frame("TEST_FAULT"); + auto frame = storage_->get_freeze_frame(rec("TEST_FAULT")); ASSERT_TRUE(frame.has_value()); EXPECT_EQ(frame->data, "{}"); } @@ -323,9 +332,9 @@ TEST_F(SnapshotCaptureTest, CaptureWritesFreezeFrameFromConfiguredTopic) { } ASSERT_GT(node_->count_publishers("/plc/pressure"), 0u); - capture.capture("PLC_PRESSURE_HIGH"); + capture.capture(rec("PLC_PRESSURE_HIGH")); - auto frame = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto frame = storage_->get_freeze_frame(rec("PLC_PRESSURE_HIGH")); ASSERT_TRUE(frame.has_value()); auto parsed = nlohmann::json::parse(frame->data); ASSERT_TRUE(parsed.contains("/plc/pressure")); @@ -362,10 +371,10 @@ TEST_F(SnapshotCaptureTest, EmptyRecaptureKeepsRetainedFreezeFrame) { } ASSERT_GT(node_->count_publishers("/plc/flow"), 0u); - capture.capture("PLC_FLOW_LOW"); + capture.capture(rec("PLC_FLOW_LOW")); } - auto frame = storage_->get_freeze_frame("PLC_FLOW_LOW"); + auto frame = storage_->get_freeze_frame(rec("PLC_FLOW_LOW")); ASSERT_TRUE(frame.has_value()); ASSERT_NE(frame->data, "{}"); @@ -378,9 +387,9 @@ TEST_F(SnapshotCaptureTest, EmptyRecaptureKeepsRetainedFreezeFrame) { } ASSERT_EQ(node_->count_publishers("/plc/flow"), 0u); - capture.capture("PLC_FLOW_LOW"); + capture.capture(rec("PLC_FLOW_LOW")); - auto retained = storage_->get_freeze_frame("PLC_FLOW_LOW"); + auto retained = storage_->get_freeze_frame(rec("PLC_FLOW_LOW")); ASSERT_TRUE(retained.has_value()); auto parsed = nlohmann::json::parse(retained->data); ASSERT_TRUE(parsed.contains("/plc/flow")); @@ -418,8 +427,8 @@ TEST_F(SnapshotCaptureTest, BackgroundCaptureCachesFreezeFrame) { auto start = std::chrono::steady_clock::now(); while (!got_frame && std::chrono::steady_clock::now() - start < std::chrono::seconds(10)) { spin_some_once(node_); - capture.capture("PLC_TEMP_HIGH"); - auto frame = storage_->get_freeze_frame("PLC_TEMP_HIGH"); + capture.capture(rec("PLC_TEMP_HIGH")); + auto frame = storage_->get_freeze_frame(rec("PLC_TEMP_HIGH")); got_frame = frame.has_value() && frame->data != "{}"; if (!got_frame) { std::this_thread::sleep_for(std::chrono::milliseconds(20)); @@ -427,14 +436,14 @@ TEST_F(SnapshotCaptureTest, BackgroundCaptureCachesFreezeFrame) { } ASSERT_TRUE(got_frame); - auto frame = storage_->get_freeze_frame("PLC_TEMP_HIGH"); + auto frame = storage_->get_freeze_frame(rec("PLC_TEMP_HIGH")); ASSERT_TRUE(frame.has_value()); auto parsed = nlohmann::json::parse(frame->data); ASSERT_TRUE(parsed.contains("/plc/temperature")); EXPECT_DOUBLE_EQ(parsed["/plc/temperature"]["data"].get(), 91.5); // The cache path also stores a per-topic snapshot. - auto snapshots = storage_->get_snapshots("PLC_TEMP_HIGH"); + auto snapshots = storage_->get_snapshots(rec("PLC_TEMP_HIGH")); ASSERT_FALSE(snapshots.empty()); EXPECT_EQ(snapshots.back().topic, "/plc/temperature"); EXPECT_EQ(snapshots.back().message_type, "std_msgs/msg/Float64"); @@ -466,8 +475,8 @@ TEST_F(SnapshotCaptureTest, BackgroundCaptureSubscribesToATopicThatAppearsAfterS const auto start = std::chrono::steady_clock::now(); while (!got_frame && std::chrono::steady_clock::now() - start < std::chrono::seconds(10)) { spin_some_once(node_); - capture.capture("PLC_LATE_TEMP_HIGH"); - auto frame = storage_->get_freeze_frame("PLC_LATE_TEMP_HIGH"); + capture.capture(rec("PLC_LATE_TEMP_HIGH")); + auto frame = storage_->get_freeze_frame(rec("PLC_LATE_TEMP_HIGH")); got_frame = frame.has_value() && frame->data != "{}"; if (!got_frame) { std::this_thread::sleep_for(std::chrono::milliseconds(20)); @@ -475,7 +484,7 @@ TEST_F(SnapshotCaptureTest, BackgroundCaptureSubscribesToATopicThatAppearsAfterS } ASSERT_TRUE(got_frame) << "the background cache never carried " << topic << ", which appeared after startup"; - auto parsed = nlohmann::json::parse(storage_->get_freeze_frame("PLC_LATE_TEMP_HIGH")->data); + auto parsed = nlohmann::json::parse(storage_->get_freeze_frame(rec("PLC_LATE_TEMP_HIGH"))->data); ASSERT_TRUE(parsed.contains(topic)); EXPECT_DOUBLE_EQ(parsed[topic]["data"].get(), 77.25); @@ -493,9 +502,9 @@ TEST_F(SnapshotCaptureTest, UnconfiguredFaultWritesNoFreezeFrame) { // No default_topics: an unrelated fault code resolves to an empty capture set. SnapshotCapture capture(node_.get(), storage_.get(), config); - capture.capture("UNMAPPED_FAULT"); + capture.capture(rec("UNMAPPED_FAULT")); - EXPECT_FALSE(storage_->get_freeze_frame("UNMAPPED_FAULT").has_value()); + EXPECT_FALSE(storage_->get_freeze_frame(rec("UNMAPPED_FAULT")).has_value()); } // @verifies REQ_INTEROP_088 @@ -505,9 +514,9 @@ TEST_F(SnapshotCaptureTest, DisabledCaptureWritesNoFreezeFrame) { config.default_topics = {"/plc/pressure"}; SnapshotCapture capture(node_.get(), storage_.get(), config); - capture.capture("ANY_FAULT"); + capture.capture(rec("ANY_FAULT")); - EXPECT_FALSE(storage_->get_freeze_frame("ANY_FAULT").has_value()); + EXPECT_FALSE(storage_->get_freeze_frame(rec("ANY_FAULT")).has_value()); } // Entity-default (zero-config) capture tests. The reporting source is this @@ -518,9 +527,14 @@ class EntityDefaultCaptureTest : public SnapshotCaptureTest { /// Store a CONFIRMED fault whose reporting source is this test node. void store_fault_from_this_node(const std::string & fault_code) { ros2_medkit_fault_manager::DebounceConfig debounce; // threshold -1: first FAILED confirms - storage_->report_fault_event(fault_code, fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "test fault", node_->get_fully_qualified_name(), rclcpp::Clock().now().nanoseconds(), - debounce); + storage_->report_fault_event(ros2_medkit_fault_manager::FaultId{fault_code, node_->get_fully_qualified_name()}, + fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "test fault", + node_->get_fully_qualified_name(), rclcpp::Clock().now().nanoseconds(), debounce); + } + + /// The record this node owns: the entity-default scope is resolved from the owner. + ros2_medkit_fault_manager::FaultId id(const std::string & fault_code) const { + return ros2_medkit_fault_manager::FaultId{fault_code, node_->get_fully_qualified_name()}; } /// Wait until this node's publisher on @p topic is visible on the graph. @@ -557,9 +571,9 @@ TEST_F(EntityDefaultCaptureTest, EntityDefaultCapturesSourceNodeTopics) { }); wait_for_publisher("/entity/own_metric"); - capture.capture("ENTITY_FAULT"); + capture.capture(id("ENTITY_FAULT")); - auto frame = storage_->get_freeze_frame("ENTITY_FAULT"); + auto frame = storage_->get_freeze_frame(id("ENTITY_FAULT")); ASSERT_TRUE(frame.has_value()); auto parsed = nlohmann::json::parse(frame->data); ASSERT_TRUE(parsed.contains("/entity/own_metric")); @@ -581,9 +595,9 @@ TEST_F(EntityDefaultCaptureTest, EntityDefaultDisabledWritesNoFreezeFrame) { store_fault_from_this_node("OPTED_OUT_FAULT"); wait_for_publisher("/entity/opted_out"); - capture.capture("OPTED_OUT_FAULT"); + capture.capture(id("OPTED_OUT_FAULT")); - EXPECT_FALSE(storage_->get_freeze_frame("OPTED_OUT_FAULT").has_value()); + EXPECT_FALSE(storage_->get_freeze_frame(id("OPTED_OUT_FAULT")).has_value()); } // @verifies REQ_INTEROP_088 @@ -611,9 +625,9 @@ TEST_F(EntityDefaultCaptureTest, ExplicitConfigWinsOverEntityDefault) { }); wait_for_publisher("/entity/configured"); - capture.capture("CONFIGURED_FAULT"); + capture.capture(id("CONFIGURED_FAULT")); - auto frame = storage_->get_freeze_frame("CONFIGURED_FAULT"); + auto frame = storage_->get_freeze_frame(id("CONFIGURED_FAULT")); ASSERT_TRUE(frame.has_value()); auto parsed = nlohmann::json::parse(frame->data); // Only the explicitly configured topic is captured, never the node's other topics. @@ -649,11 +663,11 @@ TEST_F(EntityDefaultCaptureTest, ExplicitEmptyConfigEntryOptsOutOfEntityDefault) }); wait_for_publisher("/entity/own_optout"); - capture.capture("OPTED_OUT_SPECIFIC"); - capture.capture("OPTED_OUT_PATTERN_X"); + capture.capture(id("OPTED_OUT_SPECIFIC")); + capture.capture(id("OPTED_OUT_PATTERN_X")); - EXPECT_FALSE(storage_->get_freeze_frame("OPTED_OUT_SPECIFIC").has_value()); - EXPECT_FALSE(storage_->get_freeze_frame("OPTED_OUT_PATTERN_X").has_value()); + EXPECT_FALSE(storage_->get_freeze_frame(id("OPTED_OUT_SPECIFIC")).has_value()); + EXPECT_FALSE(storage_->get_freeze_frame(id("OPTED_OUT_PATTERN_X")).has_value()); } // @verifies REQ_INTEROP_088 @@ -664,12 +678,13 @@ TEST_F(EntityDefaultCaptureTest, UnresolvableSourceWritesNoRow) { // Plugin-style bare entity id: not a live node, resolves to no topics. ros2_medkit_fault_manager::DebounceConfig debounce; - storage_->report_fault_event("PLC_FAULT", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "plc fault", "beckhoff_plc_app", rclcpp::Clock().now().nanoseconds(), debounce); + storage_->report_fault_event(ros2_medkit_fault_manager::FaultId{"PLC_FAULT", "plc_app"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "plc fault", "plc_app", + rclcpp::Clock().now().nanoseconds(), debounce); - capture.capture("PLC_FAULT"); + capture.capture({"PLC_FAULT", "plc_app"}); - EXPECT_FALSE(storage_->get_freeze_frame("PLC_FAULT").has_value()); + EXPECT_FALSE(storage_->get_freeze_frame({"PLC_FAULT", "plc_app"}).has_value()); } // @verifies REQ_INTEROP_088 @@ -686,8 +701,9 @@ TEST_F(EntityDefaultCaptureTest, BarePluginIdNeverMatchesSameNamedNode) { SnapshotCapture capture(node_.get(), storage_.get(), config); ros2_medkit_fault_manager::DebounceConfig debounce; - storage_->report_fault_event("BARE_ID_FAULT", fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, - "plugin fault", node_->get_name(), rclcpp::Clock().now().nanoseconds(), debounce); + storage_->report_fault_event(ros2_medkit_fault_manager::FaultId{"BARE_ID_FAULT", node_->get_name()}, + fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "plugin fault", + node_->get_name(), rclcpp::Clock().now().nanoseconds(), debounce); ScopedPublisherThread pub_thread([&pub](std::atomic & stop) { while (!stop.load()) { @@ -699,9 +715,9 @@ TEST_F(EntityDefaultCaptureTest, BarePluginIdNeverMatchesSameNamedNode) { }); wait_for_publisher("/entity/bare_id_metric"); - capture.capture("BARE_ID_FAULT"); + capture.capture({"BARE_ID_FAULT", node_->get_name()}); - EXPECT_FALSE(storage_->get_freeze_frame("BARE_ID_FAULT").has_value()); + EXPECT_FALSE(storage_->get_freeze_frame({"BARE_ID_FAULT", node_->get_name()}).has_value()); } namespace { @@ -719,8 +735,9 @@ void await_publisher(const std::shared_ptr & node, const std::stri /// Put the fault in the store as CONFIRMED, the way a real confirmation would. void confirm_fault(ros2_medkit_fault_manager::FaultStorage * storage, const std::string & code) { rclcpp::Clock clock; - storage->report_fault_event(code, fd::ReportKind::kFailed, ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "test", - "/node", clock.now().nanoseconds(), ros2_medkit_fault_manager::DebounceConfig{}); + storage->report_fault_event(ros2_medkit_fault_manager::FaultId{code, "/node"}, fd::ReportKind::kFailed, + ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR, "test", "/node", clock.now().nanoseconds(), + ros2_medkit_fault_manager::DebounceConfig{}); } } // namespace @@ -763,9 +780,9 @@ TEST_F(SnapshotCaptureTest, CaptureIdContinuesFromWhatStorageAlreadyHolds) { await_publisher(node_, "/plc/seeded"); confirm_fault(storage_.get(), "SEEDED_FAULT"); - capture.capture("SEEDED_FAULT"); + capture.capture(rec("SEEDED_FAULT")); - const auto rows = storage_->get_snapshots("SEEDED_FAULT"); + const auto rows = storage_->get_snapshots(rec("SEEDED_FAULT")); ASSERT_FALSE(rows.empty()); EXPECT_GT(rows.front().capture_id, 41) << "the new capture outranks everything already stored"; } @@ -824,12 +841,12 @@ TEST_F(SnapshotCaptureTest, CaptureIdIsSeededOnTheFirstCaptureWhenTheStoreWasUnr confirm_fault(&storage, "LATE_SEED_FAULT"); // Still unreachable: no capture runs with an unseeded counter. - capture->capture("LATE_SEED_FAULT"); - EXPECT_TRUE(storage.get_snapshots("LATE_SEED_FAULT").empty()); + capture->capture(rec("LATE_SEED_FAULT")); + EXPECT_TRUE(storage.get_snapshots(rec("LATE_SEED_FAULT")).empty()); storage.reachable = true; - capture->capture("LATE_SEED_FAULT"); - const auto rows = storage.get_snapshots("LATE_SEED_FAULT"); + capture->capture(rec("LATE_SEED_FAULT")); + const auto rows = storage.get_snapshots(rec("LATE_SEED_FAULT")); ASSERT_FALSE(rows.empty()); EXPECT_GT(rows.front().capture_id, 41) << "the new capture outranks everything already stored"; } @@ -858,11 +875,11 @@ TEST_F(SnapshotCaptureTest, ACaptureFinishingAfterAcknowledgementIsNotStored) { await_publisher(node_, "/plc/acked"); confirm_fault(storage_.get(), "ACKED_FAULT"); - ASSERT_TRUE(storage_->clear_fault("ACKED_FAULT")); + ASSERT_TRUE(storage_->clear_fault(rec("ACKED_FAULT"))); - capture.capture("ACKED_FAULT"); + capture.capture(rec("ACKED_FAULT")); - EXPECT_TRUE(storage_->get_snapshots("ACKED_FAULT").empty()) + EXPECT_TRUE(storage_->get_snapshots(rec("ACKED_FAULT")).empty()) << "acknowledgement promised these were gone; the batch must not resurrect them"; } @@ -891,11 +908,11 @@ TEST_F(SnapshotCaptureTest, ACaptureFinishingAfterAcknowledgementIsStoredWhenEvi await_publisher(node_, "/plc/retained"); confirm_fault(storage_.get(), "RETAINED_FAULT"); - ASSERT_TRUE(storage_->clear_fault("RETAINED_FAULT")); + ASSERT_TRUE(storage_->clear_fault(rec("RETAINED_FAULT"))); - capture.capture("RETAINED_FAULT"); + capture.capture(rec("RETAINED_FAULT")); - EXPECT_FALSE(storage_->get_snapshots("RETAINED_FAULT").empty()); + EXPECT_FALSE(storage_->get_snapshots(rec("RETAINED_FAULT")).empty()); } int main(int argc, char ** argv) { diff --git a/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp b/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp index 509f1a3ea..b668f9eff 100644 --- a/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp +++ b/src/ros2_medkit_fault_manager/test/test_sqlite_storage.cpp @@ -16,22 +16,30 @@ #include +#include +#include +#include #include #include #include #include #include +#include #include #include +#include +#include #include #include "rclcpp/rclcpp.hpp" +#include "rcutils/logging.h" #include "ros2_medkit_fault_manager/sqlite_fault_storage.hpp" #include "ros2_medkit_msgs/msg/fault.hpp" #include "ros2_medkit_msgs/srv/report_fault.hpp" #include "test_log_sink.hpp" using ros2_medkit_fault_manager::DebounceConfig; +using ros2_medkit_fault_manager::FaultId; using ros2_medkit_fault_manager::RosbagFileInfo; using ros2_medkit_fault_manager::SqliteFaultStorage; using ros2_medkit_msgs::msg::Fault; @@ -66,25 +74,82 @@ class SqliteFaultStorageTest : public ::testing::Test { std::unique_ptr storage_; }; +namespace { + +/// The detail column of every row EXPLAIN QUERY PLAN returns for @p sql, joined by newlines. +std::string query_plan(const std::string & db_path, const std::string & sql) { + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path.c_str(), &raw), SQLITE_OK); + std::string plan; + sqlite3_stmt * stmt = nullptr; + EXPECT_EQ(sqlite3_prepare_v2(raw, ("EXPLAIN QUERY PLAN " + sql).c_str(), -1, &stmt, nullptr), SQLITE_OK); + while (stmt != nullptr && sqlite3_step(stmt) == SQLITE_ROW) { + plan += reinterpret_cast(sqlite3_column_text(stmt, 3)); + plan += "\n"; + } + sqlite3_finalize(stmt); + sqlite3_close(raw); + return plan; +} + +} // namespace + +// The per-record trims and reads of snapshots and near misses filter on (fault_code, owner): they must +// be served by an index on exactly those columns, and a store that lacks them gets them on open. +TEST_F(SqliteFaultStorageTest, RecordIndexesServeTheTrims) { + const std::string path = temp_db_path_.string(); + const std::string snapshots_trim = "SELECT MAX(capture_id) FROM snapshots WHERE fault_code = 'C' AND owner = 'O'"; + const std::string snapshots_count = "SELECT COUNT(*) FROM snapshots WHERE fault_code = 'C' AND owner = 'O'"; + const std::string near_miss_trim = + "SELECT id FROM near_misses WHERE fault_code = 'C' AND owner = 'O' ORDER BY id DESC LIMIT 5"; + + const auto expect_served = [&path](const std::string & sql, const std::string & index) { + const std::string plan = query_plan(path, sql); + EXPECT_NE(plan.find(index), std::string::npos) << sql << "\n" << plan; + EXPECT_EQ(plan.find("SCAN"), std::string::npos) << sql << "\n" << plan; + }; + expect_served(snapshots_trim, "idx_snapshots_record"); + expect_served(snapshots_count, "idx_snapshots_record"); + expect_served(near_miss_trim, "idx_near_misses_record"); + + // A store written without them gets them when it is opened. + storage_.reset(); + { + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(path.c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, "DROP INDEX idx_snapshots_record; DROP INDEX idx_near_misses_record;", nullptr, nullptr, + nullptr), + SQLITE_OK); + sqlite3_close(raw); + } + EXPECT_EQ(query_plan(path, snapshots_trim).find("idx_snapshots_record"), std::string::npos) + << "the fixture must start without them"; + EXPECT_EQ(query_plan(path, near_miss_trim).find("idx_near_misses_record"), std::string::npos); + storage_ = std::make_unique(path, test_log_sink()); + expect_served(snapshots_trim, "idx_snapshots_record"); + expect_served(near_miss_trim, "idx_near_misses_record"); +} + TEST_F(SqliteFaultStorageTest, ReportNewFaultEvent) { rclcpp::Clock clock; auto timestamp = clock.now(); - bool is_new = storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "Motor temperature exceeded threshold", "/powertrain/motor", - timestamp.nanoseconds(), default_config()); + bool is_new = storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Motor temperature exceeded threshold", + "/powertrain/motor", timestamp.nanoseconds(), default_config()); EXPECT_TRUE(is_new); EXPECT_EQ(storage_->size(), 1u); - EXPECT_TRUE(storage_->contains("MOTOR_OVERHEAT")); + EXPECT_TRUE(storage_->contains({"MOTOR_OVERHEAT", "/powertrain/motor"})); } TEST_F(SqliteFaultStorageTest, PassedEventForNonExistentFaultIgnored) { rclcpp::Clock clock; auto timestamp = clock.now(); - bool is_new = storage_->report_fault_event("NON_EXISTENT", fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "Test", - "/node1", timestamp.nanoseconds(), default_config()); + bool is_new = + storage_->report_fault_event(FaultId{"NON_EXISTENT", "/node1"}, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, + "Test", "/node1", timestamp.nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_->size(), 0u); @@ -95,23 +160,76 @@ TEST_F(SqliteFaultStorageTest, ReportExistingFaultEventUpdates) { auto timestamp1 = clock.now(); auto timestamp2 = clock.now(); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Initial report", - "/powertrain/motor1", timestamp1.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "Initial report", "/powertrain/motor1", timestamp1.nanoseconds(), + default_config()); - bool is_new = - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Second report", - "/powertrain/motor2", timestamp2.nanoseconds(), default_config()); + bool is_new = storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Second report", "/powertrain/motor1", + timestamp2.nanoseconds(), default_config()); EXPECT_FALSE(is_new); EXPECT_EQ(storage_->size(), 1u); - auto fault = storage_->get_fault("MOTOR_OVERHEAT"); + auto fault = storage_->get_fault({"MOTOR_OVERHEAT", "/powertrain/motor1"}); ASSERT_TRUE(fault.has_value()); // Still the same continuous occurrence (not CLEARED in between): occurrence_count - // does not bump on every report, only severity/sources/description update. + // does not bump on every report, only severity/description update. EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->severity, Fault::SEVERITY_ERROR); // Updated to higher severity - EXPECT_EQ(fault->reporting_sources.size(), 2u); + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources.front(), "/powertrain/motor1"); +} + +// SQLite twin of the in-memory identity test: the composite index has to keep the two +// records apart in the table the way the map does in memory. +TEST_F(SqliteFaultStorageTest, TwoSourcesReportingOneCodeAreTwoRecords) { + DebounceConfig config; + config.confirmation_threshold = -2; + + const rclcpp::Time a_time(1000, 0, RCL_SYSTEM_TIME); + const rclcpp::Time b_time(2000, 0, RCL_SYSTEM_TIME); + + EXPECT_TRUE(storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "from a", "/powertrain/motor1", a_time.nanoseconds(), + config)); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor1"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "from a", "/powertrain/motor1", a_time.nanoseconds(), config); + EXPECT_TRUE(storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor2"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "from b", "/powertrain/motor2", b_time.nanoseconds(), + config)); + + EXPECT_EQ(storage_->size(), 2u); + + auto a = storage_->get_fault({"MOTOR_OVERHEAT", "/powertrain/motor1"}); + auto b = storage_->get_fault({"MOTOR_OVERHEAT", "/powertrain/motor2"}); + ASSERT_TRUE(a.has_value()); + ASSERT_TRUE(b.has_value()); + EXPECT_EQ(a->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(b->status, Fault::STATUS_PREFAILED); + EXPECT_EQ(a->severity, Fault::SEVERITY_WARN) << "B's ERROR must not escalate A's record"; + EXPECT_EQ(b->severity, Fault::SEVERITY_ERROR); + EXPECT_EQ(a->description, "from a"); + EXPECT_EQ(b->description, "from b"); + EXPECT_EQ(a->first_occurred_ns, a_time.nanoseconds()); + EXPECT_EQ(b->first_occurred_ns, b_time.nanoseconds()); +} + +TEST_F(SqliteFaultStorageTest, GetIdsByCodeReturnsEveryOwnerAndGetFaultExactlyOne) { + rclcpp::Clock clock; + storage_->report_fault_event(FaultId{"SHARED", "/owner_b"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_b", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"SHARED", "/owner_a"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "a", + "/owner_a", clock.now().nanoseconds(), default_config()); + + const auto shared = storage_->get_ids_by_code("SHARED"); + ASSERT_EQ(shared.size(), 2u); + EXPECT_EQ(shared[0].owner, "/owner_a") << "ordered by owner, like the in-memory backend"; + EXPECT_EQ(shared[1].owner, "/owner_b"); + + EXPECT_TRUE(storage_->get_ids_by_code("NEVER_REPORTED").empty()); + ASSERT_TRUE(storage_->get_fault({"SHARED", "/owner_a"}).has_value()); + EXPECT_FALSE(storage_->get_fault({"SHARED", "/nobody"}).has_value()); } TEST_F(SqliteFaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCount) { @@ -120,11 +238,11 @@ TEST_F(SqliteFaultStorageTest, ContinuouslyActiveFaultDoesNotInflateOccurrenceCo // Simulate a level-triggered poller re-reporting the same still-true condition // every cycle (issue #11): occurrence_count must stay at 1, not grow per poll. for (int i = 0; i < 25; ++i) { - storage_->report_fault_event("TANK_OVERFILL", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "level = 95 > 80", - "/tank", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"TANK_OVERFILL", "/tank"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "level = 95 > 80", "/tank", clock.now().nanoseconds(), default_config()); } - auto fault = storage_->get_fault("TANK_OVERFILL"); + auto fault = storage_->get_fault({"TANK_OVERFILL", "/tank"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -135,8 +253,8 @@ TEST_F(SqliteFaultStorageTest, ListFaultsDefaultReturnsConfirmedOnly) { auto timestamp = clock.now(); // With default threshold=-1, single report confirms immediately - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); // Default query should return the CONFIRMED fault auto faults = storage_->list_faults(false, 0, {}); @@ -153,8 +271,8 @@ TEST_F(SqliteFaultStorageTest, ListFaultsWithPrefailedStatus) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), config); // Query with PREFAILED status auto faults = storage_->list_faults(false, 0, {Fault::STATUS_PREFAILED}); @@ -168,10 +286,10 @@ TEST_F(SqliteFaultStorageTest, ListFaultsFilterBySeverity) { auto timestamp = clock.now(); // With default threshold=-1, faults are immediately CONFIRMED - storage_->report_fault_event("FAULT_INFO", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Info", "/node1", - timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_ERROR", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Error", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_INFO", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Info", + "/node1", timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_ERROR", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Error", "/node1", timestamp.nanoseconds(), default_config()); // Filter by ERROR severity (query CONFIRMED since that's the default status now) auto faults = storage_->list_faults(true, Fault::SEVERITY_ERROR, {Fault::STATUS_CONFIRMED}); @@ -183,19 +301,19 @@ TEST_F(SqliteFaultStorageTest, ClearFault) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Test", "/node1", timestamp.nanoseconds(), default_config()); - bool cleared = storage_->clear_fault("MOTOR_OVERHEAT"); + bool cleared = storage_->clear_fault({"MOTOR_OVERHEAT", "/node1"}); EXPECT_TRUE(cleared); - auto fault = storage_->get_fault("MOTOR_OVERHEAT"); + auto fault = storage_->get_fault({"MOTOR_OVERHEAT", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); } TEST_F(SqliteFaultStorageTest, ClearNonExistentFault) { - bool cleared = storage_->clear_fault("NON_EXISTENT"); + bool cleared = storage_->clear_fault({"NON_EXISTENT", "/node1"}); EXPECT_FALSE(cleared); } @@ -206,12 +324,12 @@ TEST_F(SqliteFaultStorageTest, PassedEventDoesNotAdvanceLastOccurred) { const rclcpp::Time failed_at(1000, 0, RCL_SYSTEM_TIME); const rclcpp::Time passed_at(9000, 0, RCL_SYSTEM_TIME); - storage_->report_fault_event("FAULT_LO", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - failed_at.nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_LO", fd::ReportKind::kPassed, 0, "", "/node1", passed_at.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"FAULT_LO", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", failed_at.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_LO", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + passed_at.nanoseconds(), default_config()); - auto fault = storage_->get_fault("FAULT_LO"); + auto fault = storage_->get_fault({"FAULT_LO", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // healing disabled: latched, by design EXPECT_EQ(fault->last_occurred_ns, failed_at.nanoseconds()); @@ -225,8 +343,8 @@ TEST_F(SqliteFaultStorageTest, ReopenRepairsLastOccurredInflatedByOldPassedBug) // again would keep the wrong timestamp forever. Opening the storage must // repair them from last_failed_ns. const rclcpp::Time failed_at(1000, 0, RCL_SYSTEM_TIME); - storage_->report_fault_event("FAULT_MIG", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - failed_at.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_MIG", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", failed_at.nanoseconds(), default_config()); storage_.reset(); // Simulate the old bug: a PASSED event at t=9000s re-dated the row. @@ -239,7 +357,7 @@ TEST_F(SqliteFaultStorageTest, ReopenRepairsLastOccurredInflatedByOldPassedBug) storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - auto fault = storage_->get_fault("FAULT_MIG"); + auto fault = storage_->get_fault({"FAULT_MIG", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->last_occurred_ns, failed_at.nanoseconds()); } @@ -248,9 +366,9 @@ TEST_F(SqliteFaultStorageTest, GetClearedFaults) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); - storage_->clear_fault("FAULT_1"); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); + storage_->clear_fault({"FAULT_1", "/node1"}); // Query cleared faults auto faults = storage_->list_faults(false, 0, {Fault::STATUS_CLEARED}); @@ -263,8 +381,8 @@ TEST_F(SqliteFaultStorageTest, InvalidStatusDefaultsToConfirmed) { auto timestamp = clock.now(); // With default threshold=-1, fault is immediately CONFIRMED - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", timestamp.nanoseconds(), default_config()); // Query with invalid status - defaults to CONFIRMED, which now matches our fault auto faults = storage_->list_faults(false, 0, {"INVALID_STATUS"}); @@ -279,11 +397,11 @@ TEST_F(SqliteFaultStorageTest, PersistenceAcrossRestarts) { // With default threshold=-1, faults are immediately CONFIRMED // Report some faults - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Persistent fault 1", - "/node1", timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_2", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "Persistent fault 2", "/node2", - timestamp.nanoseconds(), default_config()); - storage_->clear_fault("FAULT_2"); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Persistent fault 1", "/node1", timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_2", "/node2"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, + "Persistent fault 2", "/node2", timestamp.nanoseconds(), default_config()); + storage_->clear_fault({"FAULT_2", "/node2"}); // Verify initial state EXPECT_EQ(storage_->size(), 2u); @@ -296,16 +414,16 @@ TEST_F(SqliteFaultStorageTest, PersistenceAcrossRestarts) { // Verify faults persisted EXPECT_EQ(storage_->size(), 2u); - EXPECT_TRUE(storage_->contains("FAULT_1")); - EXPECT_TRUE(storage_->contains("FAULT_2")); + EXPECT_TRUE(storage_->contains({"FAULT_1", "/node1"})); + EXPECT_TRUE(storage_->contains({"FAULT_2", "/node2"})); - auto fault1 = storage_->get_fault("FAULT_1"); + auto fault1 = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault1.has_value()); EXPECT_EQ(fault1->severity, Fault::SEVERITY_ERROR); EXPECT_EQ(fault1->status, Fault::STATUS_CONFIRMED); // Immediately confirmed with threshold=-1 EXPECT_EQ(fault1->description, "Persistent fault 1"); - auto fault2 = storage_->get_fault("FAULT_2"); + auto fault2 = storage_->get_fault({"FAULT_2", "/node2"}); ASSERT_TRUE(fault2.has_value()); EXPECT_EQ(fault2->status, Fault::STATUS_CLEARED); } @@ -316,10 +434,10 @@ TEST_F(SqliteFaultStorageTest, TimestampPrecision) { int64_t test_ns = 1735312456123456789LL; // Specific nanosecond timestamp rclcpp::Time timestamp(test_ns, RCL_SYSTEM_TIME); - storage_->report_fault_event("FAULT_TS", fd::ReportKind::kFailed, Fault::SEVERITY_INFO, "Timestamp test", "/node1", - timestamp.nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_TS", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_INFO, + "Timestamp test", "/node1", timestamp.nanoseconds(), default_config()); - auto fault = storage_->get_fault("FAULT_TS"); + auto fault = storage_->get_fault({"FAULT_TS", "/node1"}); ASSERT_TRUE(fault.has_value()); // Wrap the stored nanoseconds in rclcpp::Time for comparison @@ -337,11 +455,11 @@ TEST(SqliteInMemoryTest, InMemoryDatabase) { rclcpp::Clock clock; auto timestamp = clock.now(); - storage.report_fault_event("MEM_FAULT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "In-memory test", "/test", - timestamp.nanoseconds(), default_config()); + storage.report_fault_event(FaultId{"MEM_FAULT", "/test"}, fd::ReportKind::kFailed, Fault::SEVERITY_WARN, + "In-memory test", "/test", timestamp.nanoseconds(), default_config()); EXPECT_EQ(storage.size(), 1u); - EXPECT_TRUE(storage.contains("MEM_FAULT")); + EXPECT_TRUE(storage.contains({"MEM_FAULT", "/test"})); } // Test reporting sources JSON handling @@ -349,23 +467,26 @@ TEST_F(SqliteFaultStorageTest, ReportingSourcesJsonHandling) { rclcpp::Clock clock; auto timestamp = clock.now(); - // Add multiple sources for the same fault - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", - "/node/path/with/slashes", timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", - "/another/node", timestamp.nanoseconds(), default_config()); - storage_->report_fault_event("MULTI_SRC", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Multi-source", - "/special\"chars", timestamp.nanoseconds(), default_config()); + // Three awkward source ids, each owning its own record of the same code. Owners are + // an identity column now, so an embedded quote has to survive the round trip through + // the key rather than through a JSON array. + const std::vector owners = {"/node/path/with/slashes", "/another/node", "/special\"chars"}; + for (const auto & owner : owners) { + storage_->report_fault_event(FaultId{"MULTI_SRC", owner}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Multi-source", owner, timestamp.nanoseconds(), default_config()); + } - auto fault = storage_->get_fault("MULTI_SRC"); - ASSERT_TRUE(fault.has_value()); - EXPECT_EQ(fault->reporting_sources.size(), 3u); + EXPECT_EQ(storage_->size(), owners.size()); - // Verify all sources are present (order may vary due to set) - std::set sources(fault->reporting_sources.begin(), fault->reporting_sources.end()); - EXPECT_TRUE(sources.count("/node/path/with/slashes") > 0); - EXPECT_TRUE(sources.count("/another/node") > 0); - EXPECT_TRUE(sources.count("/special\"chars") > 0); + std::set seen; + for (const auto & owner : owners) { + auto fault = storage_->get_fault({"MULTI_SRC", owner}); + ASSERT_TRUE(fault.has_value()) << owner; + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources.front(), owner); + seen.insert(fault->reporting_sources.front()); + } + EXPECT_EQ(seen.size(), owners.size()); } // Test database path accessor @@ -406,12 +527,13 @@ TEST_F(SqliteFaultStorageTest, FaultStaysPrefailedAboveThreshold) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); + // Both reports from one source: the counter belongs to that source's record. + for (int i = 0; i < 2; ++i) { + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Still debouncing towards confirmation, same occurrence EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); @@ -425,14 +547,12 @@ TEST_F(SqliteFaultStorageTest, FaultConfirmsAtThreshold) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), config); + for (int i = 0; i < 3; ++i) { + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); // Debounce build-up is still one occurrence EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -445,10 +565,10 @@ TEST_F(SqliteFaultStorageTest, ImmediateConfirmationWithThresholdZero) { storage_->set_debounce_config(config); // Single report should confirm immediately - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -458,10 +578,10 @@ TEST_F(SqliteFaultStorageTest, CriticalSeverityBypassesDebounce) { rclcpp::Clock clock; // CRITICAL severity should confirm immediately - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "Critical test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, + "Critical test", "/node1", clock.now().nanoseconds(), default_config()); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); @@ -472,34 +592,36 @@ TEST_F(SqliteFaultStorageTest, ClearedFaultCanBeReactivated) { // Report to confirm (with default threshold=-1, single report confirms) auto first_ts = clock.now(); - bool is_new = storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Initial", - "/node1", first_ts.nanoseconds(), default_config()); + bool is_new = + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Initial", "/node1", first_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(fault->occurrence_count, 1u); EXPECT_EQ(fault->first_occurred_ns, first_ts.nanoseconds()); - // Clear the fault - storage_->clear_fault("FAULT_1"); - fault = storage_->get_fault("FAULT_1"); + // Clear the record + storage_->clear_fault({"FAULT_1", "/node1"}); + fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); - // Report again after a gap - should reactivate as a new cycle + // The same owner reports again after a gap - reactivates its own record rclcpp::Time second_ts(first_ts.nanoseconds() + 1'000'000'000LL); // +1s - is_new = storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Reactivated", - "/node2", second_ts.nanoseconds(), default_config()); + is_new = storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Reactivated", "/node1", second_ts.nanoseconds(), default_config()); EXPECT_TRUE(is_new); // Should return true like a new fault - fault = storage_->get_fault("FAULT_1"); + fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Should be reconfirmed EXPECT_EQ(fault->occurrence_count, 2u); // Should increment - EXPECT_EQ(fault->reporting_sources.size(), 2u); // Both sources - EXPECT_EQ(fault->description, "Reactivated"); // Updated description + ASSERT_EQ(fault->reporting_sources.size(), 1u); + EXPECT_EQ(fault->reporting_sources.front(), "/node1"); + EXPECT_EQ(fault->description, "Reactivated"); // Updated description // #25: first_occurred must reflect the new cycle, not the outage that already cleared. EXPECT_EQ(fault->first_occurred_ns, second_ts.nanoseconds()); } @@ -508,18 +630,18 @@ TEST_F(SqliteFaultStorageTest, PassedEventForClearedFaultIgnored) { rclcpp::Clock clock; // Report and confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); // Clear the fault - storage_->clear_fault("FAULT_1"); + storage_->clear_fault({"FAULT_1", "/node1"}); // PASSED event should be ignored for CLEARED fault - bool result = storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", + bool result = storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), default_config()); EXPECT_FALSE(result); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CLEARED); // Should stay cleared } @@ -532,36 +654,34 @@ TEST_F(SqliteFaultStorageTest, ClearedFaultReactivationRestartsDebounce) { config.confirmation_threshold = -3; storage_->set_debounce_config(config); - // Report 3 times to confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), config); + // One source reports 3 times to confirm its own record + for (int i = 0; i < 3; ++i) { + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); - // Clear the fault - storage_->clear_fault("FAULT_1"); + // Clear the record + storage_->clear_fault({"FAULT_1", "/node1"}); // Reactivate - should start in PREFAILED with counter=-1 - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node4", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); - fault = storage_->get_fault("FAULT_1"); + fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREFAILED); // Not yet confirmed, needs 2 more FAILED // Report 2 more times to re-confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node5", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node6", - clock.now().nanoseconds(), config); + for (int i = 0; i < 2; ++i) { + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + } - fault = storage_->get_fault("FAULT_1"); + fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); // Now confirmed } @@ -569,20 +689,18 @@ TEST_F(SqliteFaultStorageTest, ClearedFaultReactivationRestartsDebounce) { TEST_F(SqliteFaultStorageTest, ConfirmationPersistsAfterReopen) { rclcpp::Clock clock; - // Report 3 times to confirm - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node3", - clock.now().nanoseconds(), default_config()); + // One source reports 3 times to confirm its own record + for (int i = 0; i < 3; ++i) { + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), default_config()); + } // Close and reopen storage storage_.reset(); storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); // Verify status persisted - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -594,19 +712,19 @@ TEST_F(SqliteFaultStorageTest, PassedEventIncrementsCounter) { DebounceConfig config; config.confirmation_threshold = -3; - // Report 2 FAILED events (counter -2, PREFAILED) - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node2", - clock.now().nanoseconds(), config); + // Report 2 FAILED events from one source (its counter -2, PREFAILED) + for (int i = 0; i < 2; ++i) { + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + } // Report 3 PASSED events (counter -2 -> +1) for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_PREPASSED); // Counter > 0, never confirmed so not latched } @@ -622,29 +740,30 @@ TEST_F(SqliteFaultStorageTest, HeartbeatHealClampedAndStatusLatched) { config.healing_enabled = true; config.healing_threshold = 3; - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + ASSERT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); // Long heal heartbeat: counter clamps at healing_threshold instead of running off. for (int i = 0; i < 100; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } - auto healed = storage_->get_fault("FAULT_1"); + auto healed = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(healed.has_value()); EXPECT_EQ(healed->status, Fault::STATUS_HEALED); // Re-confirmation is now bounded to (healing_threshold - confirmation_threshold) // reports, and the healed status latches until the counter walks all the way down. for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_HEALED) + << "latch broke after " << (i + 1); } - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - auto reconfirmed = storage_->get_fault("FAULT_1"); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + auto reconfirmed = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(reconfirmed.has_value()); EXPECT_EQ(reconfirmed->status, Fault::STATUS_CONFIRMED); } @@ -658,21 +777,22 @@ TEST_F(SqliteFaultStorageTest, ConfirmedFaultSurvivesHealHeartbeat) { config.healing_enabled = false; config.healing_threshold = 3; // upper clamp on the counter, even with healing disabled - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + ASSERT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 20; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); - EXPECT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED) << "un-confirmed after " << (i + 1); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED) + << "un-confirmed after " << (i + 1); } // The counter is now positive (clamped at +3). One FAILED must keep it CONFIRMED, not flip it to // PREPASSED via the `counter > 0` branch - that was the asymmetric-latch bug. - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("FAULT_1")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault({"FAULT_1", "/node1"})->status, Fault::STATUS_CONFIRMED); } // Latch must hold in BOTH directions and identically on both backends (regression for the @@ -685,18 +805,19 @@ TEST_F(SqliteFaultStorageTest, ConfirmedLatchSurvivesFailedAtPositiveCounter) { config.critical_immediate_confirm = true; // CRITICAL confirms immediately at counter -1. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", - clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); + ASSERT_EQ(storage_->get_fault({"F", ros2_medkit_fault_manager::kSharedOwner})->status, Fault::STATUS_CONFIRMED); // PASSED events raise the counter into positive territory; latch keeps it CONFIRMED. for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kPassed, 0, "", + "/n", clock.now().nanoseconds(), config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault({"F", ros2_medkit_fault_manager::kSharedOwner})->status, Fault::STATUS_CONFIRMED); // One normal FAILED: counter still positive, must NOT become PREPASSED. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", - clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", ros2_medkit_fault_manager::kSharedOwner}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); + EXPECT_EQ(storage_->get_fault({"F", ros2_medkit_fault_manager::kSharedOwner})->status, Fault::STATUS_CONFIRMED); } // A HEALED fault with a negative counter stays HEALED on PASSED (opposite direction of the bug above). @@ -707,21 +828,23 @@ TEST_F(SqliteFaultStorageTest, HealedLatchSurvivesPassedAtNegativeCounter) { config.healing_enabled = true; config.healing_threshold = 3; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { // counter -1 -> +3 -> HEALED - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); // FAILED events drive the counter negative while HEALED is latched (not yet at confirmation -3). for (int i = 0; i < 5; ++i) { // +3 -> -2 - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); + ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED) << "latch broke after " << (i + 1); } // One PASSED: counter goes -2 -> -1, still negative; must stay HEALED, not flip to PREFAILED. - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } TEST_F(SqliteFaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePassed) { @@ -732,24 +855,27 @@ TEST_F(SqliteFaultStorageTest, CriticalConfirmFromHealedDoesNotHealOnSinglePasse config.healing_threshold = 3; config.critical_immediate_confirm = true; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { // -1 -> +3 -> HEALED - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); // CRITICAL confirms immediately from HEALED; counter must not stay at +2 where one PASSED heals it. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, "crit", "/n", clock.now().nanoseconds(), config); - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); for (int i = 0; i < 3; ++i) { // -1 -> +2, still CONFIRMED - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); } - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); } // A database written by an older build can hold a runaway counter. It must be clamped back on first @@ -760,7 +886,7 @@ TEST_F(SqliteFaultStorageTest, RunawayCounterFromOldDbRecovers) { config.confirmation_threshold = -1; config.healing_threshold = 3; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); // Simulate the old bug: poke a huge counter directly into the DB behind the storage's back. { @@ -774,10 +900,10 @@ TEST_F(SqliteFaultStorageTest, RunawayCounterFromOldDbRecovers) { // The counter is clamped back into range on first touch, so a bounded number of FAILED events // re-confirms (healing_threshold - confirmation_threshold = 4), not the ~100000 the runaway implied. for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); } - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); } TEST_F(SqliteFaultStorageTest, ReclassifyHealedAsCleared) { @@ -786,17 +912,18 @@ TEST_F(SqliteFaultStorageTest, ReclassifyHealedAsCleared) { config.healing_enabled = true; config.healing_threshold = 3; - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("F", fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } - ASSERT_EQ(storage_->get_fault("F")->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_HEALED); const auto reclassified = storage_->reclassify_healed_as_cleared(); ASSERT_EQ(reclassified.size(), 1u); - EXPECT_EQ(reclassified[0], "F"); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(reclassified[0], (ros2_medkit_fault_manager::FaultId{"F", "/n"})); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CLEARED); EXPECT_TRUE(storage_->reclassify_healed_as_cleared().empty()); } @@ -807,21 +934,22 @@ TEST_F(SqliteFaultStorageTest, ReclassifyKeepsCodesTheCallerKeepsHealed) { config.healing_threshold = 3; for (const std::string code : {"KEEP", "FLIP"}) { - storage_->report_fault_event(code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + storage_->report_fault_event(FaultId{code, "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", clock.now().nanoseconds(), config); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event(code, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{code, "/n"}, fd::ReportKind::kPassed, 0, "", "/n", clock.now().nanoseconds(), + config); } - ASSERT_EQ(storage_->get_fault(code)->status, Fault::STATUS_HEALED); + ASSERT_EQ(storage_->get_fault({code, "/n"})->status, Fault::STATUS_HEALED); } const auto reclassified = storage_->reclassify_healed_as_cleared([](const std::string & code) { return code == "KEEP"; }); ASSERT_EQ(reclassified.size(), 1u); - EXPECT_EQ(reclassified[0], "FLIP"); - EXPECT_EQ(storage_->get_fault("KEEP")->status, Fault::STATUS_HEALED); - EXPECT_EQ(storage_->get_fault("FLIP")->status, Fault::STATUS_CLEARED); + EXPECT_EQ(reclassified[0], (ros2_medkit_fault_manager::FaultId{"FLIP", "/n"})); + EXPECT_EQ(storage_->get_fault({"KEEP", "/n"})->status, Fault::STATUS_HEALED); + EXPECT_EQ(storage_->get_fault({"FLIP", "/n"})->status, Fault::STATUS_CLEARED); } TEST_F(SqliteFaultStorageTest, HealingWhenEnabled) { @@ -832,16 +960,16 @@ TEST_F(SqliteFaultStorageTest, HealingWhenEnabled) { storage_->set_debounce_config(config); // Report 1 FAILED event - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - clock.now().nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", clock.now().nanoseconds(), config); // Report 4 PASSED events (counter = -1 + 4 = +3, reaches healing threshold) for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("FAULT_1", fd::ReportKind::kPassed, 0, "", "/node1", clock.now().nanoseconds(), - config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kPassed, 0, "", "/node1", + clock.now().nanoseconds(), config); } - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_HEALED); } @@ -854,8 +982,8 @@ TEST_F(SqliteFaultStorageTest, TimeBasedConfirmationWhenEnabled) { storage_->set_debounce_config(config); auto now = clock.now(); - storage_->report_fault_event("FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", "/node1", - now.nanoseconds(), config); + storage_->report_fault_event(FaultId{"FAULT_1", "/node1"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Test", + "/node1", now.nanoseconds(), config); // Check before timeout - should not confirm auto before_timeout = rclcpp::Time(now.nanoseconds() + static_cast(5e9)); @@ -866,9 +994,9 @@ TEST_F(SqliteFaultStorageTest, TimeBasedConfirmationWhenEnabled) { auto after_timeout = rclcpp::Time(now.nanoseconds() + static_cast(15e9)); auto confirmed = storage_->check_time_based_confirmation(after_timeout.nanoseconds()); ASSERT_EQ(confirmed.size(), 1u); - EXPECT_EQ(confirmed[0], "FAULT_1"); + EXPECT_EQ(confirmed[0], (ros2_medkit_fault_manager::FaultId{"FAULT_1", "/node1"})); - auto fault = storage_->get_fault("FAULT_1"); + auto fault = storage_->get_fault({"FAULT_1", "/node1"}); ASSERT_TRUE(fault.has_value()); EXPECT_EQ(fault->status, Fault::STATUS_CONFIRMED); } @@ -897,15 +1025,15 @@ TEST_F(SqliteFaultStorageTest, ConfirmedAtRecordedOnImmediateConfirmation) { rclcpp::Clock clock; auto t = clock.now(); // Default config confirms on the first FAILED event. - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t.nanoseconds(), - default_config()); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t.nanoseconds(), default_config()); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(read_confirmed_at(temp_db_path_, "F"), t.nanoseconds()); // A later FAILED on an already-confirmed fault must NOT move the timestamp. auto t2 = rclcpp::Time(t.nanoseconds() + static_cast(5e9)); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t2.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t2.nanoseconds(), default_config()); EXPECT_EQ(read_confirmed_at(temp_db_path_, "F"), t.nanoseconds()); } @@ -915,15 +1043,15 @@ TEST_F(SqliteFaultStorageTest, ConfirmedAtRecordedOnDebouncedConfirmation) { config.confirmation_threshold = -2; // second FAILED event confirms auto t1 = clock.now(); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t1.nanoseconds(), - config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_PREFAILED); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t1.nanoseconds(), config); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_PREFAILED); EXPECT_EQ(read_confirmed_at(temp_db_path_, "F"), 0) << "not confirmed yet: no confirmation timestamp"; auto t2 = rclcpp::Time(t1.nanoseconds() + static_cast(1e9)); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t2.nanoseconds(), - config); - EXPECT_EQ(storage_->get_fault("F")->status, Fault::STATUS_CONFIRMED); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t2.nanoseconds(), config); + EXPECT_EQ(storage_->get_fault({"F", "/n"})->status, Fault::STATUS_CONFIRMED); EXPECT_EQ(read_confirmed_at(temp_db_path_, "F"), t2.nanoseconds()); } @@ -935,8 +1063,8 @@ TEST_F(SqliteFaultStorageTest, ConfirmedAtRecordedOnTimeBasedConfirmation) { storage_->set_debounce_config(config); auto now = clock.now(); - storage_->report_fault_event("F", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", now.nanoseconds(), - config); + storage_->report_fault_event(FaultId{"F", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + now.nanoseconds(), config); auto after_timeout = rclcpp::Time(now.nanoseconds() + static_cast(15e9)); auto confirmed = storage_->check_time_based_confirmation(after_timeout.nanoseconds()); @@ -965,16 +1093,179 @@ TEST_F(SqliteFaultStorageTest, ConfirmedAtColumnMigratedIntoOldDatabase) { sqlite3_close(raw); } storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - EXPECT_TRUE(storage_->contains("OLD")); + EXPECT_TRUE(storage_->contains({"OLD", ros2_medkit_fault_manager::kSharedOwner})) + << "a legacy row becomes the shared record of its code"; EXPECT_EQ(read_confirmed_at(temp_db_path_, "OLD"), 0) << "pre-migration rows carry no confirmation time"; rclcpp::Clock clock; auto t = clock.now(); - storage_->report_fault_event("NEW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", t.nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"NEW", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "e", "/n", + t.nanoseconds(), default_config()); EXPECT_EQ(read_confirmed_at(temp_db_path_, "NEW"), t.nanoseconds()); } +namespace { + +/// Whether @p table carries an index over exactly @p columns, whatever its name. +bool has_index_over(const std::filesystem::path & db_path, const std::string & table, + const std::vector & columns, bool * unique_out = nullptr) { + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path.string().c_str(), &raw), SQLITE_OK); + + std::vector> indexes; + { + sqlite3_stmt * stmt = nullptr; + const std::string sql = "PRAGMA index_list(" + table + ")"; + EXPECT_EQ(sqlite3_prepare_v2(raw, sql.c_str(), -1, &stmt, nullptr), SQLITE_OK); + while (sqlite3_step(stmt) == SQLITE_ROW) { + const auto * name = reinterpret_cast(sqlite3_column_text(stmt, 1)); + indexes.emplace_back(name != nullptr ? name : "", sqlite3_column_int(stmt, 2) != 0); + } + sqlite3_finalize(stmt); + } + + bool found = false; + for (const auto & [name, unique] : indexes) { + std::vector cols; + sqlite3_stmt * stmt = nullptr; + const std::string sql = "PRAGMA index_info(" + name + ")"; + EXPECT_EQ(sqlite3_prepare_v2(raw, sql.c_str(), -1, &stmt, nullptr), SQLITE_OK); + while (sqlite3_step(stmt) == SQLITE_ROW) { + const auto * col = reinterpret_cast(sqlite3_column_text(stmt, 2)); + cols.emplace_back(col != nullptr ? col : ""); + } + sqlite3_finalize(stmt); + if (cols == columns) { + found = true; + if (unique_out != nullptr) { + *unique_out = unique; + } + break; + } + } + sqlite3_close(raw); + return found; +} + +/// One scalar out of a legacy-schema database, read outside the storage class. +std::string read_text(const std::filesystem::path & db_path, const std::string & sql) { + sqlite3 * raw = nullptr; + EXPECT_EQ(sqlite3_open(db_path.string().c_str(), &raw), SQLITE_OK); + sqlite3_stmt * stmt = nullptr; + EXPECT_EQ(sqlite3_prepare_v2(raw, sql.c_str(), -1, &stmt, nullptr), SQLITE_OK); + std::string value; + if (sqlite3_step(stmt) == SQLITE_ROW) { + const auto * text = reinterpret_cast(sqlite3_column_text(stmt, 0)); + value = text != nullptr ? text : ""; + } + sqlite3_finalize(stmt); + sqlite3_close(raw); + return value; +} + +} // namespace + +// A database written when every fault code was one record, fed by every source. SQLite cannot +// change a PRIMARY KEY in place while CREATE TABLE IF NOT EXISTS silently skips an existing +// table, so the database is rebuilt on open: every row becomes the shared record of its code and +// every child row attaches to it. +TEST_F(SqliteFaultStorageTest, LegacyDatabaseBecomesSharedRecordsOnOpen) { + storage_.reset(); + std::filesystem::remove(temp_db_path_); + { + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec( + raw, + "CREATE TABLE faults (fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, " + "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " + "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " + "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " + "debounce_counter INTEGER NOT NULL DEFAULT 0, " + "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " + "confirmed_at_ns INTEGER NOT NULL DEFAULT 0);" + "INSERT INTO faults VALUES ('SHARED', 2, 'shared', 1, 1, 1, 'CONFIRMED', " + "'[\"/owner_a\",\"/owner_b\"]', -1, 1, 0, 0);" + "INSERT INTO faults VALUES ('SOLO', 1, 'solo', 2, 2, 1, 'CONFIRMED', '[\"/owner_c\"]', -1, 2, 0, 0);" + "CREATE TABLE freeze_frames (fault_code TEXT PRIMARY KEY, data TEXT NOT NULL, " + "captured_at_ns INTEGER NOT NULL);" + "INSERT INTO freeze_frames VALUES ('SHARED', '{\"/t\":1}', 7);" + "CREATE TABLE snapshots (id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, " + "topic TEXT NOT NULL, message_type TEXT NOT NULL, data TEXT NOT NULL, " + "captured_at_ns INTEGER NOT NULL);" + "INSERT INTO snapshots (fault_code, topic, message_type, data, captured_at_ns) " + "VALUES ('SHARED', '/t', 'std_msgs/msg/Float64', '{}', 9);" + "CREATE TABLE near_misses (id INTEGER PRIMARY KEY AUTOINCREMENT, fault_code TEXT NOT NULL, " + "occurred_at_ns INTEGER NOT NULL, debounce_counter INTEGER NOT NULL, " + "confirmation_threshold INTEGER NOT NULL, severity INTEGER NOT NULL, source_id TEXT NOT NULL, " + "resulting_status TEXT NOT NULL DEFAULT '');" + "INSERT INTO near_misses (fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, " + "severity, source_id, resulting_status) VALUES ('SHARED', 5, -1, -3, 2, '/owner_a', 'PREFAILED');" + "INSERT INTO near_misses (fault_code, occurred_at_ns, debounce_counter, confirmation_threshold, " + "severity, source_id, resulting_status) VALUES ('SHARED', 6, -2, -3, 2, '/owner_b', 'PREFAILED');" + "CREATE TABLE rosbag_files (id INTEGER PRIMARY KEY AUTOINCREMENT, " + "fault_code TEXT NOT NULL UNIQUE, file_path TEXT NOT NULL, format TEXT NOT NULL, " + "duration_sec REAL NOT NULL, size_bytes INTEGER NOT NULL, created_at_ns INTEGER NOT NULL);" + "INSERT INTO rosbag_files (fault_code, file_path, format, duration_sec, size_bytes, created_at_ns) " + "VALUES ('SHARED', '/bags/fault_SHARED_1', 'mcap', 5.0, 1024, 11);", + nullptr, nullptr, nullptr), + SQLITE_OK); + sqlite3_close(raw); + } + + storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); + + // The record identity is an index over (fault_code, owner), and it is unique. + bool unique = false; + ASSERT_TRUE(has_index_over(temp_db_path_, "faults", {"fault_code", "owner"}, &unique)); + EXPECT_TRUE(unique); + ASSERT_TRUE(has_index_over(temp_db_path_, "freeze_frames", {"fault_code", "owner"}, &unique)); + EXPECT_TRUE(unique); + ASSERT_TRUE(has_index_over(temp_db_path_, "rosbag_files", {"fault_code", "owner", "file_path"}, &unique)); + EXPECT_TRUE(unique); + + // Every row is the shared record of its code and keeps its reporting_sources as written. + EXPECT_EQ(read_text(temp_db_path_, "SELECT COUNT(*) FROM faults WHERE owner = '*'"), "2"); + EXPECT_EQ(read_text(temp_db_path_, "SELECT reporting_sources FROM faults WHERE fault_code = 'SHARED'"), + "[\"/owner_a\",\"/owner_b\"]"); + + const ros2_medkit_fault_manager::FaultId shared{"SHARED", ros2_medkit_fault_manager::kSharedOwner}; + auto migrated = storage_->get_fault(shared); + ASSERT_TRUE(migrated.has_value()); + EXPECT_EQ(migrated->status, Fault::STATUS_CONFIRMED); + EXPECT_EQ(migrated->reporting_sources, (std::vector{"/owner_a", "/owner_b"})); + EXPECT_EQ(storage_->get_ids_by_code("SHARED").size(), 1u); + EXPECT_TRUE(storage_->get_fault({"SOLO", "*"}).has_value()); + + // Every child row attached to the shared record, none lost. + ASSERT_TRUE(storage_->get_freeze_frame(shared).has_value()); + EXPECT_EQ(storage_->get_freeze_frame(shared)->captured_at_ns, 7); + ASSERT_EQ(storage_->get_snapshots(shared).size(), 1u); + EXPECT_EQ(storage_->get_snapshots(shared).front().owner, "*"); + ASSERT_TRUE(storage_->get_rosbag_file(shared).has_value()); + EXPECT_EQ(storage_->get_rosbag_file(shared)->owner, "*"); + const auto near_misses = storage_->get_near_misses(shared); + ASSERT_EQ(near_misses.size(), 2u) << "the near misses of both sources belong to the one shared record"; + EXPECT_EQ(near_misses[0].source_id, "/owner_a"); + EXPECT_EQ(near_misses[1].source_id, "/owner_b"); + EXPECT_EQ(read_text(temp_db_path_, + "SELECT COUNT(*) FROM near_misses n WHERE NOT EXISTS " + "(SELECT 1 FROM faults f WHERE f.fault_code = n.fault_code AND f.owner = n.owner)"), + "0"); + + // Re-opening a migrated database changes nothing. + storage_.reset(); + storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); + + EXPECT_EQ(storage_->size(), 2u); + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM faults WHERE fault_code = 'SHARED'"), "*"); + ASSERT_TRUE(storage_->get_freeze_frame(shared).has_value()); + EXPECT_EQ(storage_->get_snapshots(shared).size(), 1u); + ASSERT_TRUE(storage_->get_rosbag_file(shared).has_value()); + EXPECT_EQ(storage_->get_near_misses(shared).size(), 2u); + EXPECT_TRUE(has_index_over(temp_db_path_, "faults", {"fault_code", "owner"})); +} + // --- #620: many recordings per fault ----------------------------------------- namespace { @@ -999,10 +1290,15 @@ bool has_unique_constraint_index(const std::filesystem::path & db_path, const st return found; } +/// Rows in this file belong to one owner unless a test says otherwise: they exercise +/// retention and ordering, which the record identity does not change. +constexpr const char * kRowOwner = "/test_node"; + RosbagFileInfo make_rosbag(const std::string & code, const std::string & path, int64_t created_ns, size_t bytes = 1024) { RosbagFileInfo info; info.fault_code = code; + info.owner = kRowOwner; info.file_path = path; info.format = "mcap"; info.duration_sec = 6.0; @@ -1055,6 +1351,231 @@ TEST_F(SqliteFaultStorageTest, LegacyUniqueConstraintIsRebuiltAwayAndRecordingId EXPECT_EQ(all[1].recording_id, "fault_B_200"); } +namespace { + +/// One control byte, kept out of the string literals that use it: written inline, +/// "\x01" followed by a hex digit would be read as one wider escape. +constexpr char kCtrl = '\x01'; + +/// Captures rcutils log output while alive and restores the console handler on every +/// exit path, so a live capture never swallows the output of later cases in this +/// binary. Same shape as the one the rosbag suite uses: the handler is a plain C +/// function pointer with no user-data slot, so the live capture is reached through a +/// file-static, atomic because the handler is process-global. +class LogCapture { + public: + LogCapture() { + active().store(this); + rcutils_logging_set_output_handler(&LogCapture::handler); + } + ~LogCapture() { + rcutils_logging_set_output_handler(rcutils_logging_console_output_handler); + active().store(nullptr); + } + LogCapture(const LogCapture &) = delete; + LogCapture & operator=(const LogCapture &) = delete; + LogCapture(LogCapture &&) = delete; + LogCapture & operator=(LogCapture &&) = delete; + + int count(const std::string & needle) const { + std::lock_guard lk(mutex_); + return static_cast(std::count_if(lines_.begin(), lines_.end(), [&needle](const std::string & line) { + return line.find(needle) != std::string::npos; + })); + } + + private: + static std::atomic & active() { + static std::atomic current{nullptr}; + return current; + } + + static void handler(const rcutils_log_location_t * /*location*/, int /*severity*/, const char * /*name*/, + rcutils_time_point_value_t /*timestamp*/, const char * format, va_list * args) { + char buf[1024]; + va_list copy; + va_copy(copy, *args); + // The format string arrives through the handler signature, so there is no literal + // to write here. Scoped to the single call, as in the rosbag suite. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" + vsnprintf(buf, sizeof(buf), format, copy); +#pragma GCC diagnostic pop + va_end(copy); + LogCapture * capture = active().load(); + if (capture == nullptr) { + return; + } + std::lock_guard lk(capture->mutex_); + capture->lines_.emplace_back(buf); + } + + mutable std::mutex mutex_; + std::vector lines_; +}; + +/// The legacy faults table exactly as the previous releases created it. +constexpr const char * kLegacyFaultsDdl = + "CREATE TABLE faults (fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, " + "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " + "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " + "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " + "debounce_counter INTEGER NOT NULL DEFAULT 0, " + "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " + "confirmed_at_ns INTEGER NOT NULL DEFAULT 0);"; + +} // namespace + +// A rebuild that was interrupted, or a manual recovery with an external tool, can leave +// the scratch table behind. CREATE TABLE (no IF NOT EXISTS, deliberately, so a real +// collision is never silently reused) then fails on every open and the schema stays +// legacy forever. Dropping the scratch table first inside the same transaction is what +// makes the migration able to finish the job it started. +TEST_F(SqliteFaultStorageTest, LeftoverScratchTablesDoNotBlockTheMigration) { + for (const char * leftover : {"faults_new", "freeze_frames_new"}) { + storage_.reset(); + std::filesystem::remove(temp_db_path_); + { + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, kLegacyFaultsDdl, nullptr, nullptr, nullptr), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, + "INSERT INTO faults VALUES ('LEFTOVER', 2, 'legacy', 1, 1, 1, 'CONFIRMED', " + "'[\"/owner_a\"]', -1, 1, 0, 0);" + "CREATE TABLE freeze_frames (fault_code TEXT PRIMARY KEY, data TEXT NOT NULL, " + "captured_at_ns INTEGER NOT NULL);" + "INSERT INTO freeze_frames VALUES ('LEFTOVER', '{\"/t\":1}', 7);", + nullptr, nullptr, nullptr), + SQLITE_OK); + // The debris: a scratch table with a shape that does not even match the real one, + // so reusing it instead of dropping it could not go unnoticed either. + const std::string debris = std::string("CREATE TABLE ") + leftover + " (junk TEXT);"; + ASSERT_EQ(sqlite3_exec(raw, debris.c_str(), nullptr, nullptr, nullptr), SQLITE_OK); + sqlite3_close(raw); + } + + // The drop is announced, because throwing away a table an operator may have been + // in the middle of recovering by hand has to be visible in the log. + std::unique_ptr log = std::make_unique(); + ASSERT_NO_THROW(storage_ = std::make_unique(temp_db_path_.string(), test_log_sink())) + << "leftover " << leftover << " must not block the rebuild"; + const int named = log->count(std::string("scratch table '") + leftover + "'"); + log.reset(); + EXPECT_EQ(named, 1) << "the dropped table must be named in the log: " << leftover; + + const ros2_medkit_fault_manager::FaultId id{"LEFTOVER", ros2_medkit_fault_manager::kSharedOwner}; + EXPECT_TRUE(storage_->contains(id)) << "with leftover " << leftover; + EXPECT_TRUE(has_index_over(temp_db_path_, "faults", {"fault_code", "owner"})); + ASSERT_TRUE(storage_->get_freeze_frame(id).has_value()) << "with leftover " << leftover; + EXPECT_EQ(storage_->get_freeze_frame(id)->captured_at_ns, 7); + } +} + +// New rows must never reproduce the problem the migration had to recover from: a +// source_id carrying a control byte round-trips, and what lands in the column is JSON +// every reader can parse. +TEST_F(SqliteFaultStorageTest, AControlByteInASourceIdStoresAsValidJson) { + const std::string source = std::string("/dev/ttyUSB0") + kCtrl + "reader"; + rclcpp::Clock clock; + storage_->report_fault_event(FaultId{"CTRL_BYTE", source}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "d", + source, clock.now().nanoseconds(), default_config()); + + EXPECT_EQ(read_text(temp_db_path_, "SELECT json_valid(reporting_sources) FROM faults WHERE fault_code='CTRL_BYTE'"), + "1") + << "a raw control byte inside a JSON string is what made the legacy column unreadable"; + EXPECT_EQ(read_text(temp_db_path_, + "SELECT json_extract(reporting_sources, '$[0]') FROM faults " + "WHERE fault_code='CTRL_BYTE'"), + source) + << "the escape must be reversible, not lossy"; + + auto stored = storage_->get_fault({"CTRL_BYTE", source}); + ASSERT_TRUE(stored.has_value()) << "the record is keyed by the byte-exact source"; + ASSERT_EQ(stored->reporting_sources.size(), 1u); + EXPECT_EQ(stored->reporting_sources.front(), source); +} + +// The rosbag indexes that carry fault_code have to widen with the record identity, and +// widening them is the one step no rebuild does for free. LegacyDatabaseGainsTheOwner- +// ColumnOnOpen cannot show it: its fixture still carries a column UNIQUE on fault_code, +// so the earlier rosbag migration rebuilds the table, takes every index with it, and +// the widened shape appears whether or not the owner migration drops anything. The +// shape that does exercise the drops is the PREVIOUS release, where rosbag_files is +// already rebuilt and already carries the named indexes and only owner is missing. +// CREATE INDEX IF NOT EXISTS is a no-op on a name that exists, so without the drops +// the two-column indexes survive and the upsert has no unique constraint to target. +TEST_F(SqliteFaultStorageTest, PreviousReleaseRosbagIndexesWidenWithTheRecordIdentity) { + storage_.reset(); + std::filesystem::remove(temp_db_path_); + { + sqlite3 * raw = nullptr; + ASSERT_EQ(sqlite3_open(temp_db_path_.string().c_str(), &raw), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(raw, + "CREATE TABLE faults (fault_code TEXT PRIMARY KEY, severity INTEGER NOT NULL, " + "description TEXT NOT NULL, first_occurred_ns INTEGER NOT NULL, " + "last_occurred_ns INTEGER NOT NULL, occurrence_count INTEGER NOT NULL, " + "status TEXT NOT NULL, reporting_sources TEXT NOT NULL, " + "debounce_counter INTEGER NOT NULL DEFAULT 0, " + "last_failed_ns INTEGER NOT NULL DEFAULT 0, last_passed_ns INTEGER NOT NULL DEFAULT 0, " + "confirmed_at_ns INTEGER NOT NULL DEFAULT 0);" + "INSERT INTO faults VALUES ('BURST', 3, 'burst', 1, 1, 1, 'CONFIRMED', " + "'[\"/owner_a\"]', -1, 1, 0, 0);" + // rosbag_files as the previous release left it: no column UNIQUE, a + // recording_id column, and the named indexes over fault_code alone. + "CREATE TABLE rosbag_files (id INTEGER PRIMARY KEY AUTOINCREMENT, " + "fault_code TEXT NOT NULL, recording_id TEXT NOT NULL DEFAULT '', " + "file_path TEXT NOT NULL, format TEXT NOT NULL, duration_sec REAL NOT NULL, " + "size_bytes INTEGER NOT NULL, created_at_ns INTEGER NOT NULL);" + "CREATE INDEX idx_rosbag_files_fault_code ON rosbag_files(fault_code);" + "CREATE INDEX idx_rosbag_files_created_at ON rosbag_files(created_at_ns);" + "CREATE INDEX idx_rosbag_files_fault_created ON rosbag_files(fault_code, created_at_ns, id);" + "CREATE INDEX idx_rosbag_files_recording ON rosbag_files(recording_id);" + "CREATE INDEX idx_rosbag_files_path ON rosbag_files(file_path);" + "CREATE UNIQUE INDEX idx_rosbag_files_fault_path ON rosbag_files(fault_code, file_path);" + "INSERT INTO rosbag_files (fault_code, recording_id, file_path, format, duration_sec, " + "size_bytes, created_at_ns) " + "VALUES ('BURST', 'fault_BURST_1', '/bags/fault_BURST_1', 'mcap', 6.0, 512, 11);", + nullptr, nullptr, nullptr), + SQLITE_OK); + sqlite3_close(raw); + } + ASSERT_FALSE(has_unique_constraint_index(temp_db_path_, "rosbag_files")) + << "fixture must start already rebuilt, or the rebuild would widen the indexes for free"; + ASSERT_TRUE(has_index_over(temp_db_path_, "rosbag_files", {"fault_code", "file_path"})) + << "fixture must start on the previous release's two-column unique index"; + + storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); + + bool unique = false; + ASSERT_TRUE(has_index_over(temp_db_path_, "rosbag_files", {"fault_code", "owner", "file_path"}, &unique)); + EXPECT_TRUE(unique); + EXPECT_FALSE(has_index_over(temp_db_path_, "rosbag_files", {"fault_code", "file_path"})) + << "the narrow unique index must be gone, not left beside the wide one"; + EXPECT_TRUE(has_index_over(temp_db_path_, "rosbag_files", {"fault_code", "owner", "created_at_ns", "id"})) + << "the per-record read order index widened too"; + EXPECT_EQ(read_text(temp_db_path_, "SELECT owner FROM rosbag_files WHERE fault_code = 'BURST'"), "*"); + + // What the widened index buys: two records of one code link the same bag. Under the + // previous release's index the second store would collide, and the upsert cannot + // even be prepared without a unique constraint over (fault_code, owner, file_path). + RosbagFileInfo link; + link.fault_code = "BURST"; + link.file_path = "/bags/fault_BURST_1"; + link.format = "mcap"; + link.duration_sec = 6.0; + link.size_bytes = 512; + link.created_at_ns = 11; + link.owner = "*"; + storage_->store_rosbag_file(link); + link.owner = "/owner_b"; + storage_->store_rosbag_file(link); + + const auto rows = storage_->get_rosbag_files_by_recording("fault_BURST_1"); + ASSERT_EQ(rows.size(), 2u) << "one link per record, both pointing at the one bag"; + EXPECT_EQ(rows[0].owner, "*"); + EXPECT_EQ(rows[1].owner, "/owner_b"); +} + TEST_F(SqliteFaultStorageTest, ReopeningAMigratedDatabaseIsANoOp) { // The migration runs on every open, so it has to be idempotent - a second and // third open must not rebuild again, lose rows or re-backfill. @@ -1076,13 +1597,13 @@ TEST_F(SqliteFaultStorageTest, OneFaultKeepsSeveralRecordingsWhenTheCapAllows) { storage_->store_rosbag_file(make_rosbag("FLAP", "/bags/fault_FLAP_100", 100)); storage_->store_rosbag_file(make_rosbag("FLAP", "/bags/fault_FLAP_200", 200)); - const auto rows = storage_->get_rosbag_files("FLAP"); + const auto rows = storage_->get_rosbag_files({"FLAP", kRowOwner}); ASSERT_EQ(rows.size(), 2u) << "the second recording must not replace the first"; EXPECT_EQ(rows[0].recording_id, "fault_FLAP_200") << "newest first"; EXPECT_EQ(rows[1].recording_id, "fault_FLAP_100"); // get_rosbag_file is "the newest", deterministically. - const auto newest = storage_->get_rosbag_file("FLAP"); + const auto newest = storage_->get_rosbag_file({"FLAP", kRowOwner}); ASSERT_TRUE(newest.has_value()); EXPECT_EQ(newest->recording_id, "fault_FLAP_200"); } @@ -1098,7 +1619,7 @@ TEST_F(SqliteFaultStorageTest, CapKeepsTheNewestAndUnlinksTheEvictedBag) { storage_->store_rosbag_file(make_rosbag("CAP", dir_a.string(), 100)); storage_->store_rosbag_file(make_rosbag("CAP", dir_b.string(), 200)); - const auto rows = storage_->get_rosbag_files("CAP"); + const auto rows = storage_->get_rosbag_files({"CAP", kRowOwner}); ASSERT_EQ(rows.size(), 1u) << "cap 1 is the historical behaviour: one recording per fault"; EXPECT_EQ(rows[0].file_path, dir_b.string()); EXPECT_FALSE(std::filesystem::exists(dir_a)) << "the evicted bag must be unlinked"; @@ -1120,7 +1641,7 @@ TEST_F(SqliteFaultStorageTest, EvictingOneFaultsLinkKeepsABagASiblingStillRefere storage_->store_rosbag_file(make_rosbag("A", later.string(), 200)); EXPECT_TRUE(std::filesystem::exists(shared)) << "B still references it"; - const auto b_rows = storage_->get_rosbag_files("B"); + const auto b_rows = storage_->get_rosbag_files({"B", kRowOwner}); ASSERT_EQ(b_rows.size(), 1u); EXPECT_EQ(b_rows[0].file_path, shared.string()); } @@ -1143,8 +1664,8 @@ TEST_F(SqliteFaultStorageTest, DeleteRecordingRemovesEveryLinkAndTheBag) { storage_->store_rosbag_files({make_rosbag("A", dir.string(), 100), make_rosbag("B", dir.string(), 100)}); EXPECT_EQ(storage_->delete_rosbag_recording("fault_A_100"), 2u); - EXPECT_TRUE(storage_->get_rosbag_files("A").empty()); - EXPECT_TRUE(storage_->get_rosbag_files("B").empty()); + EXPECT_TRUE(storage_->get_rosbag_files({"A", kRowOwner}).empty()); + EXPECT_TRUE(storage_->get_rosbag_files({"B", kRowOwner}).empty()); EXPECT_FALSE(std::filesystem::exists(dir)); } @@ -1153,8 +1674,8 @@ TEST_F(SqliteFaultStorageTest, DeletingAFaultDropsAllItsRecordings) { storage_->store_rosbag_file(make_rosbag("A", "/bags/fault_A_100", 100)); storage_->store_rosbag_file(make_rosbag("A", "/bags/fault_A_200", 200)); - EXPECT_TRUE(storage_->delete_rosbag_file("A")); - EXPECT_TRUE(storage_->get_rosbag_files("A").empty()) << "auto_cleanup drops the fault's whole history"; + EXPECT_TRUE(storage_->delete_rosbag_file({"A", kRowOwner})); + EXPECT_TRUE(storage_->get_rosbag_files({"A", kRowOwner}).empty()) << "auto_cleanup drops the fault's whole history"; } TEST_F(SqliteFaultStorageTest, SharedRecordingStillCountsOnceTowardsStorageWithSeveralRecordings) { @@ -1173,12 +1694,13 @@ TEST_F(SqliteFaultStorageTest, StoreAndRetrieveSnapshot) { // First, create a fault to associate the snapshot with rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); // Store a snapshot SnapshotData snapshot; snapshot.fault_code = "MOTOR_OVERHEAT"; + snapshot.owner = "/motor_node"; snapshot.topic = "/motor/temperature"; snapshot.message_type = "sensor_msgs/msg/Temperature"; snapshot.data = R"({"temperature": 85.5, "variance": 0.1})"; @@ -1187,7 +1709,7 @@ TEST_F(SqliteFaultStorageTest, StoreAndRetrieveSnapshot) { storage_->store_snapshot(snapshot); // Retrieve snapshots - auto snapshots = storage_->get_snapshots("MOTOR_OVERHEAT"); + auto snapshots = storage_->get_snapshots({"MOTOR_OVERHEAT", "/motor_node"}); ASSERT_EQ(snapshots.size(), 1u); EXPECT_EQ(snapshots[0].fault_code, "MOTOR_OVERHEAT"); @@ -1202,12 +1724,13 @@ TEST_F(SqliteFaultStorageTest, MultipleSnapshotsForSameFault) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); // Store multiple snapshots for the same fault SnapshotData snapshot1; snapshot1.fault_code = "MOTOR_OVERHEAT"; + snapshot1.owner = "/motor_node"; snapshot1.topic = "/motor/temperature"; snapshot1.message_type = "sensor_msgs/msg/Temperature"; snapshot1.data = R"({"temperature": 85.5})"; @@ -1215,6 +1738,7 @@ TEST_F(SqliteFaultStorageTest, MultipleSnapshotsForSameFault) { SnapshotData snapshot2; snapshot2.fault_code = "MOTOR_OVERHEAT"; + snapshot2.owner = "/motor_node"; snapshot2.topic = "/motor/rpm"; snapshot2.message_type = "std_msgs/msg/Float64"; snapshot2.data = R"({"data": 5500.0})"; @@ -1223,7 +1747,7 @@ TEST_F(SqliteFaultStorageTest, MultipleSnapshotsForSameFault) { storage_->store_snapshot(snapshot1); storage_->store_snapshot(snapshot2); - auto snapshots = storage_->get_snapshots("MOTOR_OVERHEAT"); + auto snapshots = storage_->get_snapshots({"MOTOR_OVERHEAT", "/motor_node"}); EXPECT_EQ(snapshots.size(), 2u); } @@ -1232,11 +1756,12 @@ TEST_F(SqliteFaultStorageTest, FilterSnapshotsByTopic) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); SnapshotData snapshot1; snapshot1.fault_code = "MOTOR_OVERHEAT"; + snapshot1.owner = "/motor_node"; snapshot1.topic = "/motor/temperature"; snapshot1.message_type = "sensor_msgs/msg/Temperature"; snapshot1.data = R"({"temperature": 85.5})"; @@ -1244,6 +1769,7 @@ TEST_F(SqliteFaultStorageTest, FilterSnapshotsByTopic) { SnapshotData snapshot2; snapshot2.fault_code = "MOTOR_OVERHEAT"; + snapshot2.owner = "/motor_node"; snapshot2.topic = "/motor/rpm"; snapshot2.message_type = "std_msgs/msg/Float64"; snapshot2.data = R"({"data": 5500.0})"; @@ -1253,14 +1779,14 @@ TEST_F(SqliteFaultStorageTest, FilterSnapshotsByTopic) { storage_->store_snapshot(snapshot2); // Filter by topic - auto filtered = storage_->get_snapshots("MOTOR_OVERHEAT", "/motor/temperature"); + auto filtered = storage_->get_snapshots({"MOTOR_OVERHEAT", "/motor_node"}, "/motor/temperature"); ASSERT_EQ(filtered.size(), 1u); EXPECT_EQ(filtered[0].topic, "/motor/temperature"); } // @verifies REQ_INTEROP_088 TEST_F(SqliteFaultStorageTest, NoSnapshotsForUnknownFault) { - auto snapshots = storage_->get_snapshots("UNKNOWN_FAULT"); + auto snapshots = storage_->get_snapshots({"UNKNOWN_FAULT", "/motor_node"}); EXPECT_TRUE(snapshots.empty()); } @@ -1270,13 +1796,14 @@ TEST_F(SqliteFaultStorageTest, ClearFaultDeletesAssociatedSnapshots) { rclcpp::Clock clock; // Create a fault using report_fault_event - storage_->report_fault_event("SNAPSHOT_CLEAR_TEST", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, - "Test fault for snapshot cleanup", "/test_node", clock.now().nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"SNAPSHOT_CLEAR_TEST", "/test_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Test fault for snapshot cleanup", "/test_node", + clock.now().nanoseconds(), default_config()); // Store snapshots for this fault SnapshotData snapshot1; snapshot1.fault_code = "SNAPSHOT_CLEAR_TEST"; + snapshot1.owner = "/test_node"; snapshot1.topic = "/test/topic1"; snapshot1.message_type = "std_msgs/msg/String"; snapshot1.data = R"({"data": "test1"})"; @@ -1285,6 +1812,7 @@ TEST_F(SqliteFaultStorageTest, ClearFaultDeletesAssociatedSnapshots) { SnapshotData snapshot2; snapshot2.fault_code = "SNAPSHOT_CLEAR_TEST"; + snapshot2.owner = "/test_node"; snapshot2.topic = "/test/topic2"; snapshot2.message_type = "std_msgs/msg/String"; snapshot2.data = R"({"data": "test2"})"; @@ -1292,15 +1820,15 @@ TEST_F(SqliteFaultStorageTest, ClearFaultDeletesAssociatedSnapshots) { storage_->store_snapshot(snapshot2); // Verify snapshots exist - auto snapshots_before = storage_->get_snapshots("SNAPSHOT_CLEAR_TEST"); + auto snapshots_before = storage_->get_snapshots({"SNAPSHOT_CLEAR_TEST", "/test_node"}); ASSERT_EQ(snapshots_before.size(), 2u); // Clear the fault - bool cleared = storage_->clear_fault("SNAPSHOT_CLEAR_TEST"); + bool cleared = storage_->clear_fault({"SNAPSHOT_CLEAR_TEST", "/test_node"}); EXPECT_TRUE(cleared); // Verify snapshots are deleted - auto snapshots_after = storage_->get_snapshots("SNAPSHOT_CLEAR_TEST"); + auto snapshots_after = storage_->get_snapshots({"SNAPSHOT_CLEAR_TEST", "/test_node"}); EXPECT_TRUE(snapshots_after.empty()); } @@ -1310,12 +1838,13 @@ TEST_F(SqliteFaultStorageTest, ClearFaultKeepsSnapshotsWhenEvidenceIsRetained) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("KEEP", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "keep", "/n", + storage_->report_fault_event(FaultId{"KEEP", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "keep", "/n", clock.now().nanoseconds(), default_config()); storage_->set_retain_snapshots_on_clear(true); SnapshotData row; row.fault_code = "KEEP"; + row.owner = "/n"; row.topic = "/t"; row.message_type = "std_msgs/msg/Float64"; row.data = R"({"data": 1.0})"; @@ -1323,28 +1852,30 @@ TEST_F(SqliteFaultStorageTest, ClearFaultKeepsSnapshotsWhenEvidenceIsRetained) { row.capture_id = 1; storage_->store_snapshots({row}); - ASSERT_TRUE(storage_->clear_fault("KEEP")); + ASSERT_TRUE(storage_->clear_fault({"KEEP", "/n"})); // Recordings survive an acknowledgement once a history is configured, so the // readings captured beside them have to as well - otherwise the fault is left // holding bags whose values are gone, which is worse than losing both. - EXPECT_EQ(storage_->get_snapshots("KEEP").size(), 1u); + EXPECT_EQ(storage_->get_snapshots({"KEEP", "/n"}).size(), 1u); } TEST_F(SqliteFaultStorageTest, StoreAndRetrieveFreezeFrame) { using ros2_medkit_fault_manager::FreezeFrameData; rclcpp::Clock clock; - storage_->report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", - "/plc_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"PLC_PRESSURE_HIGH", "/plc_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Pressure high", "/plc_node", clock.now().nanoseconds(), + default_config()); FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc_node"; frame.data = R"({"/plc/pressure":{"data":8.4},"/plc/valve":{"data":true}})"; frame.captured_at_ns = clock.now().nanoseconds(); storage_->store_freeze_frame(frame); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage_->get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc_node"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->fault_code, "PLC_PRESSURE_HIGH"); EXPECT_EQ(retrieved->data, frame.data); @@ -1353,7 +1884,7 @@ TEST_F(SqliteFaultStorageTest, StoreAndRetrieveFreezeFrame) { // @verifies REQ_INTEROP_088 TEST_F(SqliteFaultStorageTest, NoFreezeFrameForUnknownFault) { - auto retrieved = storage_->get_freeze_frame("NEVER_CAPTURED"); + auto retrieved = storage_->get_freeze_frame({"NEVER_CAPTURED", "/plc_node"}); EXPECT_FALSE(retrieved.has_value()); } @@ -1364,17 +1895,19 @@ TEST_F(SqliteFaultStorageTest, FreezeFrameReplacedOnRecapture) { FreezeFrameData first; first.fault_code = "PLC_PRESSURE_HIGH"; + first.owner = "/plc_node"; first.data = R"({"/plc/pressure":{"data":8.4}})"; first.captured_at_ns = 1000; storage_->store_freeze_frame(first); FreezeFrameData second; second.fault_code = "PLC_PRESSURE_HIGH"; + second.owner = "/plc_node"; second.data = R"({"/plc/pressure":{"data":9.9}})"; second.captured_at_ns = 2000; storage_->store_freeze_frame(second); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage_->get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc_node"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, second.data); EXPECT_EQ(retrieved->captured_at_ns, 2000); @@ -1386,12 +1919,14 @@ TEST_F(SqliteFaultStorageTest, FreezeFrameSurvivesClearFault) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("PLC_PRESSURE_HIGH", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Pressure high", - "/plc_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"PLC_PRESSURE_HIGH", "/plc_node"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Pressure high", "/plc_node", clock.now().nanoseconds(), + default_config()); // A per-topic snapshot (removed on clear) plus a freeze-frame (retained on clear). SnapshotData snapshot; snapshot.fault_code = "PLC_PRESSURE_HIGH"; + snapshot.owner = "/plc_node"; snapshot.topic = "/plc/pressure"; snapshot.message_type = "std_msgs/msg/Float64"; snapshot.data = R"({"data":8.4})"; @@ -1400,15 +1935,16 @@ TEST_F(SqliteFaultStorageTest, FreezeFrameSurvivesClearFault) { FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc_node"; frame.data = R"({"/plc/pressure":{"data":8.4}})"; frame.captured_at_ns = clock.now().nanoseconds(); storage_->store_freeze_frame(frame); - ASSERT_TRUE(storage_->clear_fault("PLC_PRESSURE_HIGH")); + ASSERT_TRUE(storage_->clear_fault({"PLC_PRESSURE_HIGH", "/plc_node"})); // Snapshots are wiped on clear, the freeze-frame is retained and still retrievable. - EXPECT_TRUE(storage_->get_snapshots("PLC_PRESSURE_HIGH").empty()); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + EXPECT_TRUE(storage_->get_snapshots({"PLC_PRESSURE_HIGH", "/plc_node"}).empty()); + auto retrieved = storage_->get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc_node"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, frame.data); } @@ -1419,6 +1955,7 @@ TEST_F(SqliteFaultStorageTest, FreezeFramePersistsAcrossReopen) { FreezeFrameData frame; frame.fault_code = "PLC_PRESSURE_HIGH"; + frame.owner = "/plc_node"; frame.data = R"({"/plc/pressure":{"data":8.4}})"; frame.captured_at_ns = 4242; storage_->store_freeze_frame(frame); @@ -1427,7 +1964,7 @@ TEST_F(SqliteFaultStorageTest, FreezeFramePersistsAcrossReopen) { storage_.reset(); storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - auto retrieved = storage_->get_freeze_frame("PLC_PRESSURE_HIGH"); + auto retrieved = storage_->get_freeze_frame({"PLC_PRESSURE_HIGH", "/plc_node"}); ASSERT_TRUE(retrieved.has_value()); EXPECT_EQ(retrieved->data, frame.data); EXPECT_EQ(retrieved->captured_at_ns, 4242); @@ -1441,17 +1978,19 @@ TEST_F(SqliteFaultStorageTest, ListRosbagsForEntityFiltersCorrectly) { rclcpp::Clock clock; // Create fault with reporting source for entity - storage_->report_fault_event("ENTITY_FAULT_1", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Fault from entity", - "/powertrain/motor", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"ENTITY_FAULT_1", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "Fault from entity", "/powertrain/motor", + clock.now().nanoseconds(), default_config()); // Create another fault with different reporting source - storage_->report_fault_event("ENTITY_FAULT_2", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, - "Fault from other entity", "/chassis/brake", clock.now().nanoseconds(), - default_config()); + storage_->report_fault_event(FaultId{"ENTITY_FAULT_2", "/chassis/brake"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "Fault from other entity", "/chassis/brake", + clock.now().nanoseconds(), default_config()); // Store rosbags for both faults RosbagFileInfo info1; info1.fault_code = "ENTITY_FAULT_1"; + info1.owner = "/powertrain/motor"; info1.file_path = "/tmp/entity1.mcap"; info1.format = "mcap"; info1.duration_sec = 5.0; @@ -1461,6 +2000,7 @@ TEST_F(SqliteFaultStorageTest, ListRosbagsForEntityFiltersCorrectly) { RosbagFileInfo info2; info2.fault_code = "ENTITY_FAULT_2"; + info2.owner = "/chassis/brake"; info2.file_path = "/tmp/entity2.mcap"; info2.format = "mcap"; info2.duration_sec = 3.0; @@ -1497,6 +2037,7 @@ TEST_F(SqliteFaultStorageTest, AFailingRowDeleteKeepsTheBagOnDisk) { RosbagFileInfo info; info.fault_code = "DELETE_FAILS"; + info.owner = kRowOwner; info.file_path = bag_dir.string(); info.format = "mcap"; info.duration_sec = 1.0; @@ -1515,9 +2056,10 @@ TEST_F(SqliteFaultStorageTest, AFailingRowDeleteKeepsTheBagOnDisk) { SQLITE_OK); sqlite3_close(raw); - EXPECT_THROW(storage_->delete_rosbag_file("DELETE_FAILS"), std::runtime_error); + EXPECT_THROW(storage_->delete_rosbag_file({"DELETE_FAILS", kRowOwner}), std::runtime_error); - EXPECT_TRUE(storage_->get_rosbag_file("DELETE_FAILS").has_value()) << "the DELETE failed, so its row must remain"; + EXPECT_TRUE(storage_->get_rosbag_file({"DELETE_FAILS", kRowOwner}).has_value()) + << "the DELETE failed, so its row must remain"; EXPECT_TRUE(std::filesystem::exists(bag_dir)) << "the bag was unlinked before its row was deleted, so the surviving row now points at nothing"; @@ -1531,6 +2073,7 @@ TEST_F(SqliteFaultStorageTest, GetAllRosbagFilesReturnsSortedByCreatedAt) { RosbagFileInfo info1; info1.fault_code = "FAULT_A"; + info1.owner = kRowOwner; info1.file_path = "/tmp/a.mcap"; info1.format = "mcap"; info1.duration_sec = 1.0; @@ -1540,6 +2083,7 @@ TEST_F(SqliteFaultStorageTest, GetAllRosbagFilesReturnsSortedByCreatedAt) { RosbagFileInfo info2; info2.fault_code = "FAULT_B"; + info2.owner = kRowOwner; info2.file_path = "/tmp/b.mcap"; info2.format = "mcap"; info2.duration_sec = 2.0; @@ -1567,6 +2111,7 @@ TEST_F(SqliteFaultStorageTest, SharedRosbagSurvivesUntilTheLastFaultIsDeleted) { RosbagFileInfo info; info.fault_code = "ROOT_CAUSE"; + info.owner = kRowOwner; info.file_path = bag_path; info.format = "mcap"; info.duration_sec = 5.0; @@ -1575,13 +2120,14 @@ TEST_F(SqliteFaultStorageTest, SharedRosbagSurvivesUntilTheLastFaultIsDeleted) { storage_->store_rosbag_file(info); info.fault_code = "CORRELATED"; + info.owner = kRowOwner; storage_->store_rosbag_file(info); - EXPECT_TRUE(storage_->delete_rosbag_file("CORRELATED")); + EXPECT_TRUE(storage_->delete_rosbag_file({"CORRELATED", kRowOwner})); EXPECT_TRUE(std::filesystem::exists(bag_path)); - EXPECT_TRUE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); + EXPECT_TRUE(storage_->get_rosbag_file({"ROOT_CAUSE", kRowOwner}).has_value()); - EXPECT_TRUE(storage_->delete_rosbag_file("ROOT_CAUSE")); + EXPECT_TRUE(storage_->delete_rosbag_file({"ROOT_CAUSE", kRowOwner})); EXPECT_FALSE(std::filesystem::exists(bag_path)); } @@ -1595,6 +2141,7 @@ TEST_F(SqliteFaultStorageTest, SharedRosbagCountsOnceTowardsStorageTotal) { auto store = [this](const char * code, const char * path, size_t bytes) { RosbagFileInfo info; info.fault_code = code; + info.owner = kRowOwner; info.file_path = path; info.format = "mcap"; info.duration_sec = 5.0; @@ -1623,6 +2170,7 @@ TEST_F(SqliteFaultStorageTest, RestoreWithNewPathUnlinksTheOldExclusiveBag) { RosbagFileInfo info; info.fault_code = "X"; + info.owner = kRowOwner; info.file_path = old_path; info.format = "mcap"; info.duration_sec = 5.0; @@ -1634,7 +2182,7 @@ TEST_F(SqliteFaultStorageTest, RestoreWithNewPathUnlinksTheOldExclusiveBag) { storage_->store_rosbag_file(info); EXPECT_FALSE(std::filesystem::exists(old_path)) << "nobody references the old bag, it must be unlinked"; - auto row = storage_->get_rosbag_file("X"); + auto row = storage_->get_rosbag_file({"X", kRowOwner}); ASSERT_TRUE(row.has_value()); EXPECT_EQ(row->file_path, old_path + "_new"); } @@ -1652,16 +2200,19 @@ TEST_F(SqliteFaultStorageTest, RestoreWithNewPathKeepsTheBagASiblingStillReferen info.size_bytes = 100; info.created_at_ns = 1000; info.fault_code = "X"; + info.owner = kRowOwner; storage_->store_rosbag_file(info); info.fault_code = "Y"; + info.owner = kRowOwner; storage_->store_rosbag_file(info); info.fault_code = "X"; + info.owner = kRowOwner; info.file_path = shared_path + "_new"; storage_->store_rosbag_file(info); EXPECT_TRUE(std::filesystem::exists(shared_path)) << "the sibling fault still owns the shared bag"; - auto sibling = storage_->get_rosbag_file("Y"); + auto sibling = storage_->get_rosbag_file({"Y", kRowOwner}); ASSERT_TRUE(sibling.has_value()); EXPECT_EQ(sibling->file_path, shared_path); @@ -1684,12 +2235,13 @@ TEST_F(SqliteFaultStorageTest, BulkStoreRegistersEveryFaultOfTheBurst) { std::vector rows; for (const char * code : {"ROOT_CAUSE", "CORRELATED_A", "CORRELATED_B"}) { info.fault_code = code; + info.owner = kRowOwner; rows.push_back(info); } storage_->store_rosbag_files(rows); for (const char * code : {"ROOT_CAUSE", "CORRELATED_A", "CORRELATED_B"}) { - auto row = storage_->get_rosbag_file(code); + auto row = storage_->get_rosbag_file({code, kRowOwner}); ASSERT_TRUE(row.has_value()) << code; EXPECT_EQ(row->file_path, "/tmp/burst_bag"); } @@ -1711,18 +2263,21 @@ TEST_F(SqliteFaultStorageTest, BulkDeleteRemovesTheBurstAndUnlinksTheBagOnce) { std::vector rows; for (const char * code : {"ROOT_CAUSE", "CORRELATED_A", "CORRELATED_B"}) { info.fault_code = code; + info.owner = kRowOwner; rows.push_back(info); } storage_->store_rosbag_files(rows); // A partial delete leaves the bag on disk for the remaining fault. - EXPECT_EQ(storage_->delete_rosbag_files({"CORRELATED_A", "CORRELATED_B", "NEVER_STORED"}), 2u); + EXPECT_EQ(storage_->delete_rosbag_files( + {{"CORRELATED_A", kRowOwner}, {"CORRELATED_B", kRowOwner}, {"NEVER_STORED", kRowOwner}}), + 2u); EXPECT_TRUE(std::filesystem::exists(bag_path)); - EXPECT_TRUE(storage_->get_rosbag_file("ROOT_CAUSE").has_value()); - EXPECT_FALSE(storage_->get_rosbag_file("CORRELATED_A").has_value()); + EXPECT_TRUE(storage_->get_rosbag_file({"ROOT_CAUSE", kRowOwner}).has_value()); + EXPECT_FALSE(storage_->get_rosbag_file({"CORRELATED_A", kRowOwner}).has_value()); // The last reference going away unlinks the directory. - EXPECT_EQ(storage_->delete_rosbag_files({"ROOT_CAUSE"}), 1u); + EXPECT_EQ(storage_->delete_rosbag_files({{"ROOT_CAUSE", kRowOwner}}), 1u); EXPECT_FALSE(std::filesystem::exists(bag_path)); EXPECT_EQ(storage_->get_total_rosbag_storage_bytes(), 0u); } @@ -1735,8 +2290,8 @@ TEST_F(SqliteFaultStorageTest, SnapshotCapKeepsTheNewestCaptureWholeAndDropsTheO using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "Motor overheated", - "/motor_node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/motor_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, + "Motor overheated", "/motor_node", clock.now().nanoseconds(), default_config()); // Two topics per capture, room for two captures. storage_->set_max_snapshots_per_fault(4); @@ -1746,6 +2301,7 @@ TEST_F(SqliteFaultStorageTest, SnapshotCapKeepsTheNewestCaptureWholeAndDropsTheO for (const char * topic : {"/motor/temp", "/motor/rpm"}) { SnapshotData row; row.fault_code = "MOTOR_OVERHEAT"; + row.owner = "/motor_node"; row.topic = topic; row.message_type = "std_msgs/msg/Float64"; row.data = R"({"data": 1.0})"; @@ -1760,7 +2316,7 @@ TEST_F(SqliteFaultStorageTest, SnapshotCapKeepsTheNewestCaptureWholeAndDropsTheO storage_->store_snapshots(capture(2, 2000)); storage_->store_snapshots(capture(3, 3000)); - auto snapshots = storage_->get_snapshots("MOTOR_OVERHEAT"); + auto snapshots = storage_->get_snapshots({"MOTOR_OVERHEAT", "/motor_node"}); // The third capture is stored WHOLE and the first goes whole. The old rule // counted rows and rejected the new one once full, so capture 3 landed with one @@ -1786,14 +2342,15 @@ TEST_F(SqliteFaultStorageTest, ACaptureLargerThanTheCapIsKeptWholeRatherThanTorn using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("WIDE", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "many topics", "/n", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"WIDE", "/n"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "many topics", + "/n", clock.now().nanoseconds(), default_config()); storage_->set_max_snapshots_per_fault(2); std::vector rows; for (int i = 0; i < 4; ++i) { SnapshotData row; row.fault_code = "WIDE"; + row.owner = "/n"; row.topic = "/t" + std::to_string(i); row.message_type = "std_msgs/msg/Float64"; row.data = "{}"; @@ -1806,15 +2363,15 @@ TEST_F(SqliteFaultStorageTest, ACaptureLargerThanTheCapIsKeptWholeRatherThanTorn // The cap is smaller than this fault's topic count. Trimming to it would mean // storing the reading with holes, which is the failure being fixed; the capture // stays whole and the operator can see the cap is too small. - EXPECT_EQ(storage_->get_snapshots("WIDE").size(), 4u); + EXPECT_EQ(storage_->get_snapshots({"WIDE", "/n"}).size(), 4u); } TEST_F(SqliteFaultStorageTest, SnapshotLimitZeroMeansUnlimited) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("FAULT_A", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", "/node", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_A", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", + "/node", clock.now().nanoseconds(), default_config()); // Default (0) = unlimited storage_->set_max_snapshots_per_fault(0); @@ -1822,6 +2379,7 @@ TEST_F(SqliteFaultStorageTest, SnapshotLimitZeroMeansUnlimited) { for (int i = 0; i < 20; ++i) { SnapshotData snap; snap.fault_code = "FAULT_A"; + snap.owner = "/node"; snap.topic = "/topic"; snap.message_type = "std_msgs/msg/String"; snap.data = "{}"; @@ -1829,7 +2387,7 @@ TEST_F(SqliteFaultStorageTest, SnapshotLimitZeroMeansUnlimited) { storage_->store_snapshot(snap); } - auto snapshots = storage_->get_snapshots("FAULT_A"); + auto snapshots = storage_->get_snapshots({"FAULT_A", "/node"}); EXPECT_EQ(snapshots.size(), 20u) << "Unlimited mode should store all snapshots"; } @@ -1837,15 +2395,16 @@ TEST_F(SqliteFaultStorageTest, SnapshotLimitPerFaultNotGlobal) { using ros2_medkit_fault_manager::SnapshotData; rclcpp::Clock clock; - storage_->report_fault_event("FAULT_A", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", "/node", - clock.now().nanoseconds(), default_config()); - storage_->report_fault_event("FAULT_B", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", "/node", - clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_A", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", + "/node", clock.now().nanoseconds(), default_config()); + storage_->report_fault_event(FaultId{"FAULT_B", "/node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "desc", + "/node", clock.now().nanoseconds(), default_config()); storage_->set_max_snapshots_per_fault(1); SnapshotData snap_a; snap_a.fault_code = "FAULT_A"; + snap_a.owner = "/node"; snap_a.topic = "/topic"; snap_a.message_type = "std_msgs/msg/String"; snap_a.data = "{}"; @@ -1853,13 +2412,14 @@ TEST_F(SqliteFaultStorageTest, SnapshotLimitPerFaultNotGlobal) { SnapshotData snap_b = snap_a; snap_b.fault_code = "FAULT_B"; + snap_b.owner = "/node"; storage_->store_snapshot(snap_a); storage_->store_snapshot(snap_b); // Both faults should have 1 snapshot each (limit is per-fault) - EXPECT_EQ(storage_->get_snapshots("FAULT_A").size(), 1u); - EXPECT_EQ(storage_->get_snapshots("FAULT_B").size(), 1u); + EXPECT_EQ(storage_->get_snapshots({"FAULT_A", "/node"}).size(), 1u); + EXPECT_EQ(storage_->get_snapshots({"FAULT_B", "/node"}).size(), 1u); } // --- Near-miss series --- @@ -1886,15 +2446,16 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesIsAppendedNotOverwritten) { const auto config = four_strike_config(); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - auto fault = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_NE(fault->status, Fault::STATUS_CONFIRMED) << "test setup: these reports must not confirm"; - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "each near miss must append an entry, not overwrite the last"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[1].debounce_counter, -2); @@ -1911,30 +2472,32 @@ TEST_F(SqliteFaultStorageTest, ConfirmingReportIsNotANearMiss) { const auto config = four_strike_config(); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - auto fault = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); // The fourth report is the fault happening, not nearly happening. - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); } TEST_F(SqliteFaultStorageTest, NearMissSeriesSurvivesClearFault) { const auto config = four_strike_config(); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); - ASSERT_TRUE(storage_->clear_fault("PUMP_PRESSURE_LOW")); + ASSERT_TRUE(storage_->clear_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "acknowledging the fault destroyed the near-miss record"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[2].debounce_counter, -3); @@ -1944,16 +2507,18 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesContinuesAcrossReactivation) { const auto config = four_strike_config(); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - ASSERT_TRUE(storage_->clear_fault("PUMP_PRESSURE_LOW")); + ASSERT_TRUE(storage_->clear_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); // A new outage cycle starts: the reactivating report resets the counter to -1 without confirming. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, - "pressure dipping again", "/hydraulics/pump", nth_report_time(10).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping again", "/hydraulics/pump", + nth_report_time(10).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 4u) << "the series must span fault cycles, one entry per occurrence"; EXPECT_EQ(series[3].debounce_counter, -1); EXPECT_EQ(series[3].occurred_at_ns, nth_report_time(10).nanoseconds()); @@ -1963,15 +2528,16 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesSurvivesReopen) { const auto config = four_strike_config(); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - ASSERT_TRUE(storage_->clear_fault("PUMP_PRESSURE_LOW")); + ASSERT_TRUE(storage_->clear_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"})); storage_.reset(); storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "the series must outlive the process, not just the fault cycle"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[2].debounce_counter, -3); @@ -1981,29 +2547,31 @@ TEST_F(SqliteFaultStorageTest, PassedReportIsNotANearMiss) { const auto config = four_strike_config(); for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); + ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); // A PASSED report moves the counter in the healing direction: the fault receding, not nearing. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); } TEST_F(SqliteFaultStorageTest, CriticalImmediateConfirmIsNotANearMiss) { DebounceConfig config = four_strike_config(); config.critical_immediate_confirm = true; - storage_->report_fault_event("BATTERY_THERMAL_RUNAWAY", fd::ReportKind::kFailed, Fault::SEVERITY_CRITICAL, - "cell over temperature", "/power/bms", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"BATTERY_THERMAL_RUNAWAY", "/power/bms"}, fd::ReportKind::kFailed, + Fault::SEVERITY_CRITICAL, "cell over temperature", "/power/bms", + nth_report_time(0).nanoseconds(), config); - auto fault = storage_->get_fault("BATTERY_THERMAL_RUNAWAY"); + auto fault = storage_->get_fault({"BATTERY_THERMAL_RUNAWAY", "/power/bms"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); - EXPECT_TRUE(storage_->get_near_misses("BATTERY_THERMAL_RUNAWAY").empty()); + EXPECT_TRUE(storage_->get_near_misses({"BATTERY_THERMAL_RUNAWAY", "/power/bms"}).empty()); } TEST_F(SqliteFaultStorageTest, ImmediateConfirmThresholdRecordsNoNearMiss) { @@ -2011,13 +2579,14 @@ TEST_F(SqliteFaultStorageTest, ImmediateConfirmThresholdRecordsNoNearMiss) { // so a fault under this config never has a near miss to record. const auto config = default_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure lost", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "pressure lost", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); - auto fault = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto fault = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(fault.has_value()); ASSERT_EQ(fault->status, Fault::STATUS_CONFIRMED); - EXPECT_TRUE(storage_->get_near_misses("PUMP_PRESSURE_LOW").empty()); + EXPECT_TRUE(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).empty()); } TEST_F(SqliteFaultStorageTest, FailedReportUnderHealedLatchIsNearMiss) { @@ -2025,23 +2594,26 @@ TEST_F(SqliteFaultStorageTest, FailedReportUnderHealedLatchIsNearMiss) { config.healing_enabled = true; config.healing_threshold = 1; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); + ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); for (int i = 1; i <= 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), + config); } - auto healed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; // The latch keeps the status at HEALED, but the counter moved toward confirmation: a near miss. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(3).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(3).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[1].debounce_counter, 0); } @@ -2052,11 +2624,12 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesBoundedKeepingNewest) { storage_->set_max_near_misses_per_fault(3); for (int i = 0; i < 5; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u); // Oldest-first eviction: a series frozen at boot would say nothing about a trend. EXPECT_EQ(series[0].debounce_counter, -3); @@ -2070,11 +2643,12 @@ TEST_F(SqliteFaultStorageTest, NearMissBoundOfOneKeepsLatestOnly) { storage_->set_max_near_misses_per_fault(1); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 1u); EXPECT_EQ(series[0].debounce_counter, -4); } @@ -2085,14 +2659,16 @@ TEST_F(SqliteFaultStorageTest, NearMissBoundIsPerFaultCode) { storage_->set_max_near_misses_per_fault(2); for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + nth_report_time(i).nanoseconds(), config); } - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 2u); - EXPECT_EQ(storage_->get_near_misses("MOTOR_OVERHEAT").size(), 2u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 2u); + EXPECT_EQ(storage_->get_near_misses({"MOTOR_OVERHEAT", "/powertrain/motor"}).size(), 2u); } TEST_F(SqliteFaultStorageTest, NearMissBoundZeroIsUnlimited) { @@ -2101,19 +2677,21 @@ TEST_F(SqliteFaultStorageTest, NearMissBoundZeroIsUnlimited) { storage_->set_max_near_misses_per_fault(0); for (int i = 0; i < 150; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 150u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 150u); } TEST_F(SqliteFaultStorageTest, NearMissTableCreatedOnDatabaseFromOlderBuild) { // The appliances this matters for already have a faults.db written by a build with no // near_misses table. Opening one must add the table, not fail and not skip recording. const auto config = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); storage_.reset(); { @@ -2124,11 +2702,12 @@ TEST_F(SqliteFaultStorageTest, NearMissTableCreatedOnDatabaseFromOlderBuild) { } storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - EXPECT_TRUE(storage_->get_near_misses("PUMP_PRESSURE_LOW").empty()); + EXPECT_TRUE(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).empty()); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(1).nanoseconds(), config); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(1).nanoseconds(), config); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); } TEST_F(SqliteFaultStorageTest, NearMissSeriesSurvivesHealedReclassification) { @@ -2138,20 +2717,22 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesSurvivesHealedReclassification) { config.healing_enabled = true; config.healing_threshold = 1; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); for (int i = 1; i <= 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), + config); } - auto healed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); ASSERT_EQ(storage_->reclassify_healed_as_cleared().size(), 1u); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 1u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 1u); } TEST_F(SqliteFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { @@ -2162,14 +2743,16 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { storage_->set_max_near_misses_per_fault(2); for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(10 + i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(10 + i).nanoseconds(), config); } // Arrives third, but carries the earliest timestamp of the three. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[0].occurred_at_ns, nth_report_time(11).nanoseconds()); EXPECT_EQ(series[1].occurred_at_ns, nth_report_time(0).nanoseconds()) @@ -2178,42 +2761,67 @@ TEST_F(SqliteFaultStorageTest, NearMissSeriesUsesArrivalOrderNotTimestamps) { } TEST_F(SqliteFaultStorageTest, NearMissEntriesDescribeTheirOwnReport) { - // Each entry must describe the report that produced it, not the fault's current state, or a - // series spanning a threshold change or a new reporting source reads as if nothing changed. + // Each entry must describe the report that produced it, not the record's current state, or a + // series spanning a threshold change reads as if nothing changed. DebounceConfig first = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), first); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), first); DebounceConfig second = four_strike_config(); second.confirmation_threshold = -8; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "pressure dipping", - "/hydraulics/backup_pump", nth_report_time(1).nanoseconds(), second); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "pressure dipping", "/hydraulics/pump", + nth_report_time(1).nanoseconds(), second); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u); EXPECT_EQ(series[0].confirmation_threshold, -4); EXPECT_EQ(series[0].severity, Fault::SEVERITY_WARN); EXPECT_EQ(series[0].source_id, "/hydraulics/pump"); EXPECT_EQ(series[1].confirmation_threshold, -8); EXPECT_EQ(series[1].severity, Fault::SEVERITY_ERROR); - EXPECT_EQ(series[1].source_id, "/hydraulics/backup_pump"); + EXPECT_EQ(series[1].source_id, "/hydraulics/pump"); +} + +// The series is per RECORD: another source reporting the same code keeps its own, +// so a plot of one reporter's approaches to confirmation is not diluted by another's. +TEST_F(SqliteFaultStorageTest, NearMissSeriesIsPerRecord) { + const auto config = four_strike_config(); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "dip", "/hydraulics/pump", nth_report_time(0).nanoseconds(), + config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_ERROR, "dip", "/hydraulics/backup_pump", + nth_report_time(1).nanoseconds(), config); + + const auto pump = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); + const auto backup = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/backup_pump"}); + ASSERT_EQ(pump.size(), 1u); + ASSERT_EQ(backup.size(), 1u); + EXPECT_EQ(pump.front().source_id, "/hydraulics/pump"); + EXPECT_EQ(pump.front().severity, Fault::SEVERITY_WARN); + EXPECT_EQ(backup.front().source_id, "/hydraulics/backup_pump"); + EXPECT_EQ(backup.front().severity, Fault::SEVERITY_ERROR); } TEST_F(SqliteFaultStorageTest, NearMissSeriesContinuesAfterReopen) { const auto config = four_strike_config(); for (int i = 0; i < 2; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } storage_.reset(); storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(2).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(2).nanoseconds(), config); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 3u) << "a restart must extend the series, not restart or reorder it"; EXPECT_EQ(series[0].debounce_counter, -1); EXPECT_EQ(series[1].debounce_counter, -2); @@ -2230,20 +2838,23 @@ TEST_F(SqliteFaultStorageTest, ApplyingSmallerBoundTrimsExistingSeries) { storage_->set_max_near_misses_per_fault(0); for (int i = 0; i < 5; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + nth_report_time(i).nanoseconds(), config); } - ASSERT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 5u); + ASSERT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 5u); storage_->set_max_near_misses_per_fault(2); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 2u) << "applying the bound left the stored series over it"; EXPECT_EQ(series[0].debounce_counter, -4); EXPECT_EQ(series[1].debounce_counter, -5); - EXPECT_EQ(storage_->get_near_misses("MOTOR_OVERHEAT").size(), 2u) << "the bound is applied per fault code"; + EXPECT_EQ(storage_->get_near_misses({"MOTOR_OVERHEAT", "/powertrain/motor"}).size(), 2u) + << "the bound is applied per record"; } TEST_F(SqliteFaultStorageTest, ApplyingUnlimitedBoundKeepsExistingSeries) { @@ -2252,13 +2863,14 @@ TEST_F(SqliteFaultStorageTest, ApplyingUnlimitedBoundKeepsExistingSeries) { storage_->set_max_near_misses_per_fault(4); for (int i = 0; i < 4; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } storage_->set_max_near_misses_per_fault(0); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 4u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 4u); } TEST_F(SqliteFaultStorageTest, UnlimitedBoundSpeltAsSizeMaxKeepsTheSeries) { @@ -2268,16 +2880,18 @@ TEST_F(SqliteFaultStorageTest, UnlimitedBoundSpeltAsSizeMaxKeepsTheSeries) { config.confirmation_threshold = -20; for (int i = 0; i < 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } EXPECT_EQ(storage_->set_max_near_misses_per_fault(std::numeric_limits::max()), 0u); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 3u); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 3u); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(3).nanoseconds(), config); - EXPECT_EQ(storage_->get_near_misses("PUMP_PRESSURE_LOW").size(), 4u); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(3).nanoseconds(), config); + EXPECT_EQ(storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}).size(), 4u); } TEST_F(SqliteFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { @@ -2286,13 +2900,15 @@ TEST_F(SqliteFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { storage_->set_max_near_misses_per_fault(0); for (int i = 0; i < 5; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); - storage_->report_fault_event("MOTOR_OVERHEAT", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "temperature rising", - "/powertrain/motor", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"MOTOR_OVERHEAT", "/powertrain/motor"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "temperature rising", "/powertrain/motor", + nth_report_time(i).nanoseconds(), config); } - // Two codes, five entries each, bound of 2: three dropped per code. + // Two records, five entries each, bound of 2: three dropped per record. EXPECT_EQ(storage_->set_max_near_misses_per_fault(2), 6u); // Applying the same bound again has nothing left to drop. EXPECT_EQ(storage_->set_max_near_misses_per_fault(2), 0u); @@ -2301,10 +2917,11 @@ TEST_F(SqliteFaultStorageTest, ApplyingBoundReportsHowManyEntriesItDropped) { TEST_F(SqliteFaultStorageTest, PassedReportOnUnknownFaultWritesNothing) { // A heal heartbeat for a fault that does not exist must stay a read: it writes no row and must // not take the writer lock on the way to finding that out. - EXPECT_FALSE(storage_->report_fault_event("NEVER_REPORTED", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/test_node", nth_report_time(0).nanoseconds(), default_config())); + EXPECT_FALSE(storage_->report_fault_event(FaultId{"NEVER_REPORTED", "/test_node"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/test_node", nth_report_time(0).nanoseconds(), + default_config())); EXPECT_EQ(storage_->size(), 0u); - EXPECT_TRUE(storage_->get_near_misses("NEVER_REPORTED").empty()); + EXPECT_TRUE(storage_->get_near_misses({"NEVER_REPORTED", "/test_node"}).empty()); } TEST_F(SqliteFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) { @@ -2316,13 +2933,15 @@ TEST_F(SqliteFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) config.healing_threshold = 2; // An approach that recedes: counter moves, nothing is latched. - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); for (int i = 1; i <= 3; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kPassed, Fault::SEVERITY_WARN, "", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kPassed, + Fault::SEVERITY_WARN, "", "/hydraulics/pump", nth_report_time(i).nanoseconds(), + config); } - auto healed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto healed = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; @@ -2330,14 +2949,15 @@ TEST_F(SqliteFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) // From the healing threshold (+2) it takes six FAILED reports to reach the confirmation // threshold (-4). for (int i = 4; i <= 9; ++i) { - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(i).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(i).nanoseconds(), config); } - auto confirmed = storage_->get_fault("PUMP_PRESSURE_LOW"); + auto confirmed = storage_->get_fault({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_TRUE(confirmed.has_value()); ASSERT_EQ(confirmed->status, Fault::STATUS_CONFIRMED) << "test setup: the ramp must end in a real fault"; - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_FALSE(series.empty()); EXPECT_EQ(series[0].resulting_status, Fault::STATUS_PREFAILED) << "the first report was a genuine approach"; @@ -2353,13 +2973,14 @@ TEST_F(SqliteFaultStorageTest, NearMissSeparatesApproachFromRampBackIntoAFault) TEST_F(SqliteFaultStorageTest, NearMissResultingStatusSurvivesReopen) { const auto config = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); storage_.reset(); storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 1u); EXPECT_EQ(series[0].resulting_status, Fault::STATUS_PREFAILED); } @@ -2367,8 +2988,9 @@ TEST_F(SqliteFaultStorageTest, NearMissResultingStatusSurvivesReopen) { TEST_F(SqliteFaultStorageTest, NearMissResultingStatusColumnAddedToOlderTable) { // A database written by an earlier build of this branch has the table without the column. const auto config = four_strike_config(); - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(0).nanoseconds(), config); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(0).nanoseconds(), config); storage_.reset(); { @@ -2381,19 +3003,20 @@ TEST_F(SqliteFaultStorageTest, NearMissResultingStatusColumnAddedToOlderTable) { storage_ = std::make_unique(temp_db_path_.string(), test_log_sink()); - auto series = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + auto series = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(series.size(), 1u) << "the migration must keep the rows it already had"; EXPECT_TRUE(series[0].resulting_status.empty()) << "an unrecorded latch state reads as empty, not as a status"; - storage_->report_fault_event("PUMP_PRESSURE_LOW", fd::ReportKind::kFailed, Fault::SEVERITY_WARN, "pressure dipping", - "/hydraulics/pump", nth_report_time(1).nanoseconds(), config); - auto extended = storage_->get_near_misses("PUMP_PRESSURE_LOW"); + storage_->report_fault_event(FaultId{"PUMP_PRESSURE_LOW", "/hydraulics/pump"}, fd::ReportKind::kFailed, + Fault::SEVERITY_WARN, "pressure dipping", "/hydraulics/pump", + nth_report_time(1).nanoseconds(), config); + auto extended = storage_->get_near_misses({"PUMP_PRESSURE_LOW", "/hydraulics/pump"}); ASSERT_EQ(extended.size(), 2u); EXPECT_EQ(extended[1].resulting_status, Fault::STATUS_PREFAILED); } TEST_F(SqliteFaultStorageTest, NearMissSeriesEmptyForUnknownFault) { - EXPECT_TRUE(storage_->get_near_misses("NEVER_REPORTED").empty()); + EXPECT_TRUE(storage_->get_near_misses({"NEVER_REPORTED", "/hydraulics/pump"}).empty()); } // --- Snapshot retention through the startup reclassification --- @@ -2407,6 +3030,7 @@ static void store_snapshots_for(ros2_medkit_fault_manager::FaultStorage & storag for (int i = 0; i < count; ++i) { ros2_medkit_fault_manager::SnapshotData snapshot; snapshot.fault_code = fault_code; + snapshot.owner = "/test_node"; snapshot.topic = "/test/topic" + std::to_string(i); snapshot.message_type = "std_msgs/msg/String"; snapshot.data = R"({"data": "value"})"; @@ -2421,13 +3045,13 @@ static void drive_to_healed(ros2_medkit_fault_manager::FaultStorage & storage, c config.healing_enabled = true; config.healing_threshold = 1; - storage.report_fault_event(fault_code, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", "/test_node", - nth_report_time(0).nanoseconds(), config); + storage.report_fault_event(FaultId{fault_code, "/test_node"}, fd::ReportKind::kFailed, Fault::SEVERITY_ERROR, "fault", + "/test_node", nth_report_time(0).nanoseconds(), config); store_snapshots_for(storage, fault_code, 2); // Two PASSED reports: the first only lifts the counter to 0, which the CONFIRMED latch holds. for (int i = 1; i <= 2; ++i) { - storage.report_fault_event(fault_code, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", "/test_node", - nth_report_time(i).nanoseconds(), config); + storage.report_fault_event(FaultId{fault_code, "/test_node"}, fd::ReportKind::kPassed, Fault::SEVERITY_ERROR, "", + "/test_node", nth_report_time(i).nanoseconds(), config); } } @@ -2435,14 +3059,14 @@ TEST_F(SqliteFaultStorageTest, SnapshotsRetainedThroughHealedReclassification) { storage_->set_retain_snapshots_on_clear(true); drive_to_healed(*storage_, "SNAPSHOT_RETAIN_TEST"); - auto healed = storage_->get_fault("SNAPSHOT_RETAIN_TEST"); + auto healed = storage_->get_fault({"SNAPSHOT_RETAIN_TEST", "/test_node"}); ASSERT_TRUE(healed.has_value()); ASSERT_EQ(healed->status, Fault::STATUS_HEALED) << "test setup: the fault must be latched HEALED"; - ASSERT_EQ(storage_->get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u); + ASSERT_EQ(storage_->get_snapshots({"SNAPSHOT_RETAIN_TEST", "/test_node"}).size(), 2u); ASSERT_EQ(storage_->reclassify_healed_as_cleared().size(), 1u); - EXPECT_EQ(storage_->get_snapshots("SNAPSHOT_RETAIN_TEST").size(), 2u) + EXPECT_EQ(storage_->get_snapshots({"SNAPSHOT_RETAIN_TEST", "/test_node"}).size(), 2u) << "the restart reclassification deleted snapshots the configuration asked to keep"; } @@ -2450,7 +3074,7 @@ TEST_F(SqliteFaultStorageTest, SnapshotsDroppedByHealedReclassificationByDefault drive_to_healed(*storage_, "SNAPSHOT_DROP_TEST"); ASSERT_EQ(storage_->reclassify_healed_as_cleared().size(), 1u); - EXPECT_TRUE(storage_->get_snapshots("SNAPSHOT_DROP_TEST").empty()); + EXPECT_TRUE(storage_->get_snapshots({"SNAPSHOT_DROP_TEST", "/test_node"}).empty()); } int main(int argc, char ** argv) { diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index 26a034d71..34ba4ce23 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -1125,7 +1125,7 @@ Snapshots are configured via FaultManager parameters: | `snapshots.max_message_size` | int | `65536` | Maximum message size in bytes (larger messages skipped) | | `snapshots.default_topics` | string[] | `[]` | Topics to capture for all faults | | `snapshots.config_file` | string | `""` | Path to YAML config file for `fault_specific` and `patterns` | -| `snapshots.recapture_cooldown_sec` | double | `60.0` | Min seconds between captures for the same fault code. | +| `snapshots.recapture_cooldown_sec` | double | `60.0` | Min seconds between captures for the same fault record. | | `snapshots.max_per_fault` | int | `10` | Max snapshots retained per fault. | | `snapshots.capture_pool_size` | int | `2` | Max concurrent capture threads under a fault storm (>= 1). Parallelizes snapshot capture only; rosbag stays single-writer (one fault at a time). | | `snapshots.capture_queue_depth` | int | `16` | Max pending captures before the full-queue policy applies (>= 1). | diff --git a/src/ros2_medkit_integration_tests/README.md b/src/ros2_medkit_integration_tests/README.md index 104326cf2..5b14b364d 100644 --- a/src/ros2_medkit_integration_tests/README.md +++ b/src/ros2_medkit_integration_tests/README.md @@ -28,6 +28,7 @@ ros2_medkit_integration_tests/ coverage.py # GCOV_PREFIX helpers for CI gateway_test_case.py # GatewayTestCase base class launch_helpers.py # create_test_launch(), DEMO_NODE_REGISTRY + service_round_trip.py # answer_once_each(): one answer per client before a test acts test/ features/ # Focused API feature tests scenarios/ # End-to-end workflow tests diff --git a/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/service_round_trip.py b/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/service_round_trip.py new file mode 100644 index 000000000..8f0803fbe --- /dev/null +++ b/src/ros2_medkit_integration_tests/ros2_medkit_test_utils/service_round_trip.py @@ -0,0 +1,55 @@ +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +First round trips to a service server that has just started. + +wait_for_service shows that the client found the server. The server can match the +client's reply reader later, and a reply sent before that is lost. +""" + +import time + +import rclpy + +# A request with no answer after this long is sent again. +ANSWER_WAIT_SEC = 1.0 + + +def call_until_answered(node, client, request, timeout_sec, *, spin=True, + answer_wait_sec=ANSWER_WAIT_SEC): + """ + Send until gets an answer and return it; must be repeatable. + + With spin=False another thread spins , and this only waits for the answer. + """ + deadline = time.monotonic() + timeout_sec + while time.monotonic() < deadline: + future = client.call_async(request) + if spin: + rclpy.spin_until_future_complete(node, future, timeout_sec=answer_wait_sec) + else: + settle = time.monotonic() + answer_wait_sec + while not future.done() and time.monotonic() < settle: + time.sleep(0.02) + if future.done(): + return future.result() + client.remove_pending_request(future) + raise AssertionError(f'{client.srv_name} did not answer within {timeout_sec} s') + + +def answer_once_each(node, clients, timeout_sec, *, spin=True): + """Get one answer on each client. The default request names no fault, so it changes nothing.""" + for client in clients: + call_until_answered(node, client, client.srv_type.Request(), timeout_sec, spin=spin) diff --git a/src/ros2_medkit_integration_tests/test/features/test_correlation_root_clear.test.py b/src/ros2_medkit_integration_tests/test/features/test_correlation_root_clear.test.py new file mode 100644 index 000000000..b13929135 --- /dev/null +++ b/src/ros2_medkit_integration_tests/test/features/test_correlation_root_clear.test.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Integration test: clearing a root cause through its app route unmutes its symptoms. + +The gateway's per-entity DELETE sends skip_correlation_auto_clear, so it clears the root cause +alone. The symptom the root cause muted must then show on its own app again, and stay a +CONFIRMED fault. +""" + +import atexit +import os +import tempfile +import time +import unittest + +import launch_testing +import rclpy +from rclpy.node import Node +import requests +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ListFaults, ReportFault + +from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES +from ros2_medkit_test_utils.gateway_test_case import GatewayTestCase +from ros2_medkit_test_utils.launch_helpers import create_test_launch +from ros2_medkit_test_utils.service_round_trip import answer_once_each + +ROOT_CODE = 'LIDAR_RANGE_INVALID' +SYMPTOM_CODE = 'ROOT_CLEAR_SYMPTOM' +SYMPTOM_SOURCE = '/powertrain/engine/temp_sensor' + +CORRELATION = """\ +correlation: + enabled: true + rules: + - id: lidar_cascade + name: "Lidar cascade" + mode: hierarchical + root_cause: + codes: ["LIDAR_RANGE_INVALID"] + symptoms: + - codes: ["ROOT_CLEAR_SYMPTOM"] + window_ms: 60000 + mute_symptoms: true + auto_clear_with_root: true +""" + + +def generate_test_description(): + """Launch gateway, fault manager with a cascade rule, a faulty lidar and a temp sensor.""" + rules = tempfile.NamedTemporaryFile( + 'w', suffix='.yaml', prefix='root_clear_rules_', delete=False) + rules.write(CORRELATION) + rules.close() + atexit.register(os.unlink, rules.name) + return create_test_launch( + demo_nodes=['lidar_sensor', 'temp_sensor'], + gateway_params={}, + fault_manager=True, + fault_manager_params={ + 'confirmation_threshold': -1, + 'correlation.config_file': rules.name, + }, + lidar_faulty=True, + ) + + +class TestCorrelationRootClear(GatewayTestCase): + """DELETE of a root cause on its app route ends its relations.""" + + MIN_EXPECTED_APPS = 2 + REQUIRED_APPS = {'lidar_sensor', 'temp_sensor'} + + @classmethod + def setUpClass(cls): + super().setUpClass() + rclpy.init() + cls._node = Node('root_clear_reporter') + cls._report = cls._node.create_client(ReportFault, '/fault_manager/report_fault') + cls._list = cls._node.create_client(ListFaults, '/fault_manager/list_faults') + assert cls._report.wait_for_service(timeout_sec=30.0) + assert cls._list.wait_for_service(timeout_sec=30.0) + answer_once_each(cls._node, (cls._report, cls._list), 30.0) + + @classmethod + def tearDownClass(cls): + cls._node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self._node, future, timeout_sec=10.0) + self.assertTrue(future.done(), 'service call timed out') + return future.result() + + def _muted_codes(self): + request = ListFaults.Request() + request.include_muted = True + return {m.fault_code for m in self._call(self._list, request).muted_faults} + + def test_deleting_the_root_cause_unmutes_its_symptom(self): + self.wait_for_fault('/apps/lidar_sensor', ROOT_CODE) + + request = ReportFault.Request() + request.fault_code = SYMPTOM_CODE + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = 'symptom of the lidar' + request.source_id = SYMPTOM_SOURCE + self.assertTrue(self._call(self._report, request).accepted) + + deadline = time.monotonic() + 30.0 + while SYMPTOM_CODE not in self._muted_codes(): + self.assertLess(time.monotonic(), deadline, 'the symptom was never muted') + rclpy.spin_once(self._node, timeout_sec=0.2) + resp = requests.get(f'{self.BASE_URL}/apps/temp_sensor/faults', timeout=10) + self.assertNotIn(SYMPTOM_CODE, [f['fault_code'] for f in resp.json().get('items', [])]) + + resp = requests.delete( + f'{self.BASE_URL}/apps/lidar_sensor/faults/{ROOT_CODE}', timeout=10) + self.assertEqual(resp.status_code, 204, resp.text) + + fault = self.wait_for_fault('/apps/temp_sensor', SYMPTOM_CODE, max_wait=30.0) + self.assertEqual(fault['status'], 'CONFIRMED') + self.assertNotIn(SYMPTOM_CODE, self._muted_codes()) + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + + def test_exit_codes(self, proc_info): + """Check all processes exited cleanly.""" + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'{info.process_name} exited with code {info.returncode}', + ) diff --git a/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py b/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py new file mode 100644 index 000000000..bdf2d8af4 --- /dev/null +++ b/src/ros2_medkit_integration_tests/test/features/test_fault_severity_consistency.test.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A fault record shows one severity on every gateway surface. + +Real fault manager (SQLite, confirmation threshold -3), real gateway, reports +through the ReportFault service. Both codes are per_source, so a record belongs +to one owner and the gateway's code-addressed routes reach it. A CRITICAL fault is acknowledged +and then reported again as WARN. The record keeps CRITICAL, so the WARN report +confirms it at once, and GET /faults, the fault detail, the /faults/stream +frame and the payload of an OnChange trigger all carry the same severity and +status. +""" + +import json +import os +import shutil +import tempfile +import threading +import time +import unittest + +import launch_testing +import launch_testing.actions +import rclpy +from rclpy.node import Node +import requests +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ReportFault + +from ros2_medkit_test_utils.constants import ALLOWED_EXIT_CODES, API_BASE_PATH, FAULT_TIMEOUT +from ros2_medkit_test_utils.gateway_test_case import GatewayTestCase +from ros2_medkit_test_utils.launch_helpers import create_test_launch +from ros2_medkit_test_utils.service_round_trip import answer_once_each + + +APP_ID = 'temp_sensor' +OWNER = '/powertrain/engine/temp_sensor' +CODE = 'SEV_CONSISTENCY' +PRIME_CODE = 'SEV_CONSISTENCY_PRIME' +DB_DIR = tempfile.mkdtemp(prefix='medkit_severity_e2e_') +THRESHOLDS_PATH = os.path.join(DB_DIR, 'fault_thresholds.yaml') + + +def generate_test_description(): + with open(THRESHOLDS_PATH, 'w') as f: + for code in (CODE, PRIME_CODE): + f.write(f'{code}:\n per_source: true\n') + return create_test_launch( + demo_nodes=['temp_sensor'], + fault_manager=True, + fault_manager_params={ + 'storage_type': 'sqlite', + 'database_path': os.path.join(DB_DIR, 'faults.db'), + 'confirmation_threshold': -3, + 'fault_thresholds.config_file': THRESHOLDS_PATH, + 'snapshots.rosbag.enabled': False, + }, + gateway_params={ + # Two SSE streams at once: /faults/stream and one trigger stream. + 'sse.max_clients': 4, + 'server.http_thread_pool_size': 10, + # A stream reader only sees its stop flag when a line arrives. + 'sse.keepalive_interval_sec': 1, + }, + ) + + +class SseCollector: + """Collect SSE frames from one stream on a background thread.""" + + def __init__(self, url): + self.frames = [] + self._stop = threading.Event() + self._response = requests.get(url, stream=True, timeout=(5, 120)) + self.status_code = self._response.status_code + self._thread = threading.Thread(target=self._pump, daemon=True) + self._thread.start() + + def _pump(self): + current = {} + try: + for line in self._response.iter_lines(decode_unicode=True): + if self._stop.is_set(): + break + if line is None: + continue + if line == '': + if current: + self.frames.append(current) + current = {} + continue + if line.startswith(':'): + continue + key, _, value = line.partition(':') + current[key.strip()] = value.strip() + except Exception: # noqa: BLE001 - closed socket on teardown + pass + + def payloads(self): + return [json.loads(f['data']) for f in list(self.frames) if 'data' in f] + + def close(self): + self._stop.set() + self._response.close() + self._thread.join(timeout=5) + + +class TestFaultSeverityConsistency(GatewayTestCase): + """Subscribers are told the severity that GET shows.""" + + MIN_EXPECTED_APPS = 1 + REQUIRED_APPS = {APP_ID} + + @classmethod + def setUpClass(cls): + super().setUpClass() + rclpy.init() + cls._node = Node('fault_severity_consistency_reporter') + cls._report_client = cls._node.create_client(ReportFault, '/fault_manager/report_fault') + assert cls._report_client.wait_for_service(timeout_sec=15.0), \ + 'report_fault service not available' + answer_once_each(cls._node, (cls._report_client,), 15.0) + + @classmethod + def tearDownClass(cls): + cls._node.destroy_node() + rclpy.shutdown() + + def _failed(self, code, severity, description): + req = ReportFault.Request() + req.fault_code = code + req.event_type = ReportFault.Request.EVENT_FAILED + req.severity = severity + req.description = description + req.source_id = OWNER + future = self._report_client.call_async(req) + deadline = time.monotonic() + 5.0 + while not future.done() and time.monotonic() < deadline: + rclpy.spin_once(self._node, timeout_sec=0.1) + self.assertTrue(future.done(), 'report_fault timed out') + self.assertTrue(future.result().accepted) + + def _wait_for(self, predicate, what): + deadline = time.monotonic() + FAULT_TIMEOUT + while time.monotonic() < deadline: + found = predicate() + if found is not None: + return found + time.sleep(0.2) + self.fail(f'timed out waiting for {what}') + + def _prime_fault_stream(self, stream): + """Report a sacrificial CRITICAL fault until its frame arrives on ``stream``.""" + deadline = time.monotonic() + FAULT_TIMEOUT + while time.monotonic() < deadline: + self._failed(PRIME_CODE, Fault.SEVERITY_CRITICAL, 'prime') + settle = min(time.monotonic() + 1.0, deadline) + while time.monotonic() < settle: + if any(p.get('fault', {}).get('fault_code') == PRIME_CODE + for p in stream.payloads()): + return + time.sleep(0.1) + self.fail(f'no frame for {PRIME_CODE} on /faults/stream within {FAULT_TIMEOUT}s') + + def _list_item(self): + r = requests.get(f'{self.BASE_URL}/faults', params={'status': 'all'}, timeout=10) + self.assertEqual(r.status_code, 200, r.text) + return next((i for i in r.json()['items'] if i['fault_code'] == CODE), None) + + def _detail(self): + r = requests.get(f'{self.BASE_URL}/apps/{APP_ID}/faults/{CODE}', timeout=10) + self.assertEqual(r.status_code, 200, r.text) + return r.json() + + def _create_trigger(self): + body = { + 'resource': f'{API_BASE_PATH}/apps/{APP_ID}/faults/{CODE}', + 'trigger_condition': {'condition_type': 'OnChange'}, + 'multishot': True, + 'lifetime': 300, + } + r = requests.post(f'{self.BASE_URL}/apps/{APP_ID}/triggers', json=body, timeout=5) + self.assertEqual(r.status_code, 201, r.text) + return r.json() + + def test_one_severity_on_every_surface(self): + trig = self._create_trigger() + trigger_stream = SseCollector( + f'{self.BASE_URL}{trig["event_source"].removeprefix(API_BASE_PATH)}') + fault_stream = SseCollector(f'{self.BASE_URL}/faults/stream') + try: + self._check_surfaces(trigger_stream, fault_stream) + finally: + # Deleting the trigger ends its stream on the server side. + requests.delete(f'{self.BASE_URL}/apps/{APP_ID}/triggers/{trig["id"]}', timeout=5) + trigger_stream.close() + fault_stream.close() + + def _check_surfaces(self, trigger_stream, fault_stream): + self.assertEqual(trigger_stream.status_code, 200) + self.assertEqual(fault_stream.status_code, 200) + self._prime_fault_stream(fault_stream) + + def frame(event, occurrence): + def find(): + for f in list(fault_stream.frames): + if f.get('event') != event or 'data' not in f: + continue + fault = json.loads(f['data'])['fault'] + if fault['fault_code'] == CODE and fault['occurrence_count'] == occurrence: + return fault + return None + return find + + def trigger_payload(event_type, occurrence): + def find(): + for frame_data in trigger_stream.payloads(): + p = frame_data.get('payload', {}) + key = (p.get('event_type'), p.get('occurrence_count')) + if key == (event_type, occurrence): + return p + return None + return find + + # Occurrence 1: CRITICAL confirms at once. The owner acknowledges it. + self._failed(CODE, Fault.SEVERITY_CRITICAL, 'occurrence 1') + self._wait_for(frame('fault_confirmed', 1), 'fault_confirmed of occurrence 1') + r = requests.delete(f'{self.BASE_URL}/apps/{APP_ID}/faults/{CODE}', timeout=10) + self.assertEqual(r.status_code, 204, r.text) + self._wait_for(frame('fault_cleared', 1), 'fault_cleared of occurrence 1') + + # Occurrence 2 is reported as WARN; one report confirms it. + self._failed(CODE, Fault.SEVERITY_WARN, 'occurrence 2') + surfaces = {} + with self.subTest(surface='/faults/stream'): + surfaces['stream'] = self._wait_for( + frame('fault_confirmed', 2), 'fault_confirmed of occurrence 2') + with self.subTest(surface='OnChange trigger'): + surfaces['trigger'] = self._wait_for( + trigger_payload('fault_confirmed', 2), 'trigger event of occurrence 2') + with self.subTest(surface='GET /faults'): + surfaces['list'] = self._list_item() + self.assertIsNotNone(surfaces['list']) + self.assertEqual(surfaces['list']['occurrence_count'], 2) + with self.subTest(surface='fault detail'): + detail = self._detail() + surfaces['detail'] = { + 'severity': detail['item']['severity'], + 'severity_label': detail['x-medkit']['severity_label'], + 'status': detail['x-medkit']['status_raw'], + } + + for name, shown in surfaces.items(): + with self.subTest(agree=name): + self.assertIsNotNone(shown) + self.assertEqual(shown['severity'], Fault.SEVERITY_CRITICAL, shown) + self.assertEqual(shown['severity_label'], 'CRITICAL', shown) + self.assertEqual(shown['status'], 'CONFIRMED', shown) + + # A lower report in the occurrence updates the fault; every surface still shows CRITICAL. + self._failed(CODE, Fault.SEVERITY_ERROR, 'occurrence 2, lower') + updated = self._wait_for(frame('fault_updated', 2), 'fault_updated of occurrence 2') + fired = self._wait_for( + trigger_payload('fault_updated', 2), 'trigger event for the update') + item = self._list_item() + for name, shown in {'stream': updated, 'trigger': fired, 'list': item}.items(): + with self.subTest(after_update=name): + self.assertEqual(shown['severity'], Fault.SEVERITY_CRITICAL, shown) + self.assertEqual(shown['description'], 'occurrence 2, lower', shown) + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + + def test_exit_codes(self, proc_info): + shutil.rmtree(DB_DIR, ignore_errors=True) + for info in proc_info: + self.assertIn( + info.returncode, ALLOWED_EXIT_CODES, + f'{info.process_name} exited with code {info.returncode}' + ) diff --git a/src/ros2_medkit_msgs/README.md b/src/ros2_medkit_msgs/README.md index fe8cf92a7..04c940470 100644 --- a/src/ros2_medkit_msgs/README.md +++ b/src/ros2_medkit_msgs/README.md @@ -14,7 +14,18 @@ This package provides the interface definitions used by the fault management com ### Fault.msg -Core fault data model representing an aggregated fault condition with AUTOSAR DEM-style debounce filtering. +Core fault data model representing one fault record with AUTOSAR DEM-style debounce filtering. + +A record is identified by the pair (`fault_code`, owner). A code listed with `per_source: true` +in `fault_thresholds.yaml` has one record per reporting source, owned by the `source_id` of the +`ReportFault` call. Every other code has one record shared by all its sources, owned by the +reserved id `*`, which `ReportFault` refuses as a `source_id`. Each record has its own status, +debounce counter, `occurrence_count`, severity and timestamps, and is cleared on its own. The +services that act on a single record (`ClearFault`, `GetFault`, `GetSnapshots`, `GetRosbag`) carry +a `source_id` request field. It names the owner (`*` for a shared record). A reporting source that +owns no record reaches the record it reports into. An empty `source_id` is unscoped: the call +applies only when exactly one record carries the `fault_code`, and fails as ambiguous when +several do. | Field | Type | Description | |-------|------|-------------| @@ -26,7 +37,7 @@ Core fault data model representing an aggregated fault condition with AUTOSAR DE | `last_passed` | builtin_interfaces/Time | When fault last reported PASSED (zero = never) | | `occurrence_count` | uint32 | Times this fault has occurred, counted on edges (first FAILED, then each FAILED that arrives while CLEARED). Repeats within one occurrence do not increment it | | `status` | string | Current status (see STATUS_* constants) | -| `reporting_sources` | string[] | List of source identifiers that reported this fault | +| `reporting_sources` | string[] | Every source that reported this record: one for a per-source record, all of them for a shared one | **Severity Levels:** | Constant | Value | Description | @@ -121,12 +132,14 @@ Query faults with optional filtering. ### ClearFault.srv -Clear/acknowledge a fault. Cleared faults are retained and queryable with `statuses=["CLEARED"]`. +Clear/acknowledge one fault record. Cleared records are retained and queryable with +`statuses=["CLEARED"]`. **Request:** | Field | Type | Description | |-------|------|-------------| -| `fault_code` | string | The fault to clear | +| `fault_code` | string | The fault code to clear | +| `source_id` | string | Owner of the record (`*` for a shared record), or a source that reports into it. Empty means unscoped: the call applies only when exactly one record carries the `fault_code`, otherwise it fails with a message beginning `ambiguous:` and clears nothing | | `skip_correlation_auto_clear` | bool | When `true`, only the requested fault_code is cleared; symptom faults that the correlation engine would normally auto-clear via `auto_clear_with_root` rules are left untouched. Default `false` (cascade clear). The gateway sets this to `true` on per-entity `DELETE /{entity-path}/faults/{fault_code}` so that an operator with access to one entity cannot cascade-clear correlated symptoms reported by apps in other entities. | **Response:** @@ -136,7 +149,16 @@ Clear/acknowledge a fault. Cleared faults are retained and queryable with `statu | `message` | string | Status or error message | | `auto_cleared_codes` | string[] | Symptom fault codes auto-cleared with the root cause (empty when `skip_correlation_auto_clear=true`) | -> **Note:** `skip_correlation_auto_clear` was added in `ros2_medkit_msgs` post-0.4.0. Adding a request field changes the service type hash, so out-of-tree callers that invoke `/fault_manager/clear_fault` directly (via `ros2 service call` or a generated client) must rebuild against the new `ros2_medkit_msgs` release to keep talking to `fault_manager`. +> **Note:** `skip_correlation_auto_clear` was added in `ros2_medkit_msgs` post-0.4.0, and `source_id` +> after it. `source_id` was added the same way to `GetFault`, `GetSnapshots` and `GetRosbag`. Adding a +> field changes the type hash, and a field added inside an array element changes the wire layout, so +> every client of the fault manager's services and topics must be rebuilt against this +> `ros2_medkit_msgs`. This includes out-of-tree callers that use `ros2 service call` or a generated +> client. The case that breaks without an error is `ListFaults`: `MutedFaultInfo.source_id` and +> `ClusterInfo.fault_owners` / `representative_owner` sit inside the elements of the `muted_faults` +> and `clusters` arrays, CDR has no per-element length, and a client built against an earlier +> `ros2_medkit_msgs` misreads any response that carries a muted or clustered entry. Trailing fields +> do not keep old clients working. ## Usage diff --git a/src/ros2_medkit_msgs/msg/ClusterInfo.msg b/src/ros2_medkit_msgs/msg/ClusterInfo.msg index d95fb9c2d..ce92abf6c 100644 --- a/src/ros2_medkit_msgs/msg/ClusterInfo.msg +++ b/src/ros2_medkit_msgs/msg/ClusterInfo.msg @@ -47,3 +47,11 @@ builtin_interfaces/Time first_at # Timestamp of the last fault in the cluster builtin_interfaces/Time last_at + +# Owner of each member record, one entry per fault_codes entry in the same order. A cluster holds +# the matching records of any owner, so (fault_codes[i], fault_owners[i]) names a record: pass the +# owner as source_id to the single-record services. +string[] fault_owners + +# Owner of the representative record: (representative_code, representative_owner) names it. +string representative_owner diff --git a/src/ros2_medkit_msgs/msg/Fault.msg b/src/ros2_medkit_msgs/msg/Fault.msg index aa24ec629..ae36bbba6 100644 --- a/src/ros2_medkit_msgs/msg/Fault.msg +++ b/src/ros2_medkit_msgs/msg/Fault.msg @@ -14,16 +14,19 @@ # # Fault.msg - Core fault data model for ros2_medkit fault management system. # -# A Fault represents an aggregated fault condition identified by a global fault_code. -# Multiple sources can report the same fault_code, and they are aggregated into a -# single Fault with tracked occurrence_count and reporting_sources. +# A Fault is one fault record, identified by the pair (fault_code, owner). A code listed +# with per_source: true in fault_thresholds.yaml has one record per reporting source, +# owned by the source_id of the ReportFault call. Every other code has one record shared +# by all its sources, owned by the reserved id "*". Each record has its own status, +# debounce counter, occurrence_count, severity and timestamps, and is cleared on its own. # # Debounce model (AUTOSAR DEM-style): # - FAILED events decrement internal counter (towards confirmation) # - PASSED events increment internal counter (towards healing) # - Status reflects the current debounce state -# Global fault identifier (e.g., "MOTOR_OVERHEAT", "SENSOR_FAILURE_001") +# Global fault identifier (e.g., "MOTOR_OVERHEAT", "SENSOR_FAILURE_001"). +# Half of the record identity: the other half is the owner. string fault_code # Fault severity level (use SEVERITY_* constants) @@ -53,9 +56,15 @@ uint32 occurrence_count # Current fault status (use STATUS_* constants) string status -# List of source identifiers that have reported this fault +# Every source that reported this record: one for a per-source record, all of them for a +# shared one. Services that act on a single record take a source_id request field naming +# the owner, or any source listed here. string[] reporting_sources +# Owner of the record this Fault describes: the reporting source of a per-source record, +# "*" for a shared record. The source_id that addresses exactly this record. +string owner + # Severity level constants # Informational message: No action required, used for logging or tracking. uint8 SEVERITY_INFO = 0 diff --git a/src/ros2_medkit_msgs/msg/FaultEvent.msg b/src/ros2_medkit_msgs/msg/FaultEvent.msg index 40cebe6bb..c1afbe37d 100644 --- a/src/ros2_medkit_msgs/msg/FaultEvent.msg +++ b/src/ros2_medkit_msgs/msg/FaultEvent.msg @@ -35,7 +35,8 @@ string EVENT_CONFIRMED = "fault_confirmed" # the two apart. string EVENT_CLEARED = "fault_cleared" # Emitted when fault data changes without status transition (e.g., last_occurred -# advanced, severity escalated, new source added to reporting_sources). +# advanced, severity escalated, new source added to reporting_sources). A report from a +# source of a per_source code that owns no record yet creates a record of its own instead. # occurrence_count is not one of them: it only changes on an occurrence edge # (the first FAILED, or a FAILED that arrives while the fault is CLEARED). string EVENT_UPDATED = "fault_updated" diff --git a/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg b/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg index f5fa8e044..7a664c601 100644 --- a/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg +++ b/src/ros2_medkit_msgs/msg/MutedFaultInfo.msg @@ -12,11 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# MutedFaultInfo.msg - Information about a muted (correlated) fault. +# MutedFaultInfo.msg - Information about a muted (correlated) fault record. # -# When fault correlation is enabled, symptom faults are muted (not published +# When fault correlation is enabled, symptom records are muted (not published # as events) but still stored. This message provides information about -# muted faults when queried with include_muted=true. +# muted records when queried with include_muted=true. +# +# One entry per muted RECORD. A root cause mutes the matching symptom records of every +# owner, so two per-source records muted on one fault_code produce two entries carrying +# that code, told apart by source_id. # The fault code of the muted symptom string fault_code @@ -29,3 +33,8 @@ string rule_id # Time delay from root cause to this symptom [ms] uint32 delay_ms + +# Owner of the muted record: the reporting source for a per_source code, "*" for the +# shared record. Together with fault_code it identifies the one record this entry +# describes, and it is the value the single-record services take to address it. +string source_id diff --git a/src/ros2_medkit_msgs/srv/ClearFault.srv b/src/ros2_medkit_msgs/srv/ClearFault.srv index cfa0b778f..d5a52f5f9 100644 --- a/src/ros2_medkit_msgs/srv/ClearFault.srv +++ b/src/ros2_medkit_msgs/srv/ClearFault.srv @@ -14,8 +14,10 @@ # # ClearFault.srv - Clear/acknowledge a fault in the FaultManager. # -# Marks the specified fault as CLEARED. The fault record is retained for historical -# purposes and can be retrieved by calling ListFaults with statuses=["CLEARED"]. +# Marks one fault record as CLEARED. A record is identified by (fault_code, owner), so +# clearing one record never touches another record of the same code. +# The record is retained for historical purposes and can be retrieved by calling +# ListFaults with statuses=["CLEARED"]. # Cleared faults are retained indefinitely unless storage cleanup is configured. # A FaultEvent with EVENT_CLEARED is published when a fault is successfully cleared. @@ -29,6 +31,13 @@ string fault_code # The fault_code to clear # Set this from scoped per-entity DELETE routes so a viewer of one entity # cannot cascade-clear symptom faults reported by apps in other entities. bool skip_correlation_auto_clear + +# Which record. Set it to the owner of the record: the reporting source for a per_source +# code, or "*" for the record that every source of any other code shares. A reporting source +# that owns no record reaches the record it reports into, so a client that knows only the +# source it reports as still finds the shared record. Empty: the call applies only when +# exactly one record carries fault_code, otherwise it fails as ambiguous. +string source_id --- # Response fields bool success # True if the fault was found and cleared diff --git a/src/ros2_medkit_msgs/srv/GetFault.srv b/src/ros2_medkit_msgs/srv/GetFault.srv index 337c2b5f6..d0e869e04 100644 --- a/src/ros2_medkit_msgs/srv/GetFault.srv +++ b/src/ros2_medkit_msgs/srv/GetFault.srv @@ -15,13 +15,21 @@ # GetFault.srv - Get single fault by code with environment data. # # Unlike ListFaults.srv (plural) which returns a list of faults for filtering, -# this service returns a single fault with full environment data including +# this service returns a single fault record with full environment data including # snapshots and extended data records for SOVD-compliant fault responses. +# A record is identified by (fault_code, owner). # Request fields # Fault code to retrieve (e.g., "MOTOR_OVERHEAT") string fault_code + +# Which record. Set it to the owner of the record: the reporting source for a per_source +# code, or "*" for the record that every source of any other code shares. A reporting source +# that owns no record reaches the record it reports into, so a client that knows only the +# source it reports as still finds the shared record. Empty: the call applies only when +# exactly one record carries fault_code, otherwise it fails as ambiguous. +string source_id --- # Response fields diff --git a/src/ros2_medkit_msgs/srv/GetRosbag.srv b/src/ros2_medkit_msgs/srv/GetRosbag.srv index 76ef27cc0..481595076 100644 --- a/src/ros2_medkit_msgs/srv/GetRosbag.srv +++ b/src/ros2_medkit_msgs/srv/GetRosbag.srv @@ -14,7 +14,8 @@ # # GetRosbag.srv - Retrieve rosbag file information for a fault. # -# Returns the path to the rosbag file that was recorded when a fault was confirmed. +# Returns the path to the rosbag file that was recorded when a fault record was +# confirmed. A recording is linked per record, identified by (fault_code, owner). # The rosbag contains time-window recording of configured topics before and after # the fault confirmation event. @@ -29,6 +30,14 @@ string fault_code # fault_code alone keeps its historical meaning, "the newest recording of this # fault". string recording_id + +# Which record. Set it to the owner of the record: the reporting source for a per_source +# code, or "*" for the record that every source of any other code shares. A reporting source +# that owns no record reaches the record it reports into, so a client that knows only the +# source it reports as still finds the shared record. Empty: the call applies only when +# exactly one record carries fault_code, otherwise it fails as ambiguous. +# It scopes the fault_code lookup only. The recording_id path ignores it. +string source_id --- # Response fields @@ -43,8 +52,9 @@ string file_path # as the bulk-data id. string recording_id -# Every fault this recording covers - a burst of correlated faults shares one bag. -# Callers authorize a download against this set, not against a single code. +# Every fault record this recording covers - a burst of correlated faults shares one bag. +# One entry per record, so a code appears once per owner that holds it. Callers authorize a +# download against this set, not against a single code. owners is parallel to this array. string[] fault_codes # Storage format: "sqlite3" or "mcap" @@ -61,3 +71,8 @@ uint64 size_bytes # Error message if success is false. string error_message + +# Owner of each record in fault_codes, one entry per fault_codes entry in the same order: +# the reporting source for a per_source code, "*" for the record that every source of any +# other code shares. +string[] owners diff --git a/src/ros2_medkit_msgs/srv/GetSnapshots.srv b/src/ros2_medkit_msgs/srv/GetSnapshots.srv index de1bf5f71..e39089497 100644 --- a/src/ros2_medkit_msgs/srv/GetSnapshots.srv +++ b/src/ros2_medkit_msgs/srv/GetSnapshots.srv @@ -14,7 +14,8 @@ # # GetSnapshots.srv - Retrieve topic snapshots captured when a fault was confirmed. # -# Returns topic data that was captured at the moment a fault transitioned to CONFIRMED. +# Returns topic data that was captured at the moment a fault record transitioned to +# CONFIRMED. Snapshots belong to one record, identified by (fault_code, owner). # Snapshots provide debugging context by preserving system state at fault time. # Request fields @@ -25,6 +26,13 @@ string fault_code # Optional topic filter. If empty, returns snapshots for all captured topics. # If specified, returns only the snapshot for that specific topic. string topic + +# Which record. Set it to the owner of the record: the reporting source for a per_source +# code, or "*" for the record that every source of any other code shares. A reporting source +# that owns no record reaches the record it reports into, so a client that knows only the +# source it reports as still finds the shared record. Empty: the call applies only when +# exactly one record carries fault_code, otherwise it fails as ambiguous. +string source_id --- # Response fields diff --git a/src/ros2_medkit_msgs/srv/ReportFault.srv b/src/ros2_medkit_msgs/srv/ReportFault.srv index c6403ca9f..b38a77e7f 100644 --- a/src/ros2_medkit_msgs/srv/ReportFault.srv +++ b/src/ros2_medkit_msgs/srv/ReportFault.srv @@ -15,14 +15,16 @@ # ReportFault.srv - Report a fault event to the FaultManager. # # Called by FaultReporter clients or directly by nodes to report fault conditions. -# The FaultManager aggregates reports by fault_code across all sources and applies -# debounce filtering based on FAILED/PASSED event counts. +# The FaultManager keeps one record per fault_code, shared by all sources, or one record per +# (fault_code, source_id) pair for a code configured per_source in fault_thresholds.yaml. It +# applies debounce filtering per record, based on FAILED/PASSED event counts. # Request fields # Global fault identifier. Convention: UPPER_SNAKE_CASE (e.g., "MOTOR_OVERHEAT"). # Recommended max length: 64 characters. Allowed: A-Z, 0-9, underscore. -# Same fault_code from different sources = single aggregated fault. +# Same fault_code from different sources = one shared record, or one record each (filtered +# independently) for a per_source code. string fault_code # Event type indicating whether the fault condition is detected or cleared. @@ -47,7 +49,10 @@ string description # Identifier of the reporting node/entity. Recommended: fully qualified ROS 2 node name # including namespace (e.g., "/powertrain/engine/temp_sensor"). -# This value is added to the fault's reporting_sources array for multi-source tracking. +# It is added to the reporting_sources of the record the report goes into, and it owns that +# record when the code is configured per_source. +# Required: a report with an empty source_id is rejected, and so is the reserved id "*", +# which names the record that every source of a code shares. string source_id --- # Response fields diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/CMakeLists.txt b/src/ros2_medkit_plugins/ros2_medkit_opcua/CMakeLists.txt index 3adf51951..c3913e92f 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/CMakeLists.txt +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/CMakeLists.txt @@ -645,8 +645,13 @@ if(BUILD_TESTING) test/integration/test_opcua_subscription_values.test.py test/integration/test_opcua_raise_qualifier.test.py test/integration/test_opcua_comms_lost.test.py + test/integration/test_opcua_per_source_clear.test.py test/integration/test_opcua_startup_sweep_signal.test.py DESTINATION lib/${PROJECT_NAME}) + # The fault manager client the installed scripts import from their directory. + install(FILES + test/integration/fault_manager_client.py + DESTINATION lib/${PROJECT_NAME}) # ---- Read-only / write-capable behaviour over real HTTP ---------------- # The build-inspection test proves the write symbols are gone; this proves @@ -718,6 +723,15 @@ if(BUILD_TESTING) RESOURCE_LOCK opcua_tcp_4840 TIMEOUT 1200) + # A device alarm of a per-source code clears the gateway's own record only. + medkit_add_wrapped_test(test_opcua_per_source_clear + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/test/integration/test_opcua_per_source_clear.test.py" + "${CMAKE_BINARY_DIR}/test_alarm_server") + set_tests_properties(test_opcua_per_source_clear PROPERTIES + LABELS "integration" + TIMEOUT 300) + # A signal during the startup discovery sweep ends the sweep and the gateway. # Two cases, each bounded by its own 60 s start and 21 s exit deadlines. # The silent listener binds TCP 4840. diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md index 4c580db38..09c033313 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md @@ -688,6 +688,7 @@ ros2_medkit_gateway: | `auto_browse` | `false` | Boolean or object; recursively discover the SOVD tree from the live address space instead of (or in addition to) `node_map_path` - see above | | `auto_alarms` | `false` | Boolean or object; subscribe a server EventNotifier and auto-derive a fault per AlarmCondition with no per-alarm `event_alarms` mapping - the zero-config native A&C surface (see above) | | `poll_interval_ms` | `1000` | Polling interval in ms (clamped to [100, 60000]) | +| `reconnect_interval_ms` | `5000` | Wait after the first failed connect while the server is down. Each further failure doubles it, up to 60 s, or up to the discovery `interval_s` while discovery re-scans, but never below this value (out of range [100, 60000] ms or not an integer is refused with a warning and the default is kept) | | `prefer_subscriptions` | `false` | Use OPC-UA subscriptions instead of polling | | `subscription_interval_ms` | `500` | Publishing interval for OPC-UA subscriptions when `prefer_subscriptions: true`. Between two polls the client is also iterated at this interval (at least 20 ms apart, at most once per poll), so changes arrive at this cadence and not once per `poll_interval_ms` | | `condition_replay_strategy` | `auto` | Active-condition replay on reconnect: `method`, `read`, `auto`, `off` (see below) | diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp index 419b12020..bd02bf9e0 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_plugin.hpp @@ -561,10 +561,13 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, } // Build the ClearFault request for one fault code. + // ``source_id`` is the entity id the fault is reported under: it picks the + // record when more than one source reported the code. // ``skip_correlation_auto_clear`` goes on the wire verbatim (see ClearOrigin - // for who sets it and why). Static so the wire field is assertable without a + // for who sets it and why). Static so the wire fields are assertable without a // fault manager. static ros2_medkit_msgs::srv::ClearFault::Request make_clear_fault_request(const std::string & fault_code, + const std::string & source_id, bool skip_correlation_auto_clear); // One entry in the bounded buffer of fault dispatches held while the @@ -644,9 +647,11 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // Report/clear fault via ROS 2 service (private helpers, not the FaultProvider overrides) void send_report_fault(const std::string & entity_id, const std::string & fault_code, const std::string & severity_str, const std::string & message); + // ``source_id`` is the entity id the fault is reported under. // ``origin`` says why the clear is being sent, which decides both the wire // flag and how the pending buffer ranks it. See ClearOrigin. - void send_clear_fault(const std::string & fault_code, ClearOrigin origin = ClearOrigin::DeviceAlarm); + void send_clear_fault(const std::string & fault_code, const std::string & source_id, + ClearOrigin origin = ClearOrigin::DeviceAlarm); // Record that a link-state decision about PLC_COMMS_LOST is due. // @@ -670,7 +675,9 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // Apply the gate to an answer the store gave and send the clear it permits. // Poll thread only. - void apply_comms_lost_answer(bool found, const std::vector & reporting_sources); + // ``clear_source`` is the id the probe that found the record was sent under. + void apply_comms_lost_answer(bool found, const std::vector & reporting_sources, + const std::string & clear_source); // The ids a fault reported by THIS process can carry: the component id being // served, every placeholder this process assigned, and the NodeMap default. @@ -988,6 +995,21 @@ class OpcuaPlugin : public ros2_medkit_gateway::GatewayPlugin, // on the wire, so one further decision follows any that had a raise behind // it. Set on the poll thread, taken on the poll thread. std::atomic comms_lost_raise_since_decision_{false}; + // Ids this process reports under that the current decision has not probed yet, and the id the + // outstanding probe was sent under. A per-source record belongs to the id it was raised under, which + // after a restart is not the one the plugin remembers. Poll thread only. + std::vector comms_lost_untried_ids_; + std::string comms_lost_probe_id_; + + // The entity id this process last reported PLC_COMMS_LOST under, empty once that fault was cleared. + // The component can be renamed while the fault stands, and a per-source record belongs to the id it + // was reported under, so the probe and the clear use this id and not the current one. + mutable std::mutex comms_lost_entity_mutex_; + std::string comms_lost_entity_id_; + + // The id the probe and the clear of PLC_COMMS_LOST name: the one it was reported under, else the + // current component id. + std::string comms_lost_entity_id() const; // How long a probe may stay outstanding (kDefaultFaultServiceTimeout unless // ``fault_service_timeout_ms`` says otherwise). std::chrono::milliseconds fault_service_timeout_{kDefaultFaultServiceTimeout}; diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp index 9c6c617a3..373755c63 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/include/ros2_medkit_opcua/opcua_poller.hpp @@ -104,6 +104,7 @@ struct PollerConfig { bool prefer_subscriptions{false}; // poll mode by default (subscriptions need event loop) double subscription_interval_ms{500.0}; std::chrono::milliseconds poll_interval{1000}; + /// First wait after a failed connect. Config: plugins.opcua.reconnect_interval_ms. std::chrono::milliseconds reconnect_interval{5000}; /// Ceiling for the exponential reconnect backoff (it doubles from /// ``reconnect_interval`` up to this). A plugin whose reconnect arm also diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp index 226819639..81fe57d2e 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_plugin.cpp @@ -331,6 +331,16 @@ void OpcuaPlugin::configure(const nlohmann::json & config) { auto ms = std::clamp(config["poll_interval_ms"].get(), 100, 60000); poller_config_.poll_interval = std::chrono::milliseconds(ms); } + if (config.contains("reconnect_interval_ms")) { + const auto & raw = config["reconnect_interval_ms"]; + const auto ceiling = poller_config_.max_reconnect_interval.count(); + if (raw.is_number_integer() && raw.get() >= 100 && raw.get() <= ceiling) { + poller_config_.reconnect_interval = std::chrono::milliseconds(raw.get()); + } else { + log_warn("plugins.opcua.reconnect_interval_ms must be an integer in [100, " + std::to_string(ceiling) + + "] - keeping " + std::to_string(poller_config_.reconnect_interval.count()) + " ms"); + } + } if (config.contains("condition_replay_strategy")) { poller_config_.condition_replay_strategy = OpcuaPoller::parse_replay_strategy(config["condition_replay_strategy"].get()); @@ -883,7 +893,7 @@ void OpcuaPlugin::set_context(PluginContext & context) { } poller_->start(poller_config_); log_info("OPC-UA poller started (mode: " + std::string(poller_->using_subscriptions() ? "subscription" : "poll") + - ")"); + ", first reconnect wait " + std::to_string(poller_config_.reconnect_interval.count()) + " ms)"); } std::vector OpcuaPlugin::get_routes() { @@ -1324,6 +1334,8 @@ void OpcuaPlugin::on_alarm_change(const std::string & entity_id, // before it has processed this report, so the decision that follows earns // one more (see comms_lost_raise_since_decision_). comms_lost_raise_since_decision_.store(true); + std::lock_guard lock(comms_lost_entity_mutex_); + comms_lost_entity_id_ = entity_id; } send_report_fault(entity_id, signal.fault_code, signal.severity, signal.message); } else { @@ -1338,7 +1350,7 @@ void OpcuaPlugin::on_alarm_change(const std::string & entity_id, if (clear_origin_for_signal(signal.fault_code) == ClearOrigin::LinkState) { owe_comms_lost_decision(); } else { - send_clear_fault(signal.fault_code, ClearOrigin::DeviceAlarm); + send_clear_fault(signal.fault_code, entity_id, ClearOrigin::DeviceAlarm); } } } @@ -1522,7 +1534,7 @@ void OpcuaPlugin::on_event_alarm(const AlarmEventDelivery & delivery) { // this IS a resolution at the source and the correlation engine may act on // it. DeviceAlarm is also the default, spelled out here because this is // the one call site where the cascade is deliberately kept. - send_clear_fault(delivery.fault_code, ClearOrigin::DeviceAlarm); + send_clear_fault(delivery.fault_code, delivery.entity_id, ClearOrigin::DeviceAlarm); break; case AlarmAction::NoOp: break; @@ -1563,21 +1575,23 @@ void OpcuaPlugin::send_report_fault(const std::string & entity_id, const std::st } ros2_medkit_msgs::srv::ClearFault::Request OpcuaPlugin::make_clear_fault_request(const std::string & fault_code, + const std::string & source_id, bool skip_correlation_auto_clear) { ros2_medkit_msgs::srv::ClearFault::Request request; request.fault_code = fault_code; + request.source_id = source_id; request.skip_correlation_auto_clear = skip_correlation_auto_clear; return request; } -void OpcuaPlugin::send_clear_fault(const std::string & fault_code, ClearOrigin origin) { +void OpcuaPlugin::send_clear_fault(const std::string & fault_code, const std::string & source_id, ClearOrigin origin) { if (!fault_clients_->clear) { log_warn("ClearFault service client not available"); return; } auto request = std::make_shared( - make_clear_fault_request(fault_code, clear_skips_correlation(origin))); + make_clear_fault_request(fault_code, source_id, clear_skips_correlation(origin))); send_or_buffer({PendingFaultDispatch::Kind::Clear, fault_code, clear_is_link_state(origin), [this, request]() { fault_clients_->clear->async_send_request(request); @@ -1596,20 +1610,21 @@ bool OpcuaPlugin::link_state_clear_permitted(bool fault_found, const std::vector std::unordered_set OpcuaPlugin::reporting_ids_of_this_process() const { std::unordered_set ids(placeholder_ids_.begin(), placeholder_ids_.end()); - // A node map that names no component_id reports under the NodeMap default, so - // a fault carrying it is this bridge's. - ids.insert(NodeMap::kDefaultComponentId); + // A node map that names no component_id reports under the NodeMap default, which + // component_id() then returns. A map that names one does not report under the default. const std::string current = node_map_.component_id(); if (!current.empty()) { ids.insert(current); } - // A process that never reached its PLC reported under the stand-in of the - // endpoint it started on, or of the one it found and never opened a session - // on. This process derives both from the same configuration. - for (const std::string & endpoint : {start_endpoint_url_, client_ ? client_->endpoint_url() : std::string{}}) { - const std::string stand_in = derive_component_identity(OpcuaClient::DeviceInfo{}, endpoint).id; - if (!stand_in.empty()) { - ids.insert(stand_in); + // A bridge without a node map that never reached its PLC reported under the stand-in of the + // endpoint it started on, or of the one it found and never opened a session on. A bridge with a + // node map reports under its component_id only. + if (node_map_path_.empty()) { + for (const std::string & endpoint : {start_endpoint_url_, client_ ? client_->endpoint_url() : std::string{}}) { + const std::string stand_in = derive_component_identity(OpcuaClient::DeviceInfo{}, endpoint).id; + if (!stand_in.empty()) { + ids.insert(stand_in); + } } } return ids; @@ -1622,9 +1637,11 @@ void OpcuaPlugin::owe_comms_lost_decision() { comms_lost_decision_owed_.store(true); } -void OpcuaPlugin::apply_comms_lost_answer(bool found, const std::vector & reporting_sources) { +void OpcuaPlugin::apply_comms_lost_answer(bool found, const std::vector & reporting_sources, + const std::string & clear_source) { if (!found) { // No row, so nothing to clear and nothing standing to report. + log_info(std::string("no ") + kCommsLostFaultCode + " record held by this bridge; nothing to clear"); return; } const std::unordered_set my_ids = reporting_ids_of_this_process(); @@ -1645,7 +1662,19 @@ void OpcuaPlugin::apply_comms_lost_answer(bool found, const std::vector lock(comms_lost_entity_mutex_); + comms_lost_entity_id_.clear(); +} + +std::string OpcuaPlugin::comms_lost_entity_id() const { + { + std::lock_guard lock(comms_lost_entity_mutex_); + if (!comms_lost_entity_id_.empty()) { + return comms_lost_entity_id_; + } + } + return node_map_.component_id(); } void OpcuaPlugin::drive_comms_lost_decision() { @@ -1679,11 +1708,19 @@ void OpcuaPlugin::drive_comms_lost_decision() { // the store from before an outage that has since been reported. It is // dropped and the decision is taken again on this session. comms_lost_decision_owed_.store(true); + comms_lost_untried_ids_.clear(); RCLCPP_DEBUG(opcua_plugin_logger(), "%s answer belongs to an earlier session; asking again on this one", kCommsLostFaultCode); return; } - apply_comms_lost_answer(found, sources); + if (!found && !comms_lost_untried_ids_.empty()) { + // No record under that id. A per-source record belongs to the id it was raised under, so ask + // under the next id this process reports under before concluding there is nothing to clear. + comms_lost_decision_owed_.store(true); + return; + } + comms_lost_untried_ids_.clear(); + apply_comms_lost_answer(found, sources, comms_lost_probe_id_); // The store may have answered from before a ReportFault this process had // already sent, so a decision with a raise behind it earns one more. if (comms_lost_raise_since_decision_.exchange(false)) { @@ -1710,6 +1747,7 @@ void OpcuaPlugin::drive_comms_lost_decision() { } comms_lost_probe_in_flight_ = false; comms_lost_decision_owed_.store(true); + comms_lost_untried_ids_.clear(); log_warn(std::string("fault store did not answer within ") + std::to_string(fault_service_timeout_.count()) + " ms; the " + kCommsLostFaultCode + " decision stays owed"); } @@ -1727,6 +1765,19 @@ void OpcuaPlugin::drive_comms_lost_decision() { return; } + // The id this probe asks under: the next untried one, else the one the fault is reported under + // now, with every other id this process reports under queued behind it. + if (!comms_lost_untried_ids_.empty()) { + comms_lost_probe_id_ = comms_lost_untried_ids_.back(); + comms_lost_untried_ids_.pop_back(); + } else { + comms_lost_probe_id_ = comms_lost_entity_id(); + std::unordered_set others = reporting_ids_of_this_process(); + others.erase(comms_lost_probe_id_); + comms_lost_untried_ids_.assign(others.begin(), others.end()); + std::sort(comms_lost_untried_ids_.begin(), comms_lost_untried_ids_.end()); + } + uint64_t generation = 0; { std::lock_guard lock(comms_lost_probe_state_->mutex); @@ -1737,6 +1788,9 @@ void OpcuaPlugin::drive_comms_lost_decision() { std::weak_ptr weak_state = comms_lost_probe_state_; auto request = std::make_shared(); request->fault_code = kCommsLostFaultCode; + // The record this bridge reports it under, so a second source that raised the code does not make the + // probe ambiguous. + request->source_id = comms_lost_probe_id_; // rclcpp selects the response-callback overload of async_send_request only for // a callable whose parameter list is exactly (SharedFuture); a const reference // parameter matches no overload. The future is therefore taken by value. @@ -1777,6 +1831,7 @@ void OpcuaPlugin::drive_comms_lost_decision() { // the poll thread has no catch of its own and a lost owe is a fault left // standing for good. comms_lost_decision_owed_.store(true); + comms_lost_untried_ids_.clear(); RCLCPP_WARN(opcua_plugin_logger(), "fault store request failed to go out (%s); the %s decision stays owed", e.what(), kCommsLostFaultCode); } @@ -3304,7 +3359,7 @@ tl::expected OpcuaPlugin::clear_f // apps in other entities, and a plugin-owned entity must not be the hole in // that rule: the request goes through this provider instead, so the same flag // has to be set here. - send_clear_fault(fault_code, ClearOrigin::ScopedOperator); + send_clear_fault(fault_code, entity_id, ClearOrigin::ScopedOperator); return dto::FaultClearResult{ nlohmann::json{{"status", "cleared"}, {"fault_code", fault_code}, {"entity_id", entity_id}}}; } diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/fault_manager_client.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/fault_manager_client.py new file mode 100644 index 000000000..ee8446085 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/fault_manager_client.py @@ -0,0 +1,73 @@ +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A second fault source for the OPC-UA tests: one node in the test process.""" + +import time + +import rclpy +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ListFaults, ReportFault + +ALL_STATUSES = ['PREFAILED', 'PREPASSED', 'CONFIRMED', 'HEALED', 'CLEARED'] + + +class FaultManagerClient: + """ + Reports and lists faults. rclpy takes ROS_DOMAIN_ID from the environment. + + A reply can be lost while a fault manager that just started has not matched this + client, so a call with no answer after is sent again. + """ + + def __init__(self, name, answer_wait_sec, timeout_sec): + rclpy.init() + self.node = rclpy.create_node(name) + self.report_client = self.node.create_client(ReportFault, '/fault_manager/report_fault') + self.list_client = self.node.create_client(ListFaults, '/fault_manager/list_faults') + self.answer_wait_sec = answer_wait_sec + self.timeout_sec = timeout_sec + + def close(self): + self.node.destroy_node() + rclpy.shutdown() + + def call(self, client, request): + """Return the answer to , sending it again while the reply is lost.""" + end = time.monotonic() + self.timeout_sec + while time.monotonic() < end: + if not client.wait_for_service(timeout_sec=self.answer_wait_sec): + continue + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=self.answer_wait_sec) + if future.done(): + return future.result() + client.remove_pending_request(future) + raise AssertionError(f'{client.srv_name} never answered') + + def report(self, code, source): + """Report FAILED from . Return whether the fault manager accepted it.""" + request = ReportFault.Request() + request.fault_code = code + request.event_type = ReportFault.Request.EVENT_FAILED + request.severity = Fault.SEVERITY_ERROR + request.description = 'second source' + request.source_id = source + return self.call(self.report_client, request).accepted + + def faults(self): + """Return every fault the fault manager holds, in any status.""" + request = ListFaults.Request() + request.statuses = ALL_STATUSES + return self.call(self.list_client, request).faults diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py index 6b8c82920..d577491bb 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_comms_lost.test.py @@ -25,9 +25,24 @@ * foreign stand-in: a fault another endpoint's gateway raised stays. * late nameplate: a fault raised under the nameplate id stays while the device serves no nameplate, and is cleared when a later read names the device. + * per-source record: PLC_COMMS_LOST configured with one record per source and + a second source that reported it. The restored link clears this gateway's own + record and leaves the other source's alone. + * per-source record under a stand-in: the gateway raises the fault under the stand-in + of its endpoint, the PLC comes up and names the component, and the same gateway + clears the record it raised under the stand-in. + * per-source pinned stand-in: the same, with the gateway restarted in between, so it remembers + nothing and the record is found under the stand-in derived from its endpoint. + * explicit component id: a gateway whose node map names a component_id leaves the + per-source record a default-id bridge holds under 'opcua_device' alone. + * explicit component id, stand-in record: the same, for a record held under the + endpoint-derived stand-in of a config-less bridge on the same host, per-source and shared. * binding directory: a binding file whose directory cannot be looked up is reported, and the plugin stays connected. Once at start, and once on the poll thread when a rescan adopts a PLC that came up later. + * reconnect interval setting: reconnect_interval_ms at both ends of its range is + used, and a value just outside each end or a non-integer is refused with a + warning, so the default 5000 ms is used. Usage: test_opcua_comms_lost.test.py """ @@ -46,11 +61,15 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'fixtures' / 'test_alarm_server')) import alarm_fixture # noqa: E402 +from fault_manager_client import FaultManagerClient # noqa: E402 COMMS_LOST = 'PLC_COMMS_LOST' HELD_BY_OTHERS = f'{COMMS_LOST} is held by sources this bridge did not report under [' # Discovery identifies OPC-UA only on this port. DISCOVERY_PORT = 4840 +# First reconnect wait of the scenarios that set one. The production default is 5000 ms. +RECONNECT_MS = 200 +CONNECT_FAILED = 'OPC-UA connect to ' def time_scale(): @@ -97,12 +116,12 @@ def http_json(url, timeout=2): return None -def wait_log(path, needle, deadline): - """Poll a log file until appears, for a scaled deadline.""" +def wait_log(path, needle, deadline, count=1): + """Poll a log file until appears times, for a scaled deadline.""" end = time.monotonic() + deadline * SCALE while time.monotonic() < end: try: - if needle in Path(path).read_text(errors='replace'): + if Path(path).read_text(errors='replace').count(needle) >= count: return True except OSError: pass @@ -179,11 +198,12 @@ def __init__(self, proc, port, log): class Run: """The processes of one scenario. stop_all() ends every one of them.""" - def __init__(self, workdir, server_bin, plugin, env): + def __init__(self, workdir, server_bin, plugin, env, client): self.workdir = workdir self.server_bin = server_bin self.plugin = plugin self.env = env + self.client = client self.procs = [] self.logs = {} # Logs of fixtures that printed READY. A failed start printed its own. @@ -228,18 +248,25 @@ def start_server(self, name, *extra, port=None): self.server_logs.append(log) return proc, port - def start_fault_manager(self, name): + def start_fault_manager(self, name, per_source_codes=()): """Start fault_manager_node with memory storage.""" log = self.workdir / f'{name}_fault_manager.log' + args = ['-p', 'storage_type:=memory'] + if per_source_codes: + thresholds = self.workdir / f'{name}_fault_thresholds.yaml' + thresholds.write_text(''.join(f'{code}:\n per_source: true\n' + for code in per_source_codes)) + args += ['-p', f'fault_thresholds.config_file:={thresholds}'] proc = self._start(['ros2', 'run', 'ros2_medkit_fault_manager', 'fault_manager_node', - '--ros-args', '-p', 'storage_type:=memory'], log, env=self.env) + '--ros-args', *args], log, env=self.env) # Logged at the end of the constructor, after the services exist. if not wait_log(log, 'FaultManager node started (', 30) or proc.poll() is not None: raise AssertionError(f'fault_manager_node did not start (rc={proc.poll()}):\n' + log.read_text(errors='replace')) return proc - def start_gateway(self, name, endpoint=None, poll_ms=200, extra_env=None): + def start_gateway(self, name, endpoint=None, poll_ms=200, extra_env=None, node_map=None, + reconnect_ms=None): """Start gateway_node with the opcua plugin and wait for HTTP.""" port = free_port() params = self.workdir / f'{name}_gateway.yaml' @@ -256,6 +283,10 @@ def start_gateway(self, name, endpoint=None, poll_ms=200, extra_env=None): ] if endpoint is not None: lines.append(f' plugins.opcua.endpoint_url: "{endpoint}"') + if node_map is not None: + lines.append(f' plugins.opcua.node_map_path: "{node_map}"') + if reconnect_ms is not None: + lines.append(f' plugins.opcua.reconnect_interval_ms: {reconnect_ms}') params.write_text('\n'.join(lines) + '\n') env = dict(self.env) env.update(extra_env or {}) @@ -378,6 +409,164 @@ def scenario_late_nameplate(run): 'after the device named itself') +def fault_manager_client(run, *args): + """Report as a second source ('report'), or list every fault ('list').""" + if args[0] == 'report': + return {'accepted': run.client.report(args[1], args[2])} + return [{'code': f.fault_code, 'status': f.status, 'sources': list(f.reporting_sources)} + for f in run.client.faults()] + + +def comms_lost_records(run): + """Return {first reporting source: status} of every PLC_COMMS_LOST record.""" + return {f['sources'][0]: f['status'] for f in fault_manager_client(run, 'list') + if f['code'] == COMMS_LOST and f['sources']} + + +def wait_records(run, accept, deadline): + """Poll a scaled deadline until accept(records) holds. Return the last records.""" + end = time.monotonic() + deadline * SCALE + records = {} + while time.monotonic() < end: + records = comms_lost_records(run) + if accept(records): + return records + time.sleep(0.3) + return records + + +def scenario_per_source_record(run): + other = '/other_plc' + run.start_fault_manager('persrc', per_source_codes=[COMMS_LOST]) + named, port = run.start_server('persrc_named') + endpoint = f'opc.tcp://127.0.0.1:{port}' + gateway = run.start_gateway('persrc', endpoint=endpoint, reconnect_ms=RECONNECT_MS) + if not wait_log(gateway.log, 'Connected to OPC-UA server', 30): + raise AssertionError('the gateway never connected:\n' + tail(gateway)) + + # A second source holds a record of the same code before the outage. + if not fault_manager_client(run, 'report', COMMS_LOST, other)['accepted']: + raise AssertionError('the second source was refused') + + run.stop(named) + records = wait_records( + run, lambda r: len(r) == 2 and all(s == 'CONFIRMED' for s in r.values()), 60) + if len(records) != 2 or any(s != 'CONFIRMED' for s in records.values()): + raise AssertionError(f'the outage was never reported as its own record: {records}\n' + + tail(gateway)) + own = next(source for source in records if source != other) + + # The same device comes back on its port: the link is restored. + run.start_server('persrc_back', port=port) + records = wait_records(run, lambda r: r.get(own) == 'CLEARED', 60) + if records.get(own) != 'CLEARED': + raise AssertionError('the restored link did not clear this gateway record: ' + f'{records}\n' + tail(gateway)) + if records.get(other) != 'CONFIRMED': + raise AssertionError(f"the other source's record was cleared too: {records}") + + +def scenario_per_source_placeholder_rename(run): + run.start_fault_manager('rename', per_source_codes=[COMMS_LOST]) + port = free_port() + gateway = run.start_gateway('rename', endpoint=f'opc.tcp://127.0.0.1:{port}', + reconnect_ms=RECONNECT_MS) + # Four failed connects take 15 s at the default wait and well under 1 s at RECONNECT_MS. + if not wait_log(gateway.log, CONNECT_FAILED, 4, count=4): + raise AssertionError('the gateway did not retry at reconnect_interval_ms:\n' + + tail(gateway)) + records = wait_records(run, lambda r: len(r) == 1 and 'CONFIRMED' in r.values(), 60) + if len(records) != 1 or 'CONFIRMED' not in records.values(): + raise AssertionError(f'the outage was never reported: {records}\n' + tail(gateway)) + stand_in = next(iter(records)) + + # The PLC comes up on the endpoint the gateway was pinned to and names the component. + run.start_server('rename', port=port) + records = wait_records(run, lambda r: r.get(stand_in) == 'CLEARED', 60) + if records.get(stand_in) != 'CLEARED': + raise AssertionError(f'the record raised under the stand-in {stand_in} was stranded ' + f'after the component was renamed: {records}\n' + tail(gateway)) + + +def scenario_per_source_pinned_stand_in(run): + run.start_fault_manager('persrc_pinned', per_source_codes=[COMMS_LOST]) + port = free_port() + endpoint = f'opc.tcp://127.0.0.1:{port}' + gateway = run.start_gateway('persrc_pinned_before', endpoint=endpoint, + reconnect_ms=RECONNECT_MS) + records = wait_records(run, lambda r: len(r) == 1 and 'CONFIRMED' in r.values(), 60) + if len(records) != 1 or 'CONFIRMED' not in records.values(): + raise AssertionError(f'the outage was never reported: {records}\n' + tail(gateway)) + stand_in = next(iter(records)) + run.stop(gateway.proc) + + # The restarted gateway remembers nothing, and the PLC names the component. + run.start_server('persrc_pinned', port=port) + after = run.start_gateway('persrc_pinned_after', endpoint=endpoint, reconnect_ms=RECONNECT_MS) + records = wait_records(run, lambda r: r.get(stand_in) == 'CLEARED', 60) + if records.get(stand_in) != 'CLEARED': + raise AssertionError(f'the record raised under the stand-in {stand_in} stayed standing ' + f'after a restart: {records}\n' + tail(after)) + + +def scenario_explicit_component_id_keeps_the_default_id_record(run): + run.start_fault_manager('explicit_id', per_source_codes=[COMMS_LOST]) + default_id = 'opcua_device' + # A bridge whose node map names no component_id holds an outage record under the default id. + if not fault_manager_client(run, 'report', COMMS_LOST, default_id)['accepted']: + raise AssertionError('the seeded record was refused') + node_map = run.workdir / 'explicit_id_nodes.yaml' + node_map.write_text('area_id: plc_area\ncomponent_id: plc_a\nnodes: []\n') + _, port = run.start_server('explicit_id') + gateway = run.start_gateway('explicit_id', endpoint=f'opc.tcp://127.0.0.1:{port}', + node_map=node_map, reconnect_ms=RECONNECT_MS) + # The gateway asked for its own record, found none, and went through every id it reports under. + if not wait_log(gateway.log, 'nothing to clear', 30): + raise AssertionError('the gateway never decided on PLC_COMMS_LOST:\n' + tail(gateway)) + records = comms_lost_records(run) + if records != {default_id: 'CONFIRMED'}: + raise AssertionError('a bridge with an explicit component_id touched the record of a ' + f'default-id bridge: {records}\n' + tail(gateway)) + + +def scenario_explicit_component_id_keeps_a_stand_in_record(run): + stand_in = 'opcua-127_0_0_1' + run.start_fault_manager('stand_in_id', per_source_codes=[COMMS_LOST]) + # A config-less bridge on the same host holds an outage record under its stand-in. + if not fault_manager_client(run, 'report', COMMS_LOST, stand_in)['accepted']: + raise AssertionError('the seeded record was refused') + node_map = run.workdir / 'stand_in_id_nodes.yaml' + node_map.write_text('area_id: plc_area\ncomponent_id: plc_a\nnodes: []\n') + _, port = run.start_server('stand_in_id') + gateway = run.start_gateway('stand_in_id', endpoint=f'opc.tcp://127.0.0.1:{port}', + node_map=node_map, reconnect_ms=RECONNECT_MS) + if not wait_log(gateway.log, 'nothing to clear', 30): + raise AssertionError('the gateway never decided on PLC_COMMS_LOST:\n' + tail(gateway)) + records = comms_lost_records(run) + if records != {stand_in: 'CONFIRMED'}: + raise AssertionError('a bridge with a node map touched the record of a stand-in bridge: ' + f'{records}\n' + tail(gateway)) + + +def scenario_explicit_component_id_keeps_a_shared_stand_in_record(run): + stand_in = 'opcua-127_0_0_1' + run.start_fault_manager('stand_in_shared') + if not fault_manager_client(run, 'report', COMMS_LOST, stand_in)['accepted']: + raise AssertionError('the seeded record was refused') + node_map = run.workdir / 'stand_in_shared_nodes.yaml' + node_map.write_text('area_id: plc_area\ncomponent_id: plc_a\nnodes: []\n') + _, port = run.start_server('stand_in_shared') + gateway = run.start_gateway('stand_in_shared', endpoint=f'opc.tcp://127.0.0.1:{port}', + node_map=node_map, reconnect_ms=RECONNECT_MS) + if not wait_log(gateway.log, HELD_BY_OTHERS, 30): + raise AssertionError('the gateway never left the shared record standing:\n' + + tail(gateway)) + records = comms_lost_records(run) + if records != {stand_in: 'CONFIRMED'}: + raise AssertionError('a bridge with a node map cleared the shared record of a stand-in ' + f'bridge: {records}\n' + tail(gateway)) + + def start_discovering_gateway(run, name): """Start a config-less gateway whose binding file sits under a symlink loop.""" loop_a = run.workdir / 'loop_a' @@ -436,12 +625,46 @@ def scenario_binding_directory_on_reconnect(run): expect_binding_failure_survived(gateway, binding, 60) +REFUSED_INTERVAL = ('plugins.opcua.reconnect_interval_ms must be an integer in [100, 60000] ' + '- keeping 5000 ms') + + +def scenario_reconnect_interval_setting(run): + endpoint = f'opc.tcp://127.0.0.1:{free_port()}' + for value, effective in ((100, 100), (60000, 60000), (99, 5000), (60001, 5000), + (250.5, 5000)): + name = 'interval_' + str(value).replace('.', '_') + gateway = run.start_gateway(name, endpoint=endpoint, reconnect_ms=value) + started = f'OPC-UA poller started (mode: poll, first reconnect wait {effective} ms)' + if not wait_log(gateway.log, started, 30): + raise AssertionError(f'reconnect_interval_ms={value}: the poller did not start with ' + f'a {effective} ms wait:\n' + tail(gateway)) + # The value is read before the poller starts, so its warning is already logged. + warned = REFUSED_INTERVAL in gateway.log.read_text(errors='replace') + if warned != (effective != value): + raise AssertionError(f'reconnect_interval_ms={value}: refused warning ' + f'{"logged" if warned else "missing"}:\n' + tail(gateway)) + # Four failed connects take 15 s at the default wait and 0.3 s at 100 ms. + if value == 100 and not wait_log(gateway.log, CONNECT_FAILED, 4, count=4): + raise AssertionError('the gateway did not retry at 100 ms:\n' + tail(gateway)) + run.stop(gateway.proc) + + SCENARIOS = [ ('pinned stand-in', scenario_pinned_stand_in), ('foreign stand-in', scenario_foreign_stand_in), ('late nameplate', scenario_late_nameplate), + ('per-source record', scenario_per_source_record), + ('per-source record under a stand-in', scenario_per_source_placeholder_rename), + ('per-source pinned stand-in', scenario_per_source_pinned_stand_in), + ('explicit component id', scenario_explicit_component_id_keeps_the_default_id_record), + ('explicit component id, stand-in record', + scenario_explicit_component_id_keeps_a_stand_in_record), + ('explicit component id, shared stand-in record', + scenario_explicit_component_id_keeps_a_shared_stand_in_record), ('binding directory at start', scenario_binding_directory_at_start), ('binding directory on reconnect', scenario_binding_directory_on_reconnect), + ('reconnect interval setting', scenario_reconnect_interval_setting), ] @@ -465,11 +688,12 @@ def main(): env = dict(os.environ, ROS_DOMAIN_ID=ros_domain_id) only = os.environ.get('OPCUA_COMMS_LOST_SCENARIO') failures = 0 + client = FaultManagerClient('comms_lost_test_client', 1.0 * SCALE, 60 * SCALE) for name, scenario in SCENARIOS: if only and only != name: continue workdir = Path(tempfile.mkdtemp(prefix='opcua_comms_lost_')) - run = Run(workdir, server_bin, plugin, env) + run = Run(workdir, server_bin, plugin, env, client) started = time.monotonic() error = None try: @@ -486,6 +710,7 @@ def main(): print(f'FAIL {name} ({elapsed:.0f} s): {error}', file=sys.stderr) for log in run.server_logs: print(f'{log.name}:\n' + alarm_fixture.read_log(log), file=sys.stderr) + client.close() return 1 if failures else 0 diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_per_source_clear.test.py b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_per_source_clear.test.py new file mode 100644 index 000000000..d41062bce --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/integration/test_opcua_per_source_clear.test.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A device alarm of a per-source code clears the gateway's own record, through a real gateway. + +The node map gives StatusWord bit 0 a fault code that the fault manager keeps one record per +source for, and a second source holds a record of the same code. Raising the bit raises the +gateway's record and clearing it clears that record alone: the plugin names the entity it +reports under in its ClearFault, so the clear is not refused as ambiguous. + +Usage: test_opcua_per_source_clear.test.py +""" + +import json +import os +from pathlib import Path +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request + +from fault_manager_client import FaultManagerClient + +COMPONENT = 'per_source_runtime' +ENTITY = 'plc_app' +CODE = 'PS_STATUS_BIT0' +OTHER = '/other_source' +POLL_MS = 200 + +SCALE = 1.0 + + +def time_scale(): + """Return MEDKIT_TEST_TIME_SCALE, the factor the sanitizer jobs apply to wall-clock budgets.""" + try: + scale = float(os.environ.get('MEDKIT_TEST_TIME_SCALE', '1')) + except ValueError: + return 1.0 + return scale if scale >= 1.0 else 1.0 + + +SCALE = time_scale() + + +def free_port(): + """Grab an OS-assigned free TCP port on the loopback and release it.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('127.0.0.1', 0)) + return s.getsockname()[1] + + +def find_plugin(): + """Locate libros2_medkit_opcua_plugin.so under AMENT_PREFIX_PATH.""" + for prefix in os.environ.get('AMENT_PREFIX_PATH', '').split(os.pathsep): + if not prefix: + continue + for root, _dirs, files in os.walk(prefix): + if 'libros2_medkit_opcua_plugin.so' in files: + return os.path.join(root, 'libros2_medkit_opcua_plugin.so') + return None + + +def http_json(url, timeout=5): + """Return the parsed JSON of a GET, or None on any failure.""" + try: + with urllib.request.urlopen(url, timeout=timeout) as resp: + return json.loads(resp.read().decode()) + except (urllib.error.URLError, OSError, ValueError): + return None + + +def wait_log(path, needle, deadline=30, period=0.5): + """Poll a log file until appears; return True/False.""" + for _ in range(int(deadline / period) + 1): + try: + if needle in Path(path).read_text(errors='replace'): + return True + except OSError: + pass + time.sleep(period) + return False + + +def terminate(proc): + """SIGTERM then SIGKILL a child process group started with start_new_session.""" + if proc is None: + return + pgid = None + if proc.returncode is None: + try: + pgid = os.getpgid(proc.pid) + except ProcessLookupError: + pgid = None + + def signal_group(sig): + if pgid is None: + return + try: + os.killpg(pgid, sig) + except ProcessLookupError: + pass + + signal_group(signal.SIGTERM) + proc.terminate() + try: + proc.wait(timeout=8) + except subprocess.TimeoutExpired: + signal_group(signal.SIGKILL) + proc.kill() + proc.wait() + signal_group(signal.SIGKILL) + log = getattr(proc, '_log', None) + if log is not None: + log.close() + + +def start_server(server_bin, port, log_path): + """Start the fixture server and wait for its READY line.""" + log = open(log_path, 'w') + proc = subprocess.Popen( + [str(server_bin), '--port', str(port)], + stdin=subprocess.PIPE, stdout=log, stderr=subprocess.STDOUT, + text=True, start_new_session=True, + ) + proc._log = log + if not wait_log(log_path, 'READY ', deadline=25 * SCALE): + terminate(proc) + return None + return proc + + +def set_register(server, log_path, name, value): + """Write one Int32 register through the fixture and wait for its OK line.""" + needle = f'OK {name}={value}' + before = Path(log_path).read_text(errors='replace').count(needle) + server.stdin.write(f'set {name} {value}\n') + server.stdin.flush() + for _ in range(int(40 * SCALE)): + if Path(log_path).read_text(errors='replace').count(needle) > before: + return True + time.sleep(0.05) + return False + + +def start_gateway(params_file, log_path, env): + """Launch a gateway_node bound to the given params file.""" + log = open(log_path, 'w') + proc = subprocess.Popen( + ['ros2', 'run', 'ros2_medkit_gateway', 'gateway_node', + '--ros-args', '--params-file', str(params_file)], + stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True, + ) + proc._log = log + return proc + + +def start_fault_manager(log_path, env): + """Start fault_manager_node with memory storage and wait until its services exist.""" + log = open(log_path, 'w') + proc = subprocess.Popen( + ['ros2', 'run', 'ros2_medkit_fault_manager', 'fault_manager_node', + '--ros-args', '-p', 'storage_type:=memory'], + stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True, + ) + proc._log = log + # Logged at the end of the constructor, after the services exist. + started = wait_log(log_path, 'FaultManager node started (', 30 * SCALE) + if not started or proc.poll() is not None: + terminate(proc) + return None + return proc + + +def gateway_records(client): + """Return {first reporting source: status} of every CODE record.""" + return {f.reporting_sources[0]: f.status for f in client.faults() + if f.fault_code == CODE and f.reporting_sources} + + +def wait_records(client, accept, deadline): + """Poll a scaled deadline until accept(records) holds. Return the last records.""" + end = time.monotonic() + deadline * SCALE + records = {} + while time.monotonic() < end: + records = gateway_records(client) + if accept(records): + return records + time.sleep(0.3) + return records + + +def main(): + global SCALE + if len(sys.argv) < 2: + print('usage: test_opcua_per_source_clear.test.py ', file=sys.stderr) + return 2 + SCALE = time_scale() + server_bin = Path(sys.argv[1]).resolve() + ros_domain_id = os.environ.get('ROS_DOMAIN_ID') + if not ros_domain_id: + print('ROS_DOMAIN_ID is not set: run this test through CTest', file=sys.stderr) + return 1 + plugin = find_plugin() + if plugin is None or not server_bin.is_file(): + print('plugin or fixture missing', file=sys.stderr) + return 1 + + workdir = Path(tempfile.mkdtemp(prefix='opcua_per_source_clear_')) + env = dict(os.environ, ROS_DOMAIN_ID=ros_domain_id) + procs = [] + client = FaultManagerClient('per_source_clear_test_client', 1.0 * SCALE, 60 * SCALE) + try: + node_map = workdir / 'nodes.yaml' + node_map.write_text( + 'area_id: plc_systems\n' + f'component_id: {COMPONENT}\n' + 'nodes:\n' + ' - node_id: "ns=2;s=StatusWord"\n' + f' entity_id: {ENTITY}\n' + ' data_name: status_word\n' + ' data_type: int\n' + ' status_bits:\n' + ' - bit: 0\n' + f' fault_code: {CODE}\n') + manifest = workdir / 'manifest.yaml' + manifest.write_text('manifest_version: "1.0"\n') + thresholds = workdir / 'thresholds.yaml' + thresholds.write_text(f'{CODE}:\n per_source: true\n') + + server_port = free_port() + server_log = workdir / 'alarm_server.log' + server = start_server(server_bin, server_port, server_log) + if server is None: + print('fixture server did not become READY', file=sys.stderr) + return 1 + procs.append(server) + + fm_log = workdir / 'fault_manager.log' + fm_proc_log = open(fm_log, 'w') + fm = subprocess.Popen( + ['ros2', 'run', 'ros2_medkit_fault_manager', 'fault_manager_node', '--ros-args', + '-p', 'storage_type:=memory', '-p', f'fault_thresholds.config_file:={thresholds}'], + stdout=fm_proc_log, stderr=subprocess.STDOUT, env=env, start_new_session=True) + fm._log = fm_proc_log + procs.append(fm) + if not wait_log(fm_log, 'FaultManager node started (', 30 * SCALE): + print(fm_log.read_text(errors='replace'), file=sys.stderr) + return 1 + + port = free_port() + params = workdir / 'gateway.yaml' + params.write_text('\n'.join([ + 'ros2_medkit_gateway:', + ' ros__parameters:', + ' server:', + ' host: "127.0.0.1"', + f' port: {port}', + ' plugins: ["opcua"]', + f' plugins.opcua.path: "{plugin}"', + f' plugins.opcua.endpoint_url: "opc.tcp://127.0.0.1:{server_port}"', + f' plugins.opcua.node_map_path: "{node_map}"', + f' plugins.opcua.poll_interval_ms: {POLL_MS}', + ' discovery.mode: "hybrid"', + f' discovery.manifest_path: "{manifest}"', + ' discovery.manifest_strict_validation: false', + ]) + '\n') + gateway_log = workdir / 'gateway.log' + gateway = start_gateway(params, gateway_log, env) + procs.append(gateway) + status_url = f'http://127.0.0.1:{port}/api/v1/components/{COMPONENT}/x-plc-status' + end = time.monotonic() + 90 * SCALE + while time.monotonic() < end: + status = http_json(status_url) + if isinstance(status, dict) and status.get('connected') is True: + break + time.sleep(0.5) + else: + tail = gateway_log.read_text(errors='replace')[-3000:] + print('the gateway never connected:\n' + tail, file=sys.stderr) + return 1 + + # A second source holds a record of the same code. + if not client.report(CODE, OTHER): + print('the second source was refused', file=sys.stderr) + return 1 + + set_register(server, server_log, 'StatusWord', 1) + records = wait_records( + client, lambda r: r.get(ENTITY) == 'CONFIRMED' and r.get(OTHER) == 'CONFIRMED', 30) + if records.get(ENTITY) != 'CONFIRMED': + print(f'the alarm never raised the gateway record: {records}', file=sys.stderr) + return 1 + + set_register(server, server_log, 'StatusWord', 0) + records = wait_records(client, lambda r: r.get(ENTITY) == 'CLEARED', 30) + if records.get(ENTITY) != 'CLEARED': + print(f"the cleared alarm did not clear the gateway's record: {records}\n" + + fm_log.read_text(errors='replace')[-3000:], file=sys.stderr) + return 1 + if records.get(OTHER) != 'CONFIRMED': + print(f"the other source's record changed: {records}", file=sys.stderr) + return 1 + print('PASS: a device alarm clear reaches only the gateway record of a per-source code') + return 0 + finally: + for proc in reversed(procs): + terminate(proc) + client.close() + shutil.rmtree(workdir, ignore_errors=True) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp index bb481e58a..c78cd4091 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_identity.cpp @@ -1799,7 +1799,12 @@ TEST_F(OpcuaIdentityE2ETest, AnAnswerToATimedOutProbeIsNotReadAsTheNextOnes) { // A answers late. ASSERT_TRUE(store.release_one_read()); - std::this_thread::sleep_for(std::chrono::seconds(2)); + // B times out in turn and a third probe parks: the decision ran again after A's answer arrived. + const auto next_probe_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (store.parked_read_count() < 2 && std::chrono::steady_clock::now() < next_probe_deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + ASSERT_GE(store.parked_read_count(), 2u) << "no further probe was sent after the late answer"; EXPECT_TRUE(store.cleared_codes().empty()) << "an answer to a probe already given up on decided the clear"; // B answers, and that is the answer the decision is waiting for. diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp index 550ecd9e9..90949fa5d 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/test/test_opcua_plugin.cpp @@ -1903,13 +1903,15 @@ TEST(CommsLostProbeDue, AllFourConditionsHold) { EXPECT_FALSE(OpcuaPlugin::comms_lost_probe_due(true, true, false, false)) << "the store cannot be asked"; } -TEST(MakeClearFaultRequest, CarriesTheSkipFlagAndCodeVerbatim) { - const auto skipping = OpcuaPlugin::make_clear_fault_request(kCommsLostFaultCode, true); +TEST(MakeClearFaultRequest, CarriesTheSkipFlagCodeAndSourceVerbatim) { + const auto skipping = OpcuaPlugin::make_clear_fault_request(kCommsLostFaultCode, "plc_component", true); EXPECT_EQ(skipping.fault_code, kCommsLostFaultCode); + EXPECT_EQ(skipping.source_id, "plc_component"); EXPECT_TRUE(skipping.skip_correlation_auto_clear); - const auto cascading = OpcuaPlugin::make_clear_fault_request("PLC_TANK_HIGH", false); + const auto cascading = OpcuaPlugin::make_clear_fault_request("PLC_TANK_HIGH", "tank", false); EXPECT_EQ(cascading.fault_code, "PLC_TANK_HIGH"); + EXPECT_EQ(cascading.source_id, "tank"); EXPECT_FALSE(cascading.skip_correlation_auto_clear); } @@ -2780,6 +2782,7 @@ component_id: scoped_runtime std::lock_guard lock(received_mutex); ASSERT_FALSE(received.empty()) << "the scoped DELETE never reached the fault manager"; EXPECT_EQ(received.front().fault_code, "PLC_TANK_HIGH"); + EXPECT_EQ(received.front().source_id, "tank") << "the clear names the entity the route addressed"; EXPECT_TRUE(received.front().skip_correlation_auto_clear) << "a per-entity DELETE served by the plugin cascade-cleared correlated symptoms"; }