diff --git a/.github/workflows/opcua-plugin.yml b/.github/workflows/opcua-plugin.yml index a0f287ee1..1e66ddcf9 100644 --- a/.github/workflows/opcua-plugin.yml +++ b/.github/workflows/opcua-plugin.yml @@ -4,9 +4,10 @@ permissions: contents: read # Runs the OpenPLC-based Docker integration suite for ros2_medkit_opcua. -# Triggers only when the plugin itself, the gateway plugin/provider API, or -# ros2_medkit_msgs change. A nightly cron provides a safety net against silent -# breaks that slip past the path filter. +# Triggers only when the plugin itself, the gateway plugin/provider API, the +# entity freeze-frame capture and its store, the gateway node and fault scope +# code that wire the capture in, or ros2_medkit_msgs change. A nightly cron +# provides a safety net against silent breaks that slip past the path filter. on: pull_request: @@ -16,6 +17,14 @@ on: - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/plugins/**' - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/providers/**' - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/error_codes.hpp' + - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/*freeze_frame*' + - 'src/ros2_medkit_gateway/src/core/faults/*freeze_frame*' + - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp' + - 'src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp' + - 'src/ros2_medkit_gateway/src/gateway_node.cpp' + - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp' + - 'src/ros2_medkit_gateway/src/core/faults/fault_scope.cpp' + - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/fault_scope.hpp' - 'src/ros2_medkit_msgs/**' - '.github/workflows/opcua-plugin.yml' push: @@ -25,6 +34,14 @@ on: - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/plugins/**' - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/providers/**' - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/error_codes.hpp' + - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/*freeze_frame*' + - 'src/ros2_medkit_gateway/src/core/faults/*freeze_frame*' + - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp' + - 'src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp' + - 'src/ros2_medkit_gateway/src/gateway_node.cpp' + - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp' + - 'src/ros2_medkit_gateway/src/core/faults/fault_scope.cpp' + - 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/fault_scope.hpp' - 'src/ros2_medkit_msgs/**' - '.github/workflows/opcua-plugin.yml' schedule: @@ -295,6 +312,56 @@ jobs: curl -s "$api/faults" | jq . || true exit 1 + - name: The fault's freeze frame survives stop.sh and start.sh + timeout-minutes: 20 + env: + MEDKIT_OPCUA_VARIANT: write-capable + run: | + api=http://localhost:8080/api/v1 + scripts=src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts + frame_of() { + curl -sf "$api/apps/tank_process/faults/$1" | + jq -c '[.environment_data.snapshots[]? | select(.type == "freeze_frame")] | first // empty' + } + # Polls until the frame of $1 has no capture_origin, and prints it. + edge_frame_of() { + for _ in $(seq 1 60); do + f=$(frame_of "$1" || true) + if [ -n "$f" ] && jq -e '.["x-medkit"] | has("capture_origin") | not' <<<"$f" >/dev/null; then + echo "$f" + return 0 + fi + sleep 2 + done + echo "FAIL: $1 has no freeze frame from its confirmation" >&2 + return 1 + } + before=$(edge_frame_of PLC_HIGH_TEMP) + echo "before stop.sh: $before" + captured=$(jq -r '.["x-medkit"].captured_at' <<<"$before") + bash "$scripts/stop.sh" + bash "$scripts/start.sh" + reloaded="" + for _ in $(seq 1 30); do + reloaded=$(docker logs gateway 2>&1 | grep -F "Entity freeze-frame: reloaded frames for" || true) + [ -n "$reloaded" ] && break + sleep 2 + done + [ -n "$reloaded" ] || { echo "FAIL: the gateway reloaded no freeze frame"; exit 1; } + echo "$reloaded" + # A second alarm: its frame comes from the drain loop, which runs after the startup catch-up. + code=$(curl -s -o /tmp/put.json -w '%{http_code}' -X PUT \ + "$api/apps/tank_process/data/tank_level" \ + -H 'Content-Type: application/json' -d '{"value": 50.0}') + [ "$code" = "200" ] || { echo "FAIL: PUT tank_level -> $code"; cat /tmp/put.json; exit 1; } + edge_frame_of PLC_LOW_LEVEL >/dev/null + after=$(frame_of PLC_HIGH_TEMP) + echo "after start.sh: $after" + jq -e --arg c "$captured" \ + '.["x-medkit"].captured_at == $c and (.["x-medkit"] | has("capture_origin") | not)' <<<"$after" >/dev/null || + { echo "FAIL: PLC_HIGH_TEMP lost the frame from its confirmation ($captured)"; exit 1; } + echo "PLC_HIGH_TEMP keeps captured_at $captured across stop.sh and start.sh" + - name: Dump container logs on failure if: failure() run: | @@ -310,6 +377,23 @@ jobs: run: | docker rm -f gateway openplc 2>/dev/null || true docker network rm plc-demo 2>/dev/null || true + docker volume rm ros2-medkit-opcua-state 2>/dev/null || true + + freeze-frame-restart: + name: Freeze frame across gateway restarts + # A plugin entity's freeze frame across gateway restarts, against the + # test_alarm_server fixture: the store, the re-read of a frame from an earlier + # occurrence, the drop when the entity cannot answer or the fault manager no + # longer holds the fault, and each entity_freeze_frame.storage.path case. The + # script builds its own images. + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run freeze-frame restart suite + run: bash src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_freeze_frame_restart_tests.sh integration-alarms: name: Integration (AlarmConditionType, ${{ matrix.variant }}) diff --git a/docs/api/rest.rst b/docs/api/rest.rst index a27580577..e6f23513a 100644 --- a/docs/api/rest.rst +++ b/docs/api/rest.rst @@ -1527,7 +1527,9 @@ Query and manage faults. - ``freeze_frame``: Data captured at fault confirmation. Entity frames for faults that were already confirmed when the gateway started are captured at gateway start instead and carry ``"capture_origin": "startup"`` in - their ``x-medkit`` block. An entity frame also carries ``source`` in + their ``x-medkit`` block, unless a frame of the same occurrence was + reloaded from ``entity_freeze_frame.storage.path``: that frame is served + as it was captured. An entity frame also carries ``source`` in ``x-medkit``, naming the path that read the values (``plugin_data_provider`` for the owning plugin's DataProvider, ``plugin_x_plc_data_route`` for its ``x-plc-data`` route). These values diff --git a/docs/config/server.rst b/docs/config/server.rst index 3b56e4a4c..0f41c32c2 100644 --- a/docs/config/server.rst +++ b/docs/config/server.rst @@ -382,13 +382,44 @@ Configure how the gateway connects to the fault manager services and event topic by its plugin's DataProvider and merge them into the fault detail's ``environment_data.snapshots``. Faults already confirmed at gateway start are caught up at startup; those frames read the values at start time (not - confirm time) and are marked ``x-medkit.capture_origin: startup``. For an + confirm time) and are marked ``x-medkit.capture_origin: startup``. A fault + whose frame of the same occurrence was reloaded from + ``entity_freeze_frame.storage.path`` is not read at start: the reloaded + frame is served as captured. For an entity that reports its link down these are the plugin's last known - values, marked ``connected: false`` in the snapshot's ``x-medkit`` block. + values. ``connected: false`` marks them in the snapshot's ``x-medkit`` + block only when the plugin's payload carries a link flag; the OPC UA + plugin's ``DataProvider`` payload carries none. Explicit snapshot config in the fault manager always wins when present. Only active when plugins are loaded. A program that hosts the gateway core without ROS sets the same switch as ``FreezeFrameCaptureConfig::enabled`` (see the gateway README). + * - ``entity_freeze_frame.storage.path`` + - string + - ``""`` + - SQLite file the captured frames are written to, so a restart serves the + values frozen when the fault confirmed instead of reading the plant + again. When empty, the frames go in ``entity_freeze_frames.db`` in the + directory of ``triggers.storage.path``. With that empty too they stay in + memory and are lost on restart (logged at startup). A file that cannot be + opened is logged as an error and the frames stay in memory. A reloaded + frame keeps its ``captured_at`` and its ``capture_origin``, and the + startup catch-up captures only faults that have no frame. The bound of + 256 faults counts reloaded and captured frames together and drops the + frame captured longest ago first, with its stored rows. Each row stores + the fault's ``occurrence_count`` and ``first_occurred`` at capture. At + startup one fault list of every status judges each reloaded frame, and a + reloaded frame is served only after that. When the lister throws or + answers with an empty list, the frames are kept and served unjudged. A + frame whose fault lists + another occurrence is discarded with a warning that names the fault code + and the entity; when the fault is confirmed and not muted, the startup + catch-up then reads it and marks it ``capture_origin: startup``. A frame + whose fault the fault manager no longer holds in any status is dropped + with a warning that names up to 10 of those faults, but not when the + fault manager lists no fault at all. The file belongs to one gateway: two + gateways must not share it, as one gateway's start drops the other's + frames as faults its fault manager does not hold. When ``fault_manager.namespace`` is set, the gateway also subscribes to the matching fault event topic (for example ``/robot1/fault_manager/events`` instead of the default diff --git a/docs/requirements/specs/faults.rst b/docs/requirements/specs/faults.rst index bc83eae8e..6c33cb0fc 100644 --- a/docs/requirements/specs/faults.rst +++ b/docs/requirements/specs/faults.rst @@ -70,4 +70,6 @@ Faults stored as environment data associated with the fault and accessible via the bulk-data endpoints on the fault's reporting entity. For faults already confirmed when the capturing component starts (e.g. the gateway restarts while a fault is standing), an - equivalent snapshot shall be captured at startup and marked with its capture origin. + equivalent snapshot shall be captured at startup and marked with its capture origin, + unless a snapshot captured for the same occurrence of the fault was kept across the + restart; that snapshot shall be served as it was captured. diff --git a/docs/tutorials/snapshots.rst b/docs/tutorials/snapshots.rst index aa2dc16b6..fb81c1615 100644 --- a/docs/tutorials/snapshots.rst +++ b/docs/tutorials/snapshots.rst @@ -223,20 +223,71 @@ values) when the fault confirms, and merges them into the fault detail's after the entity - unless the fault manager already captured a freeze-frame for that fault (explicit config wins). When the entity reports its link down (the loss-of-comms case), the frozen values are the plugin's last known ones -and may predate the confirmation by the length of the outage; the entry's -``x-medkit`` block then carries ``connected: false`` and, when the plugin's -payload includes one, ``source_timestamp`` (the payload's own timestamp) -alongside ``captured_at``. +and may predate the confirmation by the length of the outage. The entry's +``x-medkit`` block carries ``connected: false`` only when the plugin's payload +has a link flag (the OPC UA plugin's ``DataProvider`` payload has none), and, +when the payload includes one, ``source_timestamp`` (the payload's own +timestamp) alongside ``captured_at``. + +A frame stays after the fault is cleared, and a new confirmation of the +fault reads the entity again and replaces it. When that read gets no values, +a confirmation of a new occurrence drops the previous occurrence's frame and +logs a warning, so the fault has no frame; a confirmation of the same +occurrence keeps it. With +``entity_freeze_frame.storage.path`` set, a frame also survives a gateway +restart: the gateway writes it to that SQLite file when it captures it, loads +it at start, and serves it exactly as it was captured, with its original +``captured_at`` and its ``capture_origin``. Put the file on a volume that +outlives the container, or leave the path empty and the frames go in +``entity_freeze_frames.db`` next to the trigger store +(``triggers.storage.path``). With neither set the frames are process memory +only, and a restart loses them. The file belongs to one gateway: two gateways +must not share it, as one gateway's start drops the other's frames as faults +its fault manager does not hold. + +Each stored frame carries the fault manager's occurrence of the fault at +capture: the fault's ``occurrence_count`` and ``first_occurred``. The fault +manager changes both only when a cleared fault fails again. At startup the +gateway asks the fault manager once for its faults of every status, muted ones +included, and judges every reloaded frame by that one answer. Until then the +fault detail serves no reloaded frame. When the fault manager cannot be asked +or answers with an empty list, the reloaded frames are kept and served +unjudged. + +- The fault lists the stored occurrence: the frame is kept as it is, with no + read. Clocks play no part, so a gateway clock that differs from the fault + manager's changes nothing. +- The fault lists another occurrence (it cleared and failed again while the + gateway was down): the frame is discarded, and the gateway logs a warning + with the fault code, the entity, both occurrence counts and the status. A + confirmed fault then gets a startup frame from the catch-up below; any other + fault has no frame, since a read now would not show the values at that + occurrence's confirmation. +- The fault manager no longer holds the fault in any status (its storage was + replaced or wiped): the frame is dropped, and the gateway logs one warning + that names up to 10 of those faults. When the fault manager cannot be asked, + or lists no fault at all, nothing is dropped: a fault manager that lost its + storage connection answers with an empty list. Such a frame leaves by + eviction or by the next capture of its fault. +- A frame stored without an occurrence is kept, since nothing can be told + from it. Faults that are already confirmed when the gateway starts are caught up at -startup: the gateway lists the confirmed faults and captures a frame for each -plugin-backed one, so a device standing in fault across a gateway restart -still gets a frame. Catch-up frames carry ``"capture_origin": "startup"`` in -their ``x-medkit`` block because their values were read at gateway start, not -when the fault confirmed (which may be long before, since the fault manager -persists faults); ``captured_at`` always stamps the moment the values were -read. Frames without the marker were captured on the confirm edge. Disable -with: +startup: from the same answer the gateway captures a frame for each confirmed, +plugin-backed fault that has no frame, so a device standing in fault across a +gateway restart still gets a frame. A fault whose frame of the same occurrence +was reloaded is not read: that frame is served as captured. A fault muted by +correlation is not framed at start, as live capture does not frame it either: +the fault manager publishes no confirm event for it. Catch-up frames carry +``"capture_origin": "startup"`` in their ``x-medkit`` block because their +values were read at gateway start, not when the fault confirmed (which may be +long before, since the fault manager persists faults); ``captured_at`` always +stamps the moment the values were read. Frames without the marker were +captured on the confirm edge. The gateway keeps frames for at most 256 +faults, reloaded and captured together, and evicts the fault whose frame was +captured longest ago; a fault confirmed again counts from its new capture. +The catch-up does not evict a frame to make room, and its warning names up +to 10 of the faults it left without one. Disable with: .. code-block:: bash @@ -345,7 +396,8 @@ Snapshots are included inline in the fault response as ``environment_data``: frames caught up for faults that predate the gateway are captured at gateway start instead, marked ``x-medkit.capture_origin: startup``; a plugin entity that reports its link down contributes its last known values, - marked ``connected: false`` in ``x-medkit`` + marked ``connected: false`` in ``x-medkit`` when the plugin's payload carries + a link flag - ``rosbag``: Recording file available via bulk-data endpoint (binary format). One entry per recording the fault kept, newest first, each addressed by its own ``bulk_data_uri``. With the default ``max_bags_per_fault`` of ``1`` there is at diff --git a/src/ros2_medkit_gateway/CMakeLists.txt b/src/ros2_medkit_gateway/CMakeLists.txt index 3fd25feb7..94b9f1307 100644 --- a/src/ros2_medkit_gateway/CMakeLists.txt +++ b/src/ros2_medkit_gateway/CMakeLists.txt @@ -737,6 +737,14 @@ if(BUILD_TESTING) target_link_libraries(test_entity_freeze_frame_capture gateway_ros2) medkit_target_dependencies(test_entity_freeze_frame_capture rclcpp ros2_medkit_msgs) + # Freeze-frame store and the core capture with a store (gateway_core only, no ROS node) + medkit_add_gtest(test_entity_freeze_frame_store test/test_entity_freeze_frame_store.cpp) + target_link_libraries(test_entity_freeze_frame_store gateway_core) + medkit_test_needs_no_domain(test_entity_freeze_frame_store) + medkit_add_gtest(test_freeze_frame_capture_persistence test/test_freeze_frame_capture_persistence.cpp) + target_link_libraries(test_freeze_frame_capture_persistence gateway_core) + medkit_test_needs_no_domain(test_freeze_frame_capture_persistence) + # Add update manager tests medkit_add_gtest(test_update_manager test/test_update_manager.cpp) target_link_libraries(test_update_manager gateway_ros2) diff --git a/src/ros2_medkit_gateway/README.md b/src/ros2_medkit_gateway/README.md index 26a034d71..d4d81f92a 100644 --- a/src/ros2_medkit_gateway/README.md +++ b/src/ros2_medkit_gateway/README.md @@ -1700,7 +1700,7 @@ A program that hosts the core gets the cyclic-subscription samplers and the bulk - **Samplers.** `register_builtin_samplers(services)` (`core/builtin_samplers.hpp`) registers `data`, `faults`, `configurations` and `logs` on `services.sampler_registry()`, plus `updates` when `services.update_manager()` is set. Each sampler reads through the same managers and helpers as its GET route. A `data` subscription on an entity a plugin owns reads through that plugin's `DataProvider` (or its `x-plc-data` route), and on any other entity through `services.data_access_manager()` and its `TopicDataProvider`. A collection with no manager behind it answers with the error text of its GET route, not a crash. The host passes `GatewayServices` (it must outlive the samplers) and registers a subscription transport itself, for example `SseTransportProvider`. `GET /` reports `cyclic_subscriptions` true only when a transport and at least one sampler are registered. - **Bulk data.** `BulkDataConfig` (`core/config.hpp`: `storage_dir`, `max_upload_size`, `categories`, with the defaults of the `bulk_data.*` parameters) and `make_bulk_data_store(config)` (`core/managers/bulk_data_store.hpp`) build the store. The host returns it from `GatewayServices::bulk_data_store()` before it starts `RESTServer`, which sizes the upload limit from it. The `rosbags` category is listed only when `FaultServiceTransport::serves_rosbags()` is true; otherwise a request for it answers 404 like any unknown category. The default is true; a host whose fault manager records no rosbags overrides it. `GET /` reports `bulk_data` true only when a store exists or rosbags are served. -- **Freeze frames.** `make_freeze_frame_capture(config, events, plugins)` (`core/faults/freeze_frame_capture.hpp`) builds a `FreezeFrameCapture`. On a `fault_confirmed` event it reads the data of each reporting source through the same path as `GET /{entity}/data` (`plugin_data_readers()` in `core/data/data_item_reader.hpp`: the plugin's `DataProvider`, else its `x-plc-data` route) and keeps the frames, so the fault detail's `environment_data` carries a `freeze_frame` with the values from the confirmation. The host passes a `FreezeFrameCaptureConfig` (`enabled` = `entity_freeze_frame.enabled`, default true; `max_faults`, default 256, the oldest fault's frames are evicted past it), a `FaultEventSource` (the capture subscribes on construction and its subscription ends on destruction; a source serves several subscribers, so this can be the source the host returns from `GatewayServices::fault_events()`, which the fault SSE stream subscribes to), and the `PluginManager` that owns the entities. The factory returns null when the configuration is disabled or no plugin is loaded. The host returns the capture from `GatewayServices::freeze_frames()` and keeps it alive as long as the services. Optional: a `StandingFaultLister` that frames faults already confirmed at start, and an `EventsReadyWaiter` that holds the lister until the event feed delivers. An entity no plugin serves gets no frame. The ROS 2 node feeds the same class from the fault manager's events topic (`EntityFreezeFrameCapture`). +- **Freeze frames.** `make_freeze_frame_capture(config, events, plugins)` (`core/faults/freeze_frame_capture.hpp`) builds a `FreezeFrameCapture`. On a `fault_confirmed` event it reads the data of each reporting source through the same path as `GET /{entity}/data` (`plugin_data_readers()` in `core/data/data_item_reader.hpp`: the plugin's `DataProvider`, else its `x-plc-data` route) and keeps the frames, so the fault detail's `environment_data` carries a `freeze_frame` with the values from the confirmation. The host passes a `FreezeFrameCaptureConfig` (`enabled` = `entity_freeze_frame.enabled`, default true; `max_faults`, default 256, the frames of the fault captured longest ago are evicted past it), a `FaultEventSource` (the capture subscribes on construction and its subscription ends on destruction; a source serves several subscribers, so this can be the source the host returns from `GatewayServices::fault_events()`, which the fault SSE stream subscribes to), and the `PluginManager` that owns the entities. The factory returns null when the configuration is disabled or no plugin is loaded. The host returns the capture from `GatewayServices::freeze_frames()` and keeps it alive as long as the services. Optional: a `StandingFaultLister` that lists every fault at start (any status, muted ones flagged), an `EventsReadyWaiter` that holds the lister until the event feed delivers, an `EntityFreezeFrameStore` (`core/faults/entity_freeze_frame_store.hpp`; `SqliteEntityFreezeFrameStore` writes a file). With a store, every capture is written through to it with the fault's occurrence (`occurrence_count` and `first_occurred`), and the stored frames are loaded on construction, so a restart serves the frame captured at confirmation; the bound counts reloaded frames too. The lister's one answer then judges the reloaded frames: a frame whose fault it does not list is dropped (nothing when it lists no fault or gives up), and a frame whose fault lists another occurrence is discarded. No reloaded frame is served before that. When the host gives no lister, or the lister throws or answers with an empty list, the reloaded frames are kept and served unjudged. The catch-up then frames every confirmed fault that is not muted and has no frame. An entity no plugin serves gets no frame. The ROS 2 node feeds the same class from the fault manager's events topic (`EntityFreezeFrameCapture`) and opens the store from `entity_freeze_frame.storage.path`. - **Lock expiry.** `LockReaper(locks, subscriptions, interval_sec, logger)` (`core/managers/lock_reaper.hpp`) calls `LockManager::cleanup_expired()` every `interval_sec` (`LockConfig::cleanup_interval`, default 30, raised to 1 when lower) on a thread it owns. For each expired lock whose scope is empty or includes `cyclic-subscriptions` it removes the cyclic subscriptions of the locked entity. `subscriptions` may be null. The destructor and `stop()` join the thread. Without a reaper a host keeps expired locks until it calls `cleanup_expired()` itself. The ROS 2 node runs the same class. - **Scripts.** `make_script_host(config, plugin_backend)` (`core/managers/script_manager.hpp`) returns a `ScriptManager` and, when no plugin backend is given and `ScriptsConfig::scripts_dir` is not empty, the `DefaultScriptProvider` (fork/execvp) behind it. `ScriptsConfig` (`core/script_types.hpp`) has the defaults of the `scripts.*` parameters, and `entries` carries predefined scripts. The host returns `host.manager` from `GatewayServices::script_manager()` and keeps the `ScriptHost` alive as long as the services. With neither a backend nor a directory the script routes answer 501. - **Aggregation.** `AggregationHost(config, listen_port, mdns_name, logger)` (`core/aggregation/aggregation_host.hpp`) builds the `AggregationManager` from an `AggregationConfig` (the defaults of the `aggregation.*` parameters; the host range-checks values it reads from its own configuration) and, when `announce` or `discover` is set, the mDNS discovery that adds and removes peers and ignores this gateway's own announcements. The host passes `host.manager()` to `RESTServer::set_aggregation_manager()` and calls `host.merge_peers(areas, components, apps, functions)` on every entity refresh, before it fills the entity cache. The call merges the peers' entities into the four vectors and publishes the routing table, the leaf warnings and the per-entity peer contributors. It does nothing while there is no peer. Call `host.shutdown()` before `RESTServer::stop()`. diff --git a/src/ros2_medkit_gateway/design/index.rst b/src/ros2_medkit_gateway/design/index.rst index 29543915c..70b4e7463 100644 --- a/src/ros2_medkit_gateway/design/index.rst +++ b/src/ros2_medkit_gateway/design/index.rst @@ -58,9 +58,22 @@ store from it: of each reporting source through ``plugin_data_readers()`` (``core/data/data_item_reader.hpp``, the path of ``GET /{entity}/data``: the plugin's ``DataProvider``, else its ``x-plc-data`` route) on a thread of its - own, keeps the frames up to ``FreezeFrameCaptureConfig::max_faults`` (oldest - fault evicted first), and serves them as ``FreezeFrameSource``. The host - passes the config, a ``FaultEventSource`` and the ``PluginManager``. A + own, keeps the frames up to ``FreezeFrameCaptureConfig::max_faults`` (the + fault captured longest ago evicted first), and serves them as + ``FreezeFrameSource``. A frame set + stays after ``fault_cleared`` and is replaced on the next confirmation. With + an ``EntityFreezeFrameStore`` (``core/faults/entity_freeze_frame_store.hpp``) + each capture is written through with the fault's occurrence + (``occurrence_count`` and ``first_occurred``), and the stored frames are + loaded before the first capture, so a restart serves the frame captured at + confirmation. At start one fault list of every status judges the reloaded + frames: a frame whose fault the fault manager no longer holds is dropped + (unless it lists no fault), one whose fault lists another occurrence is + discarded, and none is served before that. With no lister, or one that + throws or answers with an empty list, the reloaded frames are kept and + served unjudged. The catch-up then frames every + confirmed fault that is not muted and has no frame. The + host passes the config, a ``FaultEventSource`` and the ``PluginManager``. A source serves several subscribers, each ``FaultEventSubscription`` ending only its own delivery, so the capture and the fault SSE stream can share the source the host returns from ``GatewayServices::fault_events()``. The diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/entity_freeze_frame_store.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/entity_freeze_frame_store.hpp new file mode 100644 index 000000000..2ac771f21 --- /dev/null +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/entity_freeze_frame_store.hpp @@ -0,0 +1,111 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace ros2_medkit_gateway { + +/// One stored entity freeze-frame, keyed by (fault_code, entity_id). +/// `frame` holds what has no column of its own, so a reload reproduces the frame exactly. +struct StoredEntityFreezeFrame { + std::string fault_code; + std::string entity_id; + nlohmann::json frame; ///< {"values": {...}, "connected"?: bool, "source_timestamp"?: any} + int64_t captured_at_ns{0}; + std::string source; ///< capture path that read the values + std::string capture_origin; ///< "startup" for a catch-up frame, empty for a confirm edge + /// The fault manager's occurrence of the fault at capture. 0 and 0 mean "not known". + uint32_t occurrence_count{0}; + int64_t first_occurred_ns{0}; +}; + +/// Persistence of the entity freeze-frames, so a frame outlives the gateway process. +/// Writes replace all rows of one fault code, as the capture replaces the code's frames. +/// Implementations must be thread-safe. +class EntityFreezeFrameStore { + public: + EntityFreezeFrameStore() = default; + EntityFreezeFrameStore(const EntityFreezeFrameStore &) = delete; + EntityFreezeFrameStore & operator=(const EntityFreezeFrameStore &) = delete; + EntityFreezeFrameStore(EntityFreezeFrameStore &&) = delete; + EntityFreezeFrameStore & operator=(EntityFreezeFrameStore &&) = delete; + virtual ~EntityFreezeFrameStore() = default; + + /// Replaces every row of @p fault_code with @p frames. An empty vector leaves no rows. + [[nodiscard]] virtual tl::expected + replace_frames(const std::string & fault_code, const std::vector & frames) = 0; + + /// Deletes every row of @p fault_code. A code with no rows is not an error. + [[nodiscard]] virtual tl::expected erase_frames(const std::string & fault_code) = 0; + + /// Every row, oldest capture first, then by fault code and entity id. + /// A caller keeps the newest codes under its bound by this order. + [[nodiscard]] virtual tl::expected, std::string> load_all() = 0; +}; + +/// The store contract without a file, for tests and for hosts that want no persistence. +class InMemoryEntityFreezeFrameStore : public EntityFreezeFrameStore { + public: + tl::expected replace_frames(const std::string & fault_code, + const std::vector & frames) override { + std::lock_guard lock(mutex_); + if (frames.empty()) { + rows_.erase(fault_code); + return {}; + } + rows_[fault_code] = frames; + return {}; + } + + tl::expected erase_frames(const std::string & fault_code) override { + std::lock_guard lock(mutex_); + rows_.erase(fault_code); + return {}; + } + + tl::expected, std::string> load_all() override { + std::lock_guard lock(mutex_); + std::vector all; + for (const auto & entry : rows_) { + all.insert(all.end(), entry.second.begin(), entry.second.end()); + } + // The order of SqliteEntityFreezeFrameStore::load_all. + std::stable_sort(all.begin(), all.end(), [](const StoredEntityFreezeFrame & a, const StoredEntityFreezeFrame & b) { + if (a.captured_at_ns != b.captured_at_ns) { + return a.captured_at_ns < b.captured_at_ns; + } + if (a.fault_code != b.fault_code) { + return a.fault_code < b.fault_code; + } + return a.entity_id < b.entity_id; + }); + return all; + } + + private: + std::mutex mutex_; + std::map> rows_; +}; + +} // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/freeze_frame_capture.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/freeze_frame_capture.hpp index 70e8729fe..b12ffc654 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/freeze_frame_capture.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/freeze_frame_capture.hpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -31,6 +32,7 @@ #include "ros2_medkit_fault_detection/fault_types.hpp" #include "ros2_medkit_gateway/core/data/data_item_reader.hpp" +#include "ros2_medkit_gateway/core/faults/entity_freeze_frame_store.hpp" #include "ros2_medkit_gateway/core/faults/fault_event_source.hpp" #include "ros2_medkit_gateway/core/faults/freeze_frame_source.hpp" #include "ros2_medkit_gateway/core/logging.hpp" @@ -43,7 +45,8 @@ class PluginManager; struct FreezeFrameCaptureConfig { /// `entity_freeze_frame.enabled`: capture at all. bool enabled{true}; - /// Retained-frame bound: the oldest fault's frames are evicted past it. Not a node parameter. + /// Retained-frame bound: the oldest fault's frames are evicted past it. Reloaded and + /// captured frames count together. Not a node parameter. size_t max_faults{256}; }; @@ -56,28 +59,71 @@ struct FreezeFrameCaptureConfig { /// `GET /{entity}/data` (the plugin's `DataProvider`, else its `x-plc-data` /// route). The frames are served through `FreezeFrameSource` and merged into the /// fault detail's `environment_data`. A frame set stays after `fault_cleared` and -/// is replaced on the next confirmation. +/// is replaced on the next confirmation. A confirmation of another occurrence whose +/// read gets no values drops the frame set instead, with a warning. +/// +/// With an `EntityFreezeFrameStore` every capture is written through to the store +/// with the fault manager's occurrence of the fault, and the stored frames are loaded +/// on construction, before any capture. At start one fault list of every status +/// decides. Each reloaded frame is: +/// - kept, as it was captured, when its fault lists the stored occurrence, or when +/// either occurrence is not known; +/// - discarded with a warning when its fault lists another occurrence; +/// - discarded with a warning when the fault manager does not hold its fault, unless +/// the list is empty. +/// A reloaded frame is served only after that judgement. When the host gives no +/// lister, or the lister throws or answers with an empty list, the reloaded frames +/// are kept and served unjudged. Then every confirmed fault that is not muted and +/// has no frame is read and marked startup. /// /// The reads run on a thread of this object, never on the thread that delivers /// the event. class FreezeFrameCapture : public FreezeFrameSource { public: + /// Stored `capture_origin` of a catch-up frame. A reloaded frame keeps it. + static constexpr const char * kCaptureOriginStartup = "startup"; + /// Resolves an entity id to the `DataProvider` of its plugin (null when there is none). using DataProviderResolver = PluginDataReaders::ProviderResolver; /// Fetches an entity's current values through its plugin's `x-plc-data` route. using RouteDataFetcher = PluginDataReaders::RouteFetcher; - /// One standing fault: its code and the entities that reported it. + /// The fault manager's identity of one occurrence of a fault: the record's + /// `occurrence_count` and `first_occurred`. Both change only when a CLEARED fault + /// reactivates. `first_occurred_ns` is kept as a ListFaults reply carries it (seconds), + /// so an event and a reply of one occurrence give equal values. + struct Occurrence { + uint32_t count{0}; + int64_t first_occurred_ns{0}; + /// False when either value is missing: then nothing can be told from it. + bool known() const { + return count != 0 && first_occurred_ns != 0; + } + bool operator==(const Occurrence & other) const { + return count == other.count && first_occurred_ns == other.first_occurred_ns; + } + bool operator!=(const Occurrence & other) const { + return !(*this == other); + } + }; + + /// One fault of a ListFaults reply: its code, the entities that reported it, its + /// status, its occurrence and whether correlation mutes it. struct StandingFault { std::string fault_code; std::vector reporting_sources; + /// CONFIRMED unless set: the status of a standing fault. + std::string status{ros2_medkit::fault_detection::fault_status::kConfirmed}; + Occurrence occurrence{}; + /// A muted fault publishes no confirm event, so it is not framed at start either. + bool muted{false}; }; - /// Lists the faults that are confirmed when the capture starts, so their missed - /// confirm edge can still be framed. Called once, on the capture thread, never - /// after the destructor has joined it. A blocking wait inside must poll - /// `should_abort` and return when it turns true. + /// Every fault the fault manager holds when the capture starts, in any status and + /// muted ones included. Called once, on the capture thread, never after the + /// destructor has joined it. A blocking wait inside must poll `should_abort` and + /// return when it turns true. An empty answer, or one that gives up, judges nothing. using StandingFaultLister = std::function(const std::function & should_abort)>; /// Blocks until the event feed delivers, or `should_abort` turns true. Runs on the @@ -88,10 +134,12 @@ class FreezeFrameCapture : public FreezeFrameSource { struct Hooks { /// Both reads of an entity's data. Without them nothing is captured. PluginDataReaders readers; - /// Optional startup catch-up. + /// Optional. Judges the reloaded frames and frames the confirmed faults at start. StandingFaultLister standing_lister; /// Optional wait for the feed. EventsReadyWaiter events_ready; + /// Optional persistence. Null keeps the frames in memory only. + std::shared_ptr store; }; /// @param events Feed of fault events. Subscribed on construction; the subscription @@ -108,32 +156,67 @@ class FreezeFrameCapture : public FreezeFrameSource { /// Parses a ListFaults reply body into standing faults. Nullopt when the body has no /// "faults" array. Items that lack a string "fault_code" or an array "reporting_sources" - /// are skipped, and non-string source entries are dropped. Never throws. + /// are skipped, and non-string source entries are dropped. A fault named in + /// "muted_faults" is marked muted. A "first_occurred" that is + /// not a positive number of seconds within int64 nanoseconds, or an "occurrence_count" + /// that is not a positive uint32, leaves that part of `occurrence` at 0. Never throws. static std::optional> standing_faults_from_list_reply(const nlohmann::json & data); private: void on_fault_event(const ros2_medkit::fault_detection::FaultEventRecord & event); void capture_worker(); void capture_standing_faults(); - bool capture_for_event(const ros2_medkit::fault_detection::FaultEventRecord & event, bool startup_catchup = false); + bool capture_for_event(const ros2_medkit::fault_detection::FaultEventRecord & event, const Occurrence & occurrence, + bool startup_catchup = false); + /// The occurrence of @p record, through the ListFaults reply's encoding. + static Occurrence occurrence_of(const ros2_medkit::fault_detection::FaultRecord & record); std::optional frame_from_content(const std::string & entity_id, const std::string & fault_code, const nlohmann::json & content, const std::string & source); std::optional capture_via_route(const std::string & entity_id, const std::string & fault_code); void log_fallback_failure_once(const std::string & fault_code, const std::string & message); + /// Fills frames_ from the store, keeping the newest codes under max_faults_. + void load_persisted_frames(); + static StoredEntityFreezeFrame to_stored(const std::string & fault_code, const Frame & frame, + const Occurrence & occurrence); + /// Nullopt for a row whose blob is not shaped like a frame. + static std::optional from_stored(const StoredEntityFreezeFrame & row); + /// Writes the code's frames through to the store. The caller holds mutex_. + void persist_frames_locked(const std::string & fault_code, const std::vector & frames, + const Occurrence & occurrence); + /// Deletes the code's rows from the store. The caller holds mutex_. + void erase_persisted_locked(const std::string & fault_code); + /// Keeps or discards each reloaded frame against @p listed, then lets frames_for serve + /// the ones kept. + void judge_reloaded_frames(const std::vector & listed); + /// Erases a code's frames from memory and the store. The caller holds mutex_. + void discard_locked(const std::string & fault_code); + PluginDataReaders readers_; StandingFaultLister standing_lister_; EventsReadyWaiter events_ready_; + std::shared_ptr store_; const log::Logger logger_{"entity_freeze_frame"}; const size_t max_faults_; - /// Guards frames_, insertion_order_ and fallback_logged_. + /// Guards frames_, insertion_order_, fallback_logged_, reloaded_codes_, occurrences_, + /// reloaded_judged_ and store_write_warned_. mutable std::mutex mutex_; /// Keyed by fault code only: cross-entity isolation rests on the fault manager keeping /// reporting_sources append-only for a code and on `get_fault` gating by source scope. std::unordered_map> frames_; std::deque insertion_order_; ///< eviction order (FIFO) std::unordered_set fallback_logged_; ///< fault codes already warned about (bounded) + /// Codes whose frames came from the store and were not captured again since. Only + /// these are judged at start. + std::unordered_set reloaded_codes_; + /// The occurrence of every code in frames_, reloaded or captured. + std::unordered_map occurrences_; + /// False until the start has judged the reloaded frames or given up: until then + /// frames_for serves none of them. + bool reloaded_judged_{false}; + /// One warning per process for a failed store write or delete. + bool store_write_warned_{false}; /// Confirm events waiting for capture. Bounded: the oldest event is dropped when full. std::mutex queue_mutex_; @@ -150,12 +233,13 @@ class FreezeFrameCapture : public FreezeFrameSource { /// A host attaches the result with `GatewayServices::freeze_frames()`, and passes: /// - `events`: its fault event source, which may also feed the fault SSE stream; /// - `plugins`: the manager that owns the entities; -/// - `standing_lister` and `events_ready`: optional, see `Hooks`. +/// - `standing_lister`, `events_ready` and `store`: optional, see `Hooks`. /// /// Returns null when the configuration is disabled or `plugins` has no plugin loaded. std::unique_ptr make_freeze_frame_capture(const FreezeFrameCaptureConfig & config, FaultEventSource & events, PluginManager * plugins, FreezeFrameCapture::StandingFaultLister standing_lister = nullptr, - FreezeFrameCapture::EventsReadyWaiter events_ready = nullptr); + FreezeFrameCapture::EventsReadyWaiter events_ready = nullptr, + std::shared_ptr store = nullptr); } // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/sqlite_entity_freeze_frame_store.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/sqlite_entity_freeze_frame_store.hpp new file mode 100644 index 000000000..7b30befe3 --- /dev/null +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/sqlite_entity_freeze_frame_store.hpp @@ -0,0 +1,57 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "ros2_medkit_gateway/core/faults/entity_freeze_frame_store.hpp" + +namespace ros2_medkit_gateway { + +/// SQLite file of entity freeze-frames, table `entity_freeze_frames`. +/// Thread-safe. The table is created on open. ":memory:" opens an ephemeral database. +/// `load_all` returns a row whose `frame` column is not JSON with a null `frame`. +class SqliteEntityFreezeFrameStore : public EntityFreezeFrameStore { + public: + /// Opens or creates the database at @p db_path. + /// @throws std::runtime_error when the file cannot be opened or the table cannot be created. + explicit SqliteEntityFreezeFrameStore(const std::string & db_path); + + ~SqliteEntityFreezeFrameStore() override; + SqliteEntityFreezeFrameStore(const SqliteEntityFreezeFrameStore &) = delete; + SqliteEntityFreezeFrameStore & operator=(const SqliteEntityFreezeFrameStore &) = delete; + SqliteEntityFreezeFrameStore(SqliteEntityFreezeFrameStore &&) = delete; + SqliteEntityFreezeFrameStore & operator=(SqliteEntityFreezeFrameStore &&) = delete; + + tl::expected replace_frames(const std::string & fault_code, + const std::vector & frames) override; + tl::expected erase_frames(const std::string & fault_code) override; + tl::expected, std::string> load_all() override; + + private: + void initialize_schema(); + + /// Deletes every row of a code. The caller holds mutex_. + tl::expected delete_code_locked(const std::string & fault_code); + + sqlite3 * db_{nullptr}; + std::mutex mutex_; +}; + +} // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp index 92674f1f3..18f152169 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp @@ -108,11 +108,13 @@ class EntityFreezeFrameCapture : public FreezeFrameSource { * @param route_fetcher x-plc-data route fallback for entities whose plugin * has no DataProvider (the commercial PLC bridges); may be null * @param max_faults retained-frame bound; oldest fault's frames evicted past it - * @param standing_lister startup catch-up; may be null + * @param standing_lister every fault at start, any status; may be null + * @param store frame persistence; null keeps the frames in memory only */ EntityFreezeFrameCapture(rclcpp::Node * node, ros2_common::Ros2SubscriptionExecutor & exec, DataProviderResolver resolver, RouteDataFetcher route_fetcher = nullptr, - size_t max_faults = 256, StandingFaultLister standing_lister = nullptr); + size_t max_faults = 256, StandingFaultLister standing_lister = nullptr, + std::shared_ptr store = nullptr); ~EntityFreezeFrameCapture() override; EntityFreezeFrameCapture(const EntityFreezeFrameCapture &) = delete; diff --git a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp index fd2a31dee..6ed715c63 100644 --- a/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp +++ b/src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp @@ -408,6 +408,10 @@ class GatewayNode : public rclcpp::Node, public GatewayServices { void refresh_cache(); void start_rest_server(); + /// Opens the store at `entity_freeze_frame.storage.path`, else `entity_freeze_frames.db` + /// next to `triggers.storage.path`. Null when no path resolves or the file cannot be opened. + std::shared_ptr open_entity_freeze_frame_store(); + /// Log a one-time discovery summary shortly after startup: discovered node / /// topic / entity counts, the REST URL and a sample curl. When no application /// nodes are visible, also warn loudly with the active ROS environment diff --git a/src/ros2_medkit_gateway/src/core/faults/freeze_frame_capture.cpp b/src/ros2_medkit_gateway/src/core/faults/freeze_frame_capture.cpp index b6c117e78..785e2027b 100644 --- a/src/ros2_medkit_gateway/src/core/faults/freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/src/core/faults/freeze_frame_capture.cpp @@ -14,11 +14,14 @@ #include "ros2_medkit_gateway/core/faults/freeze_frame_capture.hpp" +#include #include #include +#include #include #include +#include "ros2_medkit_gateway/core/faults/fault_json.hpp" #include "ros2_medkit_gateway/core/plugins/plugin_manager.hpp" namespace ros2_medkit_gateway { @@ -34,13 +37,56 @@ constexpr size_t kMaxQueuedEvents = 64; /// code) instead of growing forever on churny/synthetic fault codes. constexpr size_t kMaxLoggedFaultCodes = 1024; +/// How many fault codes the catch-up truncation warning names. +constexpr size_t kMaxNamedOverCapCodes = 10; + +/// Below INT64_MAX by more than one ulp of a double at this magnitude. +constexpr double kMaxRepresentableNs = 9.2e18; + +/// The occurrence of one ListFaults reply item. +FreezeFrameCapture::Occurrence occurrence_from_item(const nlohmann::json & item) { + FreezeFrameCapture::Occurrence occurrence; + const auto count = item.find("occurrence_count"); + if (count != item.end() && count->is_number_integer()) { + const int64_t value = count->get(); + if (value > 0 && value <= std::numeric_limits::max()) { + occurrence.count = static_cast(value); + } + } + const auto first_occurred = item.find("first_occurred"); + if (first_occurred != item.end() && first_occurred->is_number()) { + // Range-checked before the cast, which is undefined outside int64. NaN fails the check. + const double nanoseconds = first_occurred->get() * 1e9; + if (nanoseconds > 0.0 && nanoseconds < kMaxRepresentableNs) { + occurrence.first_occurred_ns = static_cast(nanoseconds); + } + } + return occurrence; +} + +std::string join(const std::vector & items) { + std::string joined; + for (const auto & item : items) { + if (!joined.empty()) { + joined += ", "; + } + joined += item; + } + return joined; +} + } // namespace FreezeFrameCapture::FreezeFrameCapture(FaultEventSource & events, const FreezeFrameCaptureConfig & config, Hooks hooks) : readers_(std::move(hooks.readers)) , standing_lister_(std::move(hooks.standing_lister)) , events_ready_(std::move(hooks.events_ready)) + , store_(std::move(hooks.store)) , max_faults_(config.max_faults > 0 ? config.max_faults : 1) { + // Before the first event and the first request: the catch-up must see the reloaded frames. + load_persisted_frames(); + // With no lister nothing judges the reloaded frames, so they are served as loaded. + reloaded_judged_ = !standing_lister_; subscription_ = events.subscribe([this](const fd::FaultEventRecord & event) { on_fault_event(event); }); @@ -66,6 +112,10 @@ FreezeFrameCapture::~FreezeFrameCapture() { std::vector FreezeFrameCapture::frames_for(const std::string & fault_code) const { std::lock_guard lock(mutex_); + // Until judged, a reloaded frame may be from an earlier occurrence than the one served. + if (!reloaded_judged_ && reloaded_codes_.count(fault_code) != 0) { + return {}; + } auto it = frames_.find(fault_code); return it != frames_.end() ? it->second : std::vector{}; } @@ -80,6 +130,15 @@ FreezeFrameCapture::standing_faults_from_list_reply(const nlohmann::json & data) if (faults == data.end() || !faults->is_array()) { return std::nullopt; } + std::unordered_set muted; + const auto muted_faults = data.find("muted_faults"); + if (muted_faults != data.end() && muted_faults->is_array()) { + for (const auto & item : *muted_faults) { + if (item.is_object() && item.contains("fault_code") && item["fault_code"].is_string()) { + muted.insert(item["fault_code"].get()); + } + } + } std::vector standing; for (const auto & item : *faults) { if (!item.is_object()) { @@ -97,11 +156,260 @@ FreezeFrameCapture::standing_faults_from_list_reply(const nlohmann::json & data) fault.reporting_sources.push_back(src.get()); } } + const auto status = item.find("status"); + if (status != item.end() && status->is_string()) { + fault.status = status->get(); + } + fault.occurrence = occurrence_from_item(item); + fault.muted = muted.count(fault.fault_code) != 0; standing.push_back(std::move(fault)); } return standing; } +FreezeFrameCapture::Occurrence FreezeFrameCapture::occurrence_of(const fd::FaultRecord & record) { + // The reply's own encoding, so both sources round first_occurred the same way. + return occurrence_from_item(fault_record_to_json(record)); +} + +StoredEntityFreezeFrame FreezeFrameCapture::to_stored(const std::string & fault_code, const Frame & frame, + const Occurrence & occurrence) { + StoredEntityFreezeFrame row; + row.fault_code = fault_code; + row.entity_id = frame.entity_id; + row.frame = nlohmann::json::object(); + row.frame["values"] = frame.values; + // Absent stays absent on reload, not null. + if (frame.connected.has_value()) { + row.frame["connected"] = *frame.connected; + } + if (!frame.source_timestamp.is_null()) { + row.frame["source_timestamp"] = frame.source_timestamp; + } + row.captured_at_ns = frame.captured_at_ns; + row.source = frame.source; + row.capture_origin = frame.startup_catchup ? kCaptureOriginStartup : ""; + row.occurrence_count = occurrence.count; + row.first_occurred_ns = occurrence.first_occurred_ns; + return row; +} + +std::optional FreezeFrameCapture::from_stored(const StoredEntityFreezeFrame & row) { + if (row.entity_id.empty() || !row.frame.is_object()) { + return std::nullopt; + } + const auto values = row.frame.find("values"); + if (values == row.frame.end()) { + return std::nullopt; + } + Frame frame; + frame.entity_id = row.entity_id; + frame.values = *values; + frame.captured_at_ns = row.captured_at_ns; + frame.source = row.source; + frame.startup_catchup = row.capture_origin == kCaptureOriginStartup; + const auto connected = row.frame.find("connected"); + if (connected != row.frame.end() && connected->is_boolean()) { + frame.connected = connected->get(); + } + const auto source_timestamp = row.frame.find("source_timestamp"); + if (source_timestamp != row.frame.end()) { + frame.source_timestamp = *source_timestamp; + } + return frame; +} + +void FreezeFrameCapture::persist_frames_locked(const std::string & fault_code, const std::vector & frames, + const Occurrence & occurrence) { + if (!store_) { + return; + } + std::vector rows; + rows.reserve(frames.size()); + for (const auto & frame : frames) { + rows.push_back(to_stored(fault_code, frame, occurrence)); + } + auto written = store_->replace_frames(fault_code, rows); + if (!written && !store_write_warned_) { + // A read-only or full volume costs one line, and the frames still work in memory. + store_write_warned_ = true; + MEDKIT_LOG_WARN(logger_, "Entity freeze-frame store write failed, frames are process-local until restart: %s", + written.error().c_str()); + } +} + +void FreezeFrameCapture::erase_persisted_locked(const std::string & fault_code) { + if (!store_) { + return; + } + auto erased = store_->erase_frames(fault_code); + if (!erased && !store_write_warned_) { + store_write_warned_ = true; + MEDKIT_LOG_WARN(logger_, "Entity freeze-frame store delete failed: %s", erased.error().c_str()); + } +} + +void FreezeFrameCapture::load_persisted_frames() { + if (!store_) { + return; + } + auto rows = store_->load_all(); + if (!rows) { + MEDKIT_LOG_WARN(logger_, "Entity freeze-frame store unreadable, starting with no reloaded frames: %s", + rows.error().c_str()); + return; + } + + std::unordered_map> loaded; + std::unordered_map newest; + // One capture writes all rows of a code, so they carry one occurrence. + std::unordered_map occurrences; + size_t unreadable = 0; + for (const auto & row : *rows) { + auto frame = from_stored(row); + if (!frame) { + ++unreadable; + continue; + } + auto it = newest.find(row.fault_code); + if (it == newest.end()) { + newest.emplace(row.fault_code, row.captured_at_ns); + } else { + it->second = std::max(it->second, row.captured_at_ns); + } + occurrences[row.fault_code] = Occurrence{row.occurrence_count, row.first_occurred_ns}; + loaded[row.fault_code].push_back(std::move(*frame)); + } + + // Oldest code first: past the bound the oldest go, and FIFO eviction keeps that order. + std::vector codes; + codes.reserve(loaded.size()); + for (const auto & entry : loaded) { + codes.push_back(entry.first); + } + std::sort(codes.begin(), codes.end(), [&newest](const std::string & a, const std::string & b) { + const int64_t at_a = newest.at(a); + const int64_t at_b = newest.at(b); + return at_a != at_b ? at_a < at_b : a < b; + }); + const size_t over_cap = codes.size() > max_faults_ ? codes.size() - max_faults_ : 0; + + size_t reloaded = 0; + { + std::lock_guard lock(mutex_); + for (size_t i = 0; i < codes.size(); ++i) { + const auto & code = codes[i]; + if (i < over_cap) { + // A row past the bound would be read and dropped again on every start. + erase_persisted_locked(code); + continue; + } + insertion_order_.push_back(code); + frames_[code] = std::move(loaded[code]); + reloaded_codes_.insert(code); + occurrences_[code] = occurrences[code]; + } + reloaded = frames_.size(); + } + if (reloaded > 0) { + MEDKIT_LOG_INFO(logger_, "Entity freeze-frame: reloaded frames for %zu fault(s) from the store", reloaded); + } + if (over_cap > 0) { + MEDKIT_LOG_WARN(logger_, + "Entity freeze-frame store held %zu fault(s) beyond the retained-frame bound of %zu; " + "the oldest were dropped", + over_cap, max_faults_); + } + if (unreadable > 0) { + MEDKIT_LOG_WARN(logger_, "Entity freeze-frame store: %zu unreadable row(s) skipped", unreadable); + } +} + +void FreezeFrameCapture::judge_reloaded_frames(const std::vector & listed) { + std::unordered_map by_code; + for (const auto & fault : listed) { + by_code.emplace(fault.fault_code, &fault); + } + struct Superseded { + std::string fault_code; + std::string entities; + Occurrence stored; + const StandingFault * fault; + }; + std::vector superseded; + std::vector orphans; + bool kept_for_empty_list = false; + { + std::lock_guard lock(mutex_); + // A fault manager whose storage is unreachable answers with an empty list, so empty is "cannot tell". + kept_for_empty_list = listed.empty() && !reloaded_codes_.empty(); + for (const auto & code : reloaded_codes_) { + const auto held = occurrences_.find(code); + const Occurrence stored = held != occurrences_.end() ? held->second : Occurrence{}; + const auto entry = by_code.find(code); + if (entry == by_code.end()) { + if (!listed.empty()) { + orphans.push_back(code); + } + continue; + } + const Occurrence & now = entry->second->occurrence; + // An occurrence that is not known on either side cannot tell: the frame stays. + if (!stored.known() || !now.known() || now == stored) { + continue; + } + std::vector entities; + for (const auto & frame : frames_[code]) { + entities.push_back(frame.entity_id); + } + superseded.push_back({code, join(entities), stored, entry->second}); + } + for (const auto & code : orphans) { + discard_locked(code); + } + for (const auto & entry : superseded) { + discard_locked(entry.fault_code); + } + reloaded_judged_ = true; + } + if (kept_for_empty_list) { + MEDKIT_LOG_INFO(logger_, + "Entity freeze-frame: the fault manager lists no fault in any status, so no reloaded frame is " + "dropped"); + } + if (!orphans.empty()) { + std::sort(orphans.begin(), orphans.end()); + const size_t total = orphans.size(); + if (orphans.size() > kMaxNamedOverCapCodes) { + orphans.resize(kMaxNamedOverCapCodes); + } + MEDKIT_LOG_WARN(logger_, + "Entity freeze-frame: dropped the reloaded frames of %zu fault(s) the fault manager no longer " + "holds: %s%s", + total, join(orphans).c_str(), total > orphans.size() ? ", ..." : ""); + } + std::sort(superseded.begin(), superseded.end(), [](const Superseded & a, const Superseded & b) { + return a.fault_code < b.fault_code; + }); + for (const auto & entry : superseded) { + // Serving it would present the previous occurrence's values as this one's. + MEDKIT_LOG_WARN(logger_, + "Entity freeze-frame for fault '%s': the stored frame of entity '%s' is from an earlier occurrence " + "and was discarded (stored occurrence %u, the fault manager lists occurrence %u, %s).", + entry.fault_code.c_str(), entry.entities.c_str(), entry.stored.count, entry.fault->occurrence.count, + entry.fault->status.c_str()); + } +} + +void FreezeFrameCapture::discard_locked(const std::string & fault_code) { + frames_.erase(fault_code); + insertion_order_.erase(std::remove(insertion_order_.begin(), insertion_order_.end(), fault_code), + insertion_order_.end()); + reloaded_codes_.erase(fault_code); + occurrences_.erase(fault_code); + erase_persisted_locked(fault_code); +} + std::optional FreezeFrameCapture::frame_from_content(const std::string & entity_id, const std::string & fault_code, const nlohmann::json & content, @@ -205,9 +513,9 @@ void FreezeFrameCapture::capture_standing_faults() { // The lister blocks until the fault services answer, so it runs here rather // than on a thread of its own: this thread is joined by the destructor, and // should_abort is what lets that join interrupt the wait. - std::vector standing; + std::vector listed; try { - standing = standing_lister_(should_abort); + listed = standing_lister_(should_abort); } catch (const std::exception & e) { MEDKIT_LOG_WARN(logger_, "Entity freeze-frame startup catch-up failed: standing-fault lister threw: %s", e.what()); return; @@ -215,6 +523,8 @@ void FreezeFrameCapture::capture_standing_faults() { MEDKIT_LOG_WARN(logger_, "Entity freeze-frame startup catch-up failed: standing-fault lister threw"); return; } + // One answer decides both the reloaded frames and what is read now. + judge_reloaded_frames(listed); // Codes with a confirm already queued belong to the drain loop: capturing // them here too would read the plugin twice for one confirm. std::unordered_set queued_codes; @@ -227,44 +537,75 @@ void FreezeFrameCapture::capture_standing_faults() { queued_codes.insert(queued.fault.fault_code); } } + + // Occupancy is read once, so a fault is admitted only against a free slot and the + // FIFO never evicts a frame still wanted. + std::unordered_set already_framed; size_t framed = 0; + { + std::lock_guard lock(mutex_); + for (const auto & entry : frames_) { + already_framed.insert(entry.first); + } + framed = frames_.size(); + } + size_t captured = 0; size_t over_cap = 0; - for (const auto & fault : standing) { + std::vector over_cap_codes; + for (const auto & fault : listed) { if (should_abort()) { return; } + // What a live confirm event would frame: a confirmed fault that is not muted. + if (fault.status != fd::fault_status::kConfirmed || fault.muted) { + continue; + } if (fault.fault_code.empty() || fault.reporting_sources.empty()) { continue; } if (queued_codes.count(fault.fault_code) != 0) { continue; } + // A reloaded frame from this occurrence is the one to serve. + if (already_framed.count(fault.fault_code) != 0) { + continue; + } if (framed >= max_faults_) { - ++over_cap; // storing more would FIFO-evict this catch-up's own frames + ++over_cap; // storing more would FIFO-evict a frame that is still wanted + if (over_cap_codes.size() < kMaxNamedOverCapCodes) { + over_cap_codes.push_back(fault.fault_code); + } continue; } fd::FaultEventRecord event; event.event_type = fd::fault_event_type::kConfirmed; event.fault.fault_code = fault.fault_code; event.fault.reporting_sources = fault.reporting_sources; - if (capture_for_event(event, /*startup_catchup=*/true)) { + if (capture_for_event(event, fault.occurrence, /*startup_catchup=*/true)) { ++framed; + ++captured; } } if (over_cap > 0) { MEDKIT_LOG_WARN(logger_, "Entity freeze-frame startup catch-up truncated: %zu standing fault(s) beyond the retained-frame " - "bound of %zu", - over_cap, max_faults_); + "bound of %zu, so they have no freeze-frame: %s%s", + over_cap, max_faults_, join(over_cap_codes).c_str(), + over_cap > over_cap_codes.size() ? ", ..." : ""); } - if (framed > 0) { + if (captured > 0) { MEDKIT_LOG_INFO(logger_, "Entity freeze-frame: captured %zu fault(s) that were already confirmed at startup", - framed); + captured); } } void FreezeFrameCapture::capture_worker() { capture_standing_faults(); + { + // Also when the lister gave up or threw: then every reloaded frame is kept. + std::lock_guard lock(mutex_); + reloaded_judged_ = true; + } for (;;) { fd::FaultEventRecord event; { @@ -280,11 +621,12 @@ void FreezeFrameCapture::capture_worker() { event = std::move(queue_.front()); queue_.pop_front(); } - capture_for_event(event); + capture_for_event(event, occurrence_of(event.fault)); } } -bool FreezeFrameCapture::capture_for_event(const fd::FaultEventRecord & event, bool startup_catchup) { +bool FreezeFrameCapture::capture_for_event(const fd::FaultEventRecord & event, const Occurrence & occurrence, + bool startup_catchup) { const std::string & fault_code = event.fault.fault_code; std::vector frames; @@ -320,6 +662,22 @@ bool FreezeFrameCapture::capture_for_event(const fd::FaultEventRecord & event, b } if (frames.empty()) { + uint32_t dropped_count = 0; + { + std::lock_guard lock(mutex_); + const auto held = occurrences_.find(fault_code); + // The same occurrence, or one that cannot be told, keeps the frame from its confirmation. + if (held == occurrences_.end() || !held->second.known() || !occurrence.known() || held->second == occurrence) { + return false; + } + dropped_count = held->second.count; + discard_locked(fault_code); + } + // Serving the held frame would present the previous occurrence's values as this one's. + MEDKIT_LOG_WARN(logger_, + "Entity freeze-frame for fault '%s': occurrence %u confirmed and entity '%s' served no values, so " + "this occurrence has no freeze-frame. The frame of occurrence %u was dropped.", + fault_code.c_str(), occurrence.count, join(event.fault.reporting_sources).c_str(), dropped_count); return false; } if (startup_catchup) { @@ -329,14 +687,30 @@ bool FreezeFrameCapture::capture_for_event(const fd::FaultEventRecord & event, b } std::lock_guard lock(mutex_); - if (frames_.find(fault_code) == frames_.end()) { + if (frames_.find(fault_code) != frames_.end()) { + // A recapture is the newest frame, as its captured_at says after a reload. + insertion_order_.erase(std::remove(insertion_order_.begin(), insertion_order_.end(), fault_code), + insertion_order_.end()); + insertion_order_.push_back(fault_code); + } else { insertion_order_.push_back(fault_code); while (frames_.size() >= max_faults_ && !insertion_order_.empty()) { - frames_.erase(insertion_order_.front()); + const std::string evicted = insertion_order_.front(); + frames_.erase(evicted); insertion_order_.pop_front(); + // The rows go too, or the evicted frame comes back on the next start. + reloaded_codes_.erase(evicted); + occurrences_.erase(evicted); + erase_persisted_locked(evicted); } } - frames_[fault_code] = std::move(frames); + // A capture on this fault's own edge supersedes what was reloaded for it. + reloaded_codes_.erase(fault_code); + occurrences_[fault_code] = occurrence; + auto & stored = frames_[fault_code]; + stored = std::move(frames); + // Under the same lock as the map, so the store and what is served agree. + persist_frames_locked(fault_code, stored, occurrence); MEDKIT_LOG_DEBUG(logger_, "Captured entity freeze-frame(s) for fault '%s'", fault_code.c_str()); return true; @@ -345,7 +719,8 @@ bool FreezeFrameCapture::capture_for_event(const fd::FaultEventRecord & event, b std::unique_ptr make_freeze_frame_capture(const FreezeFrameCaptureConfig & config, FaultEventSource & events, PluginManager * plugins, FreezeFrameCapture::StandingFaultLister standing_lister, - FreezeFrameCapture::EventsReadyWaiter events_ready) { + FreezeFrameCapture::EventsReadyWaiter events_ready, + std::shared_ptr store) { // Only plugins own entities whose values the fault manager cannot capture. if (!config.enabled || plugins == nullptr || !plugins->has_plugins()) { return nullptr; @@ -354,6 +729,7 @@ std::unique_ptr make_freeze_frame_capture(const FreezeFrameC hooks.readers = plugin_data_readers(plugins); hooks.standing_lister = std::move(standing_lister); hooks.events_ready = std::move(events_ready); + hooks.store = std::move(store); return std::make_unique(events, config, std::move(hooks)); } diff --git a/src/ros2_medkit_gateway/src/core/faults/sqlite_entity_freeze_frame_store.cpp b/src/ros2_medkit_gateway/src/core/faults/sqlite_entity_freeze_frame_store.cpp new file mode 100644 index 000000000..09da4d693 --- /dev/null +++ b/src/ros2_medkit_gateway/src/core/faults/sqlite_entity_freeze_frame_store.cpp @@ -0,0 +1,241 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ros2_medkit_gateway/core/faults/sqlite_entity_freeze_frame_store.hpp" + +#include +#include +#include + +namespace ros2_medkit_gateway { + +namespace { + +/// RAII prepared statement with the 64-bit binds a nanosecond timestamp needs. +class SqliteStatement { + public: + SqliteStatement(sqlite3 * db, const char * sql) : db_(db) { + if (sqlite3_prepare_v2(db, sql, -1, &stmt_, nullptr) != SQLITE_OK) { + throw std::runtime_error(std::string("Failed to prepare statement: ") + sqlite3_errmsg(db)); + } + } + + ~SqliteStatement() { + if (stmt_) { + sqlite3_finalize(stmt_); + } + } + + SqliteStatement(const SqliteStatement &) = delete; + SqliteStatement & operator=(const SqliteStatement &) = delete; + SqliteStatement(SqliteStatement &&) = delete; + SqliteStatement & operator=(SqliteStatement &&) = delete; + + void bind_text(int index, const std::string & value) { + if (value.size() > static_cast(std::numeric_limits::max())) { + throw std::runtime_error("bind_text: value exceeds SQLite int length limit"); + } + if (sqlite3_bind_text(stmt_, index, value.c_str(), static_cast(value.size()), SQLITE_TRANSIENT) != SQLITE_OK) { + throw std::runtime_error(std::string("Failed to bind text: ") + sqlite3_errmsg(db_)); + } + } + + void bind_int64(int index, int64_t value) { + if (sqlite3_bind_int64(stmt_, index, value) != SQLITE_OK) { + throw std::runtime_error(std::string("Failed to bind int64: ") + sqlite3_errmsg(db_)); + } + } + + int step() { + return sqlite3_step(stmt_); + } + + std::string column_text(int index) { + const auto * text = reinterpret_cast(sqlite3_column_text(stmt_, index)); + return text ? std::string(text) : std::string(); + } + + int64_t column_int64(int index) { + return sqlite3_column_int64(stmt_, index); + } + + private: + sqlite3 * db_; + sqlite3_stmt * stmt_{nullptr}; +}; + +std::string take_error(char * err_msg) { + std::string error = err_msg ? err_msg : "Unknown error"; + sqlite3_free(err_msg); + return error; +} + +} // namespace + +SqliteEntityFreezeFrameStore::SqliteEntityFreezeFrameStore(const std::string & db_path) { + const int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX; + if (sqlite3_open_v2(db_path.c_str(), &db_, flags, nullptr) != SQLITE_OK) { + const std::string error = db_ ? sqlite3_errmsg(db_) : "Unknown error"; + sqlite3_close(db_); + db_ = nullptr; + throw std::runtime_error("Failed to open entity freeze-frame database '" + db_path + "': " + error); + } + try { + char * err_msg = nullptr; + if (sqlite3_exec(db_, "PRAGMA journal_mode=WAL;", nullptr, nullptr, &err_msg) != SQLITE_OK) { + throw std::runtime_error("Failed to enable WAL mode: " + take_error(err_msg)); + } + sqlite3_busy_timeout(db_, 5000); + initialize_schema(); + } catch (...) { + // The destructor does not run for a constructor that throws. + sqlite3_close(db_); + db_ = nullptr; + throw; + } +} + +SqliteEntityFreezeFrameStore::~SqliteEntityFreezeFrameStore() { + if (db_) { + sqlite3_close(db_); + } +} + +void SqliteEntityFreezeFrameStore::initialize_schema() { + // One row per entity of a fault: a fault reported by two entities freezes both. + const char * create_table = R"( + CREATE TABLE IF NOT EXISTS entity_freeze_frames ( + fault_code TEXT NOT NULL, + entity_id TEXT NOT NULL, + frame TEXT NOT NULL, + captured_at_ns INTEGER NOT NULL, + source TEXT NOT NULL DEFAULT '', + capture_origin TEXT NOT NULL DEFAULT '', + occurrence_count INTEGER NOT NULL DEFAULT 0, + first_occurred_ns INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (fault_code, entity_id) + ); + )"; + char * err_msg = nullptr; + if (sqlite3_exec(db_, create_table, nullptr, nullptr, &err_msg) != SQLITE_OK) { + throw std::runtime_error("Failed to create entity_freeze_frames table: " + take_error(err_msg)); + } +} + +tl::expected SqliteEntityFreezeFrameStore::delete_code_locked(const std::string & fault_code) { + SqliteStatement del(db_, "DELETE FROM entity_freeze_frames WHERE fault_code = ?"); + del.bind_text(1, fault_code); + if (del.step() != SQLITE_DONE) { + return tl::make_unexpected(std::string("Failed to delete entity freeze-frames: ") + sqlite3_errmsg(db_)); + } + return {}; +} + +tl::expected +SqliteEntityFreezeFrameStore::replace_frames(const std::string & fault_code, + const std::vector & frames) { + std::lock_guard lock(mutex_); + const auto rollback = [this] { + sqlite3_exec(db_, "ROLLBACK", nullptr, nullptr, nullptr); + }; + // Delete then insert in one transaction: an entity the new capture no longer + // reports loses its row, and no reader sees the code half written. + char * err_msg = nullptr; + if (sqlite3_exec(db_, "BEGIN IMMEDIATE", nullptr, nullptr, &err_msg) != SQLITE_OK) { + return tl::make_unexpected("replace_frames: BEGIN failed: " + take_error(err_msg)); + } + try { + auto deleted = delete_code_locked(fault_code); + if (!deleted) { + rollback(); + return deleted; + } + for (const auto & frame : frames) { + SqliteStatement stmt(db_, + "INSERT OR REPLACE INTO entity_freeze_frames " + "(fault_code, entity_id, frame, captured_at_ns, source, capture_origin, " + "occurrence_count, first_occurred_ns) " + "VALUES (?,?,?,?,?,?,?,?)"); + stmt.bind_text(1, fault_code); + stmt.bind_text(2, frame.entity_id); + stmt.bind_text(3, frame.frame.dump()); + stmt.bind_int64(4, frame.captured_at_ns); + stmt.bind_text(5, frame.source); + stmt.bind_text(6, frame.capture_origin); + stmt.bind_int64(7, frame.occurrence_count); + stmt.bind_int64(8, frame.first_occurred_ns); + if (stmt.step() != SQLITE_DONE) { + const std::string error = sqlite3_errmsg(db_); + rollback(); + return tl::make_unexpected("Failed to save entity freeze-frame: " + error); + } + } + if (sqlite3_exec(db_, "COMMIT", nullptr, nullptr, &err_msg) != SQLITE_OK) { + const std::string error = take_error(err_msg); + rollback(); + return tl::make_unexpected("replace_frames: COMMIT failed: " + error); + } + return {}; + } catch (const std::exception & e) { + rollback(); + return tl::make_unexpected(std::string("replace_frames: ") + e.what()); + } +} + +tl::expected SqliteEntityFreezeFrameStore::erase_frames(const std::string & fault_code) { + std::lock_guard lock(mutex_); + try { + return delete_code_locked(fault_code); + } catch (const std::exception & e) { + return tl::make_unexpected(std::string("erase_frames: ") + e.what()); + } +} + +tl::expected, std::string> SqliteEntityFreezeFrameStore::load_all() { + std::lock_guard lock(mutex_); + try { + SqliteStatement stmt(db_, + "SELECT fault_code, entity_id, frame, captured_at_ns, source, capture_origin, " + "occurrence_count, first_occurred_ns " + "FROM entity_freeze_frames " + "ORDER BY captured_at_ns ASC, fault_code ASC, entity_id ASC"); + std::vector result; + int rc = SQLITE_ROW; + while ((rc = stmt.step()) == SQLITE_ROW) { + auto parsed = nlohmann::json::parse(stmt.column_text(2), nullptr, false); + StoredEntityFreezeFrame row; + row.fault_code = stmt.column_text(0); + row.entity_id = stmt.column_text(1); + // Not JSON: a null frame, so the caller skips and counts it like any unreadable row. + row.frame = parsed.is_discarded() ? nlohmann::json() : std::move(parsed); + row.captured_at_ns = stmt.column_int64(3); + row.source = stmt.column_text(4); + row.capture_origin = stmt.column_text(5); + // Out of range reads as "not known". + const int64_t count = stmt.column_int64(6); + row.occurrence_count = + count > 0 && count <= std::numeric_limits::max() ? static_cast(count) : 0; + row.first_occurred_ns = stmt.column_int64(7); + result.push_back(std::move(row)); + } + if (rc != SQLITE_DONE) { + return tl::make_unexpected(std::string("load_all: ") + sqlite3_errmsg(db_)); + } + return result; + } catch (const std::exception & e) { + return tl::make_unexpected(std::string("load_all: ") + e.what()); + } +} + +} // namespace ros2_medkit_gateway diff --git a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp index e7ddc0054..db29e2bab 100644 --- a/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp @@ -141,7 +141,8 @@ void SlotFaultEvents::wait_for_publisher(const std::function & should_ab EntityFreezeFrameCapture::EntityFreezeFrameCapture(rclcpp::Node * node, ros2_common::Ros2SubscriptionExecutor & exec, DataProviderResolver resolver, RouteDataFetcher route_fetcher, - size_t max_faults, StandingFaultLister standing_lister) + size_t max_faults, StandingFaultLister standing_lister, + std::shared_ptr store) : logger_(node->get_logger()) , events_(node, exec) , capture_(std::make_unique( @@ -150,7 +151,8 @@ EntityFreezeFrameCapture::EntityFreezeFrameCapture(rclcpp::Node * node, ros2_com std::move(standing_lister), [&events = events_](const std::function & should_abort) { events.wait_for_publisher(should_abort); - }})) { + }, + std::move(store)})) { RCLCPP_INFO(logger_, "EntityFreezeFrameCapture initialized, subscribed to %s", events_.topic().c_str()); } diff --git a/src/ros2_medkit_gateway/src/gateway_node.cpp b/src/ros2_medkit_gateway/src/gateway_node.cpp index af9c2abae..1a1415308 100644 --- a/src/ros2_medkit_gateway/src/gateway_node.cpp +++ b/src/ros2_medkit_gateway/src/gateway_node.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include "ros2_medkit_gateway/core/discovery/merge_types.hpp" #include "ros2_medkit_gateway/core/discovery/refresh_debounce.hpp" #include "ros2_medkit_gateway/core/entity_validation.hpp" +#include "ros2_medkit_gateway/core/faults/sqlite_entity_freeze_frame_store.hpp" #include "ros2_medkit_gateway/core/thread_pool_config.hpp" #include "ros2_medkit_gateway/fault_manager_paths.hpp" #include "ros2_medkit_gateway/param_utils.hpp" @@ -233,6 +235,8 @@ GatewayNode::GatewayNode(const rclcpp::NodeOptions & options) : Node("ros2_medki // Zero-config freeze-frames for plugin-backed entities (opt-out) declare_parameter("entity_freeze_frame.enabled", true); + // SQLite file of the frames. Empty falls back to a file next to triggers.storage.path. + declare_parameter("entity_freeze_frame.storage.path", ""); // Locking parameters declare_parameter("locking.enabled", true); @@ -1504,6 +1508,7 @@ void GatewayNode::init_entity_freeze_frame_capture(ros2_common::Ros2Subscription if (!get_parameter("entity_freeze_frame.enabled").as_bool() || !plugin_mgr_ || !plugin_mgr_->has_plugins()) { return; } + auto frame_store = open_entity_freeze_frame_store(); entity_freeze_frame_capture_ = std::make_unique( this, exec, [this](const std::string & entity_id) { @@ -1520,7 +1525,8 @@ void GatewayNode::init_entity_freeze_frame_capture(ros2_common::Ros2Subscription // being wired, and a device that is in fault when the box boots confirms // immediately, to nobody. The capture asks for those on its own thread, // which it joins on destruction; should_abort lets that join interrupt - // the service wait. + // the service wait. One list of every status, muted faults included, + // also judges the frames reloaded from the store. [this](const std::function & should_abort) { std::vector standing; if (!fault_service_transport_) { @@ -1551,7 +1557,7 @@ void GatewayNode::init_entity_freeze_frame_capture(ros2_common::Ros2Subscription } return standing; } - auto result = fault_service_transport_->list_faults("", false, true, false, false, false, false); + auto result = fault_service_transport_->list_faults("", true, true, true, true, true, false); if (!result.success) { RCLCPP_WARN(get_logger(), "Standing-fault freeze-frame catch-up skipped: ListFaults failed: %s", result.error_message.c_str()); @@ -1562,11 +1568,41 @@ void GatewayNode::init_entity_freeze_frame_capture(ros2_common::Ros2Subscription RCLCPP_WARN(get_logger(), "Standing-fault freeze-frame catch-up skipped: malformed ListFaults reply"); return standing; } - if (parsed->empty()) { + const bool none_confirmed = + std::none_of(parsed->begin(), parsed->end(), [](const EntityFreezeFrameCapture::StandingFault & fault) { + return fault.status == ros2_medkit::fault_detection::fault_status::kConfirmed && !fault.muted; + }); + if (none_confirmed) { RCLCPP_INFO(get_logger(), "Standing-fault freeze-frame catch-up: no confirmed faults at startup"); } return std::move(*parsed); - }); + }, + std::move(frame_store)); +} + +std::shared_ptr GatewayNode::open_entity_freeze_frame_store() { + // An explicit path wins, else the directory an operator already gave the trigger store. + std::string path = get_parameter("entity_freeze_frame.storage.path").as_string(); + if (path.empty()) { + const std::string trigger_path = get_parameter("triggers.storage.path").as_string(); + if (trigger_path.empty()) { + RCLCPP_INFO(get_logger(), + "Entity freeze-frames are not persisted (no entity_freeze_frame.storage.path and no " + "triggers.storage.path); they are lost on restart"); + return nullptr; + } + path = (std::filesystem::path(trigger_path).parent_path() / "entity_freeze_frames.db").string(); + } + try { + auto store = std::make_shared(path); + RCLCPP_INFO(get_logger(), "Entity freeze-frames persisted in %s", path.c_str()); + return store; + } catch (const std::exception & e) { + // The capture still runs, with the frames in memory. + RCLCPP_ERROR(get_logger(), "Entity freeze-frame store '%s' could not be opened, frames stay in memory: %s", + path.c_str(), e.what()); + return nullptr; + } } EntityFreezeFrameCapture * GatewayNode::get_entity_freeze_frame_capture() const { diff --git a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp index e94e657de..e35c37ad6 100644 --- a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp +++ b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_capture.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -749,16 +750,53 @@ TEST(ValuesFromListContent, NonStringIdOrNameNeverThrows) { } TEST(StandingFaultsFromListReply, ParsesWellFormedReply) { - const json data = {{"faults", json::array({json{{"fault_code", "F1"}, {"reporting_sources", json::array({"a", "b"})}}, - json{{"fault_code", "F2"}, {"reporting_sources", json::array()}}})}, - {"count", 2}}; + // F2 is muted: the reply lists it under "muted_faults" as well. + const json data = { + {"faults", + json::array({json{{"fault_code", "F1"}, {"reporting_sources", json::array({"a", "b"})}}, + json{{"fault_code", "F2"}, {"reporting_sources", json::array()}, {"status", "CLEARED"}}})}, + {"count", 2}, + {"muted_faults", json::array({json{{"fault_code", "F2"}, {"root_cause_code", "F1"}}})}}; const auto standing = EntityFreezeFrameCapture::standing_faults_from_list_reply(data); ASSERT_TRUE(standing.has_value()); ASSERT_EQ(standing->size(), 2u); EXPECT_EQ((*standing)[0].fault_code, "F1"); EXPECT_EQ((*standing)[0].reporting_sources, (std::vector{"a", "b"})); + EXPECT_EQ((*standing)[0].status, "CONFIRMED"); // no status in the item + EXPECT_FALSE((*standing)[0].muted); EXPECT_EQ((*standing)[1].fault_code, "F2"); EXPECT_TRUE((*standing)[1].reporting_sources.empty()); + EXPECT_EQ((*standing)[1].status, "CLEARED"); + EXPECT_TRUE((*standing)[1].muted); +} + +TEST(StandingFaultsFromListReply, FirstOccurredIsReadInSecondsAndKeptInNanoseconds) { + // Seconds on the wire. Anything that is not a positive number within int64 nanoseconds + // leaves 0, which reads as "cannot tell" and keeps a stored frame. + const auto item = [](const std::string & code, const json & first_occurred) { + json fault = {{"fault_code", code}, {"reporting_sources", json::array({"a"})}}; + if (!first_occurred.is_null()) { + fault["first_occurred"] = first_occurred; + } + return fault; + }; + const json data = { + {"faults", json::array({item("SECONDS", 1788948705.5), item("ABSENT", nullptr), item("NOT_A_NUMBER", "yesterday"), + item("ZERO", 0), + // Past int64 once in nanoseconds: the cast would be undefined. + item("HUGE", 1e19), item("INFINITE", std::numeric_limits::infinity()), + item("NAN_SECONDS", std::numeric_limits::quiet_NaN())})}}; + const auto standing = EntityFreezeFrameCapture::standing_faults_from_list_reply(data); + ASSERT_TRUE(standing.has_value()); + ASSERT_EQ(standing->size(), 7u); + EXPECT_EQ((*standing)[0].fault_code, "SECONDS"); + EXPECT_EQ((*standing)[0].occurrence.first_occurred_ns, 1788948705500000000); + for (size_t i = 1; i < standing->size(); ++i) { + EXPECT_EQ((*standing)[i].occurrence.first_occurred_ns, 0) << (*standing)[i].fault_code; + } + EXPECT_EQ((*standing)[4].fault_code, "HUGE"); + EXPECT_EQ((*standing)[5].fault_code, "INFINITE"); + EXPECT_EQ((*standing)[6].fault_code, "NAN_SECONDS"); } TEST(StandingFaultsFromListReply, RepliesNotShapedLikeListFaultsYieldNullopt) { diff --git a/src/ros2_medkit_gateway/test/test_entity_freeze_frame_store.cpp b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_store.cpp new file mode 100644 index 000000000..4b154e6a0 --- /dev/null +++ b/src/ros2_medkit_gateway/test/test_entity_freeze_frame_store.cpp @@ -0,0 +1,214 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include + +#include +#include +#include +#include + +#include "ros2_medkit_gateway/core/faults/entity_freeze_frame_store.hpp" +#include "ros2_medkit_gateway/core/faults/sqlite_entity_freeze_frame_store.hpp" + +using json = nlohmann::json; +using ros2_medkit_gateway::EntityFreezeFrameStore; +using ros2_medkit_gateway::InMemoryEntityFreezeFrameStore; +using ros2_medkit_gateway::SqliteEntityFreezeFrameStore; +using ros2_medkit_gateway::StoredEntityFreezeFrame; + +namespace { + +StoredEntityFreezeFrame make_row(const std::string & fault_code, const std::string & entity_id, int64_t captured_at_ns, + json frame = json{{"values", {{"temperature", 42.5}, {"pressure", 3.2}}}, + {"connected", false}, + {"source_timestamp", "2026-09-08T17:51:40.387Z"}}) { + StoredEntityFreezeFrame row; + row.fault_code = fault_code; + row.entity_id = entity_id; + row.frame = std::move(frame); + row.captured_at_ns = captured_at_ns; + row.source = "plugin_x_plc_data_route"; + row.capture_origin = ""; + return row; +} + +/// Both backends run the same contract tests. +enum class Backend { InMemory, Sqlite }; + +class EntityFreezeFrameStoreTest : public ::testing::TestWithParam { + protected: + void SetUp() override { + db_path_ = std::filesystem::temp_directory_path() / + ("test_entity_freeze_frame_store_" + std::to_string(::getpid()) + ".db"); + std::filesystem::remove(db_path_); + store_ = open(); + } + + void TearDown() override { + store_.reset(); + std::filesystem::remove(db_path_); + } + + std::unique_ptr open() { + if (GetParam() == Backend::InMemory) { + return std::make_unique(); + } + return std::make_unique(db_path_.string()); + } + + std::filesystem::path db_path_; + std::unique_ptr store_; +}; + +} // namespace + +TEST_P(EntityFreezeFrameStoreTest, RoundTripsEveryFieldOfARow) { + auto row = make_row("JAM_INFEED", "plc_app", 1757353900387000000); + row.capture_origin = "startup"; + ASSERT_TRUE(store_->replace_frames("JAM_INFEED", {row}).has_value()); + + auto loaded = store_->load_all(); + ASSERT_TRUE(loaded.has_value()); + ASSERT_EQ(loaded->size(), 1u); + const auto & got = (*loaded)[0]; + EXPECT_EQ(got.fault_code, "JAM_INFEED"); + EXPECT_EQ(got.entity_id, "plc_app"); + EXPECT_EQ(got.frame, row.frame); + EXPECT_EQ(got.captured_at_ns, 1757353900387000000); // nanoseconds need the full 64 bits + EXPECT_EQ(got.source, "plugin_x_plc_data_route"); + EXPECT_EQ(got.capture_origin, "startup"); +} + +TEST_P(EntityFreezeFrameStoreTest, ReplaceDropsEntitiesTheNewCaptureNoLongerReports) { + ASSERT_TRUE(store_ + ->replace_frames("JAM_INFEED", + {make_row("JAM_INFEED", "plc_app", 10), make_row("JAM_INFEED", "second_app", 20)}) + .has_value()); + // A re-confirm that frames one entity leaves no row for the other. + ASSERT_TRUE(store_->replace_frames("JAM_INFEED", {make_row("JAM_INFEED", "plc_app", 30)}).has_value()); + + auto loaded = store_->load_all(); + ASSERT_TRUE(loaded.has_value()); + ASSERT_EQ(loaded->size(), 1u); + EXPECT_EQ((*loaded)[0].entity_id, "plc_app"); + EXPECT_EQ((*loaded)[0].captured_at_ns, 30); +} + +TEST_P(EntityFreezeFrameStoreTest, OneCodePerFaultRowsAreKeptApart) { + ASSERT_TRUE(store_->replace_frames("JAM_INFEED", {make_row("JAM_INFEED", "plc_app", 10)}).has_value()); + ASSERT_TRUE(store_->replace_frames("SAFETY_CURTAIN", {make_row("SAFETY_CURTAIN", "plc_app", 20)}).has_value()); + + ASSERT_TRUE(store_->erase_frames("JAM_INFEED").has_value()); + + auto loaded = store_->load_all(); + ASSERT_TRUE(loaded.has_value()); + ASSERT_EQ(loaded->size(), 1u); + EXPECT_EQ((*loaded)[0].fault_code, "SAFETY_CURTAIN"); +} + +TEST_P(EntityFreezeFrameStoreTest, LoadAllServesOldestCaptureFirst) { + // The caller keeps the newest codes under its bound by this order. + ASSERT_TRUE(store_->replace_frames("NEWEST", {make_row("NEWEST", "plc_app", 300)}).has_value()); + ASSERT_TRUE(store_->replace_frames("OLDEST", {make_row("OLDEST", "plc_app", 100)}).has_value()); + ASSERT_TRUE(store_->replace_frames("MIDDLE", {make_row("MIDDLE", "plc_app", 200)}).has_value()); + + auto loaded = store_->load_all(); + ASSERT_TRUE(loaded.has_value()); + ASSERT_EQ(loaded->size(), 3u); + EXPECT_EQ((*loaded)[0].fault_code, "OLDEST"); + EXPECT_EQ((*loaded)[1].fault_code, "MIDDLE"); + EXPECT_EQ((*loaded)[2].fault_code, "NEWEST"); +} + +TEST_P(EntityFreezeFrameStoreTest, ErasingACodeWithNoRowsIsNotAnError) { + // The caller evicts by fault code and does not track what reached the file. + EXPECT_TRUE(store_->erase_frames("NEVER_STORED").has_value()); +} + +TEST_P(EntityFreezeFrameStoreTest, AnEmptyReplaceLeavesNoRows) { + ASSERT_TRUE(store_->replace_frames("JAM_INFEED", {make_row("JAM_INFEED", "plc_app", 10)}).has_value()); + ASSERT_TRUE(store_->replace_frames("JAM_INFEED", {}).has_value()); + + auto loaded = store_->load_all(); + ASSERT_TRUE(loaded.has_value()); + EXPECT_TRUE(loaded->empty()); +} + +INSTANTIATE_TEST_SUITE_P(Backends, EntityFreezeFrameStoreTest, ::testing::Values(Backend::InMemory, Backend::Sqlite), + [](const ::testing::TestParamInfo & param_info) { + return param_info.param == Backend::InMemory ? "InMemory" : "Sqlite"; + }); + +// SQLite only: the file outlives the process. + +TEST(SqliteEntityFreezeFrameStoreFile, RowsSurviveReopen) { + const auto path = std::filesystem::temp_directory_path() / + ("test_entity_freeze_frame_reopen_" + std::to_string(::getpid()) + ".db"); + std::filesystem::remove(path); + + { + SqliteEntityFreezeFrameStore store(path.string()); + ASSERT_TRUE( + store.replace_frames("JAM_INFEED", {make_row("JAM_INFEED", "plc_app", 1757353900387000000)}).has_value()); + } + + SqliteEntityFreezeFrameStore reopened(path.string()); + auto loaded = reopened.load_all(); + ASSERT_TRUE(loaded.has_value()); + ASSERT_EQ(loaded->size(), 1u); + EXPECT_EQ((*loaded)[0].fault_code, "JAM_INFEED"); + EXPECT_EQ((*loaded)[0].captured_at_ns, 1757353900387000000); + EXPECT_EQ((*loaded)[0].frame["values"]["temperature"], 42.5); + EXPECT_EQ((*loaded)[0].frame["connected"], false); + + std::filesystem::remove(path); +} + +TEST(SqliteEntityFreezeFrameStoreFile, AnUnreadableRowCostsOnlyItself) { + const auto path = std::filesystem::temp_directory_path() / + ("test_entity_freeze_frame_corrupt_" + std::to_string(::getpid()) + ".db"); + std::filesystem::remove(path); + + { + SqliteEntityFreezeFrameStore store(path.string()); + ASSERT_TRUE(store.replace_frames("GOOD", {make_row("GOOD", "plc_app", 20)}).has_value()); + } + // One blob that is not JSON, as a hand-edited file holds. + { + sqlite3 * db = nullptr; + ASSERT_EQ(sqlite3_open(path.c_str(), &db), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(db, + "INSERT INTO entity_freeze_frames " + "(fault_code, entity_id, frame, captured_at_ns, source, capture_origin) " + "VALUES ('BROKEN','plc_app','{not json',10,'','')", + nullptr, nullptr, nullptr), + SQLITE_OK); + sqlite3_close(db); + } + + SqliteEntityFreezeFrameStore reopened(path.string()); + auto loaded = reopened.load_all(); + ASSERT_TRUE(loaded.has_value()); + // The broken row comes back with a null frame, so the reader can count it. + ASSERT_EQ(loaded->size(), 2u); + EXPECT_EQ((*loaded)[0].fault_code, "BROKEN"); + EXPECT_TRUE((*loaded)[0].frame.is_null()); + EXPECT_EQ((*loaded)[1].fault_code, "GOOD"); + EXPECT_EQ((*loaded)[1].frame["values"]["temperature"], 42.5); + + std::filesystem::remove(path); +} diff --git a/src/ros2_medkit_gateway/test/test_freeze_frame_capture_persistence.cpp b/src/ros2_medkit_gateway/test/test_freeze_frame_capture_persistence.cpp new file mode 100644 index 000000000..27dfb1406 --- /dev/null +++ b/src/ros2_medkit_gateway/test/test_freeze_frame_capture_persistence.cpp @@ -0,0 +1,950 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The core capture with a store: what a gateway restart serves, and how the startup +// catch-up treats reloaded frames. In process, because the retained-frame bound is a +// literal 256 in the gateway node and these cases sit at the bound. + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ros2_medkit_gateway/core/faults/entity_freeze_frame_store.hpp" +#include "ros2_medkit_gateway/core/faults/fault_event_source.hpp" +#include "ros2_medkit_gateway/core/faults/fault_json.hpp" +#include "ros2_medkit_gateway/core/faults/freeze_frame_capture.hpp" +#include "ros2_medkit_gateway/core/faults/sqlite_entity_freeze_frame_store.hpp" +#include "ros2_medkit_gateway/core/logging.hpp" +#include "ros2_medkit_gateway/core/providers/data_provider.hpp" +#include "ros2_medkit_gateway/http/handlers/fault_handlers.hpp" + +using json = nlohmann::json; +using namespace std::chrono_literals; +using ros2_medkit_gateway::DataProvider; +using ros2_medkit_gateway::DataProviderError; +using ros2_medkit_gateway::DataProviderErrorInfo; +using ros2_medkit_gateway::EntityFreezeFrameStore; +using ros2_medkit_gateway::FaultEventSource; +using ros2_medkit_gateway::FaultEventSubscription; +using ros2_medkit_gateway::FreezeFrameCapture; +using ros2_medkit_gateway::FreezeFrameCaptureConfig; +using ros2_medkit_gateway::InMemoryEntityFreezeFrameStore; +using ros2_medkit_gateway::PluginDataReaders; +using ros2_medkit_gateway::SqliteEntityFreezeFrameStore; +using ros2_medkit_gateway::StoredEntityFreezeFrame; +using ros2_medkit_gateway::handlers::FaultHandlers; +namespace fd = ros2_medkit::fault_detection; +namespace mlog = ros2_medkit_gateway::log; + +namespace { + +/// Log lines of the process, so a test can wait for one and assert on its text. +class RecordingSink : public mlog::Sink { + public: + bool enabled(const std::string & /*logger*/, mlog::Level level) const override { + return level >= mlog::Level::kInfo; + } + + void write(const std::string & /*logger*/, mlog::Level level, std::string_view message, + const mlog::Location & /*location*/) override { + { + std::lock_guard lock(mutex_); + text_ += level >= mlog::Level::kWarn ? "WARN " : "INFO "; + text_.append(message.data(), message.size()); + text_ += '\n'; + } + cv_.notify_all(); + } + + void clear() { + std::lock_guard lock(mutex_); + text_.clear(); + } + + std::string text() { + std::lock_guard lock(mutex_); + return text_; + } + + bool wait_for(const std::string & needle, std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + return cv_.wait_for(lock, timeout, [&] { + return text_.find(needle) != std::string::npos; + }); + } + + private: + std::mutex mutex_; + std::condition_variable cv_; + std::string text_; +}; + +/// Installed once, before the first capture thread logs. +RecordingSink & log_sink() { + static const std::shared_ptr sink = [] { + auto created = std::make_shared(); + mlog::set_sink(created); + return created; + }(); + return *sink; +} + +/// A fault event source the test drives by hand. +class FakeFaultEvents : public FaultEventSource { + public: + FaultEventSubscription subscribe(Callback callback) override { + std::lock_guard lock(mutex_); + callback_ = std::move(callback); + return FaultEventSubscription([this] { + std::lock_guard cancel_lock(mutex_); + callback_ = nullptr; + }); + } + + void deliver(const fd::FaultEventRecord & event) { + std::lock_guard lock(mutex_); + if (callback_) { + callback_(event); + } + } + + private: + std::mutex mutex_; + Callback callback_; +}; + +fd::FaultEventRecord confirmed(const std::string & fault_code, const std::vector & sources) { + fd::FaultEventRecord event; + event.event_type = fd::fault_event_type::kConfirmed; + event.fault.fault_code = fault_code; + event.fault.reporting_sources = sources; + return event; +} + +/// Plugin DataProvider of one entity, with a settable temperature. +class FakePlcDataProvider : public DataProvider { + public: + explicit FakePlcDataProvider(std::string entity_id) : entity_id_(std::move(entity_id)) { + } + + void set_temperature(double value) { + temperature_.store(value); + } + + tl::expected + list_data(const std::string & entity_id) override { + if (entity_id != entity_id_) { + return tl::make_unexpected(DataProviderErrorInfo{DataProviderError::EntityNotFound, "not found", 404}); + } + json items = json::array(); + items.push_back({{"id", "temperature"}, {"value", temperature_.load()}}); + items.push_back({{"id", "pressure"}, {"value", 3.2}}); + return ros2_medkit_gateway::dto::DataListResult{json{{"items", std::move(items)}}}; + } + + tl::expected + read_data(const std::string & /*entity_id*/, const std::string & /*resource_name*/) override { + return tl::make_unexpected(DataProviderErrorInfo{DataProviderError::Internal, "unused", 500}); + } + + tl::expected + write_data(const std::string & /*entity_id*/, const std::string & /*resource_name*/, + const json & /*value*/) override { + return tl::make_unexpected(DataProviderErrorInfo{DataProviderError::ReadOnly, "read-only", 405}); + } + + private: + std::string entity_id_; + std::atomic temperature_{42.5}; +}; + +/// Entity of the catch-up probe. It serves nothing, so its read stores no frame and evicts nothing. +constexpr const char * kProbeEntity = "probe_entity"; + +/// The x-plc-data route of the plant. Serves `level` for every entity, counts reads per +/// entity, and can make an entity unreachable. +class Plant { + public: + explicit Plant(double level) : level_(level) { + } + + void make_unreachable(const std::string & entity_id) { + std::lock_guard lock(mutex_); + unreachable_.insert(entity_id); + } + + std::optional read(const std::string & entity_id) { + std::optional answer; + { + std::lock_guard lock(mutex_); + reads_[entity_id] += 1; + if (entity_id == kProbeEntity || unreachable_.count(entity_id) != 0) { + answer = std::nullopt; + } else { + answer = json{{"connected", true}, {"items", json::array({{{"name", "level"}, {"value", level_}}})}}; + } + } + cv_.notify_all(); + return answer; + } + + int reads(const std::string & entity_id) { + std::lock_guard lock(mutex_); + auto it = reads_.find(entity_id); + return it == reads_.end() ? 0 : it->second; + } + + /// Waits until @p entity_id has been read at least @p count times. + bool wait_for_reads(const std::string & entity_id, int count, std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + return cv_.wait_for(lock, timeout, [&] { + const auto it = reads_.find(entity_id); + return it != reads_.end() && it->second >= count; + }); + } + + private: + double level_; + std::mutex mutex_; + std::condition_variable cv_; + std::map reads_; + std::set unreachable_; +}; + +constexpr int64_t kStoredAtNs = 1'000'000'000'000'000'000; + +using Occurrence = FreezeFrameCapture::Occurrence; + +/// Occurrences as the fault manager numbers them. Their times sit on the fault manager's +/// clock, so they say nothing about a frame's captured_at. +constexpr Occurrence kFirstOccurrence{1, kStoredAtNs - 120'000'000'000}; +constexpr Occurrence kSecondOccurrence{2, kStoredAtNs - 60'000'000'000}; + +/// One stored row, shaped as the capture writes it. +StoredEntityFreezeFrame make_stored_row(const std::string & fault_code, const std::string & entity_id, + int64_t captured_at_ns, double level = 7.0, Occurrence occurrence = {}) { + StoredEntityFreezeFrame row; + row.fault_code = fault_code; + row.entity_id = entity_id; + row.frame = json{{"values", {{"level", level}}}, {"connected", false}, {"source_timestamp", "2026-09-08T17:51:40Z"}}; + row.captured_at_ns = captured_at_ns; + row.source = FreezeFrameCapture::kSourceXPlcDataRoute; + row.occurrence_count = occurrence.count; + row.first_occurred_ns = occurrence.first_occurred_ns; + return row; +} + +FreezeFrameCapture::StandingFault standing_fault(const std::string & code, const std::vector & sources, + Occurrence occurrence = {}, const std::string & status = "CONFIRMED", + bool muted = false) { + FreezeFrameCapture::StandingFault fault; + fault.fault_code = code; + fault.reporting_sources = sources; + fault.status = status; + fault.occurrence = occurrence; + fault.muted = muted; + return fault; +} + +FreezeFrameCapture::StandingFaultLister listed(std::vector faults) { + return [faults = std::move(faults)](const std::function &) { + return faults; + }; +} + +std::vector stored_codes(EntityFreezeFrameStore & store) { + std::vector codes; + auto rows = store.load_all(); + if (rows) { + for (const auto & row : *rows) { + codes.push_back(row.fault_code); + } + } + std::sort(codes.begin(), codes.end()); + return codes; +} + +// The bound probe: two slots, one live reloaded frame (PLC_KEEP, the older, so the FIFO +// front), one stale reloaded frame (PLC_STALE) and one standing fault with no frame (PLC_NEW). +constexpr int64_t kKeepAtNs = kStoredAtNs; +constexpr int64_t kStaleAtNs = kStoredAtNs + 10'000'000'000; +constexpr Occurrence kKeepOccurrence{1, kKeepAtNs - 60'000'000'000}; +constexpr Occurrence kStaleStoredOccurrence{1, kStaleAtNs - 60'000'000'000}; +constexpr Occurrence kStaleOccurrence{2, kStaleAtNs + 60'000'000'000}; + +std::vector bound_probe_standing() { + return {standing_fault("PLC_NEW", {"route_new"}, {1, kStaleAtNs + 30'000'000'000}), + standing_fault("PLC_STALE", {"route_stale"}, kStaleOccurrence), // a later occurrence: stale + standing_fault("PLC_KEEP", {"route_keep"}, kKeepOccurrence)}; // the stored occurrence: live +} + +void seed_bound_probe(EntityFreezeFrameStore & store) { + ASSERT_TRUE( + store.replace_frames("PLC_KEEP", {make_stored_row("PLC_KEEP", "route_keep", kKeepAtNs, 10.0, kKeepOccurrence)})); + ASSERT_TRUE(store.replace_frames( + "PLC_STALE", {make_stored_row("PLC_STALE", "route_stale", kStaleAtNs, 11.0, kStaleStoredOccurrence)})); +} + +class FreezeFrameCapturePersistenceTest : public ::testing::Test { + protected: + void SetUp() override { + log_sink().clear(); + } + + std::unique_ptr make_capture(size_t max_faults, Plant & plant, + std::shared_ptr store, + FreezeFrameCapture::StandingFaultLister standing = nullptr, + DataProvider * provider = nullptr) { + FreezeFrameCapture::Hooks hooks; + hooks.readers = PluginDataReaders{[provider](const std::string & entity_id) -> DataProvider * { + return entity_id == "plc_app" ? provider : nullptr; + }, + [&plant](const std::string & entity_id) { + return plant.read(entity_id); + }}; + hooks.standing_lister = std::move(standing); + hooks.store = std::move(store); + return std::make_unique(events_, FreezeFrameCaptureConfig{true, max_faults}, std::move(hooks)); + } + + /// The capture drains confirm events only after the catch-up returns, so the read + /// of a confirm queued now is the sign that the catch-up has finished. + /// The drain loop handles confirms in order, so the probe's read also shows that every + /// confirm delivered before it has been handled. + bool wait_for_catch_up(Plant & plant) { + const int probes = plant.reads(kProbeEntity); + events_.deliver(confirmed("PROBE_AFTER_CATCH_UP", {kProbeEntity})); + return plant.wait_for_reads(kProbeEntity, probes + 1, 15s); + } + + /// Polls until the capture holds frames for @p fault_code. + static bool wait_for_frames(const FreezeFrameCapture & capture, const std::string & fault_code) { + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (std::chrono::steady_clock::now() < deadline) { + if (!capture.frames_for(fault_code).empty()) { + return true; + } + std::this_thread::sleep_for(5ms); + } + return false; + } + + FakeFaultEvents events_; +}; + +} // namespace + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, CaptureWritesTheFrameThroughToTheStore) { + auto store = std::make_shared(); + FakePlcDataProvider provider("plc_app"); + Plant plant(0.0); + auto capture = make_capture(256, plant, store, nullptr, &provider); + + auto event = confirmed("PLC_PERSIST", {"plc_app"}); + event.fault.occurrence_count = 3; + event.fault.first_occurred_ns = 1'788'948'705'123'456'789; + events_.deliver(event); + ASSERT_TRUE(wait_for_frames(*capture, "PLC_PERSIST")); + const auto served = capture->frames_for("PLC_PERSIST"); + ASSERT_EQ(served.size(), 1u); + + auto rows = store->load_all(); + ASSERT_TRUE(rows.has_value()); + ASSERT_EQ(rows->size(), 1u); + EXPECT_EQ((*rows)[0].fault_code, "PLC_PERSIST"); + EXPECT_EQ((*rows)[0].entity_id, "plc_app"); + EXPECT_EQ((*rows)[0].captured_at_ns, served[0].captured_at_ns); + EXPECT_EQ((*rows)[0].frame["values"], served[0].values); + EXPECT_EQ((*rows)[0].source, FreezeFrameCapture::kSourceDataProvider); + EXPECT_EQ((*rows)[0].capture_origin, ""); // captured on the confirm edge + // The occurrence as a ListFaults reply of the same record carries it, so a restart can match it. + const auto replied = FreezeFrameCapture::standing_faults_from_list_reply( + json{{"faults", json::array({ros2_medkit_gateway::fault_record_to_json(event.fault)})}}); + ASSERT_TRUE(replied.has_value()); + ASSERT_EQ(replied->size(), 1u); + EXPECT_EQ((*rows)[0].occurrence_count, 3u); + EXPECT_NE((*rows)[0].first_occurred_ns, 0); + EXPECT_EQ((*rows)[0].first_occurred_ns, (*replied)[0].occurrence.first_occurred_ns); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, ReConfirmOverwritesTheStoredRow) { + auto store = std::make_shared(); + FakePlcDataProvider provider("plc_app"); + Plant plant(0.0); + auto capture = make_capture(256, plant, store, nullptr, &provider); + + events_.deliver(confirmed("PLC_RECONFIRM", {"plc_app"})); + ASSERT_TRUE(wait_for_frames(*capture, "PLC_RECONFIRM")); + + // One frame per fault: a stale row would bring the previous occurrence back on restart. + provider.set_temperature(99.0); + events_.deliver(confirmed("PLC_RECONFIRM", {"plc_app"})); + const auto deadline = std::chrono::steady_clock::now() + 15s; + bool overwritten = false; + while (std::chrono::steady_clock::now() < deadline && !overwritten) { + auto rows = store->load_all(); + overwritten = rows.has_value() && rows->size() == 1u && + std::abs((*rows)[0].frame["values"].value("temperature", 0.0) - 99.0) < 1e-9; + if (!overwritten) { + std::this_thread::sleep_for(5ms); + } + } + EXPECT_TRUE(overwritten); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, AReloadedFrameIsServedExactlyAsItWasCaptured) { + auto store = std::make_shared(); + json first_wire; + int64_t captured_at_ns = 0; + { + Plant plant(41.0); + auto capture = make_capture(256, plant, store); + events_.deliver(confirmed("PLC_RESTART", {"route_plc_app"})); + ASSERT_TRUE(wait_for_frames(*capture, "PLC_RESTART")); + const auto frames = capture->frames_for("PLC_RESTART"); + ASSERT_EQ(frames.size(), 1u); + captured_at_ns = frames[0].captured_at_ns; + first_wire = FaultHandlers::merge_entity_freeze_frames(json{{"snapshots", json::array()}}, frames); + } + + // A second life on the same store, with the plant reading something else. + Plant moved_on(999.0); + auto restarted = make_capture(256, moved_on, store); + const auto reloaded = restarted->frames_for("PLC_RESTART"); + ASSERT_EQ(reloaded.size(), 1u); + EXPECT_EQ(reloaded[0].captured_at_ns, captured_at_ns); + EXPECT_FALSE(reloaded[0].startup_catchup); + const auto second_wire = FaultHandlers::merge_entity_freeze_frames(json{{"snapshots", json::array()}}, reloaded); + EXPECT_EQ(second_wire, first_wire); + ASSERT_EQ(second_wire["snapshots"].size(), 1u); + EXPECT_FALSE(second_wire["snapshots"][0].contains("capture_origin")); + EXPECT_EQ(moved_on.reads("route_plc_app"), 0); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, CatchUpSkipsAReloadedCodeAndFramesOneWithoutARow) { + // The stored row has no occurrence, so whether it is this occurrence's cannot be told: it is kept. + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames("PLC_HAS_FRAME", {make_stored_row("PLC_HAS_FRAME", "route_stored_app", 4242)})); + + Plant plant(7.0); + auto capture = make_capture(256, plant, store, + listed({standing_fault("PLC_HAS_FRAME", {"route_stored_app"}, kSecondOccurrence), + standing_fault("PLC_NO_FRAME", {"route_fresh_app"}, kFirstOccurrence)})); + ASSERT_TRUE(wait_for_catch_up(plant)); + + // Positive control: a standing fault with no row still gets its startup frame. + const auto fresh = capture->frames_for("PLC_NO_FRAME"); + ASSERT_EQ(fresh.size(), 1u); + EXPECT_TRUE(fresh[0].startup_catchup); + + const auto stored = capture->frames_for("PLC_HAS_FRAME"); + ASSERT_EQ(stored.size(), 1u); + EXPECT_EQ(stored[0].captured_at_ns, 4242); // the frozen moment, not this start + EXPECT_FALSE(stored[0].startup_catchup); + EXPECT_EQ(plant.reads("route_stored_app"), 0); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, ReloadDropsAFaultTheManagerNoLongerHoldsAndKeepsAClearedOne) { + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames("PLC_GONE", {make_stored_row("PLC_GONE", "route_a", 100)})); + ASSERT_TRUE(store->replace_frames("PLC_CLEARED", {make_stored_row("PLC_CLEARED", "route_b", 200)})); + // Past the 10 codes the warning names. + for (int i = 1; i <= 10; ++i) { + const std::string code = "PLC_GONE_" + std::string(i < 10 ? "0" : "") + std::to_string(i); + ASSERT_TRUE(store->replace_frames(code, {make_stored_row(code, "route_a", 100 + i)})); + } + + Plant plant(7.0); + // The fault manager reports the cleared fault and knows nothing of the other. + auto capture = make_capture(256, plant, store, listed({standing_fault("PLC_CLEARED", {"route_b"}, {}, "CLEARED")})); + ASSERT_TRUE(wait_for_catch_up(plant)); + + EXPECT_TRUE(capture->frames_for("PLC_GONE").empty()); + EXPECT_FALSE(capture->frames_for("PLC_CLEARED").empty()); // a cleared fault keeps its frame + EXPECT_EQ(stored_codes(*store), (std::vector{"PLC_CLEARED"})); + const auto logs = log_sink().text(); + EXPECT_NE(logs.find("WARN Entity freeze-frame: dropped the reloaded frames of 11 fault(s) the fault manager no " + "longer holds: PLC_GONE, PLC_GONE_01, PLC_GONE_02, PLC_GONE_03, PLC_GONE_04, PLC_GONE_05, " + "PLC_GONE_06, PLC_GONE_07, PLC_GONE_08, PLC_GONE_09, ...\n"), + std::string::npos) + << logs; +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, AFaultListerThatGivesUpDropsNothing) { + // Same seeding as the test above, whose lister can answer and drops PLC_GONE. + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames("PLC_GONE", {make_stored_row("PLC_GONE", "route_a", 100)})); + ASSERT_TRUE(store->replace_frames("PLC_CLEARED", {make_stored_row("PLC_CLEARED", "route_b", 200)})); + + Plant plant(7.0); + std::atomic asked{false}; + auto capture = make_capture( + 256, plant, store, [&asked](const std::function &) -> std::vector { + asked.store(true); + throw std::runtime_error("fault manager unreachable"); + }); + ASSERT_TRUE(wait_for_catch_up(plant)); + + EXPECT_TRUE(asked.load()); + EXPECT_FALSE(capture->frames_for("PLC_GONE").empty()); + EXPECT_FALSE(capture->frames_for("PLC_CLEARED").empty()); + EXPECT_EQ(stored_codes(*store), (std::vector{"PLC_CLEARED", "PLC_GONE"})); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, AnEmptyFaultListDropsNoReloadedFrame) { + // A fault manager whose storage is unreachable answers with an empty list. That must + // not cost frames that cannot be read again, so an empty answer is "cannot tell". + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames("PLC_A", {make_stored_row("PLC_A", "route_a", 100)})); + ASSERT_TRUE(store->replace_frames("PLC_B", {make_stored_row("PLC_B", "route_b", 200)})); + + Plant plant(7.0); + auto capture = make_capture(256, plant, store, listed({})); + ASSERT_TRUE(wait_for_catch_up(plant)); + + EXPECT_FALSE(capture->frames_for("PLC_A").empty()); + EXPECT_FALSE(capture->frames_for("PLC_B").empty()); + EXPECT_EQ(stored_codes(*store), (std::vector{"PLC_A", "PLC_B"})); + const auto logs = log_sink().text(); + EXPECT_NE(logs.find("the fault manager lists no fault in any status, so no reloaded frame is dropped"), + std::string::npos) + << logs; +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, TheRetainedFrameBoundCountsReloadedFrames) { + const auto path = + std::filesystem::temp_directory_path() / ("test_freeze_frame_bound_" + std::to_string(::getpid()) + ".db"); + std::filesystem::remove(path); + auto store = std::make_shared(path.string()); + ASSERT_TRUE(store->replace_frames("PLC_LOADED_OLDEST", {make_stored_row("PLC_LOADED_OLDEST", "route_o", 50)})); + ASSERT_TRUE(store->replace_frames("PLC_LOADED_A", {make_stored_row("PLC_LOADED_A", "route_a", 100)})); + ASSERT_TRUE(store->replace_frames("PLC_LOADED_B", {make_stored_row("PLC_LOADED_B", "route_b", 200)})); + // Two unreadable rows, one of each kind, both skipped and counted: JSON that is not a + // frame, and a column that is not JSON at all. + StoredEntityFreezeFrame unreadable = make_stored_row("PLC_UNREADABLE", "route_u", 300); + unreadable.frame = json{{"not_values", 1}}; + ASSERT_TRUE(store->replace_frames("PLC_UNREADABLE", {unreadable})); + { + sqlite3 * db = nullptr; + ASSERT_EQ(sqlite3_open(path.c_str(), &db), SQLITE_OK); + ASSERT_EQ(sqlite3_exec(db, + "INSERT INTO entity_freeze_frames " + "(fault_code, entity_id, frame, captured_at_ns, source, capture_origin) " + "VALUES ('PLC_NOT_JSON','route_n','{not json',250,'','')", + nullptr, nullptr, nullptr), + SQLITE_OK); + sqlite3_close(db); + } + + // The fault manager still holds the reloaded faults, so the list keeps their frames. + const auto standing = + listed({standing_fault("PLC_FRESH", {"route_fresh"}), standing_fault("PLC_LOADED_A", {"route_a"}, {}, "CLEARED"), + standing_fault("PLC_LOADED_B", {"route_b"}, {}, "CLEARED")}); + + { + // Bound of 2. The store holds 3 readable codes, so the oldest is dropped at load, and + // the 2 reloaded frames fill the bound: catching PLC_FRESH up would evict one of them. + Plant plant(7.0); + auto capped = make_capture(/*max_faults=*/2, plant, store, standing); + ASSERT_TRUE(wait_for_catch_up(plant)); + EXPECT_TRUE(capped->frames_for("PLC_FRESH").empty()); + EXPECT_EQ(plant.reads("route_fresh"), 0); + EXPECT_TRUE(capped->frames_for("PLC_LOADED_OLDEST").empty()); + EXPECT_FALSE(capped->frames_for("PLC_LOADED_A").empty()); + EXPECT_FALSE(capped->frames_for("PLC_LOADED_B").empty()); + const auto logs = log_sink().text(); + EXPECT_NE(logs.find("reloaded frames for 2 fault(s) from the store"), std::string::npos) << logs; + EXPECT_NE(logs.find("held 1 fault(s) beyond the retained-frame bound of 2"), std::string::npos) << logs; + EXPECT_NE(logs.find("2 unreadable row(s) skipped"), std::string::npos) << logs; + EXPECT_NE(logs.find("so they have no freeze-frame: PLC_FRESH"), std::string::npos) << logs; + } + // The rows past the bound are gone with the frames; unreadable rows stay as they are. + EXPECT_EQ(stored_codes(*store), + (std::vector{"PLC_LOADED_A", "PLC_LOADED_B", "PLC_NOT_JSON", "PLC_UNREADABLE"})); + + // Positive control: one more slot and the same catch-up frames PLC_FRESH. + { + Plant plant(7.0); + auto roomy = make_capture(/*max_faults=*/3, plant, store, standing); + ASSERT_TRUE(wait_for_catch_up(plant)); + EXPECT_FALSE(roomy->frames_for("PLC_FRESH").empty()); + } + store.reset(); + std::filesystem::remove(path); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, AnEvictedFaultLosesItsStoredRowToo) { + // Otherwise an evicted frame comes back on the next start and the file grows without bound. + auto store = std::make_shared(); + FakePlcDataProvider provider("plc_app"); + Plant plant(0.0); + auto capture = make_capture(/*max_faults=*/1, plant, store, nullptr, &provider); + + events_.deliver(confirmed("PLC_EVICT_FIRST", {"plc_app"})); + ASSERT_TRUE(wait_for_frames(*capture, "PLC_EVICT_FIRST")); + events_.deliver(confirmed("PLC_EVICT_SECOND", {"plc_app"})); + ASSERT_TRUE(wait_for_frames(*capture, "PLC_EVICT_SECOND")); + + EXPECT_TRUE(capture->frames_for("PLC_EVICT_FIRST").empty()); + EXPECT_EQ(stored_codes(*store), (std::vector{"PLC_EVICT_SECOND"})); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, ARecaptureRefreshesTheEvictionAge) { + // At the bound, a fault confirmed again is the newest frame, so the next new fault + // evicts the next-oldest one, in this life and after a reload. + auto store = std::make_shared(); + { + Plant plant(7.0); + auto capture = make_capture(/*max_faults=*/3, plant, store); + for (const char * code : {"PLC_FIRST", "PLC_SECOND", "PLC_THIRD"}) { + events_.deliver(confirmed(code, {"route_app"})); + ASSERT_TRUE(wait_for_frames(*capture, code)); + } + const int64_t first_capture = capture->frames_for("PLC_FIRST")[0].captured_at_ns; + events_.deliver(confirmed("PLC_FIRST", {"route_app"})); + const auto deadline = std::chrono::steady_clock::now() + 15s; + while (std::chrono::steady_clock::now() < deadline && + capture->frames_for("PLC_FIRST")[0].captured_at_ns == first_capture) { + std::this_thread::sleep_for(5ms); + } + ASSERT_GT(capture->frames_for("PLC_FIRST")[0].captured_at_ns, first_capture); + events_.deliver(confirmed("PLC_FOURTH", {"route_app"})); + ASSERT_TRUE(wait_for_frames(*capture, "PLC_FOURTH")); + + EXPECT_FALSE(capture->frames_for("PLC_FIRST").empty()) << "the fault just confirmed again was evicted"; + EXPECT_TRUE(capture->frames_for("PLC_SECOND").empty()); + EXPECT_EQ(stored_codes(*store), (std::vector{"PLC_FIRST", "PLC_FOURTH", "PLC_THIRD"})); + } + + Plant plant(7.0); + auto reloaded = make_capture(/*max_faults=*/3, plant, store); + events_.deliver(confirmed("PLC_FIFTH", {"route_app"})); + ASSERT_TRUE(wait_for_frames(*reloaded, "PLC_FIFTH")); + EXPECT_FALSE(reloaded->frames_for("PLC_FIRST").empty()) << "after a reload the recaptured fault was evicted"; + EXPECT_TRUE(reloaded->frames_for("PLC_THIRD").empty()); + EXPECT_EQ(stored_codes(*store), (std::vector{"PLC_FIFTH", "PLC_FIRST", "PLC_FOURTH"})); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, AReloadedFrameFromAnEarlierOccurrenceIsReReadAndMarkedStartup) { + // The fault cleared and confirmed again while the gateway was down: the stored frame + // holds the previous occurrence's values. The fault manager's clock is behind the + // gateway's, so the new occurrence's first_occurred predates the stored captured_at. + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames( + "PLC_REOCCUR", {make_stored_row("PLC_REOCCUR", "route_stored_app", kStoredAtNs, 10.0, kFirstOccurrence)})); + + Plant plant(99.0); + auto capture = + make_capture(256, plant, store, listed({standing_fault("PLC_REOCCUR", {"route_stored_app"}, kSecondOccurrence)})); + ASSERT_TRUE(wait_for_catch_up(plant)); + + const auto served = capture->frames_for("PLC_REOCCUR"); + ASSERT_EQ(served.size(), 1u); + EXPECT_DOUBLE_EQ(served[0].values.value("level", 0.0), 99.0); // this occurrence, not the last one + EXPECT_TRUE(served[0].startup_catchup); + EXPECT_GT(served[0].captured_at_ns, kStoredAtNs); + + auto rows = store->load_all(); + ASSERT_TRUE(rows.has_value()); + ASSERT_EQ(rows->size(), 1u); + EXPECT_DOUBLE_EQ((*rows)[0].frame["values"].value("level", 0.0), 99.0); + EXPECT_EQ((*rows)[0].capture_origin, "startup"); + EXPECT_EQ(plant.reads("route_stored_app"), 1); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, ANewOccurrenceWithNoValuesDropsTheFrameOfThePreviousOne) { + // PLC_RELOADED holds a reloaded frame judged valid at start, PLC_LIVE one captured in + // this life. Both then confirm as their next occurrence while the entity is down. + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames( + "PLC_RELOADED", {make_stored_row("PLC_RELOADED", "route_reloaded", kStoredAtNs, 10.0, kFirstOccurrence)})); + Plant plant(99.0); + auto capture = + make_capture(256, plant, store, listed({standing_fault("PLC_RELOADED", {"route_reloaded"}, kFirstOccurrence)})); + ASSERT_TRUE(wait_for_catch_up(plant)); + ASSERT_EQ(capture->frames_for("PLC_RELOADED").size(), 1u); + + auto live_first = confirmed("PLC_LIVE", {"route_live"}); + live_first.fault.occurrence_count = kFirstOccurrence.count; + live_first.fault.first_occurred_ns = kFirstOccurrence.first_occurred_ns; + events_.deliver(live_first); + ASSERT_TRUE(wait_for_frames(*capture, "PLC_LIVE")); + + plant.make_unreachable("route_reloaded"); + plant.make_unreachable("route_live"); + for (const char * code : {"PLC_RELOADED", "PLC_LIVE"}) { + auto second = confirmed(code, {std::string(code) == "PLC_LIVE" ? "route_live" : "route_reloaded"}); + second.fault.occurrence_count = kSecondOccurrence.count; + second.fault.first_occurred_ns = kSecondOccurrence.first_occurred_ns; + events_.deliver(second); + } + ASSERT_TRUE(wait_for_catch_up(plant)); + + EXPECT_EQ(plant.reads("route_reloaded"), 1); + EXPECT_EQ(plant.reads("route_live"), 2); + EXPECT_TRUE(capture->frames_for("PLC_RELOADED").empty()) << "occurrence 2 serves the frame of occurrence 1"; + EXPECT_TRUE(capture->frames_for("PLC_LIVE").empty()) << "occurrence 2 serves the frame of occurrence 1"; + EXPECT_TRUE(stored_codes(*store).empty()); + const auto logs = log_sink().text(); + for (const char * code : {"PLC_RELOADED", "PLC_LIVE"}) { + EXPECT_NE(logs.find(std::string("WARN Entity freeze-frame for fault '") + code + "': occurrence 2 confirmed"), + std::string::npos) + << logs; + } + EXPECT_NE(logs.find("so this occurrence has no freeze-frame"), std::string::npos) << logs; +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, TheSameOccurrenceWithNoValuesKeepsItsFrame) { + // A confirm of the occurrence the frame belongs to, or one that carries no occurrence, + // with nothing to read: the frame from the confirmation stays. + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames("PLC_SAME", + {make_stored_row("PLC_SAME", "route_same", kStoredAtNs, 10.0, kFirstOccurrence)})); + Plant plant(99.0); + auto capture = + make_capture(256, plant, store, listed({standing_fault("PLC_SAME", {"route_same"}, kFirstOccurrence)})); + ASSERT_TRUE(wait_for_catch_up(plant)); + + plant.make_unreachable("route_same"); + auto same = confirmed("PLC_SAME", {"route_same"}); + same.fault.occurrence_count = kFirstOccurrence.count; + same.fault.first_occurred_ns = kFirstOccurrence.first_occurred_ns; + events_.deliver(same); + events_.deliver(confirmed("PLC_SAME", {"route_same"})); // no occurrence + ASSERT_TRUE(wait_for_catch_up(plant)); + + EXPECT_EQ(plant.reads("route_same"), 2); + const auto served = capture->frames_for("PLC_SAME"); + ASSERT_EQ(served.size(), 1u); + EXPECT_EQ(served[0].captured_at_ns, kStoredAtNs); + EXPECT_DOUBLE_EQ(served[0].values.value("level", 0.0), 10.0); + EXPECT_EQ(stored_codes(*store), (std::vector{"PLC_SAME"})); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, AStaleFrameWhoseEntityCannotBeReReadIsDroppedWithAWarning) { + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames( + "PLC_LINK_DOWN", {make_stored_row("PLC_LINK_DOWN", "route_stored_app", kStoredAtNs, 10.0, kFirstOccurrence)})); + Plant plant(99.0); + plant.make_unreachable("route_stored_app"); + + auto capture = + make_capture(256, plant, store, + listed({standing_fault("PLC_LINK_DOWN", {"route_stored_app"}, {2, kStoredAtNs + 60'000'000'000})})); + ASSERT_TRUE(wait_for_catch_up(plant)); + + EXPECT_EQ(plant.reads("route_stored_app"), 1); // the re-read was tried + EXPECT_TRUE(capture->frames_for("PLC_LINK_DOWN").empty()); + EXPECT_TRUE(stored_codes(*store).empty()); + const auto logs = log_sink().text(); + EXPECT_NE(logs.find("WARN Entity freeze-frame for fault 'PLC_LINK_DOWN': the stored frame of entity " + "'route_stored_app' is from an earlier occurrence and was discarded (stored occurrence 1, the " + "fault manager lists occurrence 2, CONFIRMED)."), + std::string::npos) + << logs; +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, AReloadedFrameFromTheSameOccurrenceIsKeptUnmarked) { + // Control for the earlier-occurrence test: the fault never cleared, so the fault + // manager lists the stored occurrence. + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames( + "PLC_REOCCUR", {make_stored_row("PLC_REOCCUR", "route_stored_app", kStoredAtNs, 10.0, kFirstOccurrence)})); + Plant plant(99.0); + + auto capture = + make_capture(256, plant, store, listed({standing_fault("PLC_REOCCUR", {"route_stored_app"}, kFirstOccurrence)})); + ASSERT_TRUE(wait_for_catch_up(plant)); + + const auto served = capture->frames_for("PLC_REOCCUR"); + ASSERT_EQ(served.size(), 1u); + EXPECT_DOUBLE_EQ(served[0].values.value("level", 0.0), 10.0); + EXPECT_EQ(served[0].captured_at_ns, kStoredAtNs); + EXPECT_FALSE(served[0].startup_catchup); + EXPECT_EQ(plant.reads("route_stored_app"), 0); + + auto rows = store->load_all(); + ASSERT_TRUE(rows.has_value()); + ASSERT_EQ(rows->size(), 1u); + EXPECT_DOUBLE_EQ((*rows)[0].frame["values"].value("level", 0.0), 10.0); + EXPECT_EQ((*rows)[0].captured_at_ns, kStoredAtNs); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, AMutedFaultWithAnotherOccurrenceLosesItsStaleFrame) { + // A muted fault has no confirm event, so nothing frames its new occurrence. The stored + // frame is from the previous one and must not be served for it. + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames( + "PLC_MUTED", {make_stored_row("PLC_MUTED", "route_stored_app", kStoredAtNs, 10.0, kFirstOccurrence)})); + Plant plant(99.0); + + auto capture = make_capture( + 256, plant, store, + listed({standing_fault("PLC_MUTED", {"route_stored_app"}, kSecondOccurrence, "CONFIRMED", /*muted=*/true)})); + ASSERT_TRUE(wait_for_catch_up(plant)); + + EXPECT_TRUE(capture->frames_for("PLC_MUTED").empty()); + EXPECT_TRUE(stored_codes(*store).empty()); + EXPECT_EQ(plant.reads("route_stored_app"), 0); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, AFaultListedAsClearedWithAnotherOccurrenceIsNotReadAtStart) { + // It confirmed and cleared again while the gateway was down: a read now would not + // show the values at that confirmation. + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames( + "PLC_FLAP", {make_stored_row("PLC_FLAP", "route_stored_app", kStoredAtNs, 10.0, kFirstOccurrence)})); + Plant plant(99.0); + + auto capture = make_capture(256, plant, store, + listed({standing_fault("PLC_FLAP", {"route_stored_app"}, kSecondOccurrence, "CLEARED")})); + ASSERT_TRUE(wait_for_catch_up(plant)); + + EXPECT_TRUE(capture->frames_for("PLC_FLAP").empty()); + EXPECT_EQ(plant.reads("route_stored_app"), 0); + EXPECT_TRUE(stored_codes(*store).empty()); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, AReloadedFrameIsNotServedBeforeItIsJudged) { + // The REST server answers before the capture has asked the fault manager. + auto store = std::make_shared(); + ASSERT_TRUE(store->replace_frames( + "PLC_HELD", {make_stored_row("PLC_HELD", "route_stored_app", kStoredAtNs, 10.0, kFirstOccurrence)})); + Plant plant(99.0); + + std::mutex gate_mutex; + std::condition_variable gate_cv; + bool entered = false; + bool released = false; + auto capture = + make_capture(256, plant, store, + [&](const std::function & should_abort) -> std::vector { + std::unique_lock lock(gate_mutex); + entered = true; + gate_cv.notify_all(); + while (!released && !should_abort()) { + gate_cv.wait_for(lock, 10ms); + } + return {standing_fault("PLC_HELD", {"route_stored_app"}, kFirstOccurrence)}; + }); + { + std::unique_lock lock(gate_mutex); + ASSERT_TRUE(gate_cv.wait_for(lock, 15s, [&] { + return entered; + })); + } + EXPECT_TRUE(capture->frames_for("PLC_HELD").empty()) << "a reloaded frame was served before it was judged"; + + { + std::lock_guard lock(gate_mutex); + released = true; + } + gate_cv.notify_all(); + ASSERT_TRUE(wait_for_catch_up(plant)); + const auto served = capture->frames_for("PLC_HELD"); + ASSERT_EQ(served.size(), 1u); + EXPECT_EQ(served[0].captured_at_ns, kStoredAtNs); + EXPECT_EQ(plant.reads("route_stored_app"), 0); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, ASameOccurrenceFrameIsKeptWhenTheGatewayClockIsBehind) { + // The gateway's clock is behind the fault manager's, so the frame's captured_at + // predates the occurrence's first_occurred. The occurrence is the stored one. + auto store = std::make_shared(); + const Occurrence occurrence{1, kStoredAtNs + 30'000'000'000}; + ASSERT_TRUE(store->replace_frames( + "PLC_SKEWED", {make_stored_row("PLC_SKEWED", "route_stored_app", kStoredAtNs, 10.0, occurrence)})); + Plant plant(99.0); + + auto capture = + make_capture(256, plant, store, listed({standing_fault("PLC_SKEWED", {"route_stored_app"}, occurrence)})); + ASSERT_TRUE(wait_for_catch_up(plant)); + + EXPECT_EQ(plant.reads("route_stored_app"), 0); + const auto served = capture->frames_for("PLC_SKEWED"); + ASSERT_EQ(served.size(), 1u); + EXPECT_DOUBLE_EQ(served[0].values.value("level", 0.0), 10.0); + EXPECT_EQ(served[0].captured_at_ns, kStoredAtNs); + EXPECT_FALSE(served[0].startup_catchup); +} + +/// @verifies REQ_INTEROP_088 +TEST_F(FreezeFrameCapturePersistenceTest, ADiscardedStaleFrameAtTheBoundNeverCostsALiveFrame) { + // Both slots hold reloaded frames. The stale one is discarded, which frees one slot for + // the two confirmed faults with no frame: the first listed gets it, the other is left out. + auto store = std::make_shared(); + seed_bound_probe(*store); + Plant plant(99.0); + + auto capture = make_capture(/*max_faults=*/2, plant, store, listed(bound_probe_standing())); + ASSERT_TRUE(wait_for_catch_up(plant)); + + const auto keep = capture->frames_for("PLC_KEEP"); + ASSERT_EQ(keep.size(), 1u) << "the live reloaded frame was evicted"; + EXPECT_DOUBLE_EQ(keep[0].values.value("level", 0.0), 10.0); + EXPECT_EQ(keep[0].captured_at_ns, kKeepAtNs); + EXPECT_FALSE(keep[0].startup_catchup); + + const auto fresh = capture->frames_for("PLC_NEW"); + ASSERT_EQ(fresh.size(), 1u); + EXPECT_TRUE(fresh[0].startup_catchup); + EXPECT_TRUE(capture->frames_for("PLC_STALE").empty()); + EXPECT_EQ(plant.reads("route_keep"), 0); + EXPECT_EQ(plant.reads("route_new"), 1); + EXPECT_EQ(plant.reads("route_stale"), 0); + + EXPECT_EQ(stored_codes(*store), (std::vector{"PLC_KEEP", "PLC_NEW"})); + const auto logs = log_sink().text(); + EXPECT_NE(logs.find("so they have no freeze-frame: PLC_STALE"), std::string::npos) << logs; +} diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md index 58ba8904c..6c3206802 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/README.md @@ -689,7 +689,7 @@ ros2_medkit_gateway: | `auto_alarms` | `false` | Boolean or object; subscribe a server EventNotifier and auto-derive a fault per AlarmCondition with no per-alarm `event_alarms` mapping - the zero-config native A&C surface (see above) | | `poll_interval_ms` | `1000` | Polling interval in ms (clamped to [100, 60000]) | | `prefer_subscriptions` | `false` | Use OPC-UA subscriptions instead of polling | -| `subscription_interval_ms` | `500` | Publishing interval for OPC-UA subscriptions when `prefer_subscriptions: true`. Between two polls the client is also iterated at this interval (at least 20 ms apart, at most once per poll), so changes arrive at this cadence and not once per `poll_interval_ms` | +| `subscription_interval_ms` | `500` | Publishing interval for OPC-UA subscriptions when `prefer_subscriptions: true`. Between two polls the client is also iterated at this interval (at least 20 ms apart, at most once per poll), so changes arrive at this cadence and not once per `poll_interval_ms`. When the plugin creates its subscriptions at start it reads every subscribed point once, so values are served at once and not one publishing interval later. After a reconnect it does not read: the last known values are served until the first publish | | `condition_replay_strategy` | `auto` | Active-condition replay on reconnect: `method`, `read`, `auto`, `off` (see below) | | `require_confirm_for_clear` | `true` | Require both Acknowledge AND Confirm before a native alarm auto-clears. Set `false` for Confirm-less servers (e.g. Siemens S7-1500) so alarms clear on Acknowledge alone (see below) | | `comms_lost_fault_enabled` | `true` | Raise a component-scoped `PLC_COMMS_LOST` fault when the connection stays down, and clear it on every successful connect (issue #496) | @@ -1194,10 +1194,25 @@ MEDKIT_OPCUA_VARIANT=write-capable bash scripts/run_integration_tests.sh bash scripts/test_all.sh MEDKIT_OPCUA_VARIANT=write-capable bash scripts/test_all.sh -# Stop +# Stop (keeps the gateway's state) bash scripts/stop.sh ``` +`start.sh` mounts the named volume `ros2-medkit-opcua-state` at +`/var/lib/ros2_medkit` in the gateway container. It holds `faults.db`, the +entity freeze-frames (`entity_freeze_frames.db`, set by +`entity_freeze_frame.storage.path` in `docker/gateway_params.yaml`) and the +rosbags. `stop.sh` removes the containers and keeps the volume, so a fault +raised before `stop.sh` is still there after the next `start.sh`, and its +detail serves the values frozen when it confirmed, with the same +`captured_at`. Set `OPCUA_DEMO_STATE_VOLUME` to use another volume. To start +from a clean state, purge the volume: + +```bash +bash scripts/stop.sh +docker volume rm ros2-medkit-opcua-state +``` + `start.sh` runs a `fault_manager_node` inside the gateway container, on the same ROS domain. It starts before `gateway_node`, and the script waits until `/fault_manager/report_fault` is advertised. Without a fault manager, @@ -1227,6 +1242,22 @@ the gateway adopts it without a restart: bash src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_discovery_race_test.sh ``` +Another scenario covers a freeze-frame across gateway restarts, against the +`test_alarm_server` fixture. It runs `fault_manager_node` and `gateway_node` +in one container with the state on a named volume, and checks that a restart +serves the frame captured at confirmation, also for a fault cleared while the +gateway ran; that a frame from an earlier occurrence is discarded with a warning +that names the fault and the entity, and replaced by a startup frame when the +fault is confirmed and the server answers; that a fault manager which lists no +fault drops nothing, and one which no longer holds a fault drops its frame with +the warning `Entity freeze-frame: dropped the reloaded frames of fault(s) the fault manager no longer holds: `; that subscription mode with a +60 s publishing interval serves values at once; and each +`entity_freeze_frame.storage.path` case: + +```bash +bash src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_freeze_frame_restart_tests.sh +``` + ### Test Coverage | Category | read-only | write-capable | What it validates | diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml index a331166cb..61225b567 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/gateway_params.yaml @@ -12,3 +12,8 @@ ros2_medkit_gateway: allowed_origins: ["*"] plugins: ["opcua"] plugins.opcua.poll_interval_ms: 1000 + # The PLC entities' freeze-frames, next to faults.db. scripts/start.sh mounts a + # named volume at /var/lib/ros2_medkit, so a frame outlives stop.sh. + entity_freeze_frame: + storage: + path: "/var/lib/ros2_medkit/entity_freeze_frames.db" diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_freeze_frame_restart_tests.sh b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_freeze_frame_restart_tests.sh new file mode 100644 index 000000000..b49484ec1 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_freeze_frame_restart_tests.sh @@ -0,0 +1,563 @@ +#!/usr/bin/env bash +# Copyright 2026 bburda +# +# End-to-end test: a plugin entity's freeze frame across gateway restarts. +# +# Boots test_alarm_server and one gateway container that runs fault_manager_node +# and gateway_node, with the gateway's state on a named docker volume. The +# server's values are set through its stdin CLI; the frame is read from the +# fault detail over HTTP and the store file with sqlite3. +# +# Scenarios: +# 1. The gateway container is replaced: the frame keeps its values, its +# captured_at and no capture_origin. A fault cleared while the gateway ran +# keeps its frame too. +# 2. The fault is cleared and reported again while gateway_node is down: the +# stored frame is discarded with a warning, and the startup catch-up reads +# the entity and marks the frame capture_origin: startup. +# 3. A fault cleared, confirmed again and cleared again while gateway_node is +# down: the frame of the first occurrence is discarded with a warning. +# 4. Scenario 2 with the OPC-UA server unreachable: the frame is discarded with a +# warning that names the fault and the entity, and no frame replaces it. +# 5. The fault manager restarts with empty storage: nothing is dropped. With one +# unrelated fault in it, the frames of the faults it does not hold are dropped +# with a warning that names them. +# 6. entity_freeze_frame.storage.path: next to triggers.storage.path, neither +# path, and a path that cannot be opened (scenario 1 covers an explicit path). +# 7. Subscription mode with a publishing interval of 60 s: GET /data serves values +# at once, and scenario 2's startup frame holds the current value. +# +# A check that something did NOT happen first waits for a sentinel fault the +# plugin confirms after start: its frame comes from the drain loop, which runs +# only after the startup catch-up has returned. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../../../../.." && pwd)" +PREFIX=ff-restart +NET_NAME="${PREFIX}-net" +SERVER_NAME="${PREFIX}-server" +GATEWAY_NAME="${PREFIX}-gateway" +VOLUME="${PREFIX}-state" +SERVER_PORT=4842 +GATEWAY_PORT="${FF_RESTART_GATEWAY_PORT:-8089}" +API="http://localhost:${GATEWAY_PORT}/api/v1" +ENTITY=tank_process +FAULT=PLC_PUMP_OVERLOAD +STATE_DIR=/var/lib/ros2_medkit +EXPLICIT_STORE="${STATE_DIR}/entity_freeze_frames.db" +ROS_ENV="source /opt/ros/jazzy/setup.bash && source /root/ws/install/setup.bash" + +# StatusWord bits of the node map below. +BIT_FAULT=8 # bit 3: PLC_PUMP_OVERLOAD +BIT_CLEARED=16 # bit 4: PLC_VALVE_STUCK +BIT_SENTINEL_A=256 # bit 8: PLC_SENTINEL_A +BIT_SENTINEL_B=512 # bit 9: PLC_SENTINEL_B + +GW_LOG="" +CONFIG_DIR="" +SERVER_CTRL="" + +cleanup() { + local rc=$? + if [[ ${rc} -ne 0 ]]; then + echo "=== ${SERVER_NAME} logs ===" >&2 + docker logs "${SERVER_NAME}" 2>&1 | tail -40 >&2 || true + echo "=== fault manager log ===" >&2 + docker exec "${GATEWAY_NAME}" bash -c 'tail -n 60 /tmp/fault_manager.log' >&2 || true + if [[ -n "${GW_LOG}" ]]; then + echo "=== ${GW_LOG} ===" >&2 + docker exec "${GATEWAY_NAME}" cat "${GW_LOG}" >&2 || true + fi + fi + docker rm -f "${SERVER_NAME}" "${GATEWAY_NAME}" >/dev/null 2>&1 || true + docker network rm "${NET_NAME}" >/dev/null 2>&1 || true + docker volume rm "${VOLUME}" >/dev/null 2>&1 || true + if [[ -n "${SERVER_DOCKER_PID:-}" ]]; then + kill "${SERVER_DOCKER_PID}" 2>/dev/null || true + wait "${SERVER_DOCKER_PID}" 2>/dev/null || true + fi + [[ -n "${CONFIG_DIR}" ]] && rm -rf "${CONFIG_DIR}" + [[ -n "${SERVER_CTRL}" ]] && rm -rf "${SERVER_CTRL}" + return 0 +} +trap cleanup EXIT + +fail() { + echo "ASSERT FAILED: $*" >&2 + exit 1 +} + +ok() { + echo " OK $*" +} + +# Poll until is true. +wait_for() { + local url="$1" expr="$2" deadline="${3:-60}" + for _ in $(seq 1 "${deadline}"); do + if curl -sf "${url}" 2>/dev/null | jq -e "${expr}" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + curl -sf "${url}" 2>/dev/null | jq . >&2 || true + fail "timed out: ${url} ${expr}" +} + +# Runs a shell command in the gateway container with ROS sourced. +in_gw() { + docker exec "${GATEWAY_NAME}" bash -c "${ROS_ENV} && $1" +} + +# Poll the current gateway log until it holds . +wait_log() { + local text="$1" deadline="${2:-60}" + for _ in $(seq 1 "${deadline}"); do + if docker exec "${GATEWAY_NAME}" grep -qF -- "${text}" "${GW_LOG}"; then + ok "log: ${text}" + return 0 + fi + sleep 1 + done + fail "${GW_LOG} never logged: ${text}" +} + +server() { + echo "$*" >&3 +} + +# Wait until the entity serves for data item . +wait_value() { + wait_for "${API}/apps/${ENTITY}/x-plc-data" ".items[] | select(.name == \"$1\") | .value == $2" 60 +} + +fault_status() { + curl -sf "${API}/faults?status=all" 2>/dev/null | jq -r --arg code "$1" '.items[] | select(.fault_code == $code) | .status' | + head -1 +} + +wait_status() { + local code="$1" expected="$2" + for _ in $(seq 1 60); do + if [[ "$(fault_status "${code}")" == "${expected}" ]]; then + ok "${code} is ${expected}" + return 0 + fi + sleep 1 + done + fail "${code} never reached ${expected} (last: $(fault_status "${code}"))" +} + +# The entity's freeze_frame snapshot in the fault detail, or nothing. +frame_of() { + curl -sf "${API}/apps/${ENTITY}/faults/$1" 2>/dev/null | + jq -c --arg e "${ENTITY}" '[.environment_data.snapshots[]? | select(.type == "freeze_frame" and .name == $e)] | first // empty' +} + +# Poll until the frame of satisfies , then print it. +wait_frame() { + local code="$1" expr="$2" frame="" + for _ in $(seq 1 60); do + frame="$(frame_of "${code}" || true)" + if [[ -n "${frame}" ]] && jq -e "${expr}" <<<"${frame}" >/dev/null 2>&1; then + echo "${frame}" + return 0 + fi + sleep 1 + done + fail "frame of ${code} never matched ${expr} (last: ${frame:-none})" +} + +# Assert on a frame and say what it proves. +check_frame() { + local frame="$1" expr="$2" what="$3" + jq -e "${expr}" <<<"${frame}" >/dev/null || fail "${what}: ${expr} on ${frame}" + ok "${what}" +} + +stored_codes() { + docker exec "${GATEWAY_NAME}" sqlite3 "$1" "SELECT DISTINCT fault_code FROM entity_freeze_frames ORDER BY fault_code" | + tr '\n' ' ' | sed 's/ $//' +} + +# Raise a sentinel fault and wait for its frame without capture_origin: the drain +# loop wrote it, so the startup catch-up of this gateway life has returned. +catch_up_done() { + local bits="$1" code="$2" + server set StatusWord "${bits}" + wait_frame "${code}" '.["x-medkit"] | has("capture_origin") | not' >/dev/null + ok "${code} framed on its confirm edge, so the startup catch-up has returned" +} + +gw_container_start() { + docker rm -f "${GATEWAY_NAME}" >/dev/null 2>&1 || true + docker run -d --name "${GATEWAY_NAME}" --network "${NET_NAME}" -p "${GATEWAY_PORT}:8080" \ + -v "${VOLUME}:${STATE_DIR}" \ + -e ROS_DOMAIN_ID=67 \ + -e ROS_AUTOMATIC_DISCOVERY_RANGE=LOCALHOST \ + -e OPCUA_ENDPOINT_URL="opc.tcp://${SERVER_NAME}:${SERVER_PORT}" \ + -e OPCUA_NODE_MAP_PATH=/config/ff_restart_nodes.yaml \ + gateway-opcua:freeze-frame-test sleep infinity >/dev/null + docker cp "${CONFIG_DIR}/." "${GATEWAY_NAME}:/config/" +} + +# Start fault_manager_node on and wait until it answers. +fm_start() { + docker exec -d "${GATEWAY_NAME}" bash -c "${ROS_ENV} && exec \"\$(ros2 pkg prefix ros2_medkit_fault_manager)/lib/ros2_medkit_fault_manager/fault_manager_node\" --ros-args -p database_path:=$1 -p snapshots.rosbag.enabled:=false >>/tmp/fault_manager.log 2>&1" + in_gw "timeout 60 ros2 service call /fault_manager/list_faults ros2_medkit_msgs/srv/ListFaults '{}' >/dev/null" || + fail "fault manager on $1 did not answer" +} + +stop_process() { + local pattern="$1" + docker exec "${GATEWAY_NAME}" pkill -TERM -f "${pattern}" || true + for _ in $(seq 1 60); do + if ! docker exec "${GATEWAY_NAME}" pgrep -f "${pattern}" >/dev/null; then + return 0 + fi + sleep 1 + done + fail "${pattern} did not exit" +} + +fm_stop() { + stop_process '[f]ault_manager_node' +} + +# Start gateway_node as life with , and wait for the entity. +gw_start() { + GW_LOG="/tmp/gateway-$1.log" + echo " [gateway life $1, $2]" + docker exec -d "${GATEWAY_NAME}" bash -c "${ROS_ENV} && exec \"\$(ros2 pkg prefix ros2_medkit_gateway)/lib/ros2_medkit_gateway/gateway_node\" --ros-args --params-file $2 -p plugins.opcua.path:=\"\$(find /root/ws/install -name libros2_medkit_opcua_plugin.so | head -1)\" -p discovery.mode:=hybrid -p discovery.manifest_path:=/config/manifest.yaml -p discovery.manifest_strict_validation:=false >${GW_LOG} 2>&1" + wait_for "${API}/apps" '.items | map(.id) | contains(["tank_process"])' 90 +} + +gw_stop() { + stop_process '[l]ib/ros2_medkit_gateway/gateway_node' +} + +# The fault manager's own services, while gateway_node is down. +fm_clear() { + local reply + reply="$(in_gw "timeout 30 ros2 service call /fault_manager/clear_fault ros2_medkit_msgs/srv/ClearFault \"{fault_code: '$1'}\"")" + [[ "${reply}" == *"success=True"* ]] || fail "clear_fault $1: ${reply}" + ok "${1} cleared through the fault manager" +} + +fm_report() { + local reply + reply="$(in_gw "timeout 30 ros2 service call /fault_manager/report_fault ros2_medkit_msgs/srv/ReportFault \"{fault_code: '$1', event_type: 0, severity: 2, description: 'reported while gateway_node is down', source_id: '${ENTITY}'}\"")" + [[ "${reply}" == *"accepted=True"* ]] || fail "report_fault $1: ${reply}" + ok "${1} reported again through the fault manager" +} + +assert_logged() { + docker exec "${GATEWAY_NAME}" grep -qF -- "$1" "${GW_LOG}" || fail "${GW_LOG} does not hold: $1" + ok "log: $1" +} + +# A gateway life on whose fault manager holds no fault. Confirms the +# fault with FaultCode at , then moves FaultCode to +1. +# Sets CAPTURED to the frame's captured_at. +CAPTURED="" +first_life_with_frame() { + local name="$1" params="$2" marker="$3" + server set StatusWord 0 + server set FaultCode "${marker}" + gw_start "${name}-1" "${params}" + # Printed after the events subscription matched, so the confirmation below is seen. + wait_log "Standing-fault freeze-frame catch-up: no confirmed faults at startup" + wait_value fault_code "${marker}" + server set StatusWord "${BIT_FAULT}" + wait_status "${FAULT}" CONFIRMED + local frame + frame="$(wait_frame "${FAULT}" ".[\"x-medkit\"].full_data.fault_code == ${marker}")" + check_frame "${frame}" '.["x-medkit"] | has("capture_origin") | not' "${name}: frame of ${marker} captured on the confirm edge" + CAPTURED="$(jq -r '.["x-medkit"].captured_at' <<<"${frame}")" + server set FaultCode $((marker + 1)) + wait_value fault_code $((marker + 1)) +} + +cd "${REPO_ROOT}" + +docker rm -f "${SERVER_NAME}" "${GATEWAY_NAME}" >/dev/null 2>&1 || true +docker network rm "${NET_NAME}" >/dev/null 2>&1 || true +docker volume rm "${VOLUME}" >/dev/null 2>&1 || true + +echo "[1/5] Build test_alarm_server and gateway-opcua images" +docker build --network=host \ + -f src/ros2_medkit_plugins/ros2_medkit_opcua/docker/test_alarm_server/Dockerfile \ + -t ros2_medkit_alarm_test_server:freeze-frame-test . >/dev/null +docker build --network=host \ + -f src/ros2_medkit_plugins/ros2_medkit_opcua/docker/Dockerfile.gateway \ + -t gateway-opcua:freeze-frame-test . >/dev/null + +echo "[2/5] Start test_alarm_server" +docker network create "${NET_NAME}" >/dev/null +docker volume create "${VOLUME}" >/dev/null +SERVER_CTRL="$(mktemp -d)" +mkfifo "${SERVER_CTRL}/stdin" +# Read and write on FD 3, so neither end of the FIFO blocks on open. The docker +# client stays in the foreground of a background job, so the FIFO stays wired +# to the container's stdin (see run_alarm_tests.sh). +exec 3<>"${SERVER_CTRL}/stdin" +# shellcheck disable=SC2094 +docker run --rm --name "${SERVER_NAME}" --network "${NET_NAME}" \ + -i ros2_medkit_alarm_test_server:freeze-frame-test --port "${SERVER_PORT}" \ + <&3 >/dev/null 2>&1 & +SERVER_DOCKER_PID=$! +for _ in $(seq 1 60); do + if [[ "$(docker logs "${SERVER_NAME}" 2>&1)" == *"READY "* ]]; then + break + fi + sleep 1 +done +[[ "$(docker logs "${SERVER_NAME}" 2>&1)" == *"READY "* ]] || fail "test_alarm_server never printed READY" + +CONFIG_DIR="$(mktemp -d)" +cat >"${CONFIG_DIR}/ff_restart_nodes.yaml" <"${CONFIG_DIR}/manifest.yaml" <"${CONFIG_DIR}/params_next_to_triggers.yaml" +{ + base_params + cat <"${CONFIG_DIR}/params_no_store.yaml" +{ + base_params + cat <"${CONFIG_DIR}/params_unopenable.yaml" + +echo "[3/5] Restart scenarios on the demo parameters (explicit store path)" +gw_container_start +fm_start "${STATE_DIR}/faults.db" +first_life_with_frame explicit /config/gateway_params.yaml 111 +captured_1="${CAPTURED}" +assert_logged "Entity freeze-frames persisted in ${EXPLICIT_STORE}" +# A second fault, confirmed at 121 and cleared while the gateway runs. +server set FaultCode 121 +wait_value fault_code 121 +server set StatusWord $((BIT_FAULT + BIT_CLEARED)) +wait_status PLC_VALVE_STUCK CONFIRMED +frame="$(wait_frame PLC_VALVE_STUCK '.["x-medkit"].full_data.fault_code == 121')" +check_frame "${frame}" '.["x-medkit"] | has("capture_origin") | not' "PLC_VALVE_STUCK: frame of 121 captured on the confirm edge" +captured_cleared="$(jq -r '.["x-medkit"].captured_at' <<<"${frame}")" +server set StatusWord "${BIT_FAULT}" +wait_status PLC_VALVE_STUCK CLEARED +server set FaultCode 112 +wait_value fault_code 112 + +echo " [scenario 1] the gateway container is replaced" +docker rm -f "${GATEWAY_NAME}" >/dev/null +gw_container_start +fm_start "${STATE_DIR}/faults.db" +gw_start explicit-2 /config/gateway_params.yaml +catch_up_done $((BIT_FAULT + BIT_SENTINEL_A)) PLC_SENTINEL_A +frame="$(frame_of "${FAULT}")" +check_frame "${frame}" '.["x-medkit"].full_data.fault_code == 111' "the frame holds the value from the confirmation (111), not today's (112)" +check_frame "${frame}" ".[\"x-medkit\"].captured_at == \"${captured_1}\"" "the frame keeps captured_at ${captured_1}" +check_frame "${frame}" '.["x-medkit"] | has("capture_origin") | not' "the frame has no capture_origin" +frame="$(frame_of PLC_VALVE_STUCK)" +check_frame "${frame}" '.["x-medkit"].full_data.fault_code == 121' "the cleared fault still serves its frame (121)" +check_frame "${frame}" ".[\"x-medkit\"].captured_at == \"${captured_cleared}\"" "the cleared fault's frame keeps captured_at ${captured_cleared}" +assert_logged "Entity freeze-frame: reloaded frames for 2 fault(s) from the store" +[[ "$(stored_codes "${EXPLICIT_STORE}")" == "${FAULT} PLC_SENTINEL_A PLC_VALVE_STUCK" ]] || + fail "store rows: $(stored_codes "${EXPLICIT_STORE}")" +ok "the store holds the frames of ${FAULT}, PLC_SENTINEL_A and PLC_VALVE_STUCK" + +echo " [scenario 2] cleared and reported again while gateway_node was down" +gw_stop +fm_clear "${FAULT}" +fm_report "${FAULT}" +server set FaultCode 333 +gw_start reoccurred /config/gateway_params.yaml +wait_log "Entity freeze-frame for fault '${FAULT}': the stored frame of entity '${ENTITY}' is from an earlier occurrence and was discarded (stored occurrence 1, the fault manager lists occurrence 2, CONFIRMED)" +frame="$(wait_frame "${FAULT}" '.["x-medkit"].capture_origin == "startup"')" +check_frame "${frame}" '.["x-medkit"].full_data.fault_code == 333' "the frame was read again on start (333)" +check_frame "${frame}" ".[\"x-medkit\"].captured_at > \"${captured_1}\"" "the frame has a newer captured_at" +ok "the frame is marked capture_origin: startup" + +echo " [scenario 3] cleared, confirmed again and cleared again while gateway_node was down" +gw_stop +fm_report PLC_VALVE_STUCK +fm_clear PLC_VALVE_STUCK +gw_start reoccurred-and-cleared /config/gateway_params.yaml +wait_log "Entity freeze-frame for fault 'PLC_VALVE_STUCK': the stored frame of entity '${ENTITY}' is from an earlier occurrence and was discarded (stored occurrence 1, the fault manager lists occurrence 2, CLEARED)" +frame="$(frame_of PLC_VALVE_STUCK)" +if [[ -n "${frame}" ]]; then + check_frame "${frame}" ".[\"x-medkit\"].captured_at != \"${captured_cleared}\" and .[\"x-medkit\"].full_data.fault_code != 121" \ + "the detail does not serve the first occurrence's frame" +else + ok "the detail does not serve the first occurrence's frame (no frame)" +fi + +echo " [scenario 4] the same as scenario 2 with the OPC-UA server unreachable" +gw_stop +fm_clear "${FAULT}" +fm_report "${FAULT}" +docker network disconnect "${NET_NAME}" "${SERVER_NAME}" +gw_start unreachable /config/gateway_params.yaml +wait_log "Entity freeze-frame for fault '${FAULT}': the stored frame of entity '${ENTITY}' is from an earlier occurrence and was discarded (stored occurrence 2, the fault manager lists occurrence 3, CONFIRMED)" +# The catch-up read of the entity, which is the sign that the catch-up has run. +wait_log "Entity freeze-frame for fault '${FAULT}': entity '${ENTITY}' values are all null" +curl -sf "${API}/apps/${ENTITY}/faults/${FAULT}" >/dev/null || fail "no fault detail for ${FAULT}" +[[ -z "$(frame_of "${FAULT}")" ]] || fail "the fault detail still serves a frame: $(frame_of "${FAULT}")" +ok "the fault detail serves no frame" +[[ "$(stored_codes "${EXPLICIT_STORE}")" == "PLC_SENTINEL_A" ]] || fail "store rows: $(stored_codes "${EXPLICIT_STORE}")" +ok "the stored row of ${FAULT} is gone" +docker network connect "${NET_NAME}" "${SERVER_NAME}" + +echo " [scenario 5] the fault manager restarts with empty storage" +gw_stop +server set StatusWord 0 +fm_stop +fm_start "${STATE_DIR}/faults-empty.db" +gw_start empty-fault-manager /config/gateway_params.yaml +wait_log "Entity freeze-frame: the fault manager lists no fault in any status, so no reloaded frame is dropped" +[[ "$(stored_codes "${EXPLICIT_STORE}")" == "PLC_SENTINEL_A" ]] || fail "store rows: $(stored_codes "${EXPLICIT_STORE}")" +ok "an empty fault list drops nothing: the store still holds PLC_SENTINEL_A" +gw_stop +fm_report PLC_UNRELATED +gw_start one-unrelated-fault /config/gateway_params.yaml +wait_log "Entity freeze-frame: dropped the reloaded frames of 1 fault(s) the fault manager no longer holds: PLC_SENTINEL_A" +rows="$(stored_codes "${EXPLICIT_STORE}")" +[[ -z "${rows}" || "${rows}" == "PLC_UNRELATED" ]] || fail "store rows: ${rows}" +ok "the frame of PLC_SENTINEL_A, which the fault manager does not hold, is dropped (rows: ${rows:-none})" + +echo "[4/5] entity_freeze_frame.storage.path cases (scenario 6)" +storage_case() { + local name="$1" params="$2" marker="$3" logged="$4" survives="$5" + echo " [case ${name}]" + gw_stop + fm_stop + fm_start "${STATE_DIR}/faults-${name}.db" + first_life_with_frame "${name}" "${params}" "${marker}" + local captured="${CAPTURED}" + assert_logged "${logged}" + gw_stop + gw_start "${name}-2" "${params}" + local frame + if [[ "${survives}" == yes ]]; then + catch_up_done $((BIT_FAULT + BIT_SENTINEL_B)) PLC_SENTINEL_B + frame="$(frame_of "${FAULT}")" + check_frame "${frame}" ".[\"x-medkit\"].full_data.fault_code == ${marker}" "${name}: the frame survives the restart" + check_frame "${frame}" ".[\"x-medkit\"].captured_at == \"${captured}\"" "${name}: captured_at is kept" + else + frame="$(wait_frame "${FAULT}" '.["x-medkit"].capture_origin == "startup"')" + check_frame "${frame}" ".[\"x-medkit\"].full_data.fault_code == $((marker + 1))" "${name}: the frame is read again after the restart" + check_frame "${frame}" ".[\"x-medkit\"].captured_at > \"${captured}\"" "${name}: with a newer captured_at" + fi +} +in_gw "mkdir -p ${STATE_DIR}/triggers" +storage_case next-to-triggers /config/params_next_to_triggers.yaml 400 \ + "Entity freeze-frames persisted in ${STATE_DIR}/triggers/entity_freeze_frames.db" yes +storage_case no-store /config/params_no_store.yaml 500 \ + "Entity freeze-frames are not persisted (no entity_freeze_frame.storage.path and no triggers.storage.path); they are lost on restart" no +storage_case unopenable /config/params_unopenable.yaml 600 \ + "Entity freeze-frame store '${STATE_DIR}/no-such-dir/entity_freeze_frames.db' could not be opened, frames stay in memory" no + +echo "[5/5] Subscription mode with a publishing interval of 60 s (scenario 7)" +# A subscription publishes its first values one interval after it is created, so +# both checks below must pass long before 60 s. +SUB_STORE="${STATE_DIR}/sub/entity_freeze_frames.db" +sub_params() { + base_params | sed 's/plugins.opcua.poll_interval_ms: 1000/plugins.opcua.poll_interval_ms: 60000/' + cat <"${CONFIG_DIR}/params_sub_first.yaml" +sub_params >"${CONFIG_DIR}/params_sub60.yaml" +docker cp "${CONFIG_DIR}/." "${GATEWAY_NAME}:/config/" +gw_stop +fm_stop +in_gw "mkdir -p ${STATE_DIR}/sub" +fm_start "${STATE_DIR}/faults-sub.db" +first_life_with_frame sub /config/params_sub_first.yaml 900 +gw_stop +fm_clear "${FAULT}" +fm_report "${FAULT}" +server set FaultCode 950 +gw_start sub-60 /config/params_sub60.yaml +wait_for "${API}/apps/${ENTITY}/data" '.items[] | select(.id == "fault_code") | .value == 950' 15 +ok "GET /data serves the current value (950) within 15 s in subscription mode" +wait_log "Entity freeze-frame for fault '${FAULT}': the stored frame of entity '${ENTITY}' is from an earlier occurrence and was discarded (stored occurrence 1, the fault manager lists occurrence 2, CONFIRMED)" 15 +frame="$(wait_frame "${FAULT}" '.["x-medkit"].capture_origin == "startup"')" +check_frame "${frame}" '.["x-medkit"].full_data.fault_code == 950' "subscription mode: the startup frame holds the current value (950)" + +echo "All freeze-frame restart scenarios passed." +exec 3>&- diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/start.sh b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/start.sh index 48f494c2c..ddc37b653 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/start.sh +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/start.sh @@ -8,8 +8,14 @@ # # MEDKIT_OPCUA_VARIANT=write-capable brings up an image with the OPC-UA write # path compiled in. The default, read-only, is what ships. +# +# The gateway's state (faults.db, entity freeze-frames, rosbags) is on the named +# volume below, so it survives stop.sh and the next start.sh. Purge it with +# docker volume rm ros2-medkit-opcua-state set -eo pipefail +STATE_VOLUME="${OPCUA_DEMO_STATE_VOLUME:-ros2-medkit-opcua-state}" + # Printed by the gateway container once fault_manager_node advertises its # services. The host waits for it. FM_READY_MARKER="fault_manager ready: /fault_manager/report_fault" @@ -44,6 +50,8 @@ echo "" echo "=== Starting containers ===" docker rm -f openplc gateway 2>/dev/null || true docker network create plc-demo 2>/dev/null || true +# stop.sh removes the container, so state in its writable layer would go with it. +docker volume create "$STATE_VOLUME" >/dev/null docker run -d --name openplc --network plc-demo -p 4840:4840 openplc-tank echo "OpenPLC starting..." @@ -56,6 +64,7 @@ for _ in $(seq 1 45); do done docker run -d --name gateway --network plc-demo -p 8080:8080 \ + -v "$STATE_VOLUME":/var/lib/ros2_medkit \ -e ROS_DOMAIN_ID=60 \ -e OPCUA_ENDPOINT_URL="opc.tcp://openplc:4840/openplc/opcua" \ -e OPCUA_NODE_MAP_PATH="/config/tank_nodes.yaml" \ @@ -128,6 +137,7 @@ for _ in $(seq 1 30); do echo "Tests: bash scripts/run_integration_tests.sh" echo "Faults: fault_manager_node runs in the gateway container" echo " (log: docker exec gateway cat /var/lib/ros2_medkit/fault_manager.log)" + echo "State: volume '$STATE_VOLUME' (kept across stop/start)" exit 0 fi sleep 2 diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/stop.sh b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/stop.sh index fb3fa3de6..41af3db97 100755 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/stop.sh +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/stop.sh @@ -1,4 +1,10 @@ #!/usr/bin/env bash +# Stop the OpenPLC + medkit gateway demo. The gateway's state volume stays, so +# faults and their freeze-frames survive the next start.sh. +STATE_VOLUME="${OPCUA_DEMO_STATE_VOLUME:-ros2-medkit-opcua-state}" + docker rm -f gateway openplc 2>/dev/null docker network rm plc-demo 2>/dev/null echo "Stopped." +echo "State kept in volume '$STATE_VOLUME'. Purge it with:" +echo " docker volume rm $STATE_VOLUME" diff --git a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_poller.cpp b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_poller.cpp index b8d74a275..7d79ab6d3 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_poller.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_opcua/src/opcua_poller.cpp @@ -242,6 +242,12 @@ void OpcuaPoller::start(const PollerConfig & config) { // Try subscription mode first if (config_.prefer_subscriptions) { setup_subscriptions(); + // A subscription delivers its first values with its first publish, up to one + // subscription_interval_ms later. One read fills the cache now, before the + // poll thread and the event pump start, so no publish can race it. + if (using_subscriptions_.load()) { + do_poll(); + } } // Issue #386: subscribe to native AlarmConditionType events. Independent @@ -1313,6 +1319,9 @@ void OpcuaPoller::poll_loop() { } comms_lost_raised_ = false; comms_down_since_.reset(); + // No read after subscribing here: the event pump may handle a publish inside + // that read, and the older read result would then overwrite the newer value. + // The cache keeps the last known values until the first publish. if (config_.prefer_subscriptions) { setup_subscriptions(); }