Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
dc1d60a
feat(gateway): keep an entity freeze-frame across a gateway restart
bburda Nov 16, 2026
0b4860a
feat(opcua): keep the demo gateway's state across stop.sh and start.sh
bburda Nov 16, 2026
4334d82
test(opcua): freeze-frame across gateway restarts, end to end
bburda Nov 16, 2026
9e00bab
fix(gateway): a freeze-frame captured again is the newest one for evi…
bburda Nov 16, 2026
673418b
fix(gateway): count a non-JSON freeze-frame row as unreadable
bburda Nov 16, 2026
dfafde9
fix(gateway): keep reloaded freeze-frames when the fault list is empty
bburda Nov 16, 2026
ab1182d
fix(gateway): judge a reloaded freeze-frame by the fault's occurrence
bburda Nov 16, 2026
322c80a
docs: freeze-frame restart rules by occurrence, empty fault list and …
bburda Nov 16, 2026
7d53ee0
test(opcua): restart e2e for a cleared frame, a re-occurrence and an …
bburda Nov 16, 2026
e4ec163
fix(gateway): one fault list decides the freeze-frame start
bburda Nov 16, 2026
8c70e90
docs: what persistence does to the startup freeze-frame read
bburda Nov 16, 2026
a2fb883
test(opcua): restart e2e for the one-list start and subscription mode
bburda Nov 16, 2026
c3fb870
fix(opcua): read every subscribed point once after subscribing
bburda Nov 16, 2026
991b8e6
fix(gateway): a new occurrence with no values drops the previous frame
bburda Nov 16, 2026
8622895
ci(opcua): run the OPC UA workflow when the capture wiring changes
bburda Nov 16, 2026
04f399d
docs: freeze-frame docs say what the capture does
bburda Nov 16, 2026
016a68e
fix(opcua): read the subscribed points once at start only
bburda Nov 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 87 additions & 3 deletions .github/workflows/opcua-plugin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ permissions:
contents: read

# Runs the OpenPLC-based Docker integration suite for ros2_medkit_opcua.
# Triggers only when the plugin itself, the gateway plugin/provider API, or
# ros2_medkit_msgs change. A nightly cron provides a safety net against silent
# breaks that slip past the path filter.
# Triggers only when the plugin itself, the gateway plugin/provider API, the
# entity freeze-frame capture and its store, the gateway node and fault scope
# code that wire the capture in, or ros2_medkit_msgs change. A nightly cron
# provides a safety net against silent breaks that slip past the path filter.

on:
pull_request:
Expand All @@ -16,6 +17,14 @@ on:
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/plugins/**'
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/providers/**'
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/error_codes.hpp'
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/*freeze_frame*'
- 'src/ros2_medkit_gateway/src/core/faults/*freeze_frame*'
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp'
- 'src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp'
- 'src/ros2_medkit_gateway/src/gateway_node.cpp'
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp'
- 'src/ros2_medkit_gateway/src/core/faults/fault_scope.cpp'
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/fault_scope.hpp'
- 'src/ros2_medkit_msgs/**'
- '.github/workflows/opcua-plugin.yml'
push:
Expand All @@ -25,6 +34,14 @@ on:
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/plugins/**'
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/providers/**'
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/http/error_codes.hpp'
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/*freeze_frame*'
- 'src/ros2_medkit_gateway/src/core/faults/*freeze_frame*'
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/entity_freeze_frame_capture.hpp'
- 'src/ros2_medkit_gateway/src/entity_freeze_frame_capture.cpp'
- 'src/ros2_medkit_gateway/src/gateway_node.cpp'
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/gateway_node.hpp'
- 'src/ros2_medkit_gateway/src/core/faults/fault_scope.cpp'
- 'src/ros2_medkit_gateway/include/ros2_medkit_gateway/core/faults/fault_scope.hpp'
- 'src/ros2_medkit_msgs/**'
- '.github/workflows/opcua-plugin.yml'
schedule:
Expand Down Expand Up @@ -295,6 +312,56 @@ jobs:
curl -s "$api/faults" | jq . || true
exit 1

- name: The fault's freeze frame survives stop.sh and start.sh
timeout-minutes: 20
env:
MEDKIT_OPCUA_VARIANT: write-capable
run: |
api=http://localhost:8080/api/v1
scripts=src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts
frame_of() {
curl -sf "$api/apps/tank_process/faults/$1" |
jq -c '[.environment_data.snapshots[]? | select(.type == "freeze_frame")] | first // empty'
}
# Polls until the frame of $1 has no capture_origin, and prints it.
edge_frame_of() {
for _ in $(seq 1 60); do
f=$(frame_of "$1" || true)
if [ -n "$f" ] && jq -e '.["x-medkit"] | has("capture_origin") | not' <<<"$f" >/dev/null; then
echo "$f"
return 0
fi
sleep 2
done
echo "FAIL: $1 has no freeze frame from its confirmation" >&2
return 1
}
before=$(edge_frame_of PLC_HIGH_TEMP)
echo "before stop.sh: $before"
captured=$(jq -r '.["x-medkit"].captured_at' <<<"$before")
bash "$scripts/stop.sh"
bash "$scripts/start.sh"
reloaded=""
for _ in $(seq 1 30); do
reloaded=$(docker logs gateway 2>&1 | grep -F "Entity freeze-frame: reloaded frames for" || true)
[ -n "$reloaded" ] && break
sleep 2
done
[ -n "$reloaded" ] || { echo "FAIL: the gateway reloaded no freeze frame"; exit 1; }
echo "$reloaded"
# A second alarm: its frame comes from the drain loop, which runs after the startup catch-up.
code=$(curl -s -o /tmp/put.json -w '%{http_code}' -X PUT \
"$api/apps/tank_process/data/tank_level" \
-H 'Content-Type: application/json' -d '{"value": 50.0}')
[ "$code" = "200" ] || { echo "FAIL: PUT tank_level -> $code"; cat /tmp/put.json; exit 1; }
edge_frame_of PLC_LOW_LEVEL >/dev/null
after=$(frame_of PLC_HIGH_TEMP)
echo "after start.sh: $after"
jq -e --arg c "$captured" \
'.["x-medkit"].captured_at == $c and (.["x-medkit"] | has("capture_origin") | not)' <<<"$after" >/dev/null ||
{ echo "FAIL: PLC_HIGH_TEMP lost the frame from its confirmation ($captured)"; exit 1; }
echo "PLC_HIGH_TEMP keeps captured_at $captured across stop.sh and start.sh"

- name: Dump container logs on failure
if: failure()
run: |
Expand All @@ -310,6 +377,23 @@ jobs:
run: |
docker rm -f gateway openplc 2>/dev/null || true
docker network rm plc-demo 2>/dev/null || true
docker volume rm ros2-medkit-opcua-state 2>/dev/null || true

freeze-frame-restart:
name: Freeze frame across gateway restarts
# A plugin entity's freeze frame across gateway restarts, against the
# test_alarm_server fixture: the store, the re-read of a frame from an earlier
# occurrence, the drop when the entity cannot answer or the fault manager no
# longer holds the fault, and each entity_freeze_frame.storage.path case. The
# script builds its own images.
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Run freeze-frame restart suite
run: bash src/ros2_medkit_plugins/ros2_medkit_opcua/docker/scripts/run_freeze_frame_restart_tests.sh

integration-alarms:
name: Integration (AlarmConditionType, ${{ matrix.variant }})
Expand Down
4 changes: 3 additions & 1 deletion docs/api/rest.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1527,7 +1527,9 @@ Query and manage faults.
- ``freeze_frame``: Data captured at fault confirmation. Entity frames for
faults that were already confirmed when the gateway started are captured
at gateway start instead and carry ``"capture_origin": "startup"`` in
their ``x-medkit`` block. An entity frame also carries ``source`` in
their ``x-medkit`` block, unless a frame of the same occurrence was
reloaded from ``entity_freeze_frame.storage.path``: that frame is served
as it was captured. An entity frame also carries ``source`` in
``x-medkit``, naming the path that read the values
(``plugin_data_provider`` for the owning plugin's DataProvider,
``plugin_x_plc_data_route`` for its ``x-plc-data`` route). These values
Expand Down
35 changes: 33 additions & 2 deletions docs/config/server.rst
Original file line number Diff line number Diff line change
Expand Up @@ -382,13 +382,44 @@ Configure how the gateway connects to the fault manager services and event topic
by its plugin's DataProvider and merge them into the fault detail's
``environment_data.snapshots``. Faults already confirmed at gateway start
are caught up at startup; those frames read the values at start time (not
confirm time) and are marked ``x-medkit.capture_origin: startup``. For an
confirm time) and are marked ``x-medkit.capture_origin: startup``. A fault
whose frame of the same occurrence was reloaded from
``entity_freeze_frame.storage.path`` is not read at start: the reloaded
frame is served as captured. For an
entity that reports its link down these are the plugin's last known
values, marked ``connected: false`` in the snapshot's ``x-medkit`` block.
values. ``connected: false`` marks them in the snapshot's ``x-medkit``
block only when the plugin's payload carries a link flag; the OPC UA
plugin's ``DataProvider`` payload carries none.
Explicit snapshot config in the fault manager always wins when present.
Only active when plugins are loaded.
A program that hosts the gateway core without ROS sets the same switch
as ``FreezeFrameCaptureConfig::enabled`` (see the gateway README).
* - ``entity_freeze_frame.storage.path``
- string
- ``""``
- SQLite file the captured frames are written to, so a restart serves the
values frozen when the fault confirmed instead of reading the plant
again. When empty, the frames go in ``entity_freeze_frames.db`` in the
directory of ``triggers.storage.path``. With that empty too they stay in
memory and are lost on restart (logged at startup). A file that cannot be
opened is logged as an error and the frames stay in memory. A reloaded
frame keeps its ``captured_at`` and its ``capture_origin``, and the
startup catch-up captures only faults that have no frame. The bound of
256 faults counts reloaded and captured frames together and drops the
frame captured longest ago first, with its stored rows. Each row stores
the fault's ``occurrence_count`` and ``first_occurred`` at capture. At
startup one fault list of every status judges each reloaded frame, and a
reloaded frame is served only after that. When the lister throws or
answers with an empty list, the frames are kept and served unjudged. A
frame whose fault lists
another occurrence is discarded with a warning that names the fault code
and the entity; when the fault is confirmed and not muted, the startup
catch-up then reads it and marks it ``capture_origin: startup``. A frame
whose fault the fault manager no longer holds in any status is dropped
with a warning that names up to 10 of those faults, but not when the
fault manager lists no fault at all. The file belongs to one gateway: two
gateways must not share it, as one gateway's start drops the other's
frames as faults its fault manager does not hold.

When ``fault_manager.namespace`` is set, the gateway also subscribes to the matching
fault event topic (for example ``/robot1/fault_manager/events`` instead of the default
Expand Down
4 changes: 3 additions & 1 deletion docs/requirements/specs/faults.rst
Original file line number Diff line number Diff line change
Expand Up @@ -70,4 +70,6 @@ Faults
stored as environment data associated with the fault and accessible via the bulk-data
endpoints on the fault's reporting entity. For faults already confirmed when the
capturing component starts (e.g. the gateway restarts while a fault is standing), an
equivalent snapshot shall be captured at startup and marked with its capture origin.
equivalent snapshot shall be captured at startup and marked with its capture origin,
unless a snapshot captured for the same occurrence of the fault was kept across the
restart; that snapshot shall be served as it was captured.
78 changes: 65 additions & 13 deletions docs/tutorials/snapshots.rst
Original file line number Diff line number Diff line change
Expand Up @@ -223,20 +223,71 @@ values) when the fault confirms, and merges them into the fault detail's
after the entity - unless the fault manager already captured a freeze-frame
for that fault (explicit config wins). When the entity reports its link down
(the loss-of-comms case), the frozen values are the plugin's last known ones
and may predate the confirmation by the length of the outage; the entry's
``x-medkit`` block then carries ``connected: false`` and, when the plugin's
payload includes one, ``source_timestamp`` (the payload's own timestamp)
alongside ``captured_at``.
and may predate the confirmation by the length of the outage. The entry's
``x-medkit`` block carries ``connected: false`` only when the plugin's payload
has a link flag (the OPC UA plugin's ``DataProvider`` payload has none), and,
when the payload includes one, ``source_timestamp`` (the payload's own
timestamp) alongside ``captured_at``.

A frame stays after the fault is cleared, and a new confirmation of the
fault reads the entity again and replaces it. When that read gets no values,
a confirmation of a new occurrence drops the previous occurrence's frame and
logs a warning, so the fault has no frame; a confirmation of the same
occurrence keeps it. With
``entity_freeze_frame.storage.path`` set, a frame also survives a gateway
restart: the gateway writes it to that SQLite file when it captures it, loads
it at start, and serves it exactly as it was captured, with its original
``captured_at`` and its ``capture_origin``. Put the file on a volume that
outlives the container, or leave the path empty and the frames go in
``entity_freeze_frames.db`` next to the trigger store
(``triggers.storage.path``). With neither set the frames are process memory
only, and a restart loses them. The file belongs to one gateway: two gateways
must not share it, as one gateway's start drops the other's frames as faults
its fault manager does not hold.

Each stored frame carries the fault manager's occurrence of the fault at
capture: the fault's ``occurrence_count`` and ``first_occurred``. The fault
manager changes both only when a cleared fault fails again. At startup the
gateway asks the fault manager once for its faults of every status, muted ones
included, and judges every reloaded frame by that one answer. Until then the
fault detail serves no reloaded frame. When the fault manager cannot be asked
or answers with an empty list, the reloaded frames are kept and served
unjudged.

- The fault lists the stored occurrence: the frame is kept as it is, with no
read. Clocks play no part, so a gateway clock that differs from the fault
manager's changes nothing.
- The fault lists another occurrence (it cleared and failed again while the
gateway was down): the frame is discarded, and the gateway logs a warning
with the fault code, the entity, both occurrence counts and the status. A
confirmed fault then gets a startup frame from the catch-up below; any other
fault has no frame, since a read now would not show the values at that
occurrence's confirmation.
- The fault manager no longer holds the fault in any status (its storage was
replaced or wiped): the frame is dropped, and the gateway logs one warning
that names up to 10 of those faults. When the fault manager cannot be asked,
or lists no fault at all, nothing is dropped: a fault manager that lost its
storage connection answers with an empty list. Such a frame leaves by
eviction or by the next capture of its fault.
- A frame stored without an occurrence is kept, since nothing can be told
from it.

Faults that are already confirmed when the gateway starts are caught up at
startup: the gateway lists the confirmed faults and captures a frame for each
plugin-backed one, so a device standing in fault across a gateway restart
still gets a frame. Catch-up frames carry ``"capture_origin": "startup"`` in
their ``x-medkit`` block because their values were read at gateway start, not
when the fault confirmed (which may be long before, since the fault manager
persists faults); ``captured_at`` always stamps the moment the values were
read. Frames without the marker were captured on the confirm edge. Disable
with:
startup: from the same answer the gateway captures a frame for each confirmed,
plugin-backed fault that has no frame, so a device standing in fault across a
gateway restart still gets a frame. A fault whose frame of the same occurrence
was reloaded is not read: that frame is served as captured. A fault muted by
correlation is not framed at start, as live capture does not frame it either:
the fault manager publishes no confirm event for it. Catch-up frames carry
``"capture_origin": "startup"`` in their ``x-medkit`` block because their
values were read at gateway start, not when the fault confirmed (which may be
long before, since the fault manager persists faults); ``captured_at`` always
stamps the moment the values were read. Frames without the marker were
captured on the confirm edge. The gateway keeps frames for at most 256
faults, reloaded and captured together, and evicts the fault whose frame was
captured longest ago; a fault confirmed again counts from its new capture.
The catch-up does not evict a frame to make room, and its warning names up
to 10 of the faults it left without one. Disable with:

.. code-block:: bash

Expand Down Expand Up @@ -345,7 +396,8 @@ Snapshots are included inline in the fault response as ``environment_data``:
frames caught up for faults that predate the gateway are captured at
gateway start instead, marked ``x-medkit.capture_origin: startup``; a
plugin entity that reports its link down contributes its last known values,
marked ``connected: false`` in ``x-medkit``
marked ``connected: false`` in ``x-medkit`` when the plugin's payload carries
a link flag
- ``rosbag``: Recording file available via bulk-data endpoint (binary format).
One entry per recording the fault kept, newest first, each addressed by its own
``bulk_data_uri``. With the default ``max_bags_per_fault`` of ``1`` there is at
Expand Down
8 changes: 8 additions & 0 deletions src/ros2_medkit_gateway/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -737,6 +737,14 @@ if(BUILD_TESTING)
target_link_libraries(test_entity_freeze_frame_capture gateway_ros2)
medkit_target_dependencies(test_entity_freeze_frame_capture rclcpp ros2_medkit_msgs)

# Freeze-frame store and the core capture with a store (gateway_core only, no ROS node)
medkit_add_gtest(test_entity_freeze_frame_store test/test_entity_freeze_frame_store.cpp)
target_link_libraries(test_entity_freeze_frame_store gateway_core)
medkit_test_needs_no_domain(test_entity_freeze_frame_store)
medkit_add_gtest(test_freeze_frame_capture_persistence test/test_freeze_frame_capture_persistence.cpp)
target_link_libraries(test_freeze_frame_capture_persistence gateway_core)
medkit_test_needs_no_domain(test_freeze_frame_capture_persistence)

# Add update manager tests
medkit_add_gtest(test_update_manager test/test_update_manager.cpp)
target_link_libraries(test_update_manager gateway_ros2)
Expand Down
Loading
Loading