Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions docs/tutorials/fault-correlation.rst
Original file line number Diff line number Diff line change
Expand Up @@ -245,14 +245,15 @@ Hierarchical mode identifies root cause → symptom relationships.
- Hide symptom faults from normal queries
* - ``auto_clear_with_root``
- ``true``
- Auto-clear symptoms when root cause is cleared
- Auto-clear symptoms when root cause is cleared or heals

**How it works:**

1. When ``ESTOP_001`` is reported, it's marked as a root cause
2. Any ``MOTOR_*`` or ``DRIVE_*`` faults within 2000ms are marked as symptoms
3. Symptoms are muted (not shown in default fault list)
4. Clearing ``ESTOP_001`` auto-clears all its symptoms
4. Clearing ``ESTOP_001`` auto-clears all its symptoms, and so does ``ESTOP_001`` healing
through PASSED reports

Auto-Cluster Mode
-----------------
Expand Down
5 changes: 5 additions & 0 deletions src/ros2_medkit_fault_manager/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,11 @@ endif()
target_link_libraries(test_fault_manager_core_concurrency fault_manager_core)
medkit_test_needs_no_domain(test_fault_manager_core_concurrency)

# Healing and correlation together. Plain C++ core, no ROS node.
medkit_add_gtest(test_fault_manager_core_correlation test/test_fault_manager_core_correlation.cpp)
target_link_libraries(test_fault_manager_core_correlation fault_manager_core)
medkit_test_needs_no_domain(test_fault_manager_core_correlation)

# CaptureThreadPool unit tests
medkit_add_gtest(test_capture_thread_pool test/test_capture_thread_pool.cpp)
target_link_libraries(test_capture_thread_pool fault_manager_lib)
Expand Down
2 changes: 1 addition & 1 deletion src/ros2_medkit_fault_manager/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ correlation:
- pattern: drive_faults
window_ms: 1000 # Symptoms within 1s of root cause
mute_symptoms: true # Don't publish symptom events
auto_clear_with_root: true # Clear symptoms when root cause clears
auto_clear_with_root: true # Clear symptoms when root cause clears or heals

# Auto-cluster rule: Group communication errors
- id: comm_storm
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,11 @@ class FaultManagerCore {

ReportOutcome report_locked(const fd::FaultReport & report);
ClearOutcome clear_locked(const std::string & code, bool skip_correlation_auto_clear);
/// Clears the symptoms a root cause took with it: storage, audit, cooldown and capture cleanup.
/// Appends each cleared code to @p cleared. Throws FaultStorage::IgnorableConnectionException
/// when the storage is unreachable; @p cleared then holds the codes done before that.
void clear_symptoms(const std::vector<std::string> & symptom_codes, const std::string & root_code,
const char * audit_source, std::vector<std::string> & cleared);
ListOutcome list_locked(const ListQuery & query);
/// @p rosbag_paths receives (snapshot index, bag path) for each rosbag entry, for sizing after unlock.
FaultDetail get_locked(const std::string & code, std::vector<std::pair<size_t, std::string>> & rosbag_paths);
Expand Down
73 changes: 40 additions & 33 deletions src/ros2_medkit_fault_manager/src/core/fault_manager_core.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,8 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) {

// Process through correlation engine (if enabled). Only FAILED events correlate.
bool should_mute = false;
if (correlation_ && failed) {
// A FAILED that leaves the fault HEALED (hysteresis) is not an active fault yet.
if (correlation_ && failed && fault_after->status != fd::fault_status::kHealed) {
auto correlation_result =
correlation_->process_fault(report.fault_code, correlation::severity_to_string(report.severity));

Expand Down Expand Up @@ -423,7 +424,19 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) {
// consumers act on it; the payload's status = HEALED distinguishes auto-heal from
// an acknowledged clear.
if (just_healed) {
publish_event(fd::fault_event_type::kCleared, *fault_after);
// A healed fault leaves the correlation state like a cleared one. For a root cause that
// takes its symptoms with it (auto_clear_with_root), or they stay muted under a cause
// that is no longer on the list.
std::vector<std::string> healed_symptoms;
if (correlation_) {
const auto symptoms = correlation_->process_clear(report.fault_code).auto_cleared_codes;
try {
clear_symptoms(symptoms, report.fault_code, "auto_heal", healed_symptoms);
} catch (const FaultStorage::IgnorableConnectionException & e) {
connection_lost(e);
}
}
publish_event(fd::fault_event_type::kCleared, *fault_after, healed_symptoms);
}

// Append tamper-evident audit records for the transitions that just happened.
Expand Down Expand Up @@ -467,6 +480,24 @@ ReportOutcome FaultManagerCore::report_locked(const fd::FaultReport & report) {
return outcome;
}

void FaultManagerCore::clear_symptoms(const std::vector<std::string> & symptom_codes, const std::string & root_code,
const char * audit_source, std::vector<std::string> & cleared) {
for (const auto & symptom_code : symptom_codes) {
last_capture_times_.erase(symptom_code);
storage_->clear_fault(symptom_code);
if (audit_) {
auto symptom = storage_->get_fault(symptom_code);
if (symptom) {
audit_transition(kTransitionCleared, *symptom, audit_source, clock_.now_ns());
}
}
logf(log_, LogLevel::kDebug, config_.logger_name, "Auto-cleared symptom: %s (root cause: %s)", symptom_code.c_str(),
root_code.c_str());
hooks_.on_cleared(symptom_code);
cleared.push_back(symptom_code);
}
}

ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_correlation_auto_clear) {
const std::string & logger = config_.logger_name;
ClearOutcome outcome;
Expand Down Expand Up @@ -508,38 +539,14 @@ ClearOutcome FaultManagerCore::clear_locked(const std::string & code, bool skip_
return outcome;
}

// Evict cooldown tracking for the cleared fault and the auto-cleared symptoms
last_capture_times_.erase(code);
for (const auto & symptom_code : auto_cleared_codes) {
last_capture_times_.erase(symptom_code);
}

// Auto-clear correlated symptoms
for (const auto & symptom_code : auto_cleared_codes) {
try {
storage_->clear_fault(symptom_code);
} catch (const FaultStorage::IgnorableConnectionException & e) {
connection_lost(e);
outcome.message = storage_unavailable(e);
return outcome;
}
if (audit_) {
std::optional<fd::FaultRecord> symptom;
try {
symptom = storage_->get_fault(symptom_code);
} catch (const FaultStorage::IgnorableConnectionException & e) {
connection_lost(e);
outcome.message = storage_unavailable(e);
return outcome;
}
if (symptom) {
audit_transition(kTransitionCleared, *symptom, "clear_service", clock_.now_ns());
}
}
logf(log_, LogLevel::kDebug, logger, "Auto-cleared symptom: %s (root cause: %s)", symptom_code.c_str(),
code.c_str());
// Also cleanup rosbag for auto-cleared faults
hooks_.on_cleared(symptom_code);
std::vector<std::string> cleared_symptoms;
try {
clear_symptoms(auto_cleared_codes, code, "clear_service", cleared_symptoms);
} catch (const FaultStorage::IgnorableConnectionException & e) {
connection_lost(e);
outcome.message = storage_unavailable(e);
return outcome;
}

outcome.auto_cleared_codes = auto_cleared_codes;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,12 @@ ProcessClearResult CorrelationEngine::process_clear(const std::string & fault_co
fault_to_cluster_.erase(cluster_it);
}

// A cleared symptom is no longer its root's: a later recurrence outside the window is independent.
for (auto & entry : root_to_symptoms_) {
auto & symptoms = entry.second;
symptoms.erase(std::remove(symptoms.begin(), symptoms.end(), fault_code), symptoms.end());
}

// Remove from muted faults if it was a symptom
muted_faults_.erase(fault_code);

Expand Down
15 changes: 15 additions & 0 deletions src/ros2_medkit_fault_manager/test/test_correlation_engine.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,21 @@ TEST_F(CorrelationEngineTest, ClearRootCauseClearsSymptoms) {
EXPECT_EQ(0u, engine.get_muted_count());
}

TEST_F(CorrelationEngineTest, ClearedSymptomIsNotClearedAgainWithItsRoot) {
auto config = create_hierarchical_config();
CorrelationEngine engine(config);

engine.process_fault("ESTOP_001", "CRITICAL");
engine.process_fault("MOTOR_COMM_FL", "ERROR");
engine.process_fault("MOTOR_COMM_FR", "ERROR");

engine.process_clear("MOTOR_COMM_FL");
auto clear_result = engine.process_clear("ESTOP_001");

ASSERT_EQ(1u, clear_result.auto_cleared_codes.size());
EXPECT_EQ("MOTOR_COMM_FR", clear_result.auto_cleared_codes[0]);
}

TEST_F(CorrelationEngineTest, UnrelatedFaultNotCorrelated) {
auto config = create_hierarchical_config();
CorrelationEngine engine(config);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
// Copyright 2026 mfaferek93
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <gtest/gtest.h>

#include <cstdio>
#include <fstream>
#include <string>
#include <vector>

#include "ros2_medkit_fault_manager/core/fault_manager_core.hpp"
#include "ros2_medkit_fault_manager/core/fault_manager_startup.hpp"

namespace fd = ros2_medkit::fault_detection;
namespace core = ros2_medkit_fault_manager::core;

namespace {

class NullLogSink : public core::LogSink {
public:
void log(core::LogLevel /*level*/, const std::string & /*logger*/, const std::string & /*message*/) override {
}
};

class RecordingEventSink : public core::EventSink {
public:
void publish(const fd::FaultEventRecord & event) override {
events.push_back(event);
}
std::vector<fd::FaultEventRecord> events;
};

class NoCaptureHooks : public core::CaptureHooks {
public:
core::CaptureResult capture_now(const std::string & /*code*/) override {
return {};
}
void on_prefailed(const std::string & /*code*/) override {
}
void on_cleared(const std::string & /*code*/) override {
}
};

class FixedClock : public core::WallClock {
public:
int64_t now_ns() const override {
return 1000000000LL;
}
};

/// One core on in-memory storage: the first FAILED confirms, the first PASSED heals, and one
/// hierarchical rule makes SYMPTOM_* a symptom of ROOT.
struct Fixture {
std::string config_path;
NullLogSink log;
RecordingEventSink events;
NoCaptureHooks hooks;
FixedClock clock;
std::unique_ptr<core::FaultManagerCore> core;

explicit Fixture(bool auto_clear_with_root, int confirmation_threshold = -1) {
config_path = testing::TempDir() + "heal_correlation_" + (auto_clear_with_root ? "on" : "off") + ".yaml";
std::ofstream file(config_path);
file << "correlation:\n"
" enabled: true\n"
" patterns:\n"
" symptoms:\n"
" codes: [\"SYMPTOM_*\"]\n"
" rules:\n"
" - id: root_rule\n"
" mode: hierarchical\n"
" window_ms: 60000\n"
" root_cause:\n"
" codes: [\"ROOT\"]\n"
" symptoms:\n"
" - pattern: symptoms\n"
" mute_symptoms: true\n"
" auto_clear_with_root: "
<< (auto_clear_with_root ? "true" : "false") << "\n";
file.close();

core::FaultManagerConfig config;
config.storage_type = "memory";
config.confirmation_threshold = confirmation_threshold;
config.healing_enabled = true;
config.healing_threshold = 0;
config.correlation_config_file = config_path;
core = std::make_unique<core::FaultManagerCore>(config, core::build_core_parts(config, log), events, hooks, clock,
log);
}

~Fixture() {
std::remove(config_path.c_str());
}

void report(const std::string & code, fd::ReportKind kind) {
fd::FaultReport r;
r.fault_code = code;
r.kind = kind;
r.severity = fd::severity::kError;
r.description = "test";
r.source_id = "/test";
ASSERT_TRUE(core->report(r).accepted);
}

core::ListOutcome list() {
core::ListQuery query;
query.statuses = {fd::fault_status::kConfirmed};
query.include_muted = true;
return core->list(query);
}
};

TEST(FaultManagerCoreCorrelation, HealedRootClearsItsSymptoms) {
Fixture f(true);
ASSERT_NE(f.core->correlation(), nullptr);
f.report("ROOT", fd::ReportKind::kFailed);
f.report("SYMPTOM_A", fd::ReportKind::kFailed);
f.report("SYMPTOM_B", fd::ReportKind::kFailed);
ASSERT_EQ(f.list().muted_count, 2u);

f.report("ROOT", fd::ReportKind::kPassed);

auto after = f.list();
EXPECT_TRUE(after.faults.empty());
EXPECT_EQ(after.muted_count, 0u);
EXPECT_EQ(f.core->get("ROOT").fault.status, fd::fault_status::kHealed);
EXPECT_EQ(f.core->get("SYMPTOM_A").fault.status, fd::fault_status::kCleared);
EXPECT_EQ(f.core->get("SYMPTOM_B").fault.status, fd::fault_status::kCleared);

// The root's end event names the symptoms that went with it, as a clear does.
ASSERT_FALSE(f.events.events.empty());
const auto & last = f.events.events.back();
EXPECT_EQ(last.fault.fault_code, "ROOT");
EXPECT_EQ(last.event_type, fd::fault_event_type::kCleared);
EXPECT_EQ(last.auto_cleared_codes.size(), 2u);
}

TEST(FaultManagerCoreCorrelation, HealedRootWithoutAutoClearUnmutesSymptoms) {
Fixture f(false);
f.report("ROOT", fd::ReportKind::kFailed);
f.report("SYMPTOM_A", fd::ReportKind::kFailed);
ASSERT_EQ(f.list().muted_count, 1u);

f.report("ROOT", fd::ReportKind::kPassed);

auto after = f.list();
EXPECT_EQ(after.muted_count, 0u);
ASSERT_EQ(after.faults.size(), 1u);
EXPECT_EQ(after.faults[0].fault_code, "SYMPTOM_A");
EXPECT_TRUE(f.events.events.back().auto_cleared_codes.empty());
}

TEST(FaultManagerCoreCorrelation, HealedSymptomLeavesTheMutedList) {
Fixture f(true);
f.report("ROOT", fd::ReportKind::kFailed);
f.report("SYMPTOM_A", fd::ReportKind::kFailed);
ASSERT_EQ(f.list().muted_count, 1u);

f.report("SYMPTOM_A", fd::ReportKind::kPassed);

auto after = f.list();
EXPECT_EQ(after.muted_count, 0u);
ASSERT_EQ(after.faults.size(), 1u);
EXPECT_EQ(after.faults[0].fault_code, "ROOT");
}

TEST(FaultManagerCoreCorrelation, HealedSymptomIsNotClearedWithItsFormerRoot) {
Fixture f(true);
f.report("ROOT", fd::ReportKind::kFailed);
f.report("SYMPTOM_A", fd::ReportKind::kFailed);
f.report("SYMPTOM_A", fd::ReportKind::kPassed);

f.report("ROOT", fd::ReportKind::kPassed);

EXPECT_EQ(f.core->get("SYMPTOM_A").fault.status, fd::fault_status::kHealed);
EXPECT_TRUE(f.events.events.back().auto_cleared_codes.empty());
}

TEST(FaultManagerCoreCorrelation, FailedReportThatKeepsAFaultHealedDoesNotMuteIt) {
// Threshold -2: one FAILED after healing leaves the status HEALED.
Fixture f(true, -2);
f.report("ROOT", fd::ReportKind::kFailed);
f.report("ROOT", fd::ReportKind::kFailed);
f.report("SYMPTOM_A", fd::ReportKind::kFailed);
f.report("SYMPTOM_A", fd::ReportKind::kFailed);
ASSERT_EQ(f.list().muted_count, 1u);
f.report("SYMPTOM_A", fd::ReportKind::kPassed);
f.report("SYMPTOM_A", fd::ReportKind::kPassed);
ASSERT_EQ(f.core->get("SYMPTOM_A").fault.status, fd::fault_status::kHealed);
ASSERT_EQ(f.list().muted_count, 0u);

f.report("SYMPTOM_A", fd::ReportKind::kFailed);

EXPECT_EQ(f.core->get("SYMPTOM_A").fault.status, fd::fault_status::kHealed);
EXPECT_EQ(f.list().muted_count, 0u);
}

} // namespace
Loading