use std::collections::{BTreeMap, BTreeSet};
use serde::{Deserialize, Serialize};
use crate::lab::ldfi::{
CausalLineage, FaultEventId, HittingSetResult, LdfiExperimentReport, LdfiExperimentStatus,
SupportGraph,
};
use crate::trace::distributed::CausalOrder;
use crate::trace::{TraceData, TraceEvent, TraceEventKind};
use crate::types::{ObligationId, TaskId};
#[must_use]
pub const fn default_faultable(kind: TraceEventKind) -> bool {
matches!(
kind,
TraceEventKind::Wake
| TraceEventKind::CancelAck
| TraceEventKind::WorkerCancelAcknowledged
| TraceEventKind::WorkerDrainCompleted
| TraceEventKind::WorkerFinalizeCompleted
| TraceEventKind::TimerFired
| TraceEventKind::IoReady
| TraceEventKind::IoResult
| TraceEventKind::ObligationReserve
| TraceEventKind::ObligationCommit
| TraceEventKind::DownDelivered
| TraceEventKind::ExitDelivered
| TraceEventKind::ChaosInjection
)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TraceLineageConfig {
pub use_logical_time: bool,
pub correlate_resources: bool,
}
impl Default for TraceLineageConfig {
fn default() -> Self {
Self {
use_logical_time: true,
correlate_resources: true,
}
}
}
#[must_use]
pub fn build_causal_lineage(events: &[TraceEvent], config: TraceLineageConfig) -> CausalLineage {
let mut lineage = CausalLineage::new();
for ev in events {
lineage.add_event(FaultEventId::new(ev.seq), default_faultable(ev.kind));
}
let mut last_by_task: BTreeMap<TaskId, u64> = BTreeMap::new();
let mut last_by_token: BTreeMap<u64, u64> = BTreeMap::new();
let mut last_by_obligation: BTreeMap<ObligationId, u64> = BTreeMap::new();
let mut timer_origin: BTreeMap<u64, u64> = BTreeMap::new();
let mut monitor_origin: BTreeMap<u64, u64> = BTreeMap::new();
let mut link_origin: BTreeMap<u64, u64> = BTreeMap::new();
for ev in events {
let this = FaultEventId::new(ev.seq);
if let Some(task) = owning_task(&ev.data) {
if let Some(&prev) = last_by_task.get(&task) {
lineage.add_happens_before(FaultEventId::new(prev), this);
}
last_by_task.insert(task, ev.seq);
}
if config.correlate_resources {
if let Some(token) = io_token(&ev.data) {
if let Some(&prev) = last_by_token.get(&token) {
lineage.add_happens_before(FaultEventId::new(prev), this);
}
last_by_token.insert(token, ev.seq);
}
if let Some(ob) = obligation_of(&ev.data) {
if let Some(&prev) = last_by_obligation.get(&ob) {
lineage.add_happens_before(FaultEventId::new(prev), this);
}
last_by_obligation.insert(ob, ev.seq);
}
if let Some(tid) = timer_id(&ev.data) {
match ev.kind {
TraceEventKind::TimerScheduled => {
timer_origin.insert(tid, ev.seq);
}
TraceEventKind::TimerFired | TraceEventKind::TimerCancelled => {
if let Some(&origin) = timer_origin.get(&tid) {
lineage.add_happens_before(FaultEventId::new(origin), this);
}
}
_ => {}
}
}
if let Some(mref) = monitor_ref(&ev.data) {
match ev.kind {
TraceEventKind::MonitorCreated => {
monitor_origin.insert(mref, ev.seq);
}
TraceEventKind::DownDelivered | TraceEventKind::MonitorDropped => {
if let Some(&origin) = monitor_origin.get(&mref) {
lineage.add_happens_before(FaultEventId::new(origin), this);
}
}
_ => {}
}
}
if let Some(lref) = link_ref(&ev.data) {
match ev.kind {
TraceEventKind::LinkCreated => {
link_origin.insert(lref, ev.seq);
}
TraceEventKind::ExitDelivered | TraceEventKind::LinkDropped => {
if let Some(&origin) = link_origin.get(&lref) {
lineage.add_happens_before(FaultEventId::new(origin), this);
}
}
_ => {}
}
}
}
}
if config.use_logical_time {
for (j, later) in events.iter().enumerate() {
let Some(lt_later) = later.logical_time.as_ref() else {
continue;
};
for earlier in events.iter().take(j) {
let Some(lt_earlier) = earlier.logical_time.as_ref() else {
continue;
};
if lt_earlier.causal_order(lt_later) == CausalOrder::Before {
lineage.add_happens_before(
FaultEventId::new(earlier.seq),
FaultEventId::new(later.seq),
);
}
}
}
}
lineage
}
pub fn outcome_events<P>(events: &[TraceEvent], mut predicate: P) -> Vec<FaultEventId>
where
P: FnMut(&TraceEvent) -> bool,
{
events
.iter()
.filter(|ev| predicate(ev))
.map(|ev| FaultEventId::new(ev.seq))
.collect()
}
#[must_use]
pub fn support_graph_for<P>(
events: &[TraceEvent],
config: TraceLineageConfig,
predicate: P,
) -> SupportGraph
where
P: FnMut(&TraceEvent) -> bool,
{
let lineage = build_causal_lineage(events, config);
SupportGraph::from_causal_cones(&lineage, outcome_events(events, predicate))
}
pub const LDFI_REPORT_SCHEMA: &str = "ldfi-report-v1";
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LdfiReport {
pub schema: String,
pub max_depth: usize,
pub exhausted: bool,
pub unbreakable: bool,
pub hypotheses: Vec<Vec<u64>>,
pub coverage_certificate: Option<usize>,
pub blind_chaos_single_fault_experiments: usize,
pub experiment: Option<LdfiExperimentSummary>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LdfiExperimentSummary {
pub status: String,
pub experiments_run: usize,
pub violating_hypothesis: Option<Vec<u64>>,
pub refuted: Vec<Vec<u64>>,
pub remaining_hypotheses: Option<usize>,
pub max_depth: Option<usize>,
pub coverage_certificate: Option<usize>,
}
impl LdfiExperimentSummary {
#[must_use]
pub fn from_report(report: &LdfiExperimentReport) -> Self {
let (status, violating_hypothesis, remaining_hypotheses, max_depth) = match &report.status {
LdfiExperimentStatus::FoundViolation { hypothesis } => {
("found_violation", Some(event_ids(hypothesis)), None, None)
}
LdfiExperimentStatus::RefutedUpToDepth { max_depth } => {
("refuted_up_to_depth", None, None, Some(*max_depth))
}
LdfiExperimentStatus::ExperimentBudgetExhausted {
remaining_hypotheses,
} => (
"experiment_budget_exhausted",
None,
Some(*remaining_hypotheses),
None,
),
LdfiExperimentStatus::HypothesisSearchTruncated => {
("hypothesis_search_truncated", None, None, None)
}
};
Self {
status: status.to_string(),
experiments_run: report.experiments_run,
violating_hypothesis,
refuted: report.refuted.iter().map(event_ids).collect(),
remaining_hypotheses,
max_depth,
coverage_certificate: report.coverage_certificate(),
}
}
}
impl LdfiReport {
#[must_use]
pub fn with_experiment(mut self, report: &LdfiExperimentReport) -> Self {
self.experiment = Some(LdfiExperimentSummary::from_report(report));
self
}
}
#[must_use]
pub fn ldfi_report(
result: &HittingSetResult,
blind_chaos_single_fault_experiments: usize,
) -> LdfiReport {
LdfiReport {
schema: LDFI_REPORT_SCHEMA.to_string(),
max_depth: result.max_depth,
exhausted: result.exhausted,
unbreakable: result.unbreakable,
hypotheses: result.hypotheses.iter().map(event_ids).collect(),
coverage_certificate: result.coverage_certificate(),
blind_chaos_single_fault_experiments,
experiment: None,
}
}
#[must_use]
pub fn blind_chaos_single_fault_count(events: &[TraceEvent]) -> usize {
events
.iter()
.filter(|ev| default_faultable(ev.kind))
.count()
}
fn event_ids(hypothesis: &BTreeSet<FaultEventId>) -> Vec<u64> {
hypothesis.iter().map(|e| e.get()).collect()
}
fn owning_task(data: &TraceData) -> Option<TaskId> {
match data {
TraceData::Task { task, .. }
| TraceData::Cancel { task, .. }
| TraceData::Futurelock { task, .. }
| TraceData::Obligation { task, .. }
| TraceData::Worker { task, .. }
| TraceData::Budget { task, .. } => Some(*task),
_ => None,
}
}
fn obligation_of(data: &TraceData) -> Option<ObligationId> {
match data {
TraceData::Obligation { obligation, .. } | TraceData::Worker { obligation, .. } => {
Some(*obligation)
}
_ => None,
}
}
fn io_token(data: &TraceData) -> Option<u64> {
match data {
TraceData::IoRequested { token, .. }
| TraceData::IoReady { token, .. }
| TraceData::IoResult { token, .. }
| TraceData::IoError { token, .. } => Some(*token),
_ => None,
}
}
fn timer_id(data: &TraceData) -> Option<u64> {
match data {
TraceData::Timer { timer_id, .. } => Some(*timer_id),
_ => None,
}
}
fn monitor_ref(data: &TraceData) -> Option<u64> {
match data {
TraceData::Monitor { monitor_ref, .. } | TraceData::Down { monitor_ref, .. } => {
Some(*monitor_ref)
}
_ => None,
}
}
fn link_ref(data: &TraceData) -> Option<u64> {
match data {
TraceData::Link { link_ref, .. } | TraceData::Exit { link_ref, .. } => Some(*link_ref),
_ => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::lab::ldfi::HittingSetBudget;
use crate::record::ObligationKind;
use crate::remote::NodeId;
use crate::trace::distributed::{LogicalTime, VectorClock};
use crate::types::{RegionId, Time};
fn task(id: u32) -> TaskId {
TaskId::new_for_test(id, 0)
}
fn region() -> RegionId {
RegionId::new_for_test(0, 0)
}
fn ob(id: u32) -> ObligationId {
ObligationId::new_for_test(id, 0)
}
fn support(lineage: &CausalLineage, seq: u64) -> Vec<u64> {
lineage
.support_of(FaultEventId::new(seq))
.into_iter()
.map(FaultEventId::get)
.collect()
}
#[test]
fn classifier_marks_deliveries_faultable_and_structure_not() {
assert!(default_faultable(TraceEventKind::Wake));
assert!(default_faultable(TraceEventKind::TimerFired));
assert!(default_faultable(TraceEventKind::IoReady));
assert!(default_faultable(TraceEventKind::ObligationCommit));
assert!(default_faultable(TraceEventKind::DownDelivered));
assert!(!default_faultable(TraceEventKind::Spawn));
assert!(!default_faultable(TraceEventKind::Poll));
assert!(!default_faultable(TraceEventKind::Complete));
assert!(!default_faultable(TraceEventKind::IoRequested));
assert!(!default_faultable(TraceEventKind::UserTrace));
}
#[test]
fn program_order_chains_same_task() {
let events = vec![
TraceEvent::spawn(0, Time::ZERO, task(1), region()),
TraceEvent::wake(1, Time::ZERO, task(1), region()),
TraceEvent::complete(2, Time::ZERO, task(1), region()),
];
let lineage = build_causal_lineage(&events, TraceLineageConfig::default());
assert_eq!(support(&lineage, 2), vec![1]);
assert!(!lineage.is_faultable(FaultEventId::new(0)));
assert!(lineage.is_faultable(FaultEventId::new(1)));
}
#[test]
fn io_token_correlation_links_send_to_ack() {
let events = vec![
TraceEvent::io_result(0, Time::ZERO, 5, 4),
TraceEvent::io_ready(1, Time::ZERO, 5, 1),
];
let with_corr = build_causal_lineage(&events, TraceLineageConfig::default());
assert_eq!(support(&with_corr, 1), vec![0, 1]);
let no_corr = build_causal_lineage(
&events,
TraceLineageConfig {
use_logical_time: false,
correlate_resources: false,
},
);
assert_eq!(support(&no_corr, 1), vec![1]);
}
#[test]
fn obligation_lifecycle_chains_reserve_to_commit() {
let events = vec![
TraceEvent::obligation_reserve(
0,
Time::ZERO,
ob(1),
task(1),
region(),
ObligationKind::Lease,
),
TraceEvent::obligation_commit(
1,
Time::ZERO,
ob(1),
task(1),
region(),
ObligationKind::Lease,
10,
),
];
let lineage = build_causal_lineage(
&events,
TraceLineageConfig {
use_logical_time: false,
correlate_resources: true,
},
);
assert_eq!(support(&lineage, 1), vec![0, 1]);
}
#[test]
fn timer_scheduled_happens_before_fire() {
let events = vec![
TraceEvent::timer_scheduled(0, Time::ZERO, 9, Time::ZERO),
TraceEvent::timer_fired(1, Time::ZERO, 9),
];
let lineage = build_causal_lineage(
&events,
TraceLineageConfig {
use_logical_time: false,
correlate_resources: true,
},
);
assert_eq!(
lineage
.causal_cone(FaultEventId::new(1))
.into_iter()
.map(FaultEventId::get)
.collect::<Vec<_>>(),
vec![0, 1]
);
assert_eq!(support(&lineage, 1), vec![1]);
}
#[test]
fn vector_clock_adds_cross_task_edge() {
let net = NodeId::new("net");
let b = NodeId::new("b");
let mut send_vc = VectorClock::new();
send_vc.increment(&net);
let mut ack_vc = send_vc.clone();
ack_vc.increment(&b);
let events = vec![
TraceEvent::io_result(0, Time::ZERO, 10, 4)
.with_logical_time(LogicalTime::Vector(send_vc)),
TraceEvent::io_ready(1, Time::ZERO, 20, 1)
.with_logical_time(LogicalTime::Vector(ack_vc)),
];
let with_lt = build_causal_lineage(&events, TraceLineageConfig::default());
assert_eq!(support(&with_lt, 1), vec![0, 1]);
let without_lt = build_causal_lineage(
&events,
TraceLineageConfig {
use_logical_time: false,
correlate_resources: true,
},
);
assert_eq!(support(&without_lt, 1), vec![1]);
}
#[test]
fn deterministic() {
let events = vec![
TraceEvent::spawn(0, Time::ZERO, task(1), region()),
TraceEvent::wake(1, Time::ZERO, task(1), region()),
TraceEvent::io_result(2, Time::ZERO, 5, 4),
TraceEvent::io_ready(3, Time::ZERO, 5, 1),
];
let a = build_causal_lineage(&events, TraceLineageConfig::default());
let b = build_causal_lineage(&events, TraceLineageConfig::default());
assert_eq!(a, b);
}
#[test]
fn support_graph_for_pipeline_finds_shared_root() {
let net = NodeId::new("net");
let a = NodeId::new("a");
let b = NodeId::new("b");
let mut send_vc = VectorClock::new();
send_vc.increment(&net);
let mut ack_a_vc = send_vc.clone();
ack_a_vc.increment(&a);
let mut ack_b_vc = send_vc.clone();
ack_b_vc.increment(&b);
let mut ok_a_vc = ack_a_vc.clone();
ok_a_vc.increment(&a);
let mut ok_b_vc = ack_b_vc.clone();
ok_b_vc.increment(&b);
let events = vec![
TraceEvent::io_result(1, Time::ZERO, 10, 4)
.with_logical_time(LogicalTime::Vector(send_vc)),
TraceEvent::io_ready(2, Time::ZERO, 20, 1)
.with_logical_time(LogicalTime::Vector(ack_a_vc)),
TraceEvent::io_ready(3, Time::ZERO, 30, 1)
.with_logical_time(LogicalTime::Vector(ack_b_vc)),
TraceEvent::user_trace(10, Time::ZERO, "delivered-a")
.with_logical_time(LogicalTime::Vector(ok_a_vc)),
TraceEvent::user_trace(11, Time::ZERO, "delivered-b")
.with_logical_time(LogicalTime::Vector(ok_b_vc)),
];
let graph = support_graph_for(&events, TraceLineageConfig::default(), |ev| {
ev.kind == TraceEventKind::UserTrace
});
assert_eq!(graph.derivations().len(), 2);
let result = graph.minimal_hitting_sets(HittingSetBudget::default());
let smallest = result.hypotheses.first().expect("a hypothesis exists");
assert_eq!(smallest.len(), 1);
assert!(smallest.contains(&FaultEventId::new(1)));
}
}