Skip to main content

zenkey_fleet/judge/
condition.rs

1//! Conditions and the watchdog (#227) — transitions, not states.
2//!
3//! Three shipped features each hard-coded their own predicate over the
4//! observation surface: `expect` (one window), `doctor --for` (five
5//! checks), `cutover` (silence). This module is the one **closed vocabulary**
6//! they were each a spelling of: [`Condition`], evaluated to three states,
7//! never two (RFC 09 §5.1 O4/O6) — `ok` / `firing` / **`unobservable`**. The
8//! third state is the reason this exists: an alerting tool that cannot say
9//! *"I could not tell"* is the one that pages at 3am for a dropped buffer. A
10//! drop under a completeness claim yields `unobservable`, never `ok`.
11//!
12//! The vocabulary is deliberately closed — no expressions, no templating, no
13//! rules engine. A new condition is a new variant, argued for the way a new
14//! doctor check id is.
15//!
16//! The semantic core is three tiny rules — [`judge_shortfall`],
17//! [`judge_excess`], [`judge_silence`] — shared with [`crate::judge::expect`], so
18//! the watchdog and the CI assertion cannot drift about what a drop means.
19//! Since RFC 13 (v1.24; the material was RFC 09 §5.1 pre-v1.24) the rules
20//! speak the four-pole [`Judgement`] core, and [`CondState`] is this
21//! module's serde-stable **wire projection** of it — see its mapping doc.
22//!
23//! [`watchdog`] is the continuous observer over the vocabulary:
24//! **foreground, explicitly launched, single-purpose, one process per
25//! invocation, no shared state** — not the hidden, auto-started,
26//! discovery-caching daemon the redesign ledger rejected
27//! (`docs/redesign-2026-07.md` §6.1). It emits [`Transition`]s: one per
28//! genuine state change, none per unchanged tick.
29
30use std::collections::BTreeMap;
31use std::time::Duration;
32
33use crate::{Error, Result};
34
35use crate::bus::monitor::SampleView;
36use crate::model::decode::SchemaStore;
37use crate::model::registry::SliceSet;
38use crate::report::{CheckId, DoctorReport};
39use crate::report::{CondState, Judgement, Transition, WatchdogSummary};
40use sipper::{Straw, sipper};
41
42/// The closed condition vocabulary (#227), over the existing observation
43/// surface. Each variant names what *firing* means; the drop rules are in
44/// the judge functions this module documents.
45#[derive(Debug, Clone, PartialEq)]
46pub enum Condition {
47    /// Samples on `selector` rode above `hz` over the evaluation window.
48    /// Firing is positive evidence, conclusive even under drops (a drop only
49    /// hides more); `ok` under drops is unobservable — the true rate is
50    /// higher than what was counted (O6).
51    RateAbove { selector: String, hz: f64 },
52    /// Samples on `selector` rode below `hz`. A shortfall under drops is
53    /// unobservable — the dropped samples could have filled it (O6); enough
54    /// observed is conclusive `ok` regardless.
55    RateBelow { selector: String, hz: f64 },
56    /// No sample matched `selector` for at least `for_s` seconds. Silence is
57    /// a completeness claim — it counts what did NOT happen — so it is
58    /// provable only over a drop-free span at least `for_s` long (O6), and
59    /// only once the observer has watched that long (O4).
60    SilentFor { selector: String, for_s: f64 },
61    /// An observed payload on `selector` did not reach [`crate::Verdict::Valid`]
62    /// (#159) — `Invalid` and `NotValidated` both count: asking for validity
63    /// and getting "unknowable" is not valid. Scoped to what was observed
64    /// and checked; the `ok` state claims "nothing checked failed", never
65    /// "nothing invalid rode" — the drop count rides in the evidence.
66    InvalidPayload { selector: String },
67    /// An observed sample on `selector` did not ride its registry-declared
68    /// QoS profile (RFC 04 §3). Same per-observed-sample scope as
69    /// [`Condition::InvalidPayload`]; samples with no declared profile are
70    /// unjudgeable and counted in the evidence, not the state.
71    QosMismatch { selector: String },
72    /// A doctor run reported at least one finding with this check id
73    /// (the stable [`crate::report::CheckId`] vocabulary). A failed doctor run is
74    /// unobservable for every doctor condition — never `ok`.
75    DoctorCheck { check: CheckId },
76    /// The origin holds no `alive` token on the liveliness roster
77    /// (RFC 04 §5). A roster that could not be asked is unobservable —
78    /// silence is not a verdict (RFC 05 §3.1).
79    OriginDown { origin: String },
80    /// The observer itself dropped samples this window (RFC 09 §5.1 O6) —
81    /// self-knowledge, so never unobservable.
82    Dropped,
83}
84
85/// The rule grammar, spelled once for the parse error and the docs.
86const VOCABULARY: &str = "rate-above <SEL> <HZ> | rate-below <SEL> <HZ> | \
87     silent-for <SEL> <SECS> | invalid-payload <SEL> | qos-mismatch <SEL> | \
88     doctor <CHECK-ID> | origin-down <ORIGIN> | dropped";
89
90impl Condition {
91    /// Parse one rule: whitespace-separated, kind first (Zenoh key
92    /// expressions cannot contain whitespace, so the split is unambiguous).
93    /// The vocabulary is closed; anything else is an error that spells it.
94    pub fn parse(rule: &str) -> Result<Condition> {
95        let hz = |s: &str, kind: &str| -> Result<f64> {
96            let v: f64 = s
97                .parse()
98                .map_err(|_| Error::unaskable(format!("{kind} {s:?}"), "is not a number"))?;
99            if !v.is_finite() || v < 0.0 {
100                return Err(Error::unaskable(
101                    kind.to_string(),
102                    "the threshold must be a finite non-negative number",
103                ));
104            }
105            Ok(v)
106        };
107        let tokens: Vec<&str> = rule.split_whitespace().collect();
108        Ok(match tokens.as_slice() {
109            ["rate-above", sel, n] => Condition::RateAbove {
110                selector: sel.to_string(),
111                hz: hz(n, "rate-above")?,
112            },
113            ["rate-below", sel, n] => Condition::RateBelow {
114                selector: sel.to_string(),
115                hz: hz(n, "rate-below")?,
116            },
117            ["silent-for", sel, n] => {
118                let for_s = hz(n, "silent-for")?;
119                if for_s <= 0.0 {
120                    return Err(Error::unaskable(
121                        "silent-for",
122                        "the span must be a positive number of seconds",
123                    ));
124                }
125                Condition::SilentFor {
126                    selector: sel.to_string(),
127                    for_s,
128                }
129            }
130            ["invalid-payload", sel] => Condition::InvalidPayload {
131                selector: sel.to_string(),
132            },
133            ["qos-mismatch", sel] => Condition::QosMismatch {
134                selector: sel.to_string(),
135            },
136            ["doctor", check] => {
137                let Some(check) = CheckId::parse(check) else {
138                    return Err(Error::unaskable(
139                        format!("doctor {check:?}"),
140                        format!(
141                            "is not a check id — the stable vocabulary is: {}",
142                            CheckId::ALL
143                                .iter()
144                                .map(|c| c.as_str())
145                                .collect::<Vec<_>>()
146                                .join(", ")
147                        ),
148                    ));
149                };
150                Condition::DoctorCheck { check }
151            }
152            ["origin-down", origin] => Condition::OriginDown {
153                origin: origin.to_string(),
154            },
155            ["dropped"] => Condition::Dropped,
156            _ => {
157                return Err(Error::unaskable(
158                    format!("{rule:?}"),
159                    format!(
160                        "is not a rule — the vocabulary is closed (no \
161                         expressions, no templating): {VOCABULARY}"
162                    ),
163                ));
164            }
165        })
166    }
167
168    /// The wire selector this condition observes, when it observes one.
169    pub fn selector(&self) -> Option<&str> {
170        match self {
171            Condition::RateAbove { selector, .. }
172            | Condition::RateBelow { selector, .. }
173            | Condition::SilentFor { selector, .. }
174            | Condition::InvalidPayload { selector }
175            | Condition::QosMismatch { selector } => Some(selector),
176            _ => None,
177        }
178    }
179
180    /// Judge one observation window. `None` for the conditions that are not
181    /// window-scoped ([`Condition::DoctorCheck`], [`Condition::OriginDown`]).
182    /// Judge this condition against everything one tick observed.
183    ///
184    /// **The single entry point**, and why `run_watchdog` has no `expect`s
185    /// left (#352). The three judges below each returned `None` for the
186    /// variants they do not own, which forced the caller to assert a
187    /// partition the compiler could not see — four times, every one
188    /// discharging the same claim. This match *is* the partition, and each
189    /// arm hands its judge exactly the evidence that judge needs, so none of
190    /// them has a `None` to return.
191    pub fn judge(&self, ev: &TickEvidence<'_>) -> Eval {
192        match self {
193            Condition::DoctorCheck { check } => judge_doctor_check(*check, ev.doctor),
194            Condition::OriginDown { origin } => judge_origin_down(origin, ev.roster),
195            _ => self.judge_window_total(ev.window),
196        }
197    }
198
199    pub fn judge_window(&self, w: &CondWindow) -> Option<Eval> {
200        let synth = if w.synthetic > 0 {
201            format!("; {} synthetic-marked (RFC 09 §5.3)", w.synthetic)
202        } else {
203            String::new()
204        };
205        let rate = if w.window_s > 0.0 {
206            w.samples as f64 / w.window_s
207        } else {
208            0.0
209        };
210        Some(match self {
211            Condition::RateAbove { hz, .. } => {
212                let state = CondState::from(judge_excess(rate > *hz, w.dropped));
213                let evidence = match state {
214                    CondState::Unobservable => format!(
215                        "{rate:.2} Hz observed but {} sample(s) dropped — the true rate \
216                         is at least that, not exactly that (O6){synth}",
217                        w.dropped
218                    ),
219                    _ => format!(
220                        "{} sample(s) in {:.1}s = {rate:.2} Hz against the {hz:.2} Hz \
221                         bound{synth}",
222                        w.samples, w.window_s
223                    ),
224                };
225                Eval { state, evidence }
226            }
227            Condition::RateBelow { hz, .. } => {
228                let state = CondState::from(judge_shortfall(rate < *hz, w.dropped));
229                let evidence = match state {
230                    CondState::Unobservable => format!(
231                        "{rate:.2} Hz observed with {} sample(s) dropped — the drops \
232                         could have carried the difference (O6){synth}",
233                        w.dropped
234                    ),
235                    _ => format!(
236                        "{} sample(s) in {:.1}s = {rate:.2} Hz against the {hz:.2} Hz \
237                         bound{synth}",
238                        w.samples, w.window_s
239                    ),
240                };
241                Eval { state, evidence }
242            }
243            Condition::SilentFor { for_s, .. } => {
244                let ev = SilenceEvidence {
245                    sample_within: w.last_sample_ago_s.map(|ago| ago < *for_s) == Some(true),
246                    span_observed: w.observed_s >= *for_s,
247                    drop_free: w.last_drop_ago_s.map(|ago| ago >= *for_s) != Some(false),
248                };
249                let SilenceEvidence { span_observed, .. } = ev;
250                let state = CondState::from(judge_silence(ev));
251                let evidence = match state {
252                    CondState::Ok => format!(
253                        "a sample rode {:.1}s ago, inside the {for_s:.1}s span{synth}",
254                        w.last_sample_ago_s.unwrap_or(0.0)
255                    ),
256                    CondState::Firing => {
257                        format!("no sample for {for_s:.1}s, on a drop-free observer{synth}")
258                    }
259                    CondState::Unobservable if !span_observed => format!(
260                        "watched only {:.1}s of a {for_s:.1}s silence claim — not asked \
261                         is not answered (O4){synth}",
262                        w.observed_s
263                    ),
264                    CondState::Unobservable => format!(
265                        "no sample seen, but the observer dropped inside the {for_s:.1}s \
266                         span — silence is unprovable (O6){synth}"
267                    ),
268                };
269                Eval { state, evidence }
270            }
271            Condition::InvalidPayload { .. } => Eval {
272                state: if w.invalid > 0 {
273                    CondState::Firing
274                } else {
275                    CondState::Ok
276                },
277                evidence: format!(
278                    "{} of {} checked sample(s) did not reach Valid ({} observed, \
279                     {} dropped{synth})",
280                    w.invalid, w.checked, w.samples, w.dropped
281                ),
282            },
283            Condition::QosMismatch { .. } => Eval {
284                state: if w.qos_mismatched > 0 {
285                    CondState::Firing
286                } else {
287                    CondState::Ok
288                },
289                evidence: format!(
290                    "{} of {} judged sample(s) did not ride their declared profile \
291                     ({} observed, {} with no declared profile to judge, \
292                     {} dropped{synth})",
293                    w.qos_mismatched,
294                    w.qos_judged,
295                    w.samples,
296                    w.samples.saturating_sub(w.qos_judged),
297                    w.dropped
298                ),
299            },
300            Condition::Dropped => Eval {
301                state: if w.dropped > 0 {
302                    CondState::Firing
303                } else {
304                    CondState::Ok
305                },
306                evidence: format!(
307                    "the observer dropped {} sample(s) in {:.1}s (O6){synth}",
308                    w.dropped, w.window_s
309                ),
310            },
311            Condition::DoctorCheck { .. } | Condition::OriginDown { .. } => return None,
312        })
313    }
314
315    /// [`judge_window`](Self::judge_window) for the variants that *have* a
316    /// window — total, because [`judge`](Self::judge) has already routed the
317    /// other two elsewhere.
318    fn judge_window_total(&self, w: &CondWindow) -> Eval {
319        debug_assert!(
320            !matches!(
321                self,
322                Condition::DoctorCheck { .. } | Condition::OriginDown { .. }
323            ),
324            "judge() routes these two to their own evidence"
325        );
326        self.judge_window(w).unwrap_or_else(|| Eval {
327            // Unreachable through `judge`; if some future variant reaches it,
328            // "I have no window for this" is the honest answer, not a panic
329            // in a watchdog that is supposed to keep running.
330            state: CondState::Unobservable,
331            evidence: "this rule is not judged against a sample window".into(),
332        })
333    }
334
335    /// Judge a roster ask. `None` unless this is [`Condition::OriginDown`].
336    /// `Err` is the ask failing, which is unobservable — silence is not a
337    /// verdict (RFC 05 §3.1).
338    pub fn judge_roster(
339        &self,
340        roster: Result<&BTreeMap<String, Vec<String>>, &str>,
341    ) -> Option<Eval> {
342        let Condition::OriginDown { origin } = self else {
343            return None;
344        };
345        Some(match roster {
346            Err(e) => Eval {
347                state: CondState::Unobservable,
348                evidence: format!("the roster could not be asked: {e}"),
349            },
350            Ok(r) => match r.get(origin) {
351                Some(producers) => Eval {
352                    state: CondState::Ok,
353                    evidence: format!(
354                        "{origin} holds an alive token ({} producer(s))",
355                        producers.len()
356                    ),
357                },
358                None => Eval {
359                    state: CondState::Firing,
360                    evidence: format!("{origin} holds no alive token (RFC 04 §5)"),
361                },
362            },
363        })
364    }
365
366    /// Judge a doctor run. `None` unless this is [`Condition::DoctorCheck`].
367    /// A failed run is unobservable for every doctor condition — never `ok`.
368    pub fn judge_doctor(&self, outcome: Result<&DoctorReport, &str>) -> Option<Eval> {
369        let Condition::DoctorCheck { check } = self else {
370            return None;
371        };
372        Some(match outcome {
373            Err(e) => Eval {
374                state: CondState::Unobservable,
375                evidence: format!("the doctor run failed: {e}"),
376            },
377            Ok(report) => {
378                let mut hits = report.findings.iter().filter(|f| f.check == *check);
379                match hits.next() {
380                    Some(first) => Eval {
381                        state: CondState::Firing,
382                        evidence: format!(
383                            "{} finding(s); first: {} — {}",
384                            1 + hits.count(),
385                            first.subject,
386                            first.evidence
387                        ),
388                    },
389                    None => Eval {
390                        state: CondState::Ok,
391                        evidence: format!("no {check} findings"),
392                    },
393                }
394            }
395        })
396    }
397}
398
399impl std::fmt::Display for Condition {
400    /// The canonical rule spelling — [`Condition::parse`] round-trips it,
401    /// and it is the `rule` field of every [`Transition`].
402    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
403        match self {
404            Condition::RateAbove { selector, hz } => write!(f, "rate-above {selector} {hz}"),
405            Condition::RateBelow { selector, hz } => write!(f, "rate-below {selector} {hz}"),
406            Condition::SilentFor { selector, for_s } => {
407                write!(f, "silent-for {selector} {for_s}")
408            }
409            Condition::InvalidPayload { selector } => write!(f, "invalid-payload {selector}"),
410            Condition::QosMismatch { selector } => write!(f, "qos-mismatch {selector}"),
411            Condition::DoctorCheck { check } => write!(f, "doctor {check}"),
412            Condition::OriginDown { origin } => write!(f, "origin-down {origin}"),
413            Condition::Dropped => write!(f, "dropped"),
414        }
415    }
416}
417
418// ─── the judgement rules (the vocabulary's semantic core) ───────────────────
419//
420// The three judges return the four-pole [`Judgement`] core (RFC 13, v1.24;
421// RFC 09 §5.1 pre-v1.24). None of them ever answers `NotAsked` — a judge is
422// only called when the question was put — but the pole exists in the currency
423// so a caller that *skipped* a judge can say so in the same vocabulary. The
424// watchdog projects each judgement onto [`CondState`] for the wire.
425
426/// The shortfall rule ([`Condition::RateBelow`]; `expect`'s count floor and
427/// rate floor): too little was seen. Enough seen is conclusively clean even
428/// under drops — a drop can only hide *more*. A shortfall with drops is
429/// unobservable: the dropped samples could have filled it (RFC 09 §5.1 O6).
430pub fn judge_shortfall(short: bool, dropped: u64) -> Judgement {
431    match (short, dropped) {
432        (false, _) => Judgement::NotEstablished {
433            reason: "enough was seen — a drop only hides more".into(),
434        },
435        (true, 0) => Judgement::Established,
436        (true, _) => Judgement::Unobservable {
437            reason: format!("{dropped} dropped sample(s) could have filled the shortfall (O6)"),
438        },
439    }
440}
441
442/// The excess rule ([`Condition::RateAbove`]; `expect`'s rate ceiling): too
443/// much was seen. An excess is positive evidence, conclusive under drops.
444/// "No excess" is a completeness claim — it counts what did NOT happen — so
445/// under drops it is unobservable, never clean (O6).
446pub fn judge_excess(over: bool, dropped: u64) -> Judgement {
447    match (over, dropped) {
448        (true, _) => Judgement::Established,
449        (false, 0) => Judgement::NotEstablished {
450            reason: "no excess was counted, on a clean observation".into(),
451        },
452        (false, _) => Judgement::Unobservable {
453            reason: format!(
454                "{dropped} sample(s) dropped — \"did not exceed\" is a completeness \
455                 claim (O6)"
456            ),
457        },
458    }
459}
460
461/// The silence rule ([`Condition::SilentFor`]; `expect --absent`): a sample
462/// inside the span conclusively breaks the silence; silence is provable only
463/// over a span the observer actually watched (O4) drop-free (O6) — otherwise
464/// unobservable, never clean.
465/// What one silence claim rests on — three facts that are all `bool` and all
466/// about the same span.
467///
468/// A struct rather than three positional parameters, because this feeds a
469/// *judgement* and a transposition of two identically-typed booleans returns
470/// a plausible wrong verdict with no compile error (#349).
471/// `judge_shortfall`/`judge_excess` keep their positional `(bool, u64)` —
472/// not transposable, so not a hazard.
473#[derive(Debug, Clone, Copy)]
474pub struct SilenceEvidence {
475    /// A sample rode inside the claimed span — the conclusive break.
476    pub sample_within: bool,
477    /// The observer actually watched the whole span (O4). A span it did not
478    /// watch is not a span it can call silent.
479    pub span_observed: bool,
480    /// The observer dropped nothing inside the span (O6). "Nothing arrived"
481    /// under drops is a completeness claim the observation cannot carry.
482    pub drop_free: bool,
483}
484
485pub fn judge_silence(ev: SilenceEvidence) -> Judgement {
486    let SilenceEvidence {
487        sample_within,
488        span_observed,
489        drop_free,
490    } = ev;
491    if sample_within {
492        Judgement::NotEstablished {
493            reason: "a sample rode inside the span".into(),
494        }
495    } else if span_observed && drop_free {
496        Judgement::Established
497    } else if !span_observed {
498        Judgement::Unobservable {
499            reason: "the observer has not watched the whole claimed span (O4)".into(),
500        }
501    } else {
502        Judgement::Unobservable {
503            reason: "the observer dropped inside the span — silence is unprovable (O6)".into(),
504        }
505    }
506}
507
508/// Everything one watchdog tick observed, in the three shapes the conditions
509/// are judged against.
510///
511/// `doctor` and `roster` are `Option` because a tick only runs those asks if
512/// some rule wants them — and "not run this tick" is *unobservable*, which is
513/// the honest reading and the one the caller used to assert away with
514/// `.expect("a doctor rule ran the doctor")` (#352).
515pub struct TickEvidence<'e> {
516    pub window: &'e CondWindow,
517    pub doctor: Option<Result<&'e DoctorReport, &'e str>>,
518    pub roster: Option<Result<&'e BTreeMap<String, Vec<String>>, &'e str>>,
519}
520
521/// Judge one doctor check against this tick's run — total, and total in the
522/// "did not run" direction too.
523pub fn judge_doctor_check(check: CheckId, outcome: Option<Result<&DoctorReport, &str>>) -> Eval {
524    let Some(outcome) = outcome else {
525        return Eval {
526            state: CondState::Unobservable,
527            evidence: "the doctor did not run this tick".into(),
528        };
529    };
530    Condition::DoctorCheck { check }
531        .judge_doctor(outcome)
532        .expect("a DoctorCheck is judged by the doctor")
533}
534
535/// Judge one origin against this tick's roster ask — likewise total.
536pub fn judge_origin_down(
537    origin: &str,
538    roster: Option<Result<&BTreeMap<String, Vec<String>>, &str>>,
539) -> Eval {
540    let Some(roster) = roster else {
541        return Eval {
542            state: CondState::Unobservable,
543            evidence: "the roster was not asked this tick".into(),
544        };
545    };
546    Condition::OriginDown {
547        origin: origin.to_string(),
548    }
549    .judge_roster(roster)
550    .expect("an OriginDown is judged by the roster")
551}
552
553// ─── observations and evaluations ───────────────────────────────────────────
554
555/// What one evaluation window observed on one condition's selector — the
556/// facts, separated from the judgement so the judgement is pure.
557///
558/// `CondWindow` and not `Window`: this type is re-exported at the crate root
559/// beside `BudgetWindow` and `RecordBounds`, and a bare `Window` there reads
560/// as *the* window of an engine that has several. Nothing serializes the
561/// name (the type carries no `Serialize`), so the rename is Rust-side only.
562#[derive(Debug, Clone, Copy, Default)]
563pub struct CondWindow {
564    /// The span this window judges, seconds.
565    pub window_s: f64,
566    /// How long the observer has been watching in total — a claim about a
567    /// span longer than this is unobservable (O4).
568    pub observed_s: f64,
569    /// Samples matching the selector within the window.
570    pub samples: u64,
571    /// Stream drops within the window — unattributable to any one selector,
572    /// so they taint every completeness claim (O6).
573    pub dropped: u64,
574    /// Seconds since the last matching sample; `None` = none seen since the
575    /// watch began.
576    pub last_sample_ago_s: Option<f64>,
577    /// Seconds since the last stream drop; `None` = the stream never dropped.
578    pub last_drop_ago_s: Option<f64>,
579    /// Samples whose payload did not reach `Valid`, among those checked.
580    pub invalid: u64,
581    /// Samples actually decode-checked (a budget bounds the cost).
582    pub checked: u64,
583    /// Samples that did not ride their declared QoS, among those judged.
584    pub qos_mismatched: u64,
585    /// Samples with a declared profile to judge against.
586    pub qos_judged: u64,
587    /// Samples carrying the RFC 09 §5.3 synthetic-traffic marker — generated
588    /// traffic judged as real would be a self-inflicted page, so every
589    /// evidence line carries the count.
590    pub synthetic: u64,
591}
592
593/// One evaluation: the three-valued state, and the evidence for it.
594#[derive(Debug, Clone, PartialEq)]
595pub struct Eval {
596    pub state: CondState,
597    pub evidence: String,
598}
599
600/// One rule's transition detector: feed evaluations in, get a [`Transition`]
601/// back **only** when the state genuinely changed. An unchanged tick returns
602/// `None` — transitions, not states.
603#[derive(Debug, Clone)]
604pub struct RuleState {
605    /// The condition itself, not its `Display`.
606    ///
607    /// It used to hold the rendered string and clone it into every
608    /// transition, with the two representations kept equal only by a
609    /// round-trip test — a second representation of a value that was
610    /// `Clone` and in scope (#352). The rendering happens where the
611    /// `Transition` is built, once, from the one source.
612    rule: Condition,
613    state: Option<CondState>,
614}
615
616impl RuleState {
617    pub fn new(rule: Condition) -> RuleState {
618        RuleState { rule, state: None }
619    }
620
621    /// The condition this state tracks.
622    pub fn rule(&self) -> &Condition {
623        &self.rule
624    }
625
626    /// The last observed state; `None` until the first evaluation.
627    pub fn state(&self) -> Option<CondState> {
628        self.state
629    }
630
631    /// Feed one evaluation. The first ever emits (from `null` — the baseline
632    /// is said once); after that only a genuine change does.
633    pub fn observe(&mut self, eval: Eval, at: impl Into<String>) -> Option<Transition> {
634        if self.state == Some(eval.state) {
635            return None;
636        }
637        let from = self.state;
638        self.state = Some(eval.state);
639        Some(Transition {
640            rule: self.rule.to_string(),
641            from,
642            to: eval.state,
643            at: at.into(),
644            evidence: eval.evidence,
645        })
646    }
647}
648
649/// Run-over-run delta over a doctor report: one [`RuleState`] per stable
650/// check id ([`CheckId`]), fed by `doctor --transitions`. The
651/// first run states the baseline (one transition per check id); every later run yields
652/// only genuine changes. A failed run flips every check to `unobservable` —
653/// a doctor that could not run has not said the fleet is healthy.
654#[derive(Debug, Clone)]
655pub struct DoctorWatch {
656    /// One state per check. A `Vec<(Condition, RuleState)>` until #352 — the
657    /// condition was in both halves of the pair.
658    checks: Vec<RuleState>,
659}
660
661impl DoctorWatch {
662    pub fn new() -> DoctorWatch {
663        DoctorWatch {
664            checks: CheckId::ALL
665                .iter()
666                .map(|id| RuleState::new(Condition::DoctorCheck { check: *id }))
667                .collect(),
668        }
669    }
670
671    /// Feed one doctor run (or its failure) and collect the transitions.
672    pub fn observe(&mut self, outcome: Result<&DoctorReport, &str>, at: &str) -> Vec<Transition> {
673        self.checks
674            .iter_mut()
675            .filter_map(|state| {
676                let Condition::DoctorCheck { check } = *state.rule() else {
677                    // Unconstructible: `new` builds only `DoctorCheck`s.
678                    return None;
679                };
680                let eval = judge_doctor_check(check, Some(outcome));
681                state.observe(eval, at)
682            })
683            .collect()
684    }
685}
686
687impl Default for DoctorWatch {
688    fn default() -> Self {
689        DoctorWatch::new()
690    }
691}
692
693// ─── the watchdog runner ────────────────────────────────────────────────────
694
695/// What a watchdog run watches, and for how long.
696#[derive(Debug, Clone)]
697pub struct WatchdogSpec {
698    /// The rules, evaluated every tick.
699    pub rules: Vec<Condition>,
700    /// Evaluation cadence. A tick that runs long (a doctor rule's fan-in)
701    /// slides rather than backlogs; windows are measured, not nominal.
702    pub tick: Duration,
703    /// Stop after this many ticks; `None` = run until the caller stops it.
704    pub ticks: Option<u64>,
705    /// Per-ask timeout for the roster and doctor conditions.
706    pub timeout: Duration,
707}
708
709/// How many decode attempts each key gets per tick under an
710/// `invalid-payload` rule — the same budget the doctor listen phase runs,
711/// for the same reason: a watchdog must not become a load test.
712const DECODE_BUDGET: u8 = 2;
713
714/// What one tick counted on one rule's selector.
715#[derive(Default, Clone, Copy)]
716struct TickCounters {
717    samples: u64,
718    invalid: u64,
719    checked: u64,
720    qos_mismatched: u64,
721    qos_judged: u64,
722    synthetic: u64,
723}
724
725/// One rule's whole per-run state, together.
726///
727/// This was four `Vec`s held in lockstep by index — `states`,
728/// `keyexprs`, `counters`, `last_sample` — across a hundred and thirty
729/// lines, with nothing structurally preventing them from disagreeing in
730/// length, and a `counters.fill(default())` reset that could silently
731/// miss one of them (#352).
732struct RuleRuntime {
733    rule: Condition,
734    /// The rule's selector, compiled once for sample attribution.
735    keyexpr: Option<zenoh::key_expr::KeyExpr<'static>>,
736    counters: TickCounters,
737    last_sample: Option<tokio::time::Instant>,
738    state: RuleState,
739}
740
741/// The sweep a tick ran beside its drain, as the rules see it: the doctor
742/// run and the roster ask, each `None` when no rule wanted it — which is
743/// *unobservable* for the rules that would have needed it, the honest
744/// reading (#352).
745#[derive(Debug, Clone, Copy, Default)]
746pub struct SweepOutcome<'e> {
747    pub doctor: Option<Result<&'e DoctorReport, &'e str>>,
748    pub roster: Option<Result<&'e BTreeMap<String, Vec<String>>, &'e str>>,
749}
750
751/// A set of rules judged tick by tick over **one** event stream — the
752/// watchdog's per-tick body, lifted out of [`watchdog`] so a second driver
753/// can run it (#218).
754///
755/// The driver owns the stream, the drain loop and the sweep; this owns
756/// everything the rules know: per-rule counters, the last sample and drop
757/// instants, the per-tick decode budget, and the transition detectors. Feed
758/// it every sample ([`observe_sample`](Self::observe_sample)) and every drop
759/// ([`observe_drop`](Self::observe_drop)) the stream yields, then
760/// [`evaluate`](Self::evaluate) once per tick and get back only what changed.
761///
762/// **Why the seam exists.** A trigger capture (`zenctl record --on`) must
763/// judge *the same event stream it records*: one subscription, one drop
764/// ledger. Had the capture run a watchdog of its own beside its recorder,
765/// the drops the judge saw and the drops in the file would have been two
766/// different facts about two different observers — and a `{"dropped": n}`
767/// in the file would say nothing about whether the rule that fired was
768/// judged over a clean window. With the body a value, the recorder drains
769/// one stream and hands every item to both the ring and the rules.
770///
771/// Sample attribution is by key-expression intersection against each rule's
772/// selector; a sample whose key does not parse as one counts for no rule.
773/// Time is `tokio::time::Instant`, so a driver under paused time judges
774/// exact windows.
775pub struct RuleSet<'a> {
776    rules: Vec<RuleRuntime>,
777    /// The distinct selectors the rules observe, in first-seen order.
778    watched: Vec<String>,
779    base: &'a str,
780    slices: Option<&'a SliceSet>,
781    started: tokio::time::Instant,
782    last_eval: tokio::time::Instant,
783    last_drop: Option<tokio::time::Instant>,
784    dropped_tick: u64,
785    decode_budget: BTreeMap<String, u8>,
786    ticks: u64,
787    transitions: u64,
788}
789
790impl<'a> RuleSet<'a> {
791    /// Compile the rules. Fails on a selector that is not a key expression —
792    /// before anything is declared, so the `?` has nothing to tear down
793    /// (#336). The watch clock starts here: [`CondWindow::observed_s`] is
794    /// measured from construction, so build the set right before the
795    /// subscriptions are declared.
796    pub fn new(rules: &[Condition], base: &'a str, slices: Option<&'a SliceSet>) -> Result<Self> {
797        let compiled = rules
798            .iter()
799            .map(|rule| {
800                Ok(RuleRuntime {
801                    rule: rule.clone(),
802                    keyexpr: rule
803                        .selector()
804                        .map(|sel| {
805                            zenoh::key_expr::KeyExpr::try_from(sel.to_string())
806                                .map_err(|e| Error::unaskable_from(format!("{sel:?}"), e))
807                        })
808                        .transpose()?,
809                    counters: TickCounters::default(),
810                    last_sample: None,
811                    state: RuleState::new(rule.clone()),
812                })
813            })
814            .collect::<Result<Vec<_>>>()?;
815        let mut watched: Vec<String> = Vec::new();
816        for rule in rules {
817            if let Some(sel) = rule.selector()
818                && !watched.iter().any(|s| s == sel)
819            {
820                watched.push(sel.to_string());
821            }
822        }
823        let now = tokio::time::Instant::now();
824        Ok(RuleSet {
825            rules: compiled,
826            watched,
827            base,
828            slices,
829            started: now,
830            last_eval: now,
831            last_drop: None,
832            dropped_tick: 0,
833            decode_budget: BTreeMap::new(),
834            ticks: 0,
835            transitions: 0,
836        })
837    }
838
839    /// The distinct selectors the rules observe — what the driver must
840    /// subscribe to before the first window opens (O4).
841    pub fn watched(&self) -> &[String] {
842        &self.watched
843    }
844
845    /// Some rule judges a doctor run, so the driver owes one per tick.
846    pub fn wants_doctor(&self) -> bool {
847        self.rules
848            .iter()
849            .any(|r| matches!(r.rule, Condition::DoctorCheck { .. }))
850    }
851
852    /// Some rule judges the liveliness roster, so the driver owes one ask
853    /// per tick.
854    pub fn wants_roster(&self) -> bool {
855        self.rules
856            .iter()
857            .any(|r| matches!(r.rule, Condition::OriginDown { .. }))
858    }
859
860    /// Some rule judges payload validity, so the driver owes a warmed,
861    /// sealed schema store (#337) and a decode per
862    /// [`wants_verdict`](Self::wants_verdict).
863    pub fn wants_decode(&self) -> bool {
864        self.rules
865            .iter()
866            .any(|r| matches!(r.rule, Condition::InvalidPayload { .. }))
867    }
868
869    /// Whether this sample should be decoded before it is observed: an
870    /// `invalid-payload` rule matches its key and the key's per-tick decode
871    /// budget has room. Spends the budget — ask once per sample, then hand
872    /// the verdict to [`observe_sample`](Self::observe_sample). The decode
873    /// stays the driver's, because it is async and this is not.
874    pub fn wants_verdict(&mut self, s: &SampleView) -> bool {
875        let Ok(key) = zenoh::key_expr::KeyExpr::try_from(s.key.as_str()) else {
876            return false;
877        };
878        let matched = self.rules.iter().any(|rt| {
879            matches!(rt.rule, Condition::InvalidPayload { .. })
880                && rt.keyexpr.as_ref().is_some_and(|sel| sel.intersects(&key))
881        });
882        if !matched {
883            return false;
884        }
885        let budget = self.decode_budget.entry(s.key.clone()).or_default();
886        if *budget < DECODE_BUDGET {
887            *budget += 1;
888            true
889        } else {
890            false
891        }
892    }
893
894    /// Count one observed sample against every rule its key matches.
895    /// `verdict` is the decode the driver ran when
896    /// [`wants_verdict`](Self::wants_verdict) said so; `None` means the
897    /// sample was not checked, which is counted as exactly that.
898    pub fn observe_sample(
899        &mut self,
900        s: &SampleView,
901        facts_cache: &mut crate::model::facts::FactsCache,
902        verdict: Option<&crate::Verdict>,
903    ) {
904        let Ok(key) = zenoh::key_expr::KeyExpr::try_from(s.key.as_str()) else {
905            return;
906        };
907        let synthetic = s
908            .attachment
909            .as_ref()
910            .is_some_and(|a| crate::judge::common::is_synthetic_marker(&a.to_bytes()));
911        let now = tokio::time::Instant::now();
912        for rt in self.rules.iter_mut() {
913            let Some(sel) = &rt.keyexpr else { continue };
914            if !sel.intersects(&key) {
915                continue;
916            }
917            rt.counters.samples += 1;
918            if synthetic {
919                rt.counters.synthetic += 1;
920            }
921            rt.last_sample = Some(now);
922            match &rt.rule {
923                Condition::InvalidPayload { .. } => {
924                    // An `invalid-payload` rule counts every not-`Valid`
925                    // verdict the same way, so with no registry loaded
926                    // `NoRegistry` (#246) changes no transition — only the
927                    // reason the sample was not validated.
928                    if let Some(v) = verdict {
929                        rt.counters.checked += 1;
930                        if !matches!(v, crate::Verdict::Valid) {
931                            rt.counters.invalid += 1;
932                        }
933                    }
934                }
935                Condition::QosMismatch { .. } => {
936                    facts_cache.ensure(self.base, &s.key, self.slices);
937                    let facts = facts_cache.get(&s.key).expect("just ensured this key");
938                    if let crate::model::facts::Registration::Registered(sf) = &facts.registration
939                        && let Some(profile) = sf.declared_qos()
940                    {
941                        rt.counters.qos_judged += 1;
942                        if !s.qos_matches(profile) {
943                            rt.counters.qos_mismatched += 1;
944                        }
945                    }
946                }
947                _ => {}
948            }
949        }
950    }
951
952    /// The stream dropped `n` samples here (RFC 09 §5.1 O6): unattributable
953    /// to any one selector, so it taints every completeness claim this tick.
954    pub fn observe_drop(&mut self, n: u64) {
955        self.dropped_tick += n;
956        self.last_drop = Some(tokio::time::Instant::now());
957    }
958
959    /// Close the tick: judge every rule over the window measured since the
960    /// last evaluation, reset the per-tick counts, and hand back only the
961    /// genuine changes — none for an unchanged rule. `at` is the wall-clock
962    /// stamp the transitions carry.
963    pub fn evaluate(
964        &mut self,
965        now: tokio::time::Instant,
966        at: &str,
967        sweep: SweepOutcome<'_>,
968    ) -> Vec<Transition> {
969        let mut out = Vec::new();
970        for rt in self.rules.iter_mut() {
971            let window = CondWindow {
972                window_s: (now - self.last_eval).as_secs_f64(),
973                observed_s: (now - self.started).as_secs_f64(),
974                samples: rt.counters.samples,
975                dropped: self.dropped_tick,
976                last_sample_ago_s: rt.last_sample.map(|t| (now - t).as_secs_f64()),
977                last_drop_ago_s: self.last_drop.map(|t| (now - t).as_secs_f64()),
978                invalid: rt.counters.invalid,
979                checked: rt.counters.checked,
980                qos_mismatched: rt.counters.qos_mismatched,
981                qos_judged: rt.counters.qos_judged,
982                synthetic: rt.counters.synthetic,
983            };
984            let eval = rt.rule.judge(&TickEvidence {
985                window: &window,
986                doctor: sweep.doctor,
987                roster: sweep.roster,
988            });
989            if let Some(transition) = rt.state.observe(eval, at) {
990                out.push(transition);
991            }
992        }
993        // One reset, over one collection — the four-`Vec` version had a
994        // `counters.fill(..)` that could miss a sibling (#352).
995        for rt in self.rules.iter_mut() {
996            rt.counters = TickCounters::default();
997        }
998        self.dropped_tick = 0;
999        self.decode_budget.clear();
1000        self.ticks += 1;
1001        self.transitions += out.len() as u64;
1002        self.last_eval = now;
1003        out
1004    }
1005
1006    /// Ticks evaluated so far.
1007    pub fn ticks(&self) -> u64 {
1008        self.ticks
1009    }
1010
1011    /// When the last tick closed (construction, before the first): the
1012    /// driver's next deadline is measured from here, so a slow consumer of
1013    /// the transitions widens the next window rather than skipping one.
1014    pub fn last_eval(&self) -> tokio::time::Instant {
1015        self.last_eval
1016    }
1017
1018    /// Transitions emitted so far.
1019    pub fn transitions(&self) -> u64 {
1020        self.transitions
1021    }
1022}
1023
1024/// Watch the rules and yield one [`Transition`] per genuine change, none per
1025/// unchanged tick. The subscriber set is declared before the first window
1026/// opens (O4); every selector rule is judged per tick over the measured
1027/// window, doctor and roster rules by one ask per tick each.
1028///
1029/// A driver over [`RuleSet`] (#218): this function owns the monitor, the
1030/// drain loop and the per-tick sweep; the rules' state is the set's.
1031///
1032/// A [`Straw`] rather than a [`Stream`](futures_core::Stream) (#397), because
1033/// a watchdog run is a sequence **and** a final value: transitions while it
1034/// runs, a [`WatchdogSummary`] when it stops, and the acknowledged monitor
1035/// teardown (#207/#336) in between. A bare `Stream` has room for the first
1036/// only — which is why this stayed a callback through #343, and why the
1037/// callback could not fail: `emit` was infallible by construction, so a
1038/// caller whose emission *could* fail had to stash the error and answer for
1039/// it after the run. Dropping it instead let `zenctl watchdog` finish clean
1040/// having emitted nothing (#360).
1041///
1042/// Drive it with `sip` for the transitions and `await` for the summary:
1043///
1044/// ```ignore
1045/// let mut run = watchdog(&fleet, slices, &store, &spec).pin();
1046/// while let Some(transition) = run.sip().await {
1047///     writeln!(out, "{}", serde_json::to_string(&transition)?)?;
1048/// }
1049/// let summary = run.await?;
1050/// ```
1051///
1052/// The summary is the *output*, not an item, so a consumer that stops sipping
1053/// early and awaits still gets the teardown — there is no `finish` to forget.
1054pub fn watchdog<'a>(
1055    fleet: &'a crate::Fleet<'a>,
1056    slices: Option<&'a SliceSet>,
1057    store: &'a SchemaStore,
1058    spec: &'a WatchdogSpec,
1059) -> impl Straw<WatchdogSummary, Transition, Error> + 'a {
1060    sipper(async move |mut sender: sipper::Sender<Transition>| {
1061        use crate::{FleetEvent, StreamItem};
1062
1063        let (session, base) = (fleet.session(), fleet.base());
1064
1065        // Compiled *before* the monitor exists, so the `?` has nothing to tear
1066        // down (#336).
1067        let mut rules = RuleSet::new(&spec.rules, base, slices)?;
1068        let (wants_doctor, wants_roster, wants_decode) = (
1069            rules.wants_doctor(),
1070            rules.wants_roster(),
1071            rules.wants_decode(),
1072        );
1073
1074        // Warmed before the first tick and sealed for the run (#337): a decode
1075        // inside the drain loop must never become a `describe` GET, because
1076        // nothing attends the broadcast while one is in flight and the tick's
1077        // verdict is about the window that lost the samples. zenctl hands this
1078        // store over cold. Each tick's sweep re-warms whatever is still
1079        // unserved — from beside the drain, where waiting costs nothing.
1080        if wants_decode {
1081            crate::model::decode::prewarm(fleet, store, slices).await;
1082        }
1083        let _sealed = store.seal();
1084
1085        // Declared before the window opens — not-asked must never read as "no".
1086        let monitor = crate::Monitor::start(session, crate::MonitorSpec::default()).await?;
1087        let mut events = monitor.events();
1088        let monitor = monitor.watching(rules.watched()).await?;
1089
1090        // Bounded (#107): the watchdog runs until stopped, so an unbounded
1091        // per-key map here is a leak on any bus with churning keys. Evictions
1092        // ride the summary (O6).
1093        let mut facts_cache = crate::model::facts::FactsCache::default();
1094
1095        let mut closed = false;
1096        loop {
1097            let deadline = rules.last_eval() + spec.tick;
1098            // The tick's bus work runs **beside** the drain, not after it (#338).
1099            //
1100            // A roster GET, a registry sweep, per-producer describes and state
1101            // snapshots take seconds, and every one of them used to happen with
1102            // the drain loop stopped — so the broadcast overflowed, and because
1103            // `dropped_tick` was reset immediately afterwards, the loss was
1104            // billed to the *following* window. In the one tool whose entire
1105            // product is a per-window verdict.
1106            //
1107            // Now the sweep is a future the drain selects on: sampling never
1108            // stops, and a sweep that outlives the tick period simply widens this
1109            // window — `window_s` is measured from the last evaluation, never
1110            // assumed — so the drops land in the tick that incurred them.
1111            let sweep = async {
1112                let doctor = if wants_doctor {
1113                    Some(
1114                        crate::judge::doctor::run_doctor(
1115                            fleet,
1116                            slices,
1117                            &crate::judge::doctor::DoctorSpec {
1118                                deep: false,
1119                                sample: None,
1120                                timeout: spec.timeout,
1121                                listen: None,
1122                            },
1123                        )
1124                        .await
1125                        .map_err(|e| e.to_string()),
1126                    )
1127                } else {
1128                    None
1129                };
1130                let roster = if wants_roster {
1131                    Some(
1132                        crate::bus::roster::roster(fleet, spec.timeout)
1133                            .await
1134                            .map_err(|e| e.to_string()),
1135                    )
1136                } else {
1137                    None
1138                };
1139                // The schema warming rides here too (#337): still-unserved
1140                // producers are re-asked at the store's own backoff, off the
1141                // drain loop.
1142                if wants_decode {
1143                    crate::model::decode::prewarm(fleet, store, slices).await;
1144                }
1145                (doctor, roster)
1146            };
1147            let mut sweep = std::pin::pin!(sweep);
1148            let mut swept = None;
1149            // One timer per tick, not one per drained sample (#346).
1150            let tick_over = tokio::time::sleep_until(deadline);
1151            tokio::pin!(tick_over);
1152            while !closed {
1153                let item = tokio::select! {
1154                    item = events.recv() => item,
1155                    // The tick cannot close before its own sweep has landed, and
1156                    // the drain keeps running until it does.
1157                    outcome = &mut sweep, if swept.is_none() => {
1158                        swept = Some(outcome);
1159                        continue;
1160                    }
1161                    () = &mut tick_over, if swept.is_some() => break,
1162                };
1163                match item {
1164                    Some(StreamItem::Event(FleetEvent::Sample(s))) => {
1165                        // Decode once per sample (budgeted per key per tick),
1166                        // shared by every invalid-payload rule the key matches.
1167                        let verdict = if rules.wants_verdict(&s) {
1168                            Some(
1169                                crate::model::decode::decode_sample(
1170                                    fleet,
1171                                    store,
1172                                    slices,
1173                                    &s.key,
1174                                    Some(&s.encoding),
1175                                    &s.payload.to_bytes(),
1176                                )
1177                                .await
1178                                .verdict,
1179                            )
1180                        } else {
1181                            None
1182                        };
1183                        rules.observe_sample(&s, &mut facts_cache, verdict.as_ref());
1184                    }
1185                    Some(StreamItem::Dropped(n)) => rules.observe_drop(n),
1186                    Some(_) => {}
1187                    None => closed = true,
1188                }
1189            }
1190
1191            // Evaluate the tick over the measured window, then say only what
1192            // changed. The sweep has already landed unless the stream closed
1193            // under it — in which case there is nothing left to drain, and
1194            // awaiting it here costs the tick nothing.
1195            let (doctor_outcome, roster_outcome) = match swept {
1196                Some(outcome) => outcome,
1197                None => sweep.await,
1198            };
1199            let now = tokio::time::Instant::now();
1200            let at = crate::tape::record::rfc3339_now();
1201            let transitions = rules.evaluate(
1202                now,
1203                &at,
1204                SweepOutcome {
1205                    doctor: doctor_outcome
1206                        .as_ref()
1207                        .map(|o| o.as_ref().map_err(String::as_str)),
1208                    roster: roster_outcome
1209                        .as_ref()
1210                        .map(|o| o.as_ref().map_err(String::as_str)),
1211                },
1212            );
1213            for transition in transitions {
1214                // Awaits, where the callback returned: the consumer's write
1215                // now happens *here*, so its error returns from where it
1216                // happened instead of being stashed for after the run (#360).
1217                // The emission point is the tick evaluation — the drain loop
1218                // above has already ended for this tick — so a slow consumer
1219                // widens the next window rather than stalling a drain (#338).
1220                sender.send(transition).await;
1221            }
1222            if closed || spec.ticks.is_some_and(|n| rules.ticks() >= n) {
1223                break;
1224            }
1225        }
1226        monitor.shutdown().await?;
1227        Ok(WatchdogSummary {
1228            ticks: rules.ticks(),
1229            transitions: rules.transitions(),
1230            facts_evicted: facts_cache.evicted(),
1231        })
1232    })
1233}
1234
1235#[cfg(test)]
1236mod tests {
1237    use super::*;
1238    use crate::report::{DoctorFinding, DoctorSeverity};
1239
1240    fn report_with(checks: &[CheckId]) -> DoctorReport {
1241        DoctorReport {
1242            findings: checks
1243                .iter()
1244                .map(|c| DoctorFinding {
1245                    severity: DoctorSeverity::Error,
1246                    check: *c,
1247                    subject: "s".into(),
1248                    evidence: "e".into(),
1249                    citation: None,
1250                })
1251                .collect(),
1252            synced: crate::report::Asked::NotAsked,
1253            introspect_answered: 0,
1254            live_producers: 0,
1255            describe_served: 0,
1256            describe_missing: 0,
1257            routers: 0,
1258            router_version: None,
1259            deep: false,
1260            observation: None,
1261        }
1262    }
1263
1264    /// Every variant's canonical spelling parses back to itself, and a rule
1265    /// outside the vocabulary is an error that names the vocabulary — closed
1266    /// means closed.
1267    #[test]
1268    fn the_vocabulary_round_trips_and_is_closed() {
1269        let rules = [
1270            "rate-above v1/*/telemetry/** 5",
1271            "rate-below v1/h-aaaaaaaaaaaa/state/p/health 0.5",
1272            "silent-for v1/*/events/** 30",
1273            "invalid-payload v1/*/state/**",
1274            "qos-mismatch v1/*/telemetry/**",
1275            "doctor slice-sync",
1276            "origin-down h-aaaaaaaaaaaa",
1277            "dropped",
1278        ];
1279        for rule in rules {
1280            let parsed = Condition::parse(rule).expect(rule);
1281            assert_eq!(parsed.to_string(), rule, "canonical spelling round-trips");
1282        }
1283        let err = Condition::parse("if rate > 5 then page").unwrap_err();
1284        assert!(err.to_string().contains("closed"), "{err}");
1285        assert!(err.to_string().contains("rate-above"), "{err}");
1286        // A doctor rule outside the stable check-id vocabulary is refused at
1287        // parse, naming the vocabulary.
1288        let err = Condition::parse("doctor no-such-check").unwrap_err();
1289        assert!(err.to_string().contains("slice-sync"), "{err}");
1290    }
1291
1292    /// The acceptance rule of #227: a drop under a completeness claim yields
1293    /// `unobservable`, **never** `ok` — across all three core judges, now
1294    /// spoken in the [`Judgement`] core and projected onto [`CondState`]
1295    /// (RFC 13, v1.24).
1296    #[test]
1297    fn a_drop_under_a_completeness_claim_is_unobservable_never_ok() {
1298        let wire = CondState::from;
1299        // Excess: the "did not exceed" side counts what did not happen.
1300        assert!(judge_excess(false, 1).is_unobservable());
1301        assert_eq!(wire(judge_excess(false, 0)), CondState::Ok);
1302        // …while firing is positive evidence, conclusive under drops.
1303        assert_eq!(judge_excess(true, 7), Judgement::Established);
1304        // Shortfall: the drops could have carried the difference.
1305        assert!(judge_shortfall(true, 1).is_unobservable());
1306        assert_eq!(judge_shortfall(true, 0), Judgement::Established);
1307        // …while "enough seen" is conclusive: a drop only hides more.
1308        assert_eq!(wire(judge_shortfall(false, 9)), CondState::Ok);
1309        // Silence: unprovable over a dropped or unwatched span. Named fields
1310        // rather than three bare `bool`s, which is the whole of #349 — read
1311        // the old spelling `judge_silence(false, true, false)` and say which
1312        // one was the drop.
1313        let silence = |sample_within, span_observed, drop_free| {
1314            judge_silence(SilenceEvidence {
1315                sample_within,
1316                span_observed,
1317                drop_free,
1318            })
1319        };
1320        assert!(silence(false, true, false).is_unobservable());
1321        assert!(silence(false, false, true).is_unobservable());
1322        assert_eq!(silence(false, true, true), Judgement::Established);
1323        assert_eq!(wire(silence(true, true, false)), CondState::Ok);
1324    }
1325
1326    /// The wire projection's documented mapping, polarity note included:
1327    /// `NotEstablished` (established-clean) is `ok`, `Established` (the
1328    /// condition holds) is `firing`, and **both** unestablished poles land
1329    /// on `unobservable` — the wire cannot say more (RFC 13, v1.24).
1330    #[test]
1331    fn cond_state_is_the_documented_projection_of_the_judgement_core() {
1332        assert_eq!(CondState::from(Judgement::Established), CondState::Firing);
1333        assert_eq!(
1334            CondState::from(Judgement::NotEstablished {
1335                reason: "clean".into()
1336            }),
1337            CondState::Ok
1338        );
1339        assert_eq!(
1340            CondState::from(Judgement::NotAsked),
1341            CondState::Unobservable
1342        );
1343        assert_eq!(
1344            CondState::from(Judgement::Unobservable {
1345                reason: "drops".into()
1346            }),
1347            CondState::Unobservable
1348        );
1349    }
1350
1351    /// The window judges apply those rules: `rate-above` firing survives
1352    /// drops, its ok does not; a young watch cannot claim silence.
1353    #[test]
1354    fn window_judgement_applies_the_drop_rules() {
1355        let rule = Condition::parse("rate-above k/** 1").unwrap();
1356        let base = CondWindow {
1357            window_s: 10.0,
1358            observed_s: 10.0,
1359            ..CondWindow::default()
1360        };
1361        let over = CondWindow {
1362            samples: 20,
1363            dropped: 5,
1364            ..base
1365        };
1366        assert_eq!(rule.judge_window(&over).unwrap().state, CondState::Firing);
1367        let under_dropped = CondWindow {
1368            samples: 2,
1369            dropped: 5,
1370            ..base
1371        };
1372        assert_eq!(
1373            rule.judge_window(&under_dropped).unwrap().state,
1374            CondState::Unobservable
1375        );
1376
1377        let rule = Condition::parse("silent-for k/** 30").unwrap();
1378        let young = CondWindow {
1379            window_s: 5.0,
1380            observed_s: 5.0,
1381            ..CondWindow::default()
1382        };
1383        let eval = rule.judge_window(&young).unwrap();
1384        assert_eq!(eval.state, CondState::Unobservable);
1385        assert!(eval.evidence.contains("watched only"), "{}", eval.evidence);
1386        let silent = CondWindow {
1387            window_s: 5.0,
1388            observed_s: 60.0,
1389            ..CondWindow::default()
1390        };
1391        assert_eq!(rule.judge_window(&silent).unwrap().state, CondState::Firing);
1392        let recently_dropped = CondWindow {
1393            last_drop_ago_s: Some(10.0),
1394            ..silent
1395        };
1396        assert_eq!(
1397            rule.judge_window(&recently_dropped).unwrap().state,
1398            CondState::Unobservable
1399        );
1400        let spoken = CondWindow {
1401            samples: 1,
1402            last_sample_ago_s: Some(3.0),
1403            ..silent
1404        };
1405        assert_eq!(rule.judge_window(&spoken).unwrap().state, CondState::Ok);
1406    }
1407
1408    /// The synthetic-traffic marker count (RFC 09 §5.3, the #162 rider)
1409    /// rides every window evidence line when present.
1410    #[test]
1411    fn synthetic_marked_samples_are_said_out_loud() {
1412        let rule = Condition::parse("rate-above k/** 0.1").unwrap();
1413        let w = CondWindow {
1414            window_s: 10.0,
1415            observed_s: 10.0,
1416            samples: 20,
1417            synthetic: 3,
1418            ..CondWindow::default()
1419        };
1420        let eval = rule.judge_window(&w).unwrap();
1421        assert!(
1422            eval.evidence.contains("3 synthetic-marked"),
1423            "{}",
1424            eval.evidence
1425        );
1426    }
1427
1428    /// The transition machine: the first evaluation states the baseline
1429    /// (from `null`), an unchanged tick emits nothing, a genuine change
1430    /// emits exactly one line.
1431    #[test]
1432    fn transitions_fire_once_per_genuine_change_and_never_per_tick() {
1433        let eval = |state| Eval {
1434            state,
1435            evidence: "e".into(),
1436        };
1437        // The condition itself, not its rendering — which is the point of
1438        // #352: the two can no longer disagree.
1439        let mut rs = RuleState::new(Condition::Dropped);
1440        let first = rs.observe(eval(CondState::Ok), "t0").expect("baseline");
1441        assert_eq!(first.rule, "dropped", "the transition renders its rule");
1442        assert_eq!(first.from, None, "the baseline comes from null (O4)");
1443        assert_eq!(first.to, CondState::Ok);
1444        assert!(rs.observe(eval(CondState::Ok), "t1").is_none());
1445        assert!(rs.observe(eval(CondState::Ok), "t2").is_none());
1446        let change = rs.observe(eval(CondState::Firing), "t3").expect("a change");
1447        assert_eq!(change.from, Some(CondState::Ok));
1448        assert_eq!(change.to, CondState::Firing);
1449        assert!(rs.observe(eval(CondState::Firing), "t4").is_none());
1450    }
1451
1452    /// The ndjson shape of a transition is a wire contract for scripts:
1453    /// `{"rule","from","to","at","evidence"}`, states snake_case, `from`
1454    /// null on the baseline.
1455    #[test]
1456    fn transition_json_shape_is_pinned() {
1457        let t = Transition {
1458            rule: "silent-for k/** 30".into(),
1459            from: None,
1460            to: CondState::Unobservable,
1461            at: "2026-08-22T00:00:00Z".into(),
1462            evidence: "watched only 5.0s of a 30.0s silence claim".into(),
1463        };
1464        assert_eq!(
1465            serde_json::to_value(&t).unwrap(),
1466            serde_json::json!({
1467                "rule": "silent-for k/** 30",
1468                "from": null,
1469                "to": "unobservable",
1470                "at": "2026-08-22T00:00:00Z",
1471                "evidence": "watched only 5.0s of a 30.0s silence claim",
1472            })
1473        );
1474        let t = Transition {
1475            from: Some(CondState::Ok),
1476            to: CondState::Firing,
1477            ..t
1478        };
1479        let json = serde_json::to_value(&t).unwrap();
1480        assert_eq!(json["from"], "ok");
1481        assert_eq!(json["to"], "firing");
1482    }
1483
1484    /// `doctor --transitions`'s delta: the first run is a full baseline (every
1485    /// stable check id, once), an identical second run says nothing, a new
1486    /// finding transitions exactly its check — and a failed run flips every
1487    /// check to unobservable, never ok.
1488    #[test]
1489    fn doctor_watch_reports_deltas_not_states() {
1490        let mut watch = DoctorWatch::new();
1491        let clean = report_with(&[]);
1492        let baseline = watch.observe(Ok(&clean), "t0");
1493        assert_eq!(baseline.len(), CheckId::ALL.len());
1494        assert!(baseline.iter().all(|t| t.from.is_none()));
1495        assert!(baseline.iter().all(|t| t.to == CondState::Ok));
1496
1497        assert!(
1498            watch.observe(Ok(&clean), "t1").is_empty(),
1499            "an unchanged run emits nothing"
1500        );
1501
1502        let drifted = report_with(&[CheckId::SchemaDrift, CheckId::SchemaDrift]);
1503        let changes = watch.observe(Ok(&drifted), "t2");
1504        assert_eq!(changes.len(), 1, "only the changed check transitions");
1505        assert_eq!(changes[0].rule, "doctor schema-drift");
1506        assert_eq!(changes[0].to, CondState::Firing);
1507        assert!(changes[0].evidence.contains("2 finding(s)"));
1508
1509        let failed = watch.observe(Err("session lost"), "t3");
1510        assert_eq!(
1511            failed.len(),
1512            CheckId::ALL.len(),
1513            "a failed run is unobservable for every check — never ok"
1514        );
1515        assert!(failed.iter().all(|t| t.to == CondState::Unobservable));
1516    }
1517}