zenkey_fleet/judge/condition.rs
1//! Conditions and the watchdog (#227) — transitions, not states.
2//!
3//! Three shipped features each hard-coded their own predicate over the
4//! observation surface: `expect` (one window), `doctor --for` (five
5//! checks), `cutover` (silence). This module is the one **closed vocabulary**
6//! they were each a spelling of: [`Condition`], evaluated to three states,
7//! never two (RFC 09 §5.1 O4/O6) — `ok` / `firing` / **`unobservable`**. The
8//! third state is the reason this exists: an alerting tool that cannot say
9//! *"I could not tell"* is the one that pages at 3am for a dropped buffer. A
10//! drop under a completeness claim yields `unobservable`, never `ok`.
11//!
12//! The vocabulary is deliberately closed — no expressions, no templating, no
13//! rules engine. A new condition is a new variant, argued for the way a new
14//! doctor check id is.
15//!
16//! The semantic core is three tiny rules — [`judge_shortfall`],
17//! [`judge_excess`], [`judge_silence`] — shared with [`crate::judge::expect`], so
18//! the watchdog and the CI assertion cannot drift about what a drop means.
19//! Since RFC 13 (v1.24; the material was RFC 09 §5.1 pre-v1.24) the rules
20//! speak the four-pole [`Judgement`] core, and [`CondState`] is this
21//! module's serde-stable **wire projection** of it — see its mapping doc.
22//!
23//! [`watchdog`] is the continuous observer over the vocabulary:
24//! **foreground, explicitly launched, single-purpose, one process per
25//! invocation, no shared state** — not the hidden, auto-started,
26//! discovery-caching daemon the redesign ledger rejected
27//! (`docs/redesign-2026-07.md` §6.1). It emits [`Transition`]s: one per
28//! genuine state change, none per unchanged tick.
29
30use std::collections::BTreeMap;
31use std::time::Duration;
32
33use crate::{Error, Result};
34
35use crate::bus::monitor::SampleView;
36use crate::model::decode::SchemaStore;
37use crate::model::registry::SliceSet;
38use crate::report::{CheckId, DoctorReport};
39use crate::report::{CondState, Judgement, Transition, WatchdogSummary};
40use sipper::{Straw, sipper};
41
42/// The closed condition vocabulary (#227), over the existing observation
43/// surface. Each variant names what *firing* means; the drop rules are in
44/// the judge functions this module documents.
45#[derive(Debug, Clone, PartialEq)]
46pub enum Condition {
47 /// Samples on `selector` rode above `hz` over the evaluation window.
48 /// Firing is positive evidence, conclusive even under drops (a drop only
49 /// hides more); `ok` under drops is unobservable — the true rate is
50 /// higher than what was counted (O6).
51 RateAbove { selector: String, hz: f64 },
52 /// Samples on `selector` rode below `hz`. A shortfall under drops is
53 /// unobservable — the dropped samples could have filled it (O6); enough
54 /// observed is conclusive `ok` regardless.
55 RateBelow { selector: String, hz: f64 },
56 /// No sample matched `selector` for at least `for_s` seconds. Silence is
57 /// a completeness claim — it counts what did NOT happen — so it is
58 /// provable only over a drop-free span at least `for_s` long (O6), and
59 /// only once the observer has watched that long (O4).
60 SilentFor { selector: String, for_s: f64 },
61 /// An observed payload on `selector` did not reach [`crate::Verdict::Valid`]
62 /// (#159) — `Invalid` and `NotValidated` both count: asking for validity
63 /// and getting "unknowable" is not valid. Scoped to what was observed
64 /// and checked; the `ok` state claims "nothing checked failed", never
65 /// "nothing invalid rode" — the drop count rides in the evidence.
66 InvalidPayload { selector: String },
67 /// An observed sample on `selector` did not ride its registry-declared
68 /// QoS profile (RFC 04 §3). Same per-observed-sample scope as
69 /// [`Condition::InvalidPayload`]; samples with no declared profile are
70 /// unjudgeable and counted in the evidence, not the state.
71 QosMismatch { selector: String },
72 /// A doctor run reported at least one finding with this check id
73 /// (the stable [`crate::report::CheckId`] vocabulary). A failed doctor run is
74 /// unobservable for every doctor condition — never `ok`.
75 DoctorCheck { check: CheckId },
76 /// The origin holds no `alive` token on the liveliness roster
77 /// (RFC 04 §5). A roster that could not be asked is unobservable —
78 /// silence is not a verdict (RFC 05 §3.1).
79 OriginDown { origin: String },
80 /// The observer itself dropped samples this window (RFC 09 §5.1 O6) —
81 /// self-knowledge, so never unobservable.
82 Dropped,
83}
84
85/// The rule grammar, spelled once for the parse error and the docs.
86const VOCABULARY: &str = "rate-above <SEL> <HZ> | rate-below <SEL> <HZ> | \
87 silent-for <SEL> <SECS> | invalid-payload <SEL> | qos-mismatch <SEL> | \
88 doctor <CHECK-ID> | origin-down <ORIGIN> | dropped";
89
90impl Condition {
91 /// Parse one rule: whitespace-separated, kind first (Zenoh key
92 /// expressions cannot contain whitespace, so the split is unambiguous).
93 /// The vocabulary is closed; anything else is an error that spells it.
94 pub fn parse(rule: &str) -> Result<Condition> {
95 let hz = |s: &str, kind: &str| -> Result<f64> {
96 let v: f64 = s
97 .parse()
98 .map_err(|_| Error::unaskable(format!("{kind} {s:?}"), "is not a number"))?;
99 if !v.is_finite() || v < 0.0 {
100 return Err(Error::unaskable(
101 kind.to_string(),
102 "the threshold must be a finite non-negative number",
103 ));
104 }
105 Ok(v)
106 };
107 let tokens: Vec<&str> = rule.split_whitespace().collect();
108 Ok(match tokens.as_slice() {
109 ["rate-above", sel, n] => Condition::RateAbove {
110 selector: sel.to_string(),
111 hz: hz(n, "rate-above")?,
112 },
113 ["rate-below", sel, n] => Condition::RateBelow {
114 selector: sel.to_string(),
115 hz: hz(n, "rate-below")?,
116 },
117 ["silent-for", sel, n] => {
118 let for_s = hz(n, "silent-for")?;
119 if for_s <= 0.0 {
120 return Err(Error::unaskable(
121 "silent-for",
122 "the span must be a positive number of seconds",
123 ));
124 }
125 Condition::SilentFor {
126 selector: sel.to_string(),
127 for_s,
128 }
129 }
130 ["invalid-payload", sel] => Condition::InvalidPayload {
131 selector: sel.to_string(),
132 },
133 ["qos-mismatch", sel] => Condition::QosMismatch {
134 selector: sel.to_string(),
135 },
136 ["doctor", check] => {
137 let Some(check) = CheckId::parse(check) else {
138 return Err(Error::unaskable(
139 format!("doctor {check:?}"),
140 format!(
141 "is not a check id — the stable vocabulary is: {}",
142 CheckId::ALL
143 .iter()
144 .map(|c| c.as_str())
145 .collect::<Vec<_>>()
146 .join(", ")
147 ),
148 ));
149 };
150 Condition::DoctorCheck { check }
151 }
152 ["origin-down", origin] => Condition::OriginDown {
153 origin: origin.to_string(),
154 },
155 ["dropped"] => Condition::Dropped,
156 _ => {
157 return Err(Error::unaskable(
158 format!("{rule:?}"),
159 format!(
160 "is not a rule — the vocabulary is closed (no \
161 expressions, no templating): {VOCABULARY}"
162 ),
163 ));
164 }
165 })
166 }
167
168 /// The wire selector this condition observes, when it observes one.
169 pub fn selector(&self) -> Option<&str> {
170 match self {
171 Condition::RateAbove { selector, .. }
172 | Condition::RateBelow { selector, .. }
173 | Condition::SilentFor { selector, .. }
174 | Condition::InvalidPayload { selector }
175 | Condition::QosMismatch { selector } => Some(selector),
176 _ => None,
177 }
178 }
179
180 /// Judge one observation window. `None` for the conditions that are not
181 /// window-scoped ([`Condition::DoctorCheck`], [`Condition::OriginDown`]).
182 /// Judge this condition against everything one tick observed.
183 ///
184 /// **The single entry point**, and why `run_watchdog` has no `expect`s
185 /// left (#352). The three judges below each returned `None` for the
186 /// variants they do not own, which forced the caller to assert a
187 /// partition the compiler could not see — four times, every one
188 /// discharging the same claim. This match *is* the partition, and each
189 /// arm hands its judge exactly the evidence that judge needs, so none of
190 /// them has a `None` to return.
191 pub fn judge(&self, ev: &TickEvidence<'_>) -> Eval {
192 match self {
193 Condition::DoctorCheck { check } => judge_doctor_check(*check, ev.doctor),
194 Condition::OriginDown { origin } => judge_origin_down(origin, ev.roster),
195 _ => self.judge_window_total(ev.window),
196 }
197 }
198
199 pub fn judge_window(&self, w: &CondWindow) -> Option<Eval> {
200 let synth = if w.synthetic > 0 {
201 format!("; {} synthetic-marked (RFC 09 §5.3)", w.synthetic)
202 } else {
203 String::new()
204 };
205 let rate = if w.window_s > 0.0 {
206 w.samples as f64 / w.window_s
207 } else {
208 0.0
209 };
210 Some(match self {
211 Condition::RateAbove { hz, .. } => {
212 let state = CondState::from(judge_excess(rate > *hz, w.dropped));
213 let evidence = match state {
214 CondState::Unobservable => format!(
215 "{rate:.2} Hz observed but {} sample(s) dropped — the true rate \
216 is at least that, not exactly that (O6){synth}",
217 w.dropped
218 ),
219 _ => format!(
220 "{} sample(s) in {:.1}s = {rate:.2} Hz against the {hz:.2} Hz \
221 bound{synth}",
222 w.samples, w.window_s
223 ),
224 };
225 Eval { state, evidence }
226 }
227 Condition::RateBelow { hz, .. } => {
228 let state = CondState::from(judge_shortfall(rate < *hz, w.dropped));
229 let evidence = match state {
230 CondState::Unobservable => format!(
231 "{rate:.2} Hz observed with {} sample(s) dropped — the drops \
232 could have carried the difference (O6){synth}",
233 w.dropped
234 ),
235 _ => format!(
236 "{} sample(s) in {:.1}s = {rate:.2} Hz against the {hz:.2} Hz \
237 bound{synth}",
238 w.samples, w.window_s
239 ),
240 };
241 Eval { state, evidence }
242 }
243 Condition::SilentFor { for_s, .. } => {
244 let ev = SilenceEvidence {
245 sample_within: w.last_sample_ago_s.map(|ago| ago < *for_s) == Some(true),
246 span_observed: w.observed_s >= *for_s,
247 drop_free: w.last_drop_ago_s.map(|ago| ago >= *for_s) != Some(false),
248 };
249 let SilenceEvidence { span_observed, .. } = ev;
250 let state = CondState::from(judge_silence(ev));
251 let evidence = match state {
252 CondState::Ok => format!(
253 "a sample rode {:.1}s ago, inside the {for_s:.1}s span{synth}",
254 w.last_sample_ago_s.unwrap_or(0.0)
255 ),
256 CondState::Firing => {
257 format!("no sample for {for_s:.1}s, on a drop-free observer{synth}")
258 }
259 CondState::Unobservable if !span_observed => format!(
260 "watched only {:.1}s of a {for_s:.1}s silence claim — not asked \
261 is not answered (O4){synth}",
262 w.observed_s
263 ),
264 CondState::Unobservable => format!(
265 "no sample seen, but the observer dropped inside the {for_s:.1}s \
266 span — silence is unprovable (O6){synth}"
267 ),
268 };
269 Eval { state, evidence }
270 }
271 Condition::InvalidPayload { .. } => Eval {
272 state: if w.invalid > 0 {
273 CondState::Firing
274 } else {
275 CondState::Ok
276 },
277 evidence: format!(
278 "{} of {} checked sample(s) did not reach Valid ({} observed, \
279 {} dropped{synth})",
280 w.invalid, w.checked, w.samples, w.dropped
281 ),
282 },
283 Condition::QosMismatch { .. } => Eval {
284 state: if w.qos_mismatched > 0 {
285 CondState::Firing
286 } else {
287 CondState::Ok
288 },
289 evidence: format!(
290 "{} of {} judged sample(s) did not ride their declared profile \
291 ({} observed, {} with no declared profile to judge, \
292 {} dropped{synth})",
293 w.qos_mismatched,
294 w.qos_judged,
295 w.samples,
296 w.samples.saturating_sub(w.qos_judged),
297 w.dropped
298 ),
299 },
300 Condition::Dropped => Eval {
301 state: if w.dropped > 0 {
302 CondState::Firing
303 } else {
304 CondState::Ok
305 },
306 evidence: format!(
307 "the observer dropped {} sample(s) in {:.1}s (O6){synth}",
308 w.dropped, w.window_s
309 ),
310 },
311 Condition::DoctorCheck { .. } | Condition::OriginDown { .. } => return None,
312 })
313 }
314
315 /// [`judge_window`](Self::judge_window) for the variants that *have* a
316 /// window — total, because [`judge`](Self::judge) has already routed the
317 /// other two elsewhere.
318 fn judge_window_total(&self, w: &CondWindow) -> Eval {
319 debug_assert!(
320 !matches!(
321 self,
322 Condition::DoctorCheck { .. } | Condition::OriginDown { .. }
323 ),
324 "judge() routes these two to their own evidence"
325 );
326 self.judge_window(w).unwrap_or_else(|| Eval {
327 // Unreachable through `judge`; if some future variant reaches it,
328 // "I have no window for this" is the honest answer, not a panic
329 // in a watchdog that is supposed to keep running.
330 state: CondState::Unobservable,
331 evidence: "this rule is not judged against a sample window".into(),
332 })
333 }
334
335 /// Judge a roster ask. `None` unless this is [`Condition::OriginDown`].
336 /// `Err` is the ask failing, which is unobservable — silence is not a
337 /// verdict (RFC 05 §3.1).
338 pub fn judge_roster(
339 &self,
340 roster: Result<&BTreeMap<String, Vec<String>>, &str>,
341 ) -> Option<Eval> {
342 let Condition::OriginDown { origin } = self else {
343 return None;
344 };
345 Some(match roster {
346 Err(e) => Eval {
347 state: CondState::Unobservable,
348 evidence: format!("the roster could not be asked: {e}"),
349 },
350 Ok(r) => match r.get(origin) {
351 Some(producers) => Eval {
352 state: CondState::Ok,
353 evidence: format!(
354 "{origin} holds an alive token ({} producer(s))",
355 producers.len()
356 ),
357 },
358 None => Eval {
359 state: CondState::Firing,
360 evidence: format!("{origin} holds no alive token (RFC 04 §5)"),
361 },
362 },
363 })
364 }
365
366 /// Judge a doctor run. `None` unless this is [`Condition::DoctorCheck`].
367 /// A failed run is unobservable for every doctor condition — never `ok`.
368 pub fn judge_doctor(&self, outcome: Result<&DoctorReport, &str>) -> Option<Eval> {
369 let Condition::DoctorCheck { check } = self else {
370 return None;
371 };
372 Some(match outcome {
373 Err(e) => Eval {
374 state: CondState::Unobservable,
375 evidence: format!("the doctor run failed: {e}"),
376 },
377 Ok(report) => {
378 let mut hits = report.findings.iter().filter(|f| f.check == *check);
379 match hits.next() {
380 Some(first) => Eval {
381 state: CondState::Firing,
382 evidence: format!(
383 "{} finding(s); first: {} — {}",
384 1 + hits.count(),
385 first.subject,
386 first.evidence
387 ),
388 },
389 None => Eval {
390 state: CondState::Ok,
391 evidence: format!("no {check} findings"),
392 },
393 }
394 }
395 })
396 }
397}
398
399impl std::fmt::Display for Condition {
400 /// The canonical rule spelling — [`Condition::parse`] round-trips it,
401 /// and it is the `rule` field of every [`Transition`].
402 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
403 match self {
404 Condition::RateAbove { selector, hz } => write!(f, "rate-above {selector} {hz}"),
405 Condition::RateBelow { selector, hz } => write!(f, "rate-below {selector} {hz}"),
406 Condition::SilentFor { selector, for_s } => {
407 write!(f, "silent-for {selector} {for_s}")
408 }
409 Condition::InvalidPayload { selector } => write!(f, "invalid-payload {selector}"),
410 Condition::QosMismatch { selector } => write!(f, "qos-mismatch {selector}"),
411 Condition::DoctorCheck { check } => write!(f, "doctor {check}"),
412 Condition::OriginDown { origin } => write!(f, "origin-down {origin}"),
413 Condition::Dropped => write!(f, "dropped"),
414 }
415 }
416}
417
418// ─── the judgement rules (the vocabulary's semantic core) ───────────────────
419//
420// The three judges return the four-pole [`Judgement`] core (RFC 13, v1.24;
421// RFC 09 §5.1 pre-v1.24). None of them ever answers `NotAsked` — a judge is
422// only called when the question was put — but the pole exists in the currency
423// so a caller that *skipped* a judge can say so in the same vocabulary. The
424// watchdog projects each judgement onto [`CondState`] for the wire.
425
426/// The shortfall rule ([`Condition::RateBelow`]; `expect`'s count floor and
427/// rate floor): too little was seen. Enough seen is conclusively clean even
428/// under drops — a drop can only hide *more*. A shortfall with drops is
429/// unobservable: the dropped samples could have filled it (RFC 09 §5.1 O6).
430pub fn judge_shortfall(short: bool, dropped: u64) -> Judgement {
431 match (short, dropped) {
432 (false, _) => Judgement::NotEstablished {
433 reason: "enough was seen — a drop only hides more".into(),
434 },
435 (true, 0) => Judgement::Established,
436 (true, _) => Judgement::Unobservable {
437 reason: format!("{dropped} dropped sample(s) could have filled the shortfall (O6)"),
438 },
439 }
440}
441
442/// The excess rule ([`Condition::RateAbove`]; `expect`'s rate ceiling): too
443/// much was seen. An excess is positive evidence, conclusive under drops.
444/// "No excess" is a completeness claim — it counts what did NOT happen — so
445/// under drops it is unobservable, never clean (O6).
446pub fn judge_excess(over: bool, dropped: u64) -> Judgement {
447 match (over, dropped) {
448 (true, _) => Judgement::Established,
449 (false, 0) => Judgement::NotEstablished {
450 reason: "no excess was counted, on a clean observation".into(),
451 },
452 (false, _) => Judgement::Unobservable {
453 reason: format!(
454 "{dropped} sample(s) dropped — \"did not exceed\" is a completeness \
455 claim (O6)"
456 ),
457 },
458 }
459}
460
461/// The silence rule ([`Condition::SilentFor`]; `expect --absent`): a sample
462/// inside the span conclusively breaks the silence; silence is provable only
463/// over a span the observer actually watched (O4) drop-free (O6) — otherwise
464/// unobservable, never clean.
465/// What one silence claim rests on — three facts that are all `bool` and all
466/// about the same span.
467///
468/// A struct rather than three positional parameters, because this feeds a
469/// *judgement* and a transposition of two identically-typed booleans returns
470/// a plausible wrong verdict with no compile error (#349).
471/// `judge_shortfall`/`judge_excess` keep their positional `(bool, u64)` —
472/// not transposable, so not a hazard.
473#[derive(Debug, Clone, Copy)]
474pub struct SilenceEvidence {
475 /// A sample rode inside the claimed span — the conclusive break.
476 pub sample_within: bool,
477 /// The observer actually watched the whole span (O4). A span it did not
478 /// watch is not a span it can call silent.
479 pub span_observed: bool,
480 /// The observer dropped nothing inside the span (O6). "Nothing arrived"
481 /// under drops is a completeness claim the observation cannot carry.
482 pub drop_free: bool,
483}
484
485pub fn judge_silence(ev: SilenceEvidence) -> Judgement {
486 let SilenceEvidence {
487 sample_within,
488 span_observed,
489 drop_free,
490 } = ev;
491 if sample_within {
492 Judgement::NotEstablished {
493 reason: "a sample rode inside the span".into(),
494 }
495 } else if span_observed && drop_free {
496 Judgement::Established
497 } else if !span_observed {
498 Judgement::Unobservable {
499 reason: "the observer has not watched the whole claimed span (O4)".into(),
500 }
501 } else {
502 Judgement::Unobservable {
503 reason: "the observer dropped inside the span — silence is unprovable (O6)".into(),
504 }
505 }
506}
507
508/// Everything one watchdog tick observed, in the three shapes the conditions
509/// are judged against.
510///
511/// `doctor` and `roster` are `Option` because a tick only runs those asks if
512/// some rule wants them — and "not run this tick" is *unobservable*, which is
513/// the honest reading and the one the caller used to assert away with
514/// `.expect("a doctor rule ran the doctor")` (#352).
515pub struct TickEvidence<'e> {
516 pub window: &'e CondWindow,
517 pub doctor: Option<Result<&'e DoctorReport, &'e str>>,
518 pub roster: Option<Result<&'e BTreeMap<String, Vec<String>>, &'e str>>,
519}
520
521/// Judge one doctor check against this tick's run — total, and total in the
522/// "did not run" direction too.
523pub fn judge_doctor_check(check: CheckId, outcome: Option<Result<&DoctorReport, &str>>) -> Eval {
524 let Some(outcome) = outcome else {
525 return Eval {
526 state: CondState::Unobservable,
527 evidence: "the doctor did not run this tick".into(),
528 };
529 };
530 Condition::DoctorCheck { check }
531 .judge_doctor(outcome)
532 .expect("a DoctorCheck is judged by the doctor")
533}
534
535/// Judge one origin against this tick's roster ask — likewise total.
536pub fn judge_origin_down(
537 origin: &str,
538 roster: Option<Result<&BTreeMap<String, Vec<String>>, &str>>,
539) -> Eval {
540 let Some(roster) = roster else {
541 return Eval {
542 state: CondState::Unobservable,
543 evidence: "the roster was not asked this tick".into(),
544 };
545 };
546 Condition::OriginDown {
547 origin: origin.to_string(),
548 }
549 .judge_roster(roster)
550 .expect("an OriginDown is judged by the roster")
551}
552
553// ─── observations and evaluations ───────────────────────────────────────────
554
555/// What one evaluation window observed on one condition's selector — the
556/// facts, separated from the judgement so the judgement is pure.
557///
558/// `CondWindow` and not `Window`: this type is re-exported at the crate root
559/// beside `BudgetWindow` and `RecordBounds`, and a bare `Window` there reads
560/// as *the* window of an engine that has several. Nothing serializes the
561/// name (the type carries no `Serialize`), so the rename is Rust-side only.
562#[derive(Debug, Clone, Copy, Default)]
563pub struct CondWindow {
564 /// The span this window judges, seconds.
565 pub window_s: f64,
566 /// How long the observer has been watching in total — a claim about a
567 /// span longer than this is unobservable (O4).
568 pub observed_s: f64,
569 /// Samples matching the selector within the window.
570 pub samples: u64,
571 /// Stream drops within the window — unattributable to any one selector,
572 /// so they taint every completeness claim (O6).
573 pub dropped: u64,
574 /// Seconds since the last matching sample; `None` = none seen since the
575 /// watch began.
576 pub last_sample_ago_s: Option<f64>,
577 /// Seconds since the last stream drop; `None` = the stream never dropped.
578 pub last_drop_ago_s: Option<f64>,
579 /// Samples whose payload did not reach `Valid`, among those checked.
580 pub invalid: u64,
581 /// Samples actually decode-checked (a budget bounds the cost).
582 pub checked: u64,
583 /// Samples that did not ride their declared QoS, among those judged.
584 pub qos_mismatched: u64,
585 /// Samples with a declared profile to judge against.
586 pub qos_judged: u64,
587 /// Samples carrying the RFC 09 §5.3 synthetic-traffic marker — generated
588 /// traffic judged as real would be a self-inflicted page, so every
589 /// evidence line carries the count.
590 pub synthetic: u64,
591}
592
593/// One evaluation: the three-valued state, and the evidence for it.
594#[derive(Debug, Clone, PartialEq)]
595pub struct Eval {
596 pub state: CondState,
597 pub evidence: String,
598}
599
600/// One rule's transition detector: feed evaluations in, get a [`Transition`]
601/// back **only** when the state genuinely changed. An unchanged tick returns
602/// `None` — transitions, not states.
603#[derive(Debug, Clone)]
604pub struct RuleState {
605 /// The condition itself, not its `Display`.
606 ///
607 /// It used to hold the rendered string and clone it into every
608 /// transition, with the two representations kept equal only by a
609 /// round-trip test — a second representation of a value that was
610 /// `Clone` and in scope (#352). The rendering happens where the
611 /// `Transition` is built, once, from the one source.
612 rule: Condition,
613 state: Option<CondState>,
614}
615
616impl RuleState {
617 pub fn new(rule: Condition) -> RuleState {
618 RuleState { rule, state: None }
619 }
620
621 /// The condition this state tracks.
622 pub fn rule(&self) -> &Condition {
623 &self.rule
624 }
625
626 /// The last observed state; `None` until the first evaluation.
627 pub fn state(&self) -> Option<CondState> {
628 self.state
629 }
630
631 /// Feed one evaluation. The first ever emits (from `null` — the baseline
632 /// is said once); after that only a genuine change does.
633 pub fn observe(&mut self, eval: Eval, at: impl Into<String>) -> Option<Transition> {
634 if self.state == Some(eval.state) {
635 return None;
636 }
637 let from = self.state;
638 self.state = Some(eval.state);
639 Some(Transition {
640 rule: self.rule.to_string(),
641 from,
642 to: eval.state,
643 at: at.into(),
644 evidence: eval.evidence,
645 })
646 }
647}
648
649/// Run-over-run delta over a doctor report: one [`RuleState`] per stable
650/// check id ([`CheckId`]), fed by `doctor --transitions`. The
651/// first run states the baseline (one transition per check id); every later run yields
652/// only genuine changes. A failed run flips every check to `unobservable` —
653/// a doctor that could not run has not said the fleet is healthy.
654#[derive(Debug, Clone)]
655pub struct DoctorWatch {
656 /// One state per check. A `Vec<(Condition, RuleState)>` until #352 — the
657 /// condition was in both halves of the pair.
658 checks: Vec<RuleState>,
659}
660
661impl DoctorWatch {
662 pub fn new() -> DoctorWatch {
663 DoctorWatch {
664 checks: CheckId::ALL
665 .iter()
666 .map(|id| RuleState::new(Condition::DoctorCheck { check: *id }))
667 .collect(),
668 }
669 }
670
671 /// Feed one doctor run (or its failure) and collect the transitions.
672 pub fn observe(&mut self, outcome: Result<&DoctorReport, &str>, at: &str) -> Vec<Transition> {
673 self.checks
674 .iter_mut()
675 .filter_map(|state| {
676 let Condition::DoctorCheck { check } = *state.rule() else {
677 // Unconstructible: `new` builds only `DoctorCheck`s.
678 return None;
679 };
680 let eval = judge_doctor_check(check, Some(outcome));
681 state.observe(eval, at)
682 })
683 .collect()
684 }
685}
686
687impl Default for DoctorWatch {
688 fn default() -> Self {
689 DoctorWatch::new()
690 }
691}
692
693// ─── the watchdog runner ────────────────────────────────────────────────────
694
695/// What a watchdog run watches, and for how long.
696#[derive(Debug, Clone)]
697pub struct WatchdogSpec {
698 /// The rules, evaluated every tick.
699 pub rules: Vec<Condition>,
700 /// Evaluation cadence. A tick that runs long (a doctor rule's fan-in)
701 /// slides rather than backlogs; windows are measured, not nominal.
702 pub tick: Duration,
703 /// Stop after this many ticks; `None` = run until the caller stops it.
704 pub ticks: Option<u64>,
705 /// Per-ask timeout for the roster and doctor conditions.
706 pub timeout: Duration,
707}
708
709/// How many decode attempts each key gets per tick under an
710/// `invalid-payload` rule — the same budget the doctor listen phase runs,
711/// for the same reason: a watchdog must not become a load test.
712const DECODE_BUDGET: u8 = 2;
713
714/// What one tick counted on one rule's selector.
715#[derive(Default, Clone, Copy)]
716struct TickCounters {
717 samples: u64,
718 invalid: u64,
719 checked: u64,
720 qos_mismatched: u64,
721 qos_judged: u64,
722 synthetic: u64,
723}
724
725/// One rule's whole per-run state, together.
726///
727/// This was four `Vec`s held in lockstep by index — `states`,
728/// `keyexprs`, `counters`, `last_sample` — across a hundred and thirty
729/// lines, with nothing structurally preventing them from disagreeing in
730/// length, and a `counters.fill(default())` reset that could silently
731/// miss one of them (#352).
732struct RuleRuntime {
733 rule: Condition,
734 /// The rule's selector, compiled once for sample attribution.
735 keyexpr: Option<zenoh::key_expr::KeyExpr<'static>>,
736 counters: TickCounters,
737 last_sample: Option<tokio::time::Instant>,
738 state: RuleState,
739}
740
741/// The sweep a tick ran beside its drain, as the rules see it: the doctor
742/// run and the roster ask, each `None` when no rule wanted it — which is
743/// *unobservable* for the rules that would have needed it, the honest
744/// reading (#352).
745#[derive(Debug, Clone, Copy, Default)]
746pub struct SweepOutcome<'e> {
747 pub doctor: Option<Result<&'e DoctorReport, &'e str>>,
748 pub roster: Option<Result<&'e BTreeMap<String, Vec<String>>, &'e str>>,
749}
750
751/// A set of rules judged tick by tick over **one** event stream — the
752/// watchdog's per-tick body, lifted out of [`watchdog`] so a second driver
753/// can run it (#218).
754///
755/// The driver owns the stream, the drain loop and the sweep; this owns
756/// everything the rules know: per-rule counters, the last sample and drop
757/// instants, the per-tick decode budget, and the transition detectors. Feed
758/// it every sample ([`observe_sample`](Self::observe_sample)) and every drop
759/// ([`observe_drop`](Self::observe_drop)) the stream yields, then
760/// [`evaluate`](Self::evaluate) once per tick and get back only what changed.
761///
762/// **Why the seam exists.** A trigger capture (`zenctl record --on`) must
763/// judge *the same event stream it records*: one subscription, one drop
764/// ledger. Had the capture run a watchdog of its own beside its recorder,
765/// the drops the judge saw and the drops in the file would have been two
766/// different facts about two different observers — and a `{"dropped": n}`
767/// in the file would say nothing about whether the rule that fired was
768/// judged over a clean window. With the body a value, the recorder drains
769/// one stream and hands every item to both the ring and the rules.
770///
771/// Sample attribution is by key-expression intersection against each rule's
772/// selector; a sample whose key does not parse as one counts for no rule.
773/// Time is `tokio::time::Instant`, so a driver under paused time judges
774/// exact windows.
775pub struct RuleSet<'a> {
776 rules: Vec<RuleRuntime>,
777 /// The distinct selectors the rules observe, in first-seen order.
778 watched: Vec<String>,
779 base: &'a str,
780 slices: Option<&'a SliceSet>,
781 started: tokio::time::Instant,
782 last_eval: tokio::time::Instant,
783 last_drop: Option<tokio::time::Instant>,
784 dropped_tick: u64,
785 decode_budget: BTreeMap<String, u8>,
786 ticks: u64,
787 transitions: u64,
788}
789
790impl<'a> RuleSet<'a> {
791 /// Compile the rules. Fails on a selector that is not a key expression —
792 /// before anything is declared, so the `?` has nothing to tear down
793 /// (#336). The watch clock starts here: [`CondWindow::observed_s`] is
794 /// measured from construction, so build the set right before the
795 /// subscriptions are declared.
796 pub fn new(rules: &[Condition], base: &'a str, slices: Option<&'a SliceSet>) -> Result<Self> {
797 let compiled = rules
798 .iter()
799 .map(|rule| {
800 Ok(RuleRuntime {
801 rule: rule.clone(),
802 keyexpr: rule
803 .selector()
804 .map(|sel| {
805 zenoh::key_expr::KeyExpr::try_from(sel.to_string())
806 .map_err(|e| Error::unaskable_from(format!("{sel:?}"), e))
807 })
808 .transpose()?,
809 counters: TickCounters::default(),
810 last_sample: None,
811 state: RuleState::new(rule.clone()),
812 })
813 })
814 .collect::<Result<Vec<_>>>()?;
815 let mut watched: Vec<String> = Vec::new();
816 for rule in rules {
817 if let Some(sel) = rule.selector()
818 && !watched.iter().any(|s| s == sel)
819 {
820 watched.push(sel.to_string());
821 }
822 }
823 let now = tokio::time::Instant::now();
824 Ok(RuleSet {
825 rules: compiled,
826 watched,
827 base,
828 slices,
829 started: now,
830 last_eval: now,
831 last_drop: None,
832 dropped_tick: 0,
833 decode_budget: BTreeMap::new(),
834 ticks: 0,
835 transitions: 0,
836 })
837 }
838
839 /// The distinct selectors the rules observe — what the driver must
840 /// subscribe to before the first window opens (O4).
841 pub fn watched(&self) -> &[String] {
842 &self.watched
843 }
844
845 /// Some rule judges a doctor run, so the driver owes one per tick.
846 pub fn wants_doctor(&self) -> bool {
847 self.rules
848 .iter()
849 .any(|r| matches!(r.rule, Condition::DoctorCheck { .. }))
850 }
851
852 /// Some rule judges the liveliness roster, so the driver owes one ask
853 /// per tick.
854 pub fn wants_roster(&self) -> bool {
855 self.rules
856 .iter()
857 .any(|r| matches!(r.rule, Condition::OriginDown { .. }))
858 }
859
860 /// Some rule judges payload validity, so the driver owes a warmed,
861 /// sealed schema store (#337) and a decode per
862 /// [`wants_verdict`](Self::wants_verdict).
863 pub fn wants_decode(&self) -> bool {
864 self.rules
865 .iter()
866 .any(|r| matches!(r.rule, Condition::InvalidPayload { .. }))
867 }
868
869 /// Whether this sample should be decoded before it is observed: an
870 /// `invalid-payload` rule matches its key and the key's per-tick decode
871 /// budget has room. Spends the budget — ask once per sample, then hand
872 /// the verdict to [`observe_sample`](Self::observe_sample). The decode
873 /// stays the driver's, because it is async and this is not.
874 pub fn wants_verdict(&mut self, s: &SampleView) -> bool {
875 let Ok(key) = zenoh::key_expr::KeyExpr::try_from(s.key.as_str()) else {
876 return false;
877 };
878 let matched = self.rules.iter().any(|rt| {
879 matches!(rt.rule, Condition::InvalidPayload { .. })
880 && rt.keyexpr.as_ref().is_some_and(|sel| sel.intersects(&key))
881 });
882 if !matched {
883 return false;
884 }
885 let budget = self.decode_budget.entry(s.key.clone()).or_default();
886 if *budget < DECODE_BUDGET {
887 *budget += 1;
888 true
889 } else {
890 false
891 }
892 }
893
894 /// Count one observed sample against every rule its key matches.
895 /// `verdict` is the decode the driver ran when
896 /// [`wants_verdict`](Self::wants_verdict) said so; `None` means the
897 /// sample was not checked, which is counted as exactly that.
898 pub fn observe_sample(
899 &mut self,
900 s: &SampleView,
901 facts_cache: &mut crate::model::facts::FactsCache,
902 verdict: Option<&crate::Verdict>,
903 ) {
904 let Ok(key) = zenoh::key_expr::KeyExpr::try_from(s.key.as_str()) else {
905 return;
906 };
907 let synthetic = s
908 .attachment
909 .as_ref()
910 .is_some_and(|a| crate::judge::common::is_synthetic_marker(&a.to_bytes()));
911 let now = tokio::time::Instant::now();
912 for rt in self.rules.iter_mut() {
913 let Some(sel) = &rt.keyexpr else { continue };
914 if !sel.intersects(&key) {
915 continue;
916 }
917 rt.counters.samples += 1;
918 if synthetic {
919 rt.counters.synthetic += 1;
920 }
921 rt.last_sample = Some(now);
922 match &rt.rule {
923 Condition::InvalidPayload { .. } => {
924 // An `invalid-payload` rule counts every not-`Valid`
925 // verdict the same way, so with no registry loaded
926 // `NoRegistry` (#246) changes no transition — only the
927 // reason the sample was not validated.
928 if let Some(v) = verdict {
929 rt.counters.checked += 1;
930 if !matches!(v, crate::Verdict::Valid) {
931 rt.counters.invalid += 1;
932 }
933 }
934 }
935 Condition::QosMismatch { .. } => {
936 facts_cache.ensure(self.base, &s.key, self.slices);
937 let facts = facts_cache.get(&s.key).expect("just ensured this key");
938 if let crate::model::facts::Registration::Registered(sf) = &facts.registration
939 && let Some(profile) = sf.declared_qos()
940 {
941 rt.counters.qos_judged += 1;
942 if !s.qos_matches(profile) {
943 rt.counters.qos_mismatched += 1;
944 }
945 }
946 }
947 _ => {}
948 }
949 }
950 }
951
952 /// The stream dropped `n` samples here (RFC 09 §5.1 O6): unattributable
953 /// to any one selector, so it taints every completeness claim this tick.
954 pub fn observe_drop(&mut self, n: u64) {
955 self.dropped_tick += n;
956 self.last_drop = Some(tokio::time::Instant::now());
957 }
958
959 /// Close the tick: judge every rule over the window measured since the
960 /// last evaluation, reset the per-tick counts, and hand back only the
961 /// genuine changes — none for an unchanged rule. `at` is the wall-clock
962 /// stamp the transitions carry.
963 pub fn evaluate(
964 &mut self,
965 now: tokio::time::Instant,
966 at: &str,
967 sweep: SweepOutcome<'_>,
968 ) -> Vec<Transition> {
969 let mut out = Vec::new();
970 for rt in self.rules.iter_mut() {
971 let window = CondWindow {
972 window_s: (now - self.last_eval).as_secs_f64(),
973 observed_s: (now - self.started).as_secs_f64(),
974 samples: rt.counters.samples,
975 dropped: self.dropped_tick,
976 last_sample_ago_s: rt.last_sample.map(|t| (now - t).as_secs_f64()),
977 last_drop_ago_s: self.last_drop.map(|t| (now - t).as_secs_f64()),
978 invalid: rt.counters.invalid,
979 checked: rt.counters.checked,
980 qos_mismatched: rt.counters.qos_mismatched,
981 qos_judged: rt.counters.qos_judged,
982 synthetic: rt.counters.synthetic,
983 };
984 let eval = rt.rule.judge(&TickEvidence {
985 window: &window,
986 doctor: sweep.doctor,
987 roster: sweep.roster,
988 });
989 if let Some(transition) = rt.state.observe(eval, at) {
990 out.push(transition);
991 }
992 }
993 // One reset, over one collection — the four-`Vec` version had a
994 // `counters.fill(..)` that could miss a sibling (#352).
995 for rt in self.rules.iter_mut() {
996 rt.counters = TickCounters::default();
997 }
998 self.dropped_tick = 0;
999 self.decode_budget.clear();
1000 self.ticks += 1;
1001 self.transitions += out.len() as u64;
1002 self.last_eval = now;
1003 out
1004 }
1005
1006 /// Ticks evaluated so far.
1007 pub fn ticks(&self) -> u64 {
1008 self.ticks
1009 }
1010
1011 /// When the last tick closed (construction, before the first): the
1012 /// driver's next deadline is measured from here, so a slow consumer of
1013 /// the transitions widens the next window rather than skipping one.
1014 pub fn last_eval(&self) -> tokio::time::Instant {
1015 self.last_eval
1016 }
1017
1018 /// Transitions emitted so far.
1019 pub fn transitions(&self) -> u64 {
1020 self.transitions
1021 }
1022}
1023
1024/// Watch the rules and yield one [`Transition`] per genuine change, none per
1025/// unchanged tick. The subscriber set is declared before the first window
1026/// opens (O4); every selector rule is judged per tick over the measured
1027/// window, doctor and roster rules by one ask per tick each.
1028///
1029/// A driver over [`RuleSet`] (#218): this function owns the monitor, the
1030/// drain loop and the per-tick sweep; the rules' state is the set's.
1031///
1032/// A [`Straw`] rather than a [`Stream`](futures_core::Stream) (#397), because
1033/// a watchdog run is a sequence **and** a final value: transitions while it
1034/// runs, a [`WatchdogSummary`] when it stops, and the acknowledged monitor
1035/// teardown (#207/#336) in between. A bare `Stream` has room for the first
1036/// only — which is why this stayed a callback through #343, and why the
1037/// callback could not fail: `emit` was infallible by construction, so a
1038/// caller whose emission *could* fail had to stash the error and answer for
1039/// it after the run. Dropping it instead let `zenctl watchdog` finish clean
1040/// having emitted nothing (#360).
1041///
1042/// Drive it with `sip` for the transitions and `await` for the summary:
1043///
1044/// ```ignore
1045/// let mut run = watchdog(&fleet, slices, &store, &spec).pin();
1046/// while let Some(transition) = run.sip().await {
1047/// writeln!(out, "{}", serde_json::to_string(&transition)?)?;
1048/// }
1049/// let summary = run.await?;
1050/// ```
1051///
1052/// The summary is the *output*, not an item, so a consumer that stops sipping
1053/// early and awaits still gets the teardown — there is no `finish` to forget.
1054pub fn watchdog<'a>(
1055 fleet: &'a crate::Fleet<'a>,
1056 slices: Option<&'a SliceSet>,
1057 store: &'a SchemaStore,
1058 spec: &'a WatchdogSpec,
1059) -> impl Straw<WatchdogSummary, Transition, Error> + 'a {
1060 sipper(async move |mut sender: sipper::Sender<Transition>| {
1061 use crate::{FleetEvent, StreamItem};
1062
1063 let (session, base) = (fleet.session(), fleet.base());
1064
1065 // Compiled *before* the monitor exists, so the `?` has nothing to tear
1066 // down (#336).
1067 let mut rules = RuleSet::new(&spec.rules, base, slices)?;
1068 let (wants_doctor, wants_roster, wants_decode) = (
1069 rules.wants_doctor(),
1070 rules.wants_roster(),
1071 rules.wants_decode(),
1072 );
1073
1074 // Warmed before the first tick and sealed for the run (#337): a decode
1075 // inside the drain loop must never become a `describe` GET, because
1076 // nothing attends the broadcast while one is in flight and the tick's
1077 // verdict is about the window that lost the samples. zenctl hands this
1078 // store over cold. Each tick's sweep re-warms whatever is still
1079 // unserved — from beside the drain, where waiting costs nothing.
1080 if wants_decode {
1081 crate::model::decode::prewarm(fleet, store, slices).await;
1082 }
1083 let _sealed = store.seal();
1084
1085 // Declared before the window opens — not-asked must never read as "no".
1086 let monitor = crate::Monitor::start(session, crate::MonitorSpec::default()).await?;
1087 let mut events = monitor.events();
1088 let monitor = monitor.watching(rules.watched()).await?;
1089
1090 // Bounded (#107): the watchdog runs until stopped, so an unbounded
1091 // per-key map here is a leak on any bus with churning keys. Evictions
1092 // ride the summary (O6).
1093 let mut facts_cache = crate::model::facts::FactsCache::default();
1094
1095 let mut closed = false;
1096 loop {
1097 let deadline = rules.last_eval() + spec.tick;
1098 // The tick's bus work runs **beside** the drain, not after it (#338).
1099 //
1100 // A roster GET, a registry sweep, per-producer describes and state
1101 // snapshots take seconds, and every one of them used to happen with
1102 // the drain loop stopped — so the broadcast overflowed, and because
1103 // `dropped_tick` was reset immediately afterwards, the loss was
1104 // billed to the *following* window. In the one tool whose entire
1105 // product is a per-window verdict.
1106 //
1107 // Now the sweep is a future the drain selects on: sampling never
1108 // stops, and a sweep that outlives the tick period simply widens this
1109 // window — `window_s` is measured from the last evaluation, never
1110 // assumed — so the drops land in the tick that incurred them.
1111 let sweep = async {
1112 let doctor = if wants_doctor {
1113 Some(
1114 crate::judge::doctor::run_doctor(
1115 fleet,
1116 slices,
1117 &crate::judge::doctor::DoctorSpec {
1118 deep: false,
1119 sample: None,
1120 timeout: spec.timeout,
1121 listen: None,
1122 },
1123 )
1124 .await
1125 .map_err(|e| e.to_string()),
1126 )
1127 } else {
1128 None
1129 };
1130 let roster = if wants_roster {
1131 Some(
1132 crate::bus::roster::roster(fleet, spec.timeout)
1133 .await
1134 .map_err(|e| e.to_string()),
1135 )
1136 } else {
1137 None
1138 };
1139 // The schema warming rides here too (#337): still-unserved
1140 // producers are re-asked at the store's own backoff, off the
1141 // drain loop.
1142 if wants_decode {
1143 crate::model::decode::prewarm(fleet, store, slices).await;
1144 }
1145 (doctor, roster)
1146 };
1147 let mut sweep = std::pin::pin!(sweep);
1148 let mut swept = None;
1149 // One timer per tick, not one per drained sample (#346).
1150 let tick_over = tokio::time::sleep_until(deadline);
1151 tokio::pin!(tick_over);
1152 while !closed {
1153 let item = tokio::select! {
1154 item = events.recv() => item,
1155 // The tick cannot close before its own sweep has landed, and
1156 // the drain keeps running until it does.
1157 outcome = &mut sweep, if swept.is_none() => {
1158 swept = Some(outcome);
1159 continue;
1160 }
1161 () = &mut tick_over, if swept.is_some() => break,
1162 };
1163 match item {
1164 Some(StreamItem::Event(FleetEvent::Sample(s))) => {
1165 // Decode once per sample (budgeted per key per tick),
1166 // shared by every invalid-payload rule the key matches.
1167 let verdict = if rules.wants_verdict(&s) {
1168 Some(
1169 crate::model::decode::decode_sample(
1170 fleet,
1171 store,
1172 slices,
1173 &s.key,
1174 Some(&s.encoding),
1175 &s.payload.to_bytes(),
1176 )
1177 .await
1178 .verdict,
1179 )
1180 } else {
1181 None
1182 };
1183 rules.observe_sample(&s, &mut facts_cache, verdict.as_ref());
1184 }
1185 Some(StreamItem::Dropped(n)) => rules.observe_drop(n),
1186 Some(_) => {}
1187 None => closed = true,
1188 }
1189 }
1190
1191 // Evaluate the tick over the measured window, then say only what
1192 // changed. The sweep has already landed unless the stream closed
1193 // under it — in which case there is nothing left to drain, and
1194 // awaiting it here costs the tick nothing.
1195 let (doctor_outcome, roster_outcome) = match swept {
1196 Some(outcome) => outcome,
1197 None => sweep.await,
1198 };
1199 let now = tokio::time::Instant::now();
1200 let at = crate::tape::record::rfc3339_now();
1201 let transitions = rules.evaluate(
1202 now,
1203 &at,
1204 SweepOutcome {
1205 doctor: doctor_outcome
1206 .as_ref()
1207 .map(|o| o.as_ref().map_err(String::as_str)),
1208 roster: roster_outcome
1209 .as_ref()
1210 .map(|o| o.as_ref().map_err(String::as_str)),
1211 },
1212 );
1213 for transition in transitions {
1214 // Awaits, where the callback returned: the consumer's write
1215 // now happens *here*, so its error returns from where it
1216 // happened instead of being stashed for after the run (#360).
1217 // The emission point is the tick evaluation — the drain loop
1218 // above has already ended for this tick — so a slow consumer
1219 // widens the next window rather than stalling a drain (#338).
1220 sender.send(transition).await;
1221 }
1222 if closed || spec.ticks.is_some_and(|n| rules.ticks() >= n) {
1223 break;
1224 }
1225 }
1226 monitor.shutdown().await?;
1227 Ok(WatchdogSummary {
1228 ticks: rules.ticks(),
1229 transitions: rules.transitions(),
1230 facts_evicted: facts_cache.evicted(),
1231 })
1232 })
1233}
1234
1235#[cfg(test)]
1236mod tests {
1237 use super::*;
1238 use crate::report::{DoctorFinding, DoctorSeverity};
1239
1240 fn report_with(checks: &[CheckId]) -> DoctorReport {
1241 DoctorReport {
1242 findings: checks
1243 .iter()
1244 .map(|c| DoctorFinding {
1245 severity: DoctorSeverity::Error,
1246 check: *c,
1247 subject: "s".into(),
1248 evidence: "e".into(),
1249 citation: None,
1250 })
1251 .collect(),
1252 synced: crate::report::Asked::NotAsked,
1253 introspect_answered: 0,
1254 live_producers: 0,
1255 describe_served: 0,
1256 describe_missing: 0,
1257 routers: 0,
1258 router_version: None,
1259 deep: false,
1260 observation: None,
1261 }
1262 }
1263
1264 /// Every variant's canonical spelling parses back to itself, and a rule
1265 /// outside the vocabulary is an error that names the vocabulary — closed
1266 /// means closed.
1267 #[test]
1268 fn the_vocabulary_round_trips_and_is_closed() {
1269 let rules = [
1270 "rate-above v1/*/telemetry/** 5",
1271 "rate-below v1/h-aaaaaaaaaaaa/state/p/health 0.5",
1272 "silent-for v1/*/events/** 30",
1273 "invalid-payload v1/*/state/**",
1274 "qos-mismatch v1/*/telemetry/**",
1275 "doctor slice-sync",
1276 "origin-down h-aaaaaaaaaaaa",
1277 "dropped",
1278 ];
1279 for rule in rules {
1280 let parsed = Condition::parse(rule).expect(rule);
1281 assert_eq!(parsed.to_string(), rule, "canonical spelling round-trips");
1282 }
1283 let err = Condition::parse("if rate > 5 then page").unwrap_err();
1284 assert!(err.to_string().contains("closed"), "{err}");
1285 assert!(err.to_string().contains("rate-above"), "{err}");
1286 // A doctor rule outside the stable check-id vocabulary is refused at
1287 // parse, naming the vocabulary.
1288 let err = Condition::parse("doctor no-such-check").unwrap_err();
1289 assert!(err.to_string().contains("slice-sync"), "{err}");
1290 }
1291
1292 /// The acceptance rule of #227: a drop under a completeness claim yields
1293 /// `unobservable`, **never** `ok` — across all three core judges, now
1294 /// spoken in the [`Judgement`] core and projected onto [`CondState`]
1295 /// (RFC 13, v1.24).
1296 #[test]
1297 fn a_drop_under_a_completeness_claim_is_unobservable_never_ok() {
1298 let wire = CondState::from;
1299 // Excess: the "did not exceed" side counts what did not happen.
1300 assert!(judge_excess(false, 1).is_unobservable());
1301 assert_eq!(wire(judge_excess(false, 0)), CondState::Ok);
1302 // …while firing is positive evidence, conclusive under drops.
1303 assert_eq!(judge_excess(true, 7), Judgement::Established);
1304 // Shortfall: the drops could have carried the difference.
1305 assert!(judge_shortfall(true, 1).is_unobservable());
1306 assert_eq!(judge_shortfall(true, 0), Judgement::Established);
1307 // …while "enough seen" is conclusive: a drop only hides more.
1308 assert_eq!(wire(judge_shortfall(false, 9)), CondState::Ok);
1309 // Silence: unprovable over a dropped or unwatched span. Named fields
1310 // rather than three bare `bool`s, which is the whole of #349 — read
1311 // the old spelling `judge_silence(false, true, false)` and say which
1312 // one was the drop.
1313 let silence = |sample_within, span_observed, drop_free| {
1314 judge_silence(SilenceEvidence {
1315 sample_within,
1316 span_observed,
1317 drop_free,
1318 })
1319 };
1320 assert!(silence(false, true, false).is_unobservable());
1321 assert!(silence(false, false, true).is_unobservable());
1322 assert_eq!(silence(false, true, true), Judgement::Established);
1323 assert_eq!(wire(silence(true, true, false)), CondState::Ok);
1324 }
1325
1326 /// The wire projection's documented mapping, polarity note included:
1327 /// `NotEstablished` (established-clean) is `ok`, `Established` (the
1328 /// condition holds) is `firing`, and **both** unestablished poles land
1329 /// on `unobservable` — the wire cannot say more (RFC 13, v1.24).
1330 #[test]
1331 fn cond_state_is_the_documented_projection_of_the_judgement_core() {
1332 assert_eq!(CondState::from(Judgement::Established), CondState::Firing);
1333 assert_eq!(
1334 CondState::from(Judgement::NotEstablished {
1335 reason: "clean".into()
1336 }),
1337 CondState::Ok
1338 );
1339 assert_eq!(
1340 CondState::from(Judgement::NotAsked),
1341 CondState::Unobservable
1342 );
1343 assert_eq!(
1344 CondState::from(Judgement::Unobservable {
1345 reason: "drops".into()
1346 }),
1347 CondState::Unobservable
1348 );
1349 }
1350
1351 /// The window judges apply those rules: `rate-above` firing survives
1352 /// drops, its ok does not; a young watch cannot claim silence.
1353 #[test]
1354 fn window_judgement_applies_the_drop_rules() {
1355 let rule = Condition::parse("rate-above k/** 1").unwrap();
1356 let base = CondWindow {
1357 window_s: 10.0,
1358 observed_s: 10.0,
1359 ..CondWindow::default()
1360 };
1361 let over = CondWindow {
1362 samples: 20,
1363 dropped: 5,
1364 ..base
1365 };
1366 assert_eq!(rule.judge_window(&over).unwrap().state, CondState::Firing);
1367 let under_dropped = CondWindow {
1368 samples: 2,
1369 dropped: 5,
1370 ..base
1371 };
1372 assert_eq!(
1373 rule.judge_window(&under_dropped).unwrap().state,
1374 CondState::Unobservable
1375 );
1376
1377 let rule = Condition::parse("silent-for k/** 30").unwrap();
1378 let young = CondWindow {
1379 window_s: 5.0,
1380 observed_s: 5.0,
1381 ..CondWindow::default()
1382 };
1383 let eval = rule.judge_window(&young).unwrap();
1384 assert_eq!(eval.state, CondState::Unobservable);
1385 assert!(eval.evidence.contains("watched only"), "{}", eval.evidence);
1386 let silent = CondWindow {
1387 window_s: 5.0,
1388 observed_s: 60.0,
1389 ..CondWindow::default()
1390 };
1391 assert_eq!(rule.judge_window(&silent).unwrap().state, CondState::Firing);
1392 let recently_dropped = CondWindow {
1393 last_drop_ago_s: Some(10.0),
1394 ..silent
1395 };
1396 assert_eq!(
1397 rule.judge_window(&recently_dropped).unwrap().state,
1398 CondState::Unobservable
1399 );
1400 let spoken = CondWindow {
1401 samples: 1,
1402 last_sample_ago_s: Some(3.0),
1403 ..silent
1404 };
1405 assert_eq!(rule.judge_window(&spoken).unwrap().state, CondState::Ok);
1406 }
1407
1408 /// The synthetic-traffic marker count (RFC 09 §5.3, the #162 rider)
1409 /// rides every window evidence line when present.
1410 #[test]
1411 fn synthetic_marked_samples_are_said_out_loud() {
1412 let rule = Condition::parse("rate-above k/** 0.1").unwrap();
1413 let w = CondWindow {
1414 window_s: 10.0,
1415 observed_s: 10.0,
1416 samples: 20,
1417 synthetic: 3,
1418 ..CondWindow::default()
1419 };
1420 let eval = rule.judge_window(&w).unwrap();
1421 assert!(
1422 eval.evidence.contains("3 synthetic-marked"),
1423 "{}",
1424 eval.evidence
1425 );
1426 }
1427
1428 /// The transition machine: the first evaluation states the baseline
1429 /// (from `null`), an unchanged tick emits nothing, a genuine change
1430 /// emits exactly one line.
1431 #[test]
1432 fn transitions_fire_once_per_genuine_change_and_never_per_tick() {
1433 let eval = |state| Eval {
1434 state,
1435 evidence: "e".into(),
1436 };
1437 // The condition itself, not its rendering — which is the point of
1438 // #352: the two can no longer disagree.
1439 let mut rs = RuleState::new(Condition::Dropped);
1440 let first = rs.observe(eval(CondState::Ok), "t0").expect("baseline");
1441 assert_eq!(first.rule, "dropped", "the transition renders its rule");
1442 assert_eq!(first.from, None, "the baseline comes from null (O4)");
1443 assert_eq!(first.to, CondState::Ok);
1444 assert!(rs.observe(eval(CondState::Ok), "t1").is_none());
1445 assert!(rs.observe(eval(CondState::Ok), "t2").is_none());
1446 let change = rs.observe(eval(CondState::Firing), "t3").expect("a change");
1447 assert_eq!(change.from, Some(CondState::Ok));
1448 assert_eq!(change.to, CondState::Firing);
1449 assert!(rs.observe(eval(CondState::Firing), "t4").is_none());
1450 }
1451
1452 /// The ndjson shape of a transition is a wire contract for scripts:
1453 /// `{"rule","from","to","at","evidence"}`, states snake_case, `from`
1454 /// null on the baseline.
1455 #[test]
1456 fn transition_json_shape_is_pinned() {
1457 let t = Transition {
1458 rule: "silent-for k/** 30".into(),
1459 from: None,
1460 to: CondState::Unobservable,
1461 at: "2026-08-22T00:00:00Z".into(),
1462 evidence: "watched only 5.0s of a 30.0s silence claim".into(),
1463 };
1464 assert_eq!(
1465 serde_json::to_value(&t).unwrap(),
1466 serde_json::json!({
1467 "rule": "silent-for k/** 30",
1468 "from": null,
1469 "to": "unobservable",
1470 "at": "2026-08-22T00:00:00Z",
1471 "evidence": "watched only 5.0s of a 30.0s silence claim",
1472 })
1473 );
1474 let t = Transition {
1475 from: Some(CondState::Ok),
1476 to: CondState::Firing,
1477 ..t
1478 };
1479 let json = serde_json::to_value(&t).unwrap();
1480 assert_eq!(json["from"], "ok");
1481 assert_eq!(json["to"], "firing");
1482 }
1483
1484 /// `doctor --transitions`'s delta: the first run is a full baseline (every
1485 /// stable check id, once), an identical second run says nothing, a new
1486 /// finding transitions exactly its check — and a failed run flips every
1487 /// check to unobservable, never ok.
1488 #[test]
1489 fn doctor_watch_reports_deltas_not_states() {
1490 let mut watch = DoctorWatch::new();
1491 let clean = report_with(&[]);
1492 let baseline = watch.observe(Ok(&clean), "t0");
1493 assert_eq!(baseline.len(), CheckId::ALL.len());
1494 assert!(baseline.iter().all(|t| t.from.is_none()));
1495 assert!(baseline.iter().all(|t| t.to == CondState::Ok));
1496
1497 assert!(
1498 watch.observe(Ok(&clean), "t1").is_empty(),
1499 "an unchanged run emits nothing"
1500 );
1501
1502 let drifted = report_with(&[CheckId::SchemaDrift, CheckId::SchemaDrift]);
1503 let changes = watch.observe(Ok(&drifted), "t2");
1504 assert_eq!(changes.len(), 1, "only the changed check transitions");
1505 assert_eq!(changes[0].rule, "doctor schema-drift");
1506 assert_eq!(changes[0].to, CondState::Firing);
1507 assert!(changes[0].evidence.contains("2 finding(s)"));
1508
1509 let failed = watch.observe(Err("session lost"), "t3");
1510 assert_eq!(
1511 failed.len(),
1512 CheckId::ALL.len(),
1513 "a failed run is unobservable for every check — never ok"
1514 );
1515 assert!(failed.iter().all(|t| t.to == CondState::Unobservable));
1516 }
1517}