1use crate::events::{Event, EventKind};
8use crate::gate::GateVerdict;
9use crate::standards_coverage::{standards_coverage, RuleDisposition};
10use serde::{Deserialize, Serialize};
11use std::collections::{BTreeMap, BTreeSet, HashMap};
12
13pub const MINIMUM_SAMPLES: u64 = 5;
14pub const HIGH_WAIVER_SHARE: f64 = 0.30;
15pub const NEAR_CONSTANT_SCORE_RANGE: f64 = 0.01;
16pub const NEAR_THRESHOLD_DISTANCE: f64 = 0.10;
17
18#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
19#[serde(rename_all = "camelCase")]
20pub struct StandardsMetricsReport {
21 pub minimum_samples: u64,
22 pub definitions: Vec<String>,
23 pub rules: Vec<RuleMetrics>,
24}
25
26#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
27#[serde(rename_all = "camelCase")]
28pub struct RuleMetrics {
29 pub id: String,
30 pub revision: u64,
31 pub statement: String,
32 pub checker: Option<String>,
33 pub lifecycles: Vec<String>,
34 pub applicable_missions: u64,
35 pub evaluated_missions: u64,
36 pub advisory_missions: u64,
37 pub failed_missions: u64,
38 pub blocked_missions: u64,
39 pub waived_missions: u64,
40 pub not_evaluated_missions: u64,
41 pub false_green_missions: u64,
42 pub evaluation_rate: Option<f64>,
43 pub advisory_rate: Option<f64>,
44 pub failure_rate: Option<f64>,
45 pub block_rate: Option<f64>,
46 pub waiver_rate: Option<f64>,
47 pub mean_resolution_ms: Option<f64>,
48 pub score_distribution: Option<RuleScoreDistribution>,
49 pub conclusions_suppressed: bool,
50 pub smells: Vec<RuleMetricSmell>,
51}
52
53#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
54#[serde(rename_all = "camelCase")]
55pub struct RuleScoreDistribution {
56 pub samples: u64,
57 pub minimum: f64,
58 pub maximum: f64,
59 pub mean: f64,
60 pub near_threshold: u64,
61}
62
63#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
64#[serde(rename_all = "camelCase")]
65pub struct RuleMetricSmell {
66 pub kind: String,
67 pub observed: String,
68 pub definition: String,
69 pub samples: u64,
70}
71
72#[derive(Default)]
73struct Accumulator {
74 statement: String,
75 checker: Option<String>,
76 lifecycles: BTreeSet<String>,
77 applicable: u64,
78 evaluated: u64,
79 advisory: u64,
80 failed: u64,
81 blocked: u64,
82 waived: u64,
83 not_evaluated: u64,
84 false_greens: u64,
85 scores: Vec<(f64, f64)>,
86 resolutions_ms: Vec<u64>,
87}
88
89fn rate(numerator: u64, denominator: u64) -> Option<f64> {
90 (denominator > 0).then(|| numerator as f64 / denominator as f64)
91}
92
93fn mission_resolution_times(events: &[Event]) -> HashMap<(String, u64), Vec<u64>> {
94 let mut open: HashMap<(String, u64), chrono::DateTime<chrono::Utc>> = HashMap::new();
95 let mut durations: HashMap<(String, u64), Vec<u64>> = HashMap::new();
96 let mut revisions: HashMap<String, u64> = HashMap::new();
97 for event in events {
98 match &event.kind {
99 EventKind::PlanApproved { plan, .. } => {
100 revisions.clear();
101 if let Some(pin) = plan.standards_manifest.as_deref() {
102 revisions.extend(
103 pin.rules
104 .iter()
105 .map(|rule| (rule.id.clone(), rule.revision)),
106 );
107 }
108 }
109 EventKind::GateResult {
110 verdict, rule_ids, ..
111 } => {
112 for id in rule_ids {
113 let Some(revision) = revisions.get(id) else {
114 continue;
115 };
116 let key = (id.clone(), *revision);
117 match verdict {
118 GateVerdict::Fail => {
119 open.entry(key).or_insert(event.ts);
120 }
121 GateVerdict::Pass => {
122 if let Some(started) = open.remove(&key) {
123 let millis = (event.ts - started).num_milliseconds().max(0) as u64;
124 durations.entry(key).or_default().push(millis);
125 }
126 }
127 }
128 }
129 }
130 EventKind::ValidationFinding { finding, .. } => {
131 if let Some(rule) = &finding.rule {
132 open.entry((rule.id.clone(), rule.revision))
133 .or_insert(event.ts);
134 }
135 }
136 EventKind::StandardsWaiverApproved {
137 rule_id,
138 rule_revision,
139 ..
140 } => {
141 let key = (rule_id.clone(), *rule_revision);
142 if let Some(started) = open.remove(&key) {
143 let millis = (event.ts - started).num_milliseconds().max(0) as u64;
144 durations.entry(key).or_default().push(millis);
145 }
146 }
147 _ => {}
148 }
149 }
150 durations
151}
152
153pub fn aggregate(
154 missions: &[(String, Vec<Event>)],
155 traced_defects: &[crate::escalation_metrics::TracedDefect],
156) -> StandardsMetricsReport {
157 let false_green_missions: BTreeSet<&str> = traced_defects
158 .iter()
159 .map(|defect| defect.mission_id.as_str())
160 .collect();
161 let mut rows: BTreeMap<(String, u64), Accumulator> = BTreeMap::new();
162
163 for (mission_id, events) in missions {
164 let Some(coverage) = standards_coverage(mission_id, events) else {
165 continue;
166 };
167 let completed = events
168 .iter()
169 .any(|event| matches!(event.kind, EventKind::MissionCompleted {}));
170 let resolutions = mission_resolution_times(events);
171 let blocked_ids: BTreeSet<(String, u64)> = events
172 .iter()
173 .filter_map(|event| match &event.kind {
174 EventKind::ValidationFinding { finding, .. }
175 if finding.class == "standards-authoritative" =>
176 {
177 finding
178 .rule
179 .as_ref()
180 .map(|rule| (rule.id.clone(), rule.revision))
181 }
182 _ => None,
183 })
184 .collect();
185 let mut scores: HashMap<(String, u64), Vec<(f64, f64)>> = HashMap::new();
186 for event in events {
187 if let EventKind::GateResult {
188 score: Some(score),
189 threshold: Some(threshold),
190 rule_ids,
191 ..
192 } = &event.kind
193 {
194 if !score.is_finite() || !threshold.is_finite() {
195 continue;
196 }
197 for id in rule_ids {
198 if let Some(rule) = coverage.rules.iter().find(|rule| {
199 rule.id == *id && rule.disposition != RuleDisposition::NotApplicable
200 }) {
201 scores
202 .entry((id.clone(), rule.revision))
203 .or_default()
204 .push((*score, *threshold));
205 }
206 }
207 }
208 }
209
210 for rule in coverage
211 .rules
212 .iter()
213 .filter(|rule| rule.disposition != RuleDisposition::NotApplicable)
214 {
215 let key = (rule.id.clone(), rule.revision);
216 let row = rows.entry(key.clone()).or_default();
217 if row.statement.is_empty() || rule.statement < row.statement {
218 row.statement = rule.statement.clone();
219 }
220 if let Some(checker) = &rule.checker {
221 if row.checker.as_ref().is_none_or(|current| checker < current) {
222 row.checker = Some(checker.clone());
223 }
224 }
225 row.lifecycles.insert(rule.lifecycle.clone());
226 row.applicable += 1;
227 match rule.disposition {
228 RuleDisposition::Passed => row.evaluated += 1,
229 RuleDisposition::Failed => {
230 row.evaluated += 1;
231 row.failed += 1;
232 }
233 RuleDisposition::Advisory => {
234 row.evaluated += 1;
235 row.advisory += 1;
236 }
237 RuleDisposition::Waived => {
238 row.evaluated += 1;
239 row.waived += 1;
240 }
241 RuleDisposition::NotEvaluated => row.not_evaluated += 1,
242 RuleDisposition::NotApplicable => unreachable!(),
243 }
244 if blocked_ids.contains(&key) {
245 row.blocked += 1;
246 }
247 if completed
248 && false_green_missions.contains(mission_id.as_str())
249 && matches!(
250 rule.disposition,
251 RuleDisposition::Passed | RuleDisposition::Waived
252 )
253 {
254 row.false_greens += 1;
255 }
256 row.scores.extend(scores.remove(&key).unwrap_or_default());
257 row.resolutions_ms
258 .extend(resolutions.get(&key).cloned().unwrap_or_default());
259 }
260 }
261
262 let rules = rows
263 .into_iter()
264 .map(|((id, revision), row)| {
265 let score_distribution = if row.scores.is_empty() {
266 None
267 } else {
268 let minimum = row
269 .scores
270 .iter()
271 .map(|(score, _)| *score)
272 .fold(f64::INFINITY, f64::min);
273 let maximum = row
274 .scores
275 .iter()
276 .map(|(score, _)| *score)
277 .fold(f64::NEG_INFINITY, f64::max);
278 Some(RuleScoreDistribution {
279 samples: row.scores.len() as u64,
280 minimum,
281 maximum,
282 mean: row.scores.iter().map(|(score, _)| score).sum::<f64>()
283 / row.scores.len() as f64,
284 near_threshold: row
285 .scores
286 .iter()
287 .filter(|(score, threshold)| {
288 (*score - *threshold).abs() <= NEAR_THRESHOLD_DISTANCE
289 })
290 .count() as u64,
291 })
292 };
293 let mut smells = Vec::new();
294 if row.applicable >= MINIMUM_SAMPLES && row.evaluated == 0 {
295 smells.push(RuleMetricSmell {
296 kind: "never-selected".to_string(),
297 observed: format!("0 of {} applicable missions produced checker evidence", row.applicable),
298 definition: "Flagged when a rule is applicable in at least the minimum sample count but its checker is never selected/evaluated.".to_string(),
299 samples: row.applicable,
300 });
301 }
302 if row.evaluated >= MINIMUM_SAMPLES && row.failed == row.evaluated {
303 smells.push(RuleMetricSmell {
304 kind: "always-fail".to_string(),
305 observed: format!("{} of {} evaluations failed", row.failed, row.evaluated),
306 definition: "Flagged when every evaluated mission's latest rule disposition is failed.".to_string(),
307 samples: row.evaluated,
308 });
309 }
310 if row.evaluated >= MINIMUM_SAMPLES
311 && rate(row.waived, row.evaluated).is_some_and(|share| share >= HIGH_WAIVER_SHARE)
312 {
313 smells.push(RuleMetricSmell {
314 kind: "high-waiver".to_string(),
315 observed: format!("{} of {} evaluations were waived", row.waived, row.evaluated),
316 definition: format!("Flagged when waivers are at least {:.0}% of evaluated missions.", HIGH_WAIVER_SHARE * 100.0),
317 samples: row.evaluated,
318 });
319 }
320 if let Some(scores) = &score_distribution {
321 if scores.samples >= MINIMUM_SAMPLES
322 && scores.maximum - scores.minimum <= NEAR_CONSTANT_SCORE_RANGE
323 {
324 smells.push(RuleMetricSmell {
325 kind: "near-constant-score".to_string(),
326 observed: format!("score range {:.4}", scores.maximum - scores.minimum),
327 definition: format!("Flagged when at least {MINIMUM_SAMPLES} scores span no more than {NEAR_CONSTANT_SCORE_RANGE:.2}."),
328 samples: scores.samples,
329 });
330 }
331 if scores.samples >= MINIMUM_SAMPLES && scores.near_threshold == 0 {
332 smells.push(RuleMetricSmell {
333 kind: "never-near-threshold".to_string(),
334 observed: format!("0 of {} scores were within {:.2} of threshold", scores.samples, NEAR_THRESHOLD_DISTANCE),
335 definition: "Flagged when no sufficiently-sampled score approaches its gate-owned threshold; the threshold may not discriminate.".to_string(),
336 samples: scores.samples,
337 });
338 }
339 }
340 RuleMetrics {
341 id,
342 revision,
343 statement: row.statement,
344 checker: row.checker,
345 lifecycles: row.lifecycles.into_iter().collect(),
346 applicable_missions: row.applicable,
347 evaluated_missions: row.evaluated,
348 advisory_missions: row.advisory,
349 failed_missions: row.failed,
350 blocked_missions: row.blocked,
351 waived_missions: row.waived,
352 not_evaluated_missions: row.not_evaluated,
353 false_green_missions: row.false_greens,
354 evaluation_rate: rate(row.evaluated, row.applicable),
355 advisory_rate: rate(row.advisory, row.evaluated),
356 failure_rate: rate(row.failed, row.evaluated),
357 block_rate: rate(row.blocked, row.applicable),
358 waiver_rate: rate(row.waived, row.evaluated),
359 mean_resolution_ms: (!row.resolutions_ms.is_empty()).then(|| {
360 row.resolutions_ms
361 .iter()
362 .map(|millis| *millis as f64)
363 .sum::<f64>()
364 / row.resolutions_ms.len() as f64
365 }),
366 score_distribution,
367 conclusions_suppressed: if row.evaluated == 0 {
368 row.applicable < MINIMUM_SAMPLES
369 } else {
370 row.evaluated < MINIMUM_SAMPLES
371 },
372 smells,
373 }
374 })
375 .collect();
376
377 StandardsMetricsReport {
378 minimum_samples: MINIMUM_SAMPLES,
379 definitions: vec![
380 "applicable = the stable rule/revision appears in a mission's approval pin"
381 .to_string(),
382 "evaluated = rule-linked gate/finding evidence exists; absence remains not-evaluated"
383 .to_string(),
384 "block = an authoritative standards finding interrupted a mission, even if later repaired"
385 .to_string(),
386 "false green = a completed mission with passed/waived rule evidence later received a traced-from-mission defect ticket"
387 .to_string(),
388 format!("interpretive smells are suppressed below {MINIMUM_SAMPLES} samples; raw counts are never suppressed"),
389 ],
390 rules,
391 }
392}
393
394pub fn compute(repo_root: &std::path::Path) -> crate::error::Result<StandardsMetricsReport> {
395 let mut missions = Vec::new();
396 for id in crate::paths::MissionPaths::list_missions(repo_root) {
397 let path = crate::paths::MissionPaths::new(repo_root, &id).events_file();
398 if !path.is_file() {
399 continue;
400 }
401 let events = crate::event_log::EventLog::read_events(&path)?;
402 missions.push((id, events));
403 }
404 Ok(aggregate(
405 &missions,
406 &crate::escalation_metrics::traced_defects_from_tickets(repo_root),
407 ))
408}
409
410#[cfg(test)]
411mod tests {
412 use super::*;
413 use crate::gate::{GateKind, GateSurface};
414 use crate::types::{
415 MissionConfig, PinnedRule, Plan, RuleCitation, StandardsPin, StandardsPinSource,
416 };
417
418 fn event(seq: u64, seconds: i64, mission: &str, kind: EventKind) -> Event {
419 Event {
420 seq,
421 ts: chrono::DateTime::from_timestamp(1_800_000_000 + seconds, 0).unwrap(),
422 mission_id: mission.to_string(),
423 kind,
424 }
425 }
426
427 fn rule() -> PinnedRule {
428 PinnedRule {
429 id: "ZZ-METRIC-001".to_string(),
430 revision: 2,
431 rfc: "RFC-001".to_string(),
432 level: "must".to_string(),
433 effective_status: "enforced".to_string(),
434 statement: "Metric rule statement.".to_string(),
435 domains: Vec::new(),
436 stages: vec!["validation".to_string()],
437 when_paths: Vec::new(),
438 task_classes: Vec::new(),
439 checker: Some("gate:metric".to_string()),
440 waivable: true,
441 }
442 }
443
444 fn mission(id: &str, index: u64, disposition: &str) -> (String, Vec<Event>) {
445 let rule = rule();
446 let pin = StandardsPin {
447 pack_name: "zz".to_string(),
448 pack_dir: "vendor/zz".to_string(),
449 standards_root: "standards".to_string(),
450 digest: "ab".repeat(32),
451 source: StandardsPinSource::RepoTracked,
452 task_class: None,
453 touch_set: Vec::new(),
454 context_paths: Vec::new(),
455 gates: Vec::new(),
456 rules: vec![rule.clone()],
457 };
458 let plan = Plan {
459 goal: "g".to_string(),
460 validation_contract: Vec::new(),
461 milestones: Vec::new(),
462 considered_alternatives: None,
463 command_grants: Vec::new(),
464 touch_set: Vec::new(),
465 standards_manifest: Some(Box::new(pin.clone())),
466 reviewer_independence: None,
467 };
468 let mut events = vec![
469 event(
470 1,
471 index as i64 * 100,
472 id,
473 EventKind::MissionCreated {
474 goal: "g".to_string(),
475 base_branch: "main".to_string(),
476 mission_branch: format!("kranz/{id}"),
477 config: MissionConfig::default(),
478 },
479 ),
480 event(
481 2,
482 index as i64 * 100 + 1,
483 id,
484 EventKind::PlanApproved {
485 plan,
486 base_sha: Some("deadbeef".to_string()),
487 },
488 ),
489 ];
490 let verdict = if disposition == "pass" {
491 GateVerdict::Pass
492 } else {
493 GateVerdict::Fail
494 };
495 events.push(event(
496 3,
497 index as i64 * 100 + 2,
498 id,
499 EventKind::GateResult {
500 gate: "metric".to_string(),
501 surface: GateSurface::FinalGate,
502 kind: GateKind::Deterministic,
503 index: 0,
504 verdict,
505 artefact_ref: "inline".to_string(),
506 artefact_detail: None,
507 score: Some(0.9),
508 threshold: Some(0.5),
509 rule_ids: vec![rule.id.clone()],
510 },
511 ));
512 if disposition != "pass" {
513 events.push(event(
514 4,
515 index as i64 * 100 + 3,
516 id,
517 EventKind::ValidationFinding {
518 milestone_id: "ms-1".to_string(),
519 run_id: crate::reducer::ENGINE_RUN_ID.to_string(),
520 finding: crate::types::Finding {
521 subject: format!("flight-rule:{}", rule.id),
522 severity: "critical".to_string(),
523 evidence: "failed".to_string(),
524 suggested_fix: String::new(),
525 class: "standards-authoritative".to_string(),
526 rule: Some(RuleCitation {
527 id: rule.id.clone(),
528 revision: rule.revision,
529 source: "zz standards".to_string(),
530 digest: pin.digest.clone(),
531 lifecycle: "enforced".to_string(),
532 level: "must".to_string(),
533 checker: rule.checker.clone(),
534 }),
535 },
536 },
537 ));
538 }
539 if disposition == "waived" {
540 events.push(event(
541 5,
542 index as i64 * 100 + 12,
543 id,
544 EventKind::StandardsWaiverApproved {
545 rule_id: rule.id.clone(),
546 rule_revision: rule.revision,
547 manifest_digest: pin.digest,
548 approval_seq: 2,
549 finding_fingerprint: crate::standards_waiver::finding_fingerprint(
550 crate::reducer::ENGINE_RUN_ID,
551 match &events[3].kind {
552 EventKind::ValidationFinding { finding, .. } => finding,
553 _ => unreachable!(),
554 },
555 ),
556 paths: Vec::new(),
557 diff_digest: "cd".repeat(32),
558 reason: "reviewed exception".to_string(),
559 approver: crate::standards_waiver::LOCAL_OPERATOR.to_string(),
560 surface: "cli".to_string(),
561 expires_at: chrono::DateTime::from_timestamp(1_900_000_000, 0).unwrap(),
562 },
563 ));
564 }
565 if disposition == "pass" {
566 events.push(event(
567 4,
568 index as i64 * 100 + 3,
569 id,
570 EventKind::MissionCompleted {},
571 ));
572 }
573 (id.to_string(), events)
574 }
575
576 #[test]
577 fn flight_rules_metrics_keeps_revision_counts_denominators_and_false_greens_honest() {
578 let mut missions = vec![
579 mission("m-1", 0, "pass"),
580 mission("m-2", 1, "pass"),
581 mission("m-3", 2, "fail"),
582 mission("m-4", 3, "waived"),
583 mission("m-5", 4, "pass"),
584 ];
585 let (revision_id, mut revision_events) = mission("m-6", 5, "pass");
586 let EventKind::PlanApproved { plan, .. } = &mut revision_events[1].kind else {
587 unreachable!();
588 };
589 plan.standards_manifest.as_mut().unwrap().rules[0].revision = 3;
590 missions.push((revision_id, revision_events));
591 let report = aggregate(
592 &missions,
593 &[crate::escalation_metrics::TracedDefect {
594 ticket: "defect-one".to_string(),
595 mission_id: "m-1".to_string(),
596 }],
597 );
598 let row = &report.rules[0];
599 assert_eq!((row.id.as_str(), row.revision), ("ZZ-METRIC-001", 2));
600 assert_eq!(row.applicable_missions, 5);
601 assert_eq!(row.evaluated_missions, 5);
602 assert_eq!(row.failed_missions, 1);
603 assert_eq!(row.blocked_missions, 2);
604 assert_eq!(row.waived_missions, 1);
605 assert_eq!(row.false_green_missions, 1);
606 assert_eq!(row.failure_rate, Some(0.2));
607 assert_eq!(row.waiver_rate, Some(0.2));
608 assert_eq!(row.mean_resolution_ms, Some(10_000.0));
609 assert!(!row.conclusions_suppressed);
610 assert!(row
611 .smells
612 .iter()
613 .any(|smell| smell.kind == "near-constant-score"));
614 assert!(row
615 .smells
616 .iter()
617 .any(|smell| smell.kind == "never-near-threshold"));
618 assert_eq!(report.rules.len(), 2);
619 assert_eq!(report.rules[1].revision, 3);
620 assert_eq!(report.rules[1].applicable_missions, 1);
621 assert_eq!(report.rules[1].evaluated_missions, 1);
622 }
623
624 #[test]
625 fn flight_rules_metrics_suppresses_small_sample_conclusions_not_raw_counts() {
626 let report = aggregate(&[mission("m-1", 0, "fail")], &[]);
627 let row = &report.rules[0];
628 assert_eq!(row.applicable_missions, 1);
629 assert_eq!(row.failed_missions, 1);
630 assert!(row.conclusions_suppressed);
631 assert!(row.smells.is_empty());
632 }
633
634 #[test]
635 fn flight_rules_metrics_no_evidence_is_not_evaluated_never_green() {
636 let (id, mut events) = mission("m-1", 0, "pass");
637 events.retain(|event| !matches!(event.kind, EventKind::GateResult { .. }));
638 let report = aggregate(&[(id, events)], &[]);
639 let row = &report.rules[0];
640 assert_eq!(row.evaluated_missions, 0);
641 assert_eq!(row.not_evaluated_missions, 1);
642 assert_eq!(row.evaluation_rate, Some(0.0));
643 assert_eq!(row.failure_rate, None);
644 }
645}