1pub mod reasons;
15
16#[cfg(test)]
17#[path = "outcomes/reasons_tests.rs"]
18mod reasons_tests;
19
20use crate::events::{Event, EventKind};
21use chrono::{DateTime, Utc};
22use serde::{Deserialize, Serialize};
23
24#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
25#[serde(rename_all = "camelCase")]
26pub struct AutonomyRatio {
27 pub closed_missions: u64,
28 pub total_interventions: u64,
29 pub interventions_per_closed_mission: f64,
30 pub zero_intervention_missions: u64,
31 pub zero_intervention_share: f64,
32}
33
34#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
35#[serde(rename_all = "camelCase")]
36pub struct LatencyBucket {
37 pub label: String,
38 pub count: u64,
39}
40
41#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
42#[serde(rename_all = "camelCase")]
43pub struct GrantLatency {
44 pub buckets: Vec<LatencyBucket>,
45 pub total_decided: u64,
46}
47
48#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
49#[serde(rename_all = "lowercase")]
50pub enum EscalationKind {
51 Block,
52 Grant,
53 Revision,
54}
55
56impl EscalationKind {
57 pub fn as_str(&self) -> &'static str {
59 match self {
60 Self::Block => "block",
61 Self::Grant => "grant",
62 Self::Revision => "revision",
63 }
64 }
65}
66
67#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
68#[serde(rename_all = "camelCase")]
69pub struct EscalationRow {
70 pub ts: DateTime<Utc>,
71 pub mission_id: String,
72 pub kind: EscalationKind,
73 pub summary: String,
74 pub decision: String,
75 pub latency_ms: Option<u64>,
76 #[serde(default)]
83 pub rubber_stamp: Option<bool>,
84}
85
86#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
96#[serde(rename_all = "camelCase")]
97pub struct DivergenceOutcomes {
98 pub noted: u64,
101 pub diverged: u64,
103 pub agreed: u64,
106 pub resolved_selected: u64,
110 pub resolved_none: u64,
113}
114
115#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
116#[serde(rename_all = "camelCase")]
117pub struct Outcomes {
118 #[serde(default, skip_serializing_if = "Option::is_none")]
120 pub outcome_reasons: Option<reasons::Report>,
121 pub autonomy_ratio: AutonomyRatio,
122 pub grant_latency: GrantLatency,
123 pub escalations: Vec<EscalationRow>,
124 pub cost_per_change: CostPerChange,
126 pub cycle_time: CycleTime,
128 #[serde(default)]
134 pub task_classes: Vec<TaskClassRow>,
135 #[serde(default)]
140 pub context_reuse: Vec<ContextReuseRow>,
141 #[serde(default)]
144 pub rubber_stamp: RubberStampReport,
145 #[serde(default)]
156 pub gate_score_flags: crate::gate_score_flags::GateScoreFlagsReport,
157 #[serde(default, skip_serializing_if = "Option::is_none")]
162 pub divergences: Option<DivergenceOutcomes>,
163 #[serde(default, skip_serializing_if = "Option::is_none")]
172 pub comparison: Option<crate::comparison_metrics::ComparisonReport>,
173}
174
175#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
177#[serde(rename_all = "camelCase")]
178pub struct TaskClassRow {
179 pub task_class: String,
182 pub missions: u64,
183 pub closed_missions: u64,
184 pub total_cost_usd: f64,
187 pub non_meta_commits: u64,
188 pub usd_per_commit: Option<f64>,
191 pub escalations: u64,
193 pub advisor_invocations: u64,
195 pub escalations_per_mission: f64,
197 pub cycle_mean_ms: Option<f64>,
200}
201
202pub const UNCLASSIFIED_TASK_CLASS: &str = "unclassified";
205
206#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
212#[serde(rename_all = "camelCase")]
213pub struct ContextReuseRow {
214 pub backend: String,
217 pub missions: u64,
219 pub runs: u64,
221 pub fresh_input: u64,
223 pub cache_read: u64,
225 pub cache_write: Option<u64>,
228 pub reuse_share: Option<f64>,
231}
232
233#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
237#[serde(rename_all = "camelCase", default)]
238pub struct RubberStampReport {
239 pub threshold_ms: u64,
242 pub approved_decisions: u64,
244 pub flagged: u64,
247 pub share: Option<f64>,
249}
250
251impl Default for RubberStampReport {
252 fn default() -> Self {
255 Self {
256 threshold_ms: crate::types::DEFAULT_RUBBER_STAMP_THRESHOLD_MS,
257 approved_decisions: 0,
258 flagged: 0,
259 share: None,
260 }
261 }
262}
263
264#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
265#[serde(rename_all = "camelCase")]
266pub struct CostPerChange {
267 pub total_cost_usd: f64,
268 pub non_meta_commits: u64,
272 pub usd_per_commit: Option<f64>,
275}
276
277#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
278#[serde(rename_all = "camelCase")]
279pub struct CycleTime {
280 pub closed_missions: u64,
282 pub total_ms: u64,
283 pub mean_ms: Option<f64>,
285}
286
287#[derive(Debug, Clone, PartialEq)]
290pub struct MissionOutcomes {
291 pub outcome_reasons: reasons::MissionReasons,
292 pub interventions: u64,
293 pub is_closed: bool,
294 pub latencies_ms: Vec<u64>,
295 pub escalations: Vec<EscalationRow>,
296 pub cost_usd: f64,
299 pub non_meta_commits: u64,
300 pub cycle_time_ms: Option<u64>,
302 pub task_class: Option<String>,
306 pub token_sums: Vec<BackendTokenSum>,
309 pub divergences: Option<DivergenceOutcomes>,
313 pub gate_score_samples: Vec<crate::gate_score_flags::GateScoreSample>,
318 pub comparison: ComparisonInputs,
326}
327
328#[derive(Debug, Clone, PartialEq)]
332pub struct ComparisonInputs {
333 pub terminal_ts: Option<DateTime<Utc>>,
337 pub base_branch: Option<String>,
342 pub folded: Option<FoldedMissionRefs>,
349}
350
351#[derive(Debug, Clone, PartialEq)]
354pub struct FoldedMissionRefs {
355 pub status: crate::types::MissionStatus,
356 pub base_branch: String,
357 pub mission_branch: String,
358}
359
360#[derive(Debug, Clone, Copy, PartialEq)]
363pub struct BackendTokenSum {
364 pub backend: crate::types::BackendKind,
365 pub runs: u64,
366 pub fresh_input: u64,
368 pub cache_read: u64,
369 pub cache_write: u64,
370}
371
372const BUCKET_LABELS: [&str; 4] = ["<10s", "<60s", "<10m", ">=10m"];
374
375#[derive(Clone)]
387struct CachedMission {
388 len: u64,
389 mtime: std::time::SystemTime,
390 outcomes: MissionOutcomes,
391 computes: u64,
392 hits: u64,
393}
394
395static MISSION_CACHE: std::sync::OnceLock<
396 std::sync::Mutex<std::collections::HashMap<std::path::PathBuf, CachedMission>>,
397> = std::sync::OnceLock::new();
398
399#[cfg(test)]
401fn cache_entry_stats(events_path: &std::path::Path) -> Option<(u64, u64)> {
402 MISSION_CACHE
403 .get()?
404 .lock()
405 .ok()?
406 .get(events_path)
407 .map(|c| (c.computes, c.hits))
408}
409
410pub(crate) fn cached_mission_outcomes(
417 mission_id: &str,
418 events_path: &std::path::Path,
419) -> Option<MissionOutcomes> {
420 let meta = std::fs::metadata(events_path).ok()?;
421 let (len, mtime) = (meta.len(), meta.modified().ok()?);
422 let cache =
423 MISSION_CACHE.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()));
424 {
425 let mut guard = cache.lock().ok()?;
426 if let Some(hit) = guard.get_mut(events_path) {
427 if hit.len == len && hit.mtime == mtime {
428 hit.hits += 1;
429 return Some(hit.outcomes.clone());
430 }
431 }
432 }
433 let events = crate::event_log::EventLog::read_events(events_path).ok()?;
434 let outcomes = mission_outcomes(mission_id, &events);
435 if let Ok(mut guard) = cache.lock() {
436 guard
437 .entry(events_path.to_path_buf())
438 .and_modify(|entry| {
439 entry.len = len;
440 entry.mtime = mtime;
441 entry.outcomes = outcomes.clone();
442 entry.computes += 1;
443 })
444 .or_insert_with(|| CachedMission {
445 len,
446 mtime,
447 outcomes: outcomes.clone(),
448 computes: 1,
449 hits: 0,
450 });
451 }
452 Some(outcomes)
453}
454
455pub fn mission_outcomes(mission_id: &str, events: &[Event]) -> MissionOutcomes {
460 let filtered: std::borrow::Cow<'_, [Event]> =
461 if events.iter().all(|e| e.mission_id == mission_id) {
462 std::borrow::Cow::Borrowed(events)
463 } else {
464 std::borrow::Cow::Owned(
465 events
466 .iter()
467 .filter(|e| e.mission_id == mission_id)
468 .cloned()
469 .collect(),
470 )
471 };
472 let events = filtered.as_ref();
473 let mission_events: Vec<&Event> = events.iter().collect();
474
475 let is_closed = mission_events.iter().any(|e| {
476 matches!(
477 e.kind,
478 EventKind::MissionCompleted {}
479 | EventKind::MissionFailed { .. }
480 | EventKind::MissionAbandoned { .. }
481 )
482 });
483
484 let plan_approved_ts = mission_events
485 .iter()
486 .find(|e| matches!(e.kind, EventKind::PlanApproved { .. }))
487 .map(|e| e.ts);
488
489 let mut interventions: u64 = 0;
490 for e in &mission_events {
491 match &e.kind {
492 EventKind::UserMessage { .. } => {
493 if let Some(approved_ts) = plan_approved_ts {
494 if e.ts >= approved_ts {
495 interventions += 1;
496 }
497 }
498 }
499 EventKind::GrantApproved { .. }
500 | EventKind::GrantDenied { .. }
501 | EventKind::PlanRevised { .. }
502 | EventKind::PlanRevisionRejected { .. } => {
503 interventions += 1;
504 }
505 _ => {}
506 }
507 }
508
509 let mut latencies_ms = Vec::new();
510 let mut escalations = Vec::new();
511
512 let mut used_decisions = vec![false; mission_events.len()];
516 for (req_idx, req) in mission_events.iter().enumerate() {
517 let EventKind::GrantRequested { command, .. } = &req.kind else {
518 continue;
519 };
520
521 let mut matched: Option<usize> = None;
522 for (i, cand) in mission_events.iter().enumerate() {
523 if i <= req_idx || used_decisions[i] {
524 continue;
525 }
526 let cand_command = match &cand.kind {
527 EventKind::GrantApproved { command, .. }
528 | EventKind::GrantDenied { command, .. } => command,
529 _ => continue,
530 };
531 if cand_command == command {
532 matched = Some(i);
533 break;
534 }
535 }
536 if matched.is_none() {
537 for (i, cand) in mission_events.iter().enumerate() {
542 if i <= req_idx || used_decisions[i] {
543 continue;
544 }
545 if matches!(
546 cand.kind,
547 EventKind::GrantApproved { .. } | EventKind::GrantDenied { .. }
548 ) {
549 matched = Some(i);
550 break;
551 }
552 }
553 }
554
555 let (decision, latency_ms) = match matched {
556 Some(i) => {
557 used_decisions[i] = true;
558 let decided = mission_events[i];
559 let latency = (decided.ts - req.ts).num_milliseconds();
560 let latency_ms = if latency >= 0 {
561 Some(latency as u64)
562 } else {
563 None
564 };
565 if let Some(l) = latency_ms {
566 latencies_ms.push(l);
567 }
568 let decision = match &decided.kind {
569 EventKind::GrantApproved { .. } => "approved".to_string(),
570 EventKind::GrantDenied { reason, .. } => format!("denied: {reason}"),
571 _ => unreachable!(),
572 };
573 (decision, latency_ms)
574 }
575 None => ("pending".to_string(), None),
576 };
577
578 escalations.push(EscalationRow {
579 ts: req.ts,
580 mission_id: mission_id.to_string(),
581 kind: EscalationKind::Grant,
582 summary: command.clone(),
583 decision,
584 latency_ms,
585 rubber_stamp: None,
587 });
588 }
589
590 let mut used_unblocks = vec![false; mission_events.len()];
593 for (idx, e) in mission_events.iter().enumerate() {
594 let EventKind::MilestoneBlocked {
595 milestone_id,
596 reason,
597 ..
598 } = &e.kind
599 else {
600 continue;
601 };
602 let mut matched: Option<usize> = None;
603 for (i, cand) in mission_events.iter().enumerate() {
604 if i <= idx || used_unblocks[i] {
605 continue;
606 }
607 if let EventKind::MilestoneUnblocked {
608 milestone_id: mid, ..
609 } = &cand.kind
610 {
611 if mid == milestone_id {
612 matched = Some(i);
613 break;
614 }
615 }
616 }
617 let decision = match matched {
618 Some(i) => {
619 used_unblocks[i] = true;
620 let EventKind::MilestoneUnblocked {
621 reason: unblock_reason,
622 ..
623 } = &mission_events[i].kind
624 else {
625 unreachable!()
626 };
627 format!("unblocked: {unblock_reason}")
628 }
629 None => "open".to_string(),
630 };
631 escalations.push(EscalationRow {
632 ts: e.ts,
633 mission_id: mission_id.to_string(),
634 kind: EscalationKind::Block,
635 summary: reason.clone(),
636 decision,
637 latency_ms: None,
638 rubber_stamp: None,
639 });
640 }
641
642 for (idx, e) in mission_events.iter().enumerate() {
645 let EventKind::PlanRevisionProposed {
646 revision,
647 instructions,
648 ..
649 } = &e.kind
650 else {
651 continue;
652 };
653 let mut decision = "pending".to_string();
654 for cand in mission_events.iter().skip(idx + 1) {
655 match &cand.kind {
656 EventKind::PlanRevised { revision: rev, .. } if rev == revision => {
657 decision = format!("accepted (rev {rev})");
658 break;
659 }
660 EventKind::PlanRevisionRejected {
661 revision: rev,
662 reason,
663 } if rev == revision => {
664 decision = format!("rejected: {reason}");
665 break;
666 }
667 _ => {}
668 }
669 }
670 escalations.push(EscalationRow {
671 ts: e.ts,
672 mission_id: mission_id.to_string(),
673 kind: EscalationKind::Revision,
674 summary: instructions.clone(),
675 decision,
676 latency_ms: None,
677 rubber_stamp: None,
678 });
679 }
680
681 let mut non_meta_commits: u64 = 0;
685 for e in &mission_events {
686 if let EventKind::FeatureCompleted { commits, .. } = &e.kind {
687 for commit in commits {
688 let subject = commit.split_once(' ').map(|(_, s)| s).unwrap_or("");
689 if !crate::contract_sweep::is_meta_commit(subject) {
690 non_meta_commits += 1;
691 }
692 }
693 }
694 }
695
696 let config = mission_events.iter().find_map(|e| match &e.kind {
699 EventKind::MissionCreated { config, .. } => Some(config),
700 _ => None,
701 });
702 let task_class = mission_events.iter().find_map(|e| match &e.kind {
705 EventKind::MissionCreated { goal, .. } => crate::ticket::parse_task_class_from_goal(goal),
706 _ => None,
707 });
708 let mut run_models = std::collections::HashMap::new();
709 for e in &mission_events {
710 if let EventKind::WorkerSpawned {
711 run_id,
712 role,
713 model,
714 backend,
715 ..
716 } = &e.kind
717 {
718 run_models.insert(run_id.as_str(), (model.as_str(), *role, *backend));
719 }
720 }
721 let mut cost_usd = 0.0;
722 let mut token_sums: std::collections::BTreeMap<&'static str, BackendTokenSum> =
726 std::collections::BTreeMap::new();
727 for e in &mission_events {
728 if let EventKind::WorkerCompleted {
729 run_id,
730 tokens,
731 cost_usd: recorded,
732 ..
733 } = &e.kind
734 {
735 let (model, role, recorded_backend) = run_models
736 .get(run_id.as_str())
737 .copied()
738 .unwrap_or(("", crate::types::Role::Worker, None));
739 let backend = crate::cost::resolved_run_backend(recorded_backend, role, config);
740 cost_usd += crate::cost::resolved_run_cost(*recorded, tokens, model, backend);
741 let sum = token_sums
742 .entry(backend.as_str())
743 .or_insert(BackendTokenSum {
744 backend,
745 runs: 0,
746 fresh_input: 0,
747 cache_read: 0,
748 cache_write: 0,
749 });
750 sum.runs += 1;
751 sum.fresh_input += tokens.input;
752 sum.cache_read += tokens.cache_read;
753 sum.cache_write += tokens.cache_write;
754 }
755 }
756
757 let created_ts = mission_events
760 .iter()
761 .find(|e| matches!(e.kind, EventKind::MissionCreated { .. }))
762 .map(|e| e.ts);
763 let terminal_ts = mission_events.iter().find_map(|e| {
764 matches!(
765 e.kind,
766 EventKind::MissionCompleted {}
767 | EventKind::MissionFailed { .. }
768 | EventKind::MissionAbandoned { .. }
769 )
770 .then_some(e.ts)
771 });
772 let cycle_time_ms = match (created_ts, terminal_ts) {
773 (Some(start), Some(end)) => {
774 let mut paused_ms: i64 = 0;
775 let mut pause_start: Option<DateTime<Utc>> = None;
776 for e in &mission_events {
777 match &e.kind {
778 EventKind::MissionPaused {} => pause_start = Some(e.ts),
779 EventKind::MissionResumed {} => {
780 if let Some(p) = pause_start.take() {
781 paused_ms += (e.ts - p).num_milliseconds().max(0);
782 }
783 }
784 _ => {}
785 }
786 }
787 if let Some(p) = pause_start {
788 paused_ms += (end - p).num_milliseconds().max(0);
789 }
790 Some(((end - start).num_milliseconds() - paused_ms).max(0) as u64)
791 }
792 _ => None,
793 };
794
795 let mut noted: u64 = 0;
802 let mut diverged: u64 = 0;
803 let mut resolved_units: std::collections::HashSet<&str> = std::collections::HashSet::new();
804 let mut resolved_selected: u64 = 0;
805 let mut resolved_none: u64 = 0;
806 for e in &mission_events {
807 match &e.kind {
808 EventKind::DivergenceNoted { diverged: d, .. } => {
809 noted += 1;
810 if *d {
811 diverged += 1;
812 }
813 }
814 EventKind::DivergenceResolved { unit, selected, .. } => {
815 let first_for_unit = resolved_units.insert(unit.as_str());
816 match (first_for_unit, selected) {
817 (true, Some(_)) => resolved_selected += 1,
818 (true, None) => resolved_none += 1,
819 (false, _) => {}
820 }
821 }
822 _ => {}
823 }
824 }
825 let divergences = (noted > 0 || !resolved_units.is_empty()).then(|| DivergenceOutcomes {
826 noted,
827 diverged,
828 agreed: noted - diverged,
829 resolved_selected,
830 resolved_none,
831 });
832
833 let gate_score_samples = crate::gate_score_flags::collect_scored_samples(&mission_events);
838
839 let comparison = ComparisonInputs {
848 terminal_ts,
849 base_branch: mission_events.iter().find_map(|e| match &e.kind {
850 EventKind::MissionCreated { base_branch, .. } => Some(base_branch.clone()),
851 _ => None,
852 }),
853 folded: crate::reducer::fold(events)
854 .ok()
855 .map(|state| FoldedMissionRefs {
856 status: state.mission.status,
857 base_branch: state.mission.base_branch,
858 mission_branch: state.mission.mission_branch,
859 }),
860 };
861
862 let outcome_reasons = reasons::mission(
863 mission_id,
864 &mission_events,
865 comparison.folded.as_ref().map(|s| s.status),
866 task_class.as_deref(),
867 );
868 MissionOutcomes {
869 outcome_reasons,
870 interventions,
871 is_closed,
872 latencies_ms,
873 escalations,
874 cost_usd,
875 non_meta_commits,
876 cycle_time_ms,
877 task_class,
878 token_sums: token_sums.into_values().collect(),
879 divergences,
880 gate_score_samples,
881 comparison,
882 }
883}
884
885#[derive(Default)]
887struct TaskClassAcc {
888 missions: u64,
889 closed_missions: u64,
890 total_cost_usd: f64,
891 non_meta_commits: u64,
892 escalations: u64,
893 advisor_invocations: u64,
894 cycle_count: u64,
895 cycle_total_ms: u64,
896}
897
898struct ReuseAcc {
900 backend: crate::types::BackendKind,
901 missions: u64,
902 runs: u64,
903 fresh_input: u64,
904 cache_read: u64,
905 cache_write: u64,
906}
907
908#[derive(Debug, Clone, Copy, PartialEq, Eq)]
912pub struct OutcomesOptions {
913 pub reason_window: Option<(u64, DateTime<Utc>)>,
915 pub rubber_stamp_threshold_ms: u64,
918 pub comparison_window: Option<(u64, DateTime<Utc>)>,
926}
927
928impl Default for OutcomesOptions {
929 fn default() -> Self {
930 Self {
931 rubber_stamp_threshold_ms: crate::types::DEFAULT_RUBBER_STAMP_THRESHOLD_MS,
932 comparison_window: None,
933 reason_window: None,
934 }
935 }
936}
937
938impl OutcomesOptions {
939 pub fn resolve(repo_root: &std::path::Path) -> Self {
944 match crate::config::load(repo_root) {
945 Ok(cfg) => Self {
946 rubber_stamp_threshold_ms: cfg.rubber_stamp_threshold_ms,
947 ..Self::default()
948 },
949 Err(_) => Self::default(),
950 }
951 .with_comparison_window()
952 }
953
954 fn with_comparison_window(mut self) -> Self {
958 self.comparison_window = Some((DEFAULT_MERGED_CHANGE_WINDOW_DAYS, Utc::now()));
959 self.reason_window = self.comparison_window;
960 self
961 }
962}
963
964pub fn compute_outcomes(repo_root: &std::path::Path) -> anyhow::Result<Outcomes> {
971 compute_outcomes_with_options(repo_root, &OutcomesOptions::resolve(repo_root))
972}
973
974pub fn compute_outcomes_with_options(
977 repo_root: &std::path::Path,
978 options: &OutcomesOptions,
979) -> anyhow::Result<Outcomes> {
980 let index_contents = std::fs::read_to_string(
981 crate::paths::MissionPaths::new(repo_root, "_")
982 .missions_dir()
983 .join("index.md"),
984 )
985 .unwrap_or_default();
986
987 let mut ids = crate::paths::MissionPaths::list_missions(repo_root);
988 for id in crate::mission_catalog::mission_index_ids(&index_contents) {
989 if !ids.contains(&id) {
990 ids.push(id);
991 }
992 }
993 ids.sort();
994
995 let mut closed_missions: u64 = 0;
996 let mut total_interventions: u64 = 0;
997 let mut zero_intervention_missions: u64 = 0;
998 let mut all_latencies_ms = Vec::new();
999 let mut escalations = Vec::new();
1000 let mut total_cost_usd = 0.0;
1001 let mut total_non_meta_commits: u64 = 0;
1002 let mut cycle_closed: u64 = 0;
1003 let mut cycle_total_ms: u64 = 0;
1004 let mut class_accs: std::collections::BTreeMap<String, TaskClassAcc> =
1007 std::collections::BTreeMap::new();
1008 let mut reuse_accs: std::collections::BTreeMap<&'static str, ReuseAcc> =
1010 std::collections::BTreeMap::new();
1011 let mut divergence_acc: Option<DivergenceOutcomes> = None;
1014 let mut all_score_samples: Vec<crate::gate_score_flags::GateScoreSample> = Vec::new();
1017 let mut comparison_inputs: Vec<(String, ComparisonInputs)> = Vec::new();
1021 let mut reason_missions = Vec::new();
1022 let mut unavailable_logs = Vec::new();
1023
1024 for id in ids {
1025 let paths = crate::paths::MissionPaths::new(repo_root, &id);
1026 let events_path = paths.events_file();
1027 if !events_path.is_file() {
1028 unavailable_logs.push(id);
1029 continue;
1030 }
1031 if paths.require_no_follow().is_err() {
1034 unavailable_logs.push(id);
1035 continue;
1036 }
1037 let Some(out) = cached_mission_outcomes(&id, &events_path) else {
1041 unavailable_logs.push(id);
1042 continue;
1043 };
1044 if out.outcome_reasons.latest_event_seq.is_some() {
1045 reason_missions.push(out.outcome_reasons.clone());
1046 } else {
1047 unavailable_logs.push(id.clone());
1048 }
1049 comparison_inputs.push((id.clone(), out.comparison.clone()));
1050 if out.is_closed {
1051 closed_missions += 1;
1052 total_interventions += out.interventions;
1053 if out.interventions == 0 {
1054 zero_intervention_missions += 1;
1055 }
1056 }
1057 all_latencies_ms.extend(out.latencies_ms);
1058 total_cost_usd += out.cost_usd;
1059 total_non_meta_commits += out.non_meta_commits;
1060 if let Some(ms) = out.cycle_time_ms {
1061 cycle_closed += 1;
1062 cycle_total_ms += ms;
1063 }
1064
1065 let class_key = out
1067 .task_class
1068 .clone()
1069 .unwrap_or_else(|| UNCLASSIFIED_TASK_CLASS.to_string());
1070 let acc = class_accs.entry(class_key).or_default();
1071 acc.missions += 1;
1072 if out.is_closed {
1073 acc.closed_missions += 1;
1074 }
1075 acc.total_cost_usd += out.cost_usd;
1076 acc.non_meta_commits += out.non_meta_commits;
1077 if let Some(ms) = out.cycle_time_ms {
1078 acc.cycle_count += 1;
1079 acc.cycle_total_ms += ms;
1080 }
1081 acc.escalations += out.escalations.len() as u64;
1082 acc.advisor_invocations += out
1083 .escalations
1084 .iter()
1085 .filter(|r| r.kind == EscalationKind::Grant)
1086 .count() as u64;
1087
1088 for sum in &out.token_sums {
1090 let acc = reuse_accs
1091 .entry(sum.backend.as_str())
1092 .or_insert_with(|| ReuseAcc {
1093 backend: sum.backend,
1094 missions: 0,
1095 runs: 0,
1096 fresh_input: 0,
1097 cache_read: 0,
1098 cache_write: 0,
1099 });
1100 acc.missions += 1;
1101 acc.runs += sum.runs;
1102 acc.fresh_input += sum.fresh_input;
1103 acc.cache_read += sum.cache_read;
1104 acc.cache_write += sum.cache_write;
1105 }
1106
1107 if let Some(d) = &out.divergences {
1109 let acc = divergence_acc.get_or_insert_with(DivergenceOutcomes::default);
1110 acc.noted += d.noted;
1111 acc.diverged += d.diverged;
1112 acc.agreed += d.agreed;
1113 acc.resolved_selected += d.resolved_selected;
1114 acc.resolved_none += d.resolved_none;
1115 }
1116
1117 all_score_samples.extend(out.gate_score_samples);
1118
1119 escalations.extend(out.escalations);
1120 }
1121
1122 let interventions_per_closed_mission = if closed_missions > 0 {
1123 total_interventions as f64 / closed_missions as f64
1124 } else {
1125 0.0
1126 };
1127 let zero_intervention_share = if closed_missions > 0 {
1128 zero_intervention_missions as f64 / closed_missions as f64
1129 } else {
1130 0.0
1131 };
1132
1133 escalations.sort_by_key(|e| std::cmp::Reverse(e.ts));
1134
1135 let mut approved_decisions: u64 = 0;
1140 let mut flagged: u64 = 0;
1141 for row in &mut escalations {
1142 if row.kind != EscalationKind::Grant || row.decision != "approved" {
1143 continue;
1144 }
1145 let Some(latency) = row.latency_ms else {
1146 continue;
1147 };
1148 approved_decisions += 1;
1149 let is_flagged = latency < options.rubber_stamp_threshold_ms;
1150 if is_flagged {
1151 flagged += 1;
1152 }
1153 row.rubber_stamp = Some(is_flagged);
1154 }
1155
1156 let mut task_classes: Vec<TaskClassRow> = class_accs
1159 .into_iter()
1160 .map(|(task_class, acc)| TaskClassRow {
1161 escalations_per_mission: acc.escalations as f64 / acc.missions as f64,
1162 usd_per_commit: (acc.non_meta_commits > 0)
1163 .then(|| acc.total_cost_usd / acc.non_meta_commits as f64),
1164 cycle_mean_ms: (acc.cycle_count > 0)
1165 .then(|| acc.cycle_total_ms as f64 / acc.cycle_count as f64),
1166 task_class,
1167 missions: acc.missions,
1168 closed_missions: acc.closed_missions,
1169 total_cost_usd: acc.total_cost_usd,
1170 non_meta_commits: acc.non_meta_commits,
1171 escalations: acc.escalations,
1172 advisor_invocations: acc.advisor_invocations,
1173 })
1174 .collect();
1175 task_classes.sort_by_key(|row| {
1176 (
1177 row.task_class == UNCLASSIFIED_TASK_CLASS,
1178 row.task_class.clone(),
1179 )
1180 });
1181
1182 let context_reuse: Vec<ContextReuseRow> = reuse_accs
1186 .into_values()
1187 .filter(|acc| acc.backend.reports_cache_read_tokens())
1188 .map(|acc| {
1189 let cache_write = acc
1190 .backend
1191 .reports_cache_write_tokens()
1192 .then_some(acc.cache_write);
1193 let cached = acc.cache_read + cache_write.unwrap_or(0);
1194 let total = acc.fresh_input + cached;
1195 ContextReuseRow {
1196 backend: acc.backend.as_str().to_string(),
1197 missions: acc.missions,
1198 runs: acc.runs,
1199 fresh_input: acc.fresh_input,
1200 cache_read: acc.cache_read,
1201 cache_write,
1202 reuse_share: (total > 0).then(|| cached as f64 / total as f64),
1203 }
1204 })
1205 .collect();
1206
1207 let comparison = options
1215 .comparison_window
1216 .map(|(window_days, now)| {
1217 crate::comparison_metrics::comparison_report_from_inputs(
1218 repo_root,
1219 &comparison_inputs,
1220 window_days,
1221 now,
1222 )
1223 })
1224 .transpose()?;
1225
1226 Ok(Outcomes {
1227 outcome_reasons: Some(reasons::report(
1228 reason_missions,
1229 unavailable_logs,
1230 options.reason_window,
1231 )?),
1232 autonomy_ratio: AutonomyRatio {
1233 closed_missions,
1234 total_interventions,
1235 interventions_per_closed_mission,
1236 zero_intervention_missions,
1237 zero_intervention_share,
1238 },
1239 grant_latency: bucketize(&all_latencies_ms),
1240 escalations,
1241 cost_per_change: CostPerChange {
1242 total_cost_usd,
1243 non_meta_commits: total_non_meta_commits,
1244 usd_per_commit: (total_non_meta_commits > 0)
1245 .then(|| total_cost_usd / total_non_meta_commits as f64),
1246 },
1247 cycle_time: CycleTime {
1248 closed_missions: cycle_closed,
1249 total_ms: cycle_total_ms,
1250 mean_ms: (cycle_closed > 0).then(|| cycle_total_ms as f64 / cycle_closed as f64),
1251 },
1252 task_classes,
1253 context_reuse,
1254 rubber_stamp: RubberStampReport {
1255 threshold_ms: options.rubber_stamp_threshold_ms,
1256 approved_decisions,
1257 flagged,
1258 share: (approved_decisions > 0).then(|| flagged as f64 / approved_decisions as f64),
1259 },
1260 gate_score_flags: crate::gate_score_flags::score_distribution_report(&all_score_samples),
1261 divergences: divergence_acc,
1262 comparison,
1263 })
1264}
1265
1266pub const DEFAULT_MERGED_CHANGE_WINDOW_DAYS: u64 = 30;
1270
1271pub const MAX_MERGED_CHANGE_WINDOW_DAYS: u64 = 36_525;
1278
1279#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1285#[serde(rename_all = "camelCase")]
1286pub struct CostPerMergedChange {
1287 pub window_days: u64,
1289 pub closed_in_window: u64,
1292 pub total_cost_usd: f64,
1295 pub merged_changes: u64,
1297 pub usd_per_merged_change: Option<f64>,
1300 pub zero_intervention_share: Option<f64>,
1303}
1304
1305pub fn compute_cost_per_merged_change(
1314 repo_root: &std::path::Path,
1315 window_days: u64,
1316 now: DateTime<Utc>,
1317) -> anyhow::Result<CostPerMergedChange> {
1318 let index_contents = std::fs::read_to_string(
1319 crate::paths::MissionPaths::new(repo_root, "_")
1320 .missions_dir()
1321 .join("index.md"),
1322 )
1323 .unwrap_or_default();
1324
1325 let mut ids = crate::paths::MissionPaths::list_missions(repo_root);
1326 for id in crate::mission_catalog::mission_index_ids(&index_contents) {
1327 if !ids.contains(&id) {
1328 ids.push(id);
1329 }
1330 }
1331 ids.sort();
1332
1333 if window_days > MAX_MERGED_CHANGE_WINDOW_DAYS {
1340 return Err(crate::error::EngineError::InvalidState(format!(
1341 "window_days {window_days} exceeds the maximum {MAX_MERGED_CHANGE_WINDOW_DAYS} days"
1342 ))
1343 .into());
1344 }
1345 let days = i64::try_from(window_days).map_err(|_| {
1346 crate::error::EngineError::InvalidState(format!(
1347 "window_days {window_days} is out of range"
1348 ))
1349 })?;
1350 let window = chrono::Duration::try_days(days).ok_or_else(|| {
1351 crate::error::EngineError::InvalidState(format!(
1352 "window_days {window_days} is out of range"
1353 ))
1354 })?;
1355 let cutoff = now - window;
1356 let repo = crate::git_ops::GitRepo::open(repo_root).ok();
1357
1358 let mut closed_in_window: u64 = 0;
1359 let mut zero_intervention: u64 = 0;
1360 let mut total_cost_usd = 0.0;
1361 let mut merged_changes: u64 = 0;
1362
1363 for id in ids {
1364 let paths = crate::paths::MissionPaths::new(repo_root, &id);
1365 let events_path = paths.events_file();
1366 if !events_path.is_file() {
1367 continue;
1368 }
1369 if paths.require_no_follow().is_err() {
1370 continue;
1371 }
1372 let events = match crate::event_log::EventLog::read_events(&events_path) {
1373 Ok(events) => events,
1374 Err(_) => continue, };
1376 let Some(terminal_ts) = events.iter().find_map(|e| {
1379 matches!(
1380 e.kind,
1381 EventKind::MissionCompleted {}
1382 | EventKind::MissionFailed { .. }
1383 | EventKind::MissionAbandoned { .. }
1384 )
1385 .then_some(e.ts)
1386 }) else {
1387 continue; };
1389 if terminal_ts < cutoff || terminal_ts > now {
1390 continue;
1391 }
1392 let out = mission_outcomes(&id, &events);
1393 closed_in_window += 1;
1394 if out.interventions == 0 {
1395 zero_intervention += 1;
1396 }
1397 total_cost_usd += out.cost_usd;
1398
1399 if let (Some(repo), Some(folded)) = (repo.as_ref(), out.comparison.folded.as_ref()) {
1405 if folded.status == crate::types::MissionStatus::Complete
1406 && crate::merged::merged_bit_for_branches(
1407 repo,
1408 &folded.mission_branch,
1409 &folded.base_branch,
1410 ) == Some(true)
1411 {
1412 merged_changes += 1;
1413 }
1414 }
1415 }
1416
1417 Ok(CostPerMergedChange {
1418 window_days,
1419 closed_in_window,
1420 total_cost_usd,
1421 merged_changes,
1422 usd_per_merged_change: (merged_changes > 0).then(|| total_cost_usd / merged_changes as f64),
1423 zero_intervention_share: (closed_in_window > 0)
1424 .then(|| zero_intervention as f64 / closed_in_window as f64),
1425 })
1426}
1427
1428pub fn bucketize(latencies_ms: &[u64]) -> GrantLatency {
1431 let mut counts = [0u64; 4];
1432 for &latency in latencies_ms {
1433 let idx = if latency < 10_000 {
1434 0
1435 } else if latency < 60_000 {
1436 1
1437 } else if latency < 600_000 {
1438 2
1439 } else {
1440 3
1441 };
1442 counts[idx] += 1;
1443 }
1444 let buckets = BUCKET_LABELS
1445 .iter()
1446 .zip(counts)
1447 .map(|(label, count)| LatencyBucket {
1448 label: label.to_string(),
1449 count,
1450 })
1451 .collect();
1452 GrantLatency {
1453 buckets,
1454 total_decided: latencies_ms.len() as u64,
1455 }
1456}
1457
1458#[cfg(test)]
1459mod tests {
1460 use super::*;
1461 use crate::types::GrantKind;
1462
1463 fn ev(seq: u64, mission_id: &str, ts_secs: i64, kind: EventKind) -> Event {
1464 Event {
1465 seq,
1466 ts: DateTime::from_timestamp(ts_secs, 0).unwrap(),
1467 mission_id: mission_id.to_string(),
1468 kind,
1469 }
1470 }
1471
1472 fn ev_ms(seq: u64, mission_id: &str, ts_ms: i64, kind: EventKind) -> Event {
1473 Event {
1474 seq,
1475 ts: DateTime::from_timestamp_millis(ts_ms).unwrap(),
1476 mission_id: mission_id.to_string(),
1477 kind,
1478 }
1479 }
1480
1481 #[test]
1482 fn outcomes_latency_bucket_boundaries() {
1483 let latencies = vec![9_999, 10_000, 59_999, 60_000, 599_999, 600_000];
1484 let result = bucketize(&latencies);
1485 assert_eq!(result.total_decided, 6);
1486 assert_eq!(result.buckets.len(), 4);
1487 assert_eq!(result.buckets[0].label, "<10s");
1488 assert_eq!(result.buckets[0].count, 1); assert_eq!(result.buckets[1].label, "<60s");
1490 assert_eq!(result.buckets[1].count, 2); assert_eq!(result.buckets[2].label, "<10m");
1492 assert_eq!(result.buckets[2].count, 2); assert_eq!(result.buckets[3].label, ">=10m");
1494 assert_eq!(result.buckets[3].count, 1); }
1496
1497 #[test]
1498 fn outcomes_latency_empty_fills_all_zero_buckets() {
1499 let result = bucketize(&[]);
1500 assert_eq!(result.total_decided, 0);
1501 assert_eq!(result.buckets.len(), 4);
1502 assert!(result.buckets.iter().all(|b| b.count == 0));
1503 }
1504
1505 #[test]
1506 fn interventions_ignore_pre_approval_messages_but_count_post_approval() {
1507 let events = vec![
1508 ev(
1509 1,
1510 "m-1",
1511 100,
1512 EventKind::UserMessage {
1513 text: "before".into(),
1514 interrupt: false,
1515 },
1516 ),
1517 ev(
1518 2,
1519 "m-1",
1520 200,
1521 EventKind::PlanApproved {
1522 plan: sample_plan(),
1523 base_sha: None,
1524 },
1525 ),
1526 ev(
1527 3,
1528 "m-1",
1529 300,
1530 EventKind::UserMessage {
1531 text: "after".into(),
1532 interrupt: false,
1533 },
1534 ),
1535 ];
1536 let out = mission_outcomes("m-1", &events);
1537 assert_eq!(out.interventions, 1);
1538 }
1539
1540 #[test]
1541 fn interventions_count_each_decision_kind() {
1542 let events = vec![
1543 ev(
1544 1,
1545 "m-1",
1546 100,
1547 EventKind::PlanApproved {
1548 plan: sample_plan(),
1549 base_sha: None,
1550 },
1551 ),
1552 ev(
1553 2,
1554 "m-1",
1555 200,
1556 EventKind::GrantApproved {
1557 kind: GrantKind::Command,
1558 command: "cargo test".into(),
1559 },
1560 ),
1561 ev(
1562 3,
1563 "m-1",
1564 300,
1565 EventKind::GrantDenied {
1566 kind: GrantKind::Command,
1567 command: "rm -rf".into(),
1568 reason: "no".into(),
1569 },
1570 ),
1571 ev(
1572 4,
1573 "m-1",
1574 400,
1575 EventKind::PlanRevised {
1576 revision: 1,
1577 plan: sample_plan(),
1578 },
1579 ),
1580 ev(
1581 5,
1582 "m-1",
1583 500,
1584 EventKind::PlanRevisionRejected {
1585 revision: 2,
1586 reason: "bad".into(),
1587 },
1588 ),
1589 ];
1590 let out = mission_outcomes("m-1", &events);
1591 assert_eq!(out.interventions, 4);
1592 }
1593
1594 #[test]
1595 fn interventions_no_plan_approved_counts_zero_user_messages() {
1596 let events = vec![ev(
1597 1,
1598 "m-1",
1599 100,
1600 EventKind::UserMessage {
1601 text: "hi".into(),
1602 interrupt: false,
1603 },
1604 )];
1605 let out = mission_outcomes("m-1", &events);
1606 assert_eq!(out.interventions, 0);
1607 }
1608
1609 #[test]
1610 fn is_closed_true_for_each_terminal_event() {
1611 for kind in [
1612 EventKind::MissionCompleted {},
1613 EventKind::MissionFailed { reason: "x".into() },
1614 EventKind::MissionAbandoned { reason: "x".into() },
1615 ] {
1616 let events = vec![ev(1, "m-1", 100, kind)];
1617 let out = mission_outcomes("m-1", &events);
1618 assert!(out.is_closed);
1619 }
1620 }
1621
1622 #[test]
1623 fn is_closed_false_without_terminal_event() {
1624 let events = vec![ev(
1625 1,
1626 "m-1",
1627 100,
1628 EventKind::PlanApproved {
1629 plan: sample_plan(),
1630 base_sha: None,
1631 },
1632 )];
1633 let out = mission_outcomes("m-1", &events);
1634 assert!(!out.is_closed);
1635 }
1636
1637 #[test]
1638 fn escalation_grant_row_approved_denied_pending() {
1639 let events = vec![
1640 ev_ms(
1641 1,
1642 "m-1",
1643 0,
1644 EventKind::GrantRequested {
1645 milestone_id: "ms-1".into(),
1646 kind: GrantKind::Command,
1647 command: "cargo test".into(),
1648 },
1649 ),
1650 ev_ms(
1651 2,
1652 "m-1",
1653 5_000,
1654 EventKind::GrantApproved {
1655 kind: GrantKind::Command,
1656 command: "cargo test".into(),
1657 },
1658 ),
1659 ev_ms(
1660 3,
1661 "m-1",
1662 10_000,
1663 EventKind::GrantRequested {
1664 milestone_id: "ms-1".into(),
1665 kind: GrantKind::Command,
1666 command: "rm -rf".into(),
1667 },
1668 ),
1669 ev_ms(
1670 4,
1671 "m-1",
1672 15_000,
1673 EventKind::GrantDenied {
1674 kind: GrantKind::Command,
1675 command: "rm -rf".into(),
1676 reason: "unsafe".into(),
1677 },
1678 ),
1679 ev_ms(
1680 5,
1681 "m-1",
1682 20_000,
1683 EventKind::GrantRequested {
1684 milestone_id: "ms-1".into(),
1685 kind: GrantKind::Command,
1686 command: "still pending".into(),
1687 },
1688 ),
1689 ];
1690 let out = mission_outcomes("m-1", &events);
1691 let grants: Vec<_> = out
1692 .escalations
1693 .iter()
1694 .filter(|r| r.kind == EscalationKind::Grant)
1695 .collect();
1696 assert_eq!(grants.len(), 3);
1697 assert_eq!(grants[0].summary, "cargo test");
1698 assert_eq!(grants[0].decision, "approved");
1699 assert_eq!(grants[0].latency_ms, Some(5_000));
1700 assert_eq!(grants[1].summary, "rm -rf");
1701 assert_eq!(grants[1].decision, "denied: unsafe");
1702 assert_eq!(grants[1].latency_ms, Some(5_000));
1703 assert_eq!(grants[2].summary, "still pending");
1704 assert_eq!(grants[2].decision, "pending");
1705 assert_eq!(grants[2].latency_ms, None);
1706 }
1707
1708 #[test]
1709 fn escalation_block_row_open_and_unblocked() {
1710 let events = vec![
1711 ev(
1712 1,
1713 "m-1",
1714 0,
1715 EventKind::MilestoneBlocked {
1716 block_context: None,
1717 milestone_id: "ms-1".into(),
1718 reason: "waiting".into(),
1719 },
1720 ),
1721 ev(
1722 2,
1723 "m-1",
1724 10,
1725 EventKind::MilestoneUnblocked {
1726 block_context: None,
1727 milestone_id: "ms-1".into(),
1728 reason: "cap raised".into(),
1729 validator_guidance: None,
1730 },
1731 ),
1732 ev(
1733 3,
1734 "m-1",
1735 20,
1736 EventKind::MilestoneBlocked {
1737 block_context: None,
1738 milestone_id: "ms-2".into(),
1739 reason: "still stuck".into(),
1740 },
1741 ),
1742 ];
1743 let out = mission_outcomes("m-1", &events);
1744 let blocks: Vec<_> = out
1745 .escalations
1746 .iter()
1747 .filter(|r| r.kind == EscalationKind::Block)
1748 .collect();
1749 assert_eq!(blocks.len(), 2);
1750 assert_eq!(blocks[0].summary, "waiting");
1751 assert_eq!(blocks[0].decision, "unblocked: cap raised");
1752 assert_eq!(blocks[1].summary, "still stuck");
1753 assert_eq!(blocks[1].decision, "open");
1754 }
1755
1756 #[test]
1757 fn escalation_revision_row_accepted_rejected_pending() {
1758 let events = vec![
1759 ev(
1760 1,
1761 "m-1",
1762 0,
1763 EventKind::PlanRevisionProposed {
1764 revision: 1,
1765 plan: sample_plan(),
1766 instructions: "add tests".into(),
1767 },
1768 ),
1769 ev(
1770 2,
1771 "m-1",
1772 10,
1773 EventKind::PlanRevised {
1774 revision: 1,
1775 plan: sample_plan(),
1776 },
1777 ),
1778 ev(
1779 3,
1780 "m-1",
1781 20,
1782 EventKind::PlanRevisionProposed {
1783 revision: 2,
1784 plan: sample_plan(),
1785 instructions: "drop scope".into(),
1786 },
1787 ),
1788 ev(
1789 4,
1790 "m-1",
1791 30,
1792 EventKind::PlanRevisionRejected {
1793 revision: 2,
1794 reason: "too risky".into(),
1795 },
1796 ),
1797 ev(
1798 5,
1799 "m-1",
1800 40,
1801 EventKind::PlanRevisionProposed {
1802 revision: 3,
1803 plan: sample_plan(),
1804 instructions: "pending one".into(),
1805 },
1806 ),
1807 ];
1808 let out = mission_outcomes("m-1", &events);
1809 let revisions: Vec<_> = out
1810 .escalations
1811 .iter()
1812 .filter(|r| r.kind == EscalationKind::Revision)
1813 .collect();
1814 assert_eq!(revisions.len(), 3);
1815 assert_eq!(revisions[0].summary, "add tests");
1816 assert_eq!(revisions[0].decision, "accepted (rev 1)");
1817 assert_eq!(revisions[1].summary, "drop scope");
1818 assert_eq!(revisions[1].decision, "rejected: too risky");
1819 assert_eq!(revisions[2].summary, "pending one");
1820 assert_eq!(revisions[2].decision, "pending");
1821 }
1822
1823 fn sample_plan() -> crate::types::Plan {
1824 crate::types::Plan {
1825 goal: "g".into(),
1826 validation_contract: vec![],
1827 milestones: vec![],
1828 considered_alternatives: None,
1829 command_grants: vec![],
1830 touch_set: vec![],
1831 standards_manifest: None,
1832 reviewer_independence: None,
1833 }
1834 }
1835
1836 #[test]
1841 fn divergence_event_outcomes_fold_surfaces_count_and_resolution_kind() {
1842 let candidate = |run_id: &str, tree: &str| crate::types::DivergenceCandidate {
1843 run_id: run_id.into(),
1844 branch: format!("kranz/pool/m-1/f-1-1-{run_id}"),
1845 backend: "claude".into(),
1846 tree: tree.into(),
1847 };
1848 let noted = |seq: u64, unit: &str, diverged: bool| {
1849 ev(
1850 seq,
1851 "m-1",
1852 seq as i64,
1853 EventKind::DivergenceNoted {
1854 unit: unit.into(),
1855 candidates: vec![candidate("r-c0", "aaa"), candidate("r-c1", "bbb")],
1856 diverged,
1857 },
1858 )
1859 };
1860 let resolved = |seq: u64, unit: &str, selected: Option<u32>| {
1861 ev(
1862 seq,
1863 "m-1",
1864 seq as i64,
1865 EventKind::DivergenceResolved {
1866 unit: unit.into(),
1867 selected,
1868 reason: "r".into(),
1869 decided_by: "operator".into(),
1870 },
1871 )
1872 };
1873 let events = vec![
1874 noted(1, "f-1-1", true), noted(2, "f-1-2", false), resolved(3, "f-1-1", Some(1)), resolved(4, "f-1-2", None), resolved(5, "f-1-1", Some(0)), ];
1880 let out = mission_outcomes("m-1", &events);
1881 let ledger = out.divergences.expect("a pool mission has a ledger");
1882 assert_eq!(ledger.noted, 2, "two units compared");
1883 assert_eq!(ledger.diverged, 1);
1884 assert_eq!(
1885 ledger.agreed, 1,
1886 "the agreement record counts — logged, never trusted"
1887 );
1888 assert_eq!(ledger.resolved_selected, 1, "first-wins dedupes the repeat");
1889 assert_eq!(ledger.resolved_none, 1);
1890
1891 let quiet = mission_outcomes(
1893 "m-1",
1894 &[ev(
1895 1,
1896 "m-1",
1897 1,
1898 EventKind::UserMessage {
1899 text: "hi".into(),
1900 interrupt: false,
1901 },
1902 )],
1903 );
1904 assert_eq!(
1905 quiet.divergences, None,
1906 "absent for missions without pools — never a zeroed row"
1907 );
1908 }
1909
1910 mod compute_outcomes_tests {
1911 use super::*;
1912 use crate::event_log::{EventLog, LockForce};
1913 use crate::paths::MissionPaths;
1914 use crate::types::{GrantKind, MissionConfig};
1915 use std::time::Duration;
1916 use tempfile::TempDir;
1917
1918 fn seed_mission(repo_root: &std::path::Path, id: &str, kinds: Vec<EventKind>) {
1920 let paths = MissionPaths::new(repo_root, id);
1921 let mut log = EventLog::acquire(&paths, id, Duration::ZERO, LockForce::No).unwrap();
1922 for kind in kinds {
1923 log.append(kind).unwrap();
1924 }
1925 }
1926
1927 fn write_timed_log(repo_root: &std::path::Path, id: &str, events: Vec<Event>) {
1930 let paths = MissionPaths::new(repo_root, id);
1931 std::fs::create_dir_all(paths.mission_dir()).unwrap();
1932 let lines: Vec<String> = events
1933 .iter()
1934 .map(|e| serde_json::to_string(e).unwrap())
1935 .collect();
1936 std::fs::write(paths.events_file(), lines.join("\n") + "\n").unwrap();
1937 }
1938
1939 fn timed(seq: u64, secs: i64, kind: EventKind) -> Event {
1940 Event {
1941 seq,
1942 ts: chrono::DateTime::from_timestamp(secs, 0).unwrap(),
1943 mission_id: "m-cost".into(),
1944 kind,
1945 }
1946 }
1947
1948 #[test]
1949 fn cost_per_change_and_cycle_time_fold_from_events() {
1950 use crate::types::{Role, RunResult, TokenUsage};
1951
1952 let tmp = TempDir::new().unwrap();
1953 let root = tmp.path();
1954 write_timed_log(
1955 root,
1956 "m-cost",
1957 vec![
1958 timed(1, 0, created("g")),
1959 timed(
1960 2,
1961 10,
1962 EventKind::PlanApproved {
1963 plan: sample_plan(),
1964 base_sha: None,
1965 },
1966 ),
1967 timed(
1968 3,
1969 20,
1970 EventKind::WorkerSpawned {
1971 backend: None,
1972 run_id: "r-1".into(),
1973 role: Role::Worker,
1974 feature_id: Some("f-1-1".into()),
1975 milestone_id: Some("ms-1".into()),
1976 candidate: None,
1977 executor_route: None,
1978 sdk_session_id: "s".into(),
1979 model: "sonnet".into(),
1980 quant: "n/a".into(),
1981 weight_hash: None,
1982 prompt_hash: "h".into(),
1983 transcript_path: "t".into(),
1984 },
1985 ),
1986 timed(
1987 4,
1988 100,
1989 EventKind::WorkerCompleted {
1990 run_id: "r-1".into(),
1991 result: RunResult::Pass,
1992 tokens: TokenUsage {
1993 input: 1,
1994 output: 1,
1995 cache_read: 0,
1996 cache_write: 0,
1997 },
1998 cost_usd: Some(12.50),
1999 report: None,
2000 },
2001 ),
2002 timed(
2003 5,
2004 110,
2005 EventKind::FeatureCompleted {
2006 feature_id: "f-1-1".into(),
2007 commits: vec![
2008 "aaa [f-1-1] add the thing".to_string(),
2009 "bbb [kranz] mission report for m-cost".to_string(),
2010 ],
2011 },
2012 ),
2013 timed(6, 120, EventKind::MissionPaused {}),
2014 timed(7, 130, EventKind::MissionResumed {}),
2015 timed(8, 160, EventKind::MissionCompleted {}),
2016 ],
2017 );
2018
2019 let outcomes = compute_outcomes(root).unwrap();
2020 let cost = &outcomes.cost_per_change;
2021 assert_eq!(cost.total_cost_usd, 12.50);
2022 assert_eq!(cost.non_meta_commits, 1);
2024 assert_eq!(cost.usd_per_commit, Some(12.50));
2025
2026 let cycle = &outcomes.cycle_time;
2027 assert_eq!(cycle.closed_missions, 1);
2028 assert_eq!(cycle.total_ms, 150_000);
2030 assert_eq!(cycle.mean_ms, Some(150_000.0));
2031 }
2032
2033 #[test]
2034 fn cost_fallback_prices_tokens_with_spawned_model_and_local_is_zero() {
2035 use crate::types::{Role, RunResult, TokenUsage};
2036
2037 let tmp = TempDir::new().unwrap();
2038 let root = tmp.path();
2039 let mut local_cfg = MissionConfig::default();
2042 local_cfg.worker.backend = Some("local".into());
2043 let tokens = TokenUsage {
2044 input: 2_000_000,
2045 output: 100_000,
2046 cache_read: 0,
2047 cache_write: 0,
2048 };
2049 write_timed_log(
2050 root,
2051 "m-cost",
2052 vec![
2053 timed(
2054 1,
2055 0,
2056 EventKind::MissionCreated {
2057 goal: "g".into(),
2058 base_branch: "main".into(),
2059 mission_branch: "kranz/mission-x".into(),
2060 config: local_cfg,
2061 },
2062 ),
2063 timed(
2064 2,
2065 10,
2066 EventKind::WorkerSpawned {
2067 backend: None,
2068 run_id: "r-1".into(),
2069 role: Role::Worker,
2070 feature_id: None,
2071 milestone_id: Some("ms-1".into()),
2072 candidate: None,
2073 executor_route: None,
2074 sdk_session_id: "s".into(),
2075 model: "my-local-model".into(),
2076 quant: "n/a".into(),
2077 weight_hash: None,
2078 prompt_hash: "h".into(),
2079 transcript_path: "t".into(),
2080 },
2081 ),
2082 timed(
2083 3,
2084 20,
2085 EventKind::WorkerCompleted {
2086 run_id: "r-1".into(),
2087 result: RunResult::Pass,
2088 tokens,
2089 cost_usd: None,
2090 report: None,
2091 },
2092 ),
2093 timed(4, 30, EventKind::MissionCompleted {}),
2094 ],
2095 );
2096
2097 let outcomes = compute_outcomes(root).unwrap();
2098 assert_eq!(
2099 outcomes.cost_per_change.total_cost_usd, 0.0,
2100 "the local tier must price at $0, never the frontier fallback"
2101 );
2102 }
2103
2104 #[test]
2105 fn memoized_fold_skips_unchanged_logs_and_invalidates_on_new_events() {
2106 let tmp = TempDir::new().unwrap();
2107 let root = tmp.path();
2108 let events_path = MissionPaths::new(root, "m-cache").events_file();
2109
2110 seed_mission(
2111 root,
2112 "m-cache",
2113 vec![
2114 created("g"),
2115 EventKind::PlanApproved {
2116 plan: sample_plan(),
2117 base_sha: None,
2118 },
2119 EventKind::MissionCompleted {},
2120 ],
2121 );
2122 let options = OutcomesOptions::default();
2125 let first = compute_outcomes_with_options(root, &options).unwrap();
2126 assert_eq!(
2127 cache_entry_stats(&events_path),
2128 Some((1, 0)),
2129 "first fold computes once, no hits"
2130 );
2131
2132 let second = compute_outcomes_with_options(root, &options).unwrap();
2133 assert_eq!(
2134 cache_entry_stats(&events_path),
2135 Some((1, 1)),
2136 "an unchanged log is served from the cache — no re-parse"
2137 );
2138 assert_eq!(first, second);
2139
2140 seed_mission(
2143 root,
2144 "m-cache",
2145 vec![
2146 EventKind::GrantRequested {
2147 milestone_id: "ms-1".into(),
2148 kind: GrantKind::Command,
2149 command: "cargo test".into(),
2150 },
2151 EventKind::GrantApproved {
2152 kind: GrantKind::Command,
2153 command: "cargo test".into(),
2154 },
2155 ],
2156 );
2157 let third = compute_outcomes_with_options(root, &options).unwrap();
2158 assert_eq!(
2159 cache_entry_stats(&events_path),
2160 Some((2, 1)),
2161 "appended events invalidate the memo entry"
2162 );
2163 assert_ne!(third, second);
2164 }
2165
2166 #[test]
2173 fn comparison_fold_reuses_the_memoized_native_fold_scan() {
2174 let tmp = TempDir::new().unwrap();
2175 let root = tmp.path();
2176 let events_path = MissionPaths::new(root, "m-cmp").events_file();
2177 seed_mission(
2178 root,
2179 "m-cmp",
2180 vec![
2181 created("g"),
2182 EventKind::PlanApproved {
2183 plan: sample_plan(),
2184 base_sha: None,
2185 },
2186 EventKind::MissionCompleted {},
2187 ],
2188 );
2189
2190 let outcomes = compute_outcomes(root).unwrap();
2191 assert_eq!(
2192 cache_entry_stats(&events_path),
2193 Some((1, 0)),
2194 "the native fold computes once"
2195 );
2196 let attached = outcomes
2200 .comparison
2201 .as_ref()
2202 .expect("resolve() pins the comparison window");
2203 assert_eq!(
2204 attached.assisted_change_share.base_branch.as_deref(),
2205 Some("main")
2206 );
2207 assert_eq!(attached.window_days, DEFAULT_MERGED_CHANGE_WINDOW_DAYS);
2208
2209 let report = crate::comparison_metrics::compute_comparison_report(
2211 root,
2212 DEFAULT_MERGED_CHANGE_WINDOW_DAYS,
2213 chrono::Utc::now(),
2214 )
2215 .unwrap();
2216 assert_eq!(
2217 cache_entry_stats(&events_path),
2218 Some((1, 1)),
2219 "the comparison fold must ride the memoized scan, not re-read the log"
2220 );
2221 assert_eq!(&report, attached, "same inputs, same report");
2222 }
2223
2224 fn created(goal: &str) -> EventKind {
2225 EventKind::MissionCreated {
2226 goal: goal.into(),
2227 base_branch: "main".into(),
2228 mission_branch: "kranz/mission-x".into(),
2229 config: MissionConfig::default(),
2230 }
2231 }
2232
2233 #[test]
2234 fn outcomes_ratio_denominator_is_closed_missions() {
2235 let tmp = TempDir::new().unwrap();
2236 let root = tmp.path();
2237
2238 seed_mission(
2240 root,
2241 "m-closed",
2242 vec![
2243 created("closed one"),
2244 EventKind::PlanApproved {
2245 plan: sample_plan(),
2246 base_sha: None,
2247 },
2248 EventKind::GrantApproved {
2249 kind: GrantKind::Command,
2250 command: "cargo test".into(),
2251 },
2252 EventKind::GrantDenied {
2253 kind: GrantKind::Command,
2254 command: "rm -rf".into(),
2255 reason: "no".into(),
2256 },
2257 EventKind::MissionCompleted {},
2258 ],
2259 );
2260
2261 seed_mission(
2264 root,
2265 "m-open",
2266 vec![
2267 created("still running"),
2268 EventKind::PlanApproved {
2269 plan: sample_plan(),
2270 base_sha: None,
2271 },
2272 EventKind::GrantApproved {
2273 kind: GrantKind::Command,
2274 command: "echo hi".into(),
2275 },
2276 ],
2277 );
2278
2279 let outcomes = compute_outcomes(root).unwrap();
2280 let ratio = outcomes.autonomy_ratio;
2281 assert_eq!(ratio.closed_missions, 1);
2282 assert_eq!(ratio.total_interventions, 2);
2283 assert_eq!(ratio.interventions_per_closed_mission, 2.0);
2284 assert_eq!(ratio.zero_intervention_missions, 0);
2285 assert_eq!(ratio.zero_intervention_share, 0.0);
2286 }
2287
2288 #[test]
2289 fn outcomes_ratio_zero_intervention_share_counts_clean_closed_missions() {
2290 let tmp = TempDir::new().unwrap();
2291 let root = tmp.path();
2292
2293 seed_mission(
2294 root,
2295 "m-clean",
2296 vec![
2297 created("clean"),
2298 EventKind::PlanApproved {
2299 plan: sample_plan(),
2300 base_sha: None,
2301 },
2302 EventKind::MissionCompleted {},
2303 ],
2304 );
2305 seed_mission(
2306 root,
2307 "m-dirty",
2308 vec![
2309 created("dirty"),
2310 EventKind::PlanApproved {
2311 plan: sample_plan(),
2312 base_sha: None,
2313 },
2314 EventKind::GrantApproved {
2315 kind: GrantKind::Command,
2316 command: "cargo test".into(),
2317 },
2318 EventKind::MissionCompleted {},
2319 ],
2320 );
2321
2322 let outcomes = compute_outcomes(root).unwrap();
2323 let ratio = outcomes.autonomy_ratio;
2324 assert_eq!(ratio.closed_missions, 2);
2325 assert_eq!(ratio.zero_intervention_missions, 1);
2326 assert_eq!(ratio.zero_intervention_share, 0.5);
2327 }
2328
2329 #[test]
2330 fn outcomes_ledger_newest_first() {
2331 let tmp = TempDir::new().unwrap();
2332 let root = tmp.path();
2333
2334 seed_mission(
2336 root,
2337 "m-a",
2338 vec![
2339 created("a"),
2340 EventKind::GrantRequested {
2341 milestone_id: "ms-1".into(),
2342 kind: GrantKind::Command,
2343 command: "cargo test".into(),
2344 },
2345 EventKind::GrantApproved {
2346 kind: GrantKind::Command,
2347 command: "cargo test".into(),
2348 },
2349 ],
2350 );
2351 seed_mission(
2352 root,
2353 "m-b",
2354 vec![
2355 created("b"),
2356 EventKind::GrantRequested {
2357 milestone_id: "ms-1".into(),
2358 kind: GrantKind::Command,
2359 command: "npm test".into(),
2360 },
2361 EventKind::GrantDenied {
2362 kind: GrantKind::Command,
2363 command: "npm test".into(),
2364 reason: "no".into(),
2365 },
2366 ],
2367 );
2368
2369 let outcomes = compute_outcomes(root).unwrap();
2370 assert!(outcomes.escalations.len() >= 2);
2371 for pair in outcomes.escalations.windows(2) {
2372 assert!(pair[0].ts >= pair[1].ts);
2373 }
2374 let mission_order: Vec<&str> = outcomes
2377 .escalations
2378 .iter()
2379 .map(|r| r.mission_id.as_str())
2380 .collect();
2381 assert_eq!(mission_order[0], "m-b");
2382 }
2383
2384 #[test]
2385 fn outcomes_empty_repo_yields_all_zero_defaults() {
2386 let tmp = TempDir::new().unwrap();
2387 let outcomes = compute_outcomes(tmp.path()).unwrap();
2388
2389 let ratio = outcomes.autonomy_ratio;
2390 assert_eq!(ratio.closed_missions, 0);
2391 assert_eq!(ratio.interventions_per_closed_mission, 0.0);
2392 assert_eq!(ratio.zero_intervention_share, 0.0);
2393
2394 assert_eq!(outcomes.grant_latency.buckets.len(), 4);
2395 assert!(outcomes.grant_latency.buckets.iter().all(|b| b.count == 0));
2396 assert_eq!(outcomes.grant_latency.total_decided, 0);
2397
2398 assert!(outcomes.escalations.is_empty());
2399 }
2400
2401 #[test]
2407 fn window_days_bound_over_max_errors_instead_of_panicking() {
2408 let tmp = TempDir::new().unwrap();
2409 let now = Utc::now();
2410 let err = compute_cost_per_merged_change(tmp.path(), u64::MAX, now)
2411 .expect_err("u64::MAX must error, never wrap or panic");
2412 assert!(err.to_string().contains("exceeds the maximum"), "{err}");
2413 let err =
2414 compute_cost_per_merged_change(tmp.path(), MAX_MERGED_CHANGE_WINDOW_DAYS + 1, now)
2415 .expect_err("just over the bound errors");
2416 assert!(err.to_string().contains("exceeds the maximum"), "{err}");
2417 let report =
2418 compute_cost_per_merged_change(tmp.path(), MAX_MERGED_CHANGE_WINDOW_DAYS, now)
2419 .expect("the documented maximum computes");
2420 assert_eq!(report.window_days, MAX_MERGED_CHANGE_WINDOW_DAYS);
2421 assert_eq!(report.closed_in_window, 0);
2422 }
2423
2424 #[test]
2425 fn outcomes_skips_mission_with_corrupt_event_log() {
2426 let tmp = TempDir::new().unwrap();
2427 let root = tmp.path();
2428
2429 seed_mission(
2430 root,
2431 "m-good",
2432 vec![
2433 created("good"),
2434 EventKind::PlanApproved {
2435 plan: sample_plan(),
2436 base_sha: None,
2437 },
2438 EventKind::MissionCompleted {},
2439 ],
2440 );
2441
2442 let bad_paths = MissionPaths::new(root, "m-bad");
2444 std::fs::create_dir_all(bad_paths.mission_dir()).unwrap();
2445 std::fs::write(bad_paths.events_file(), "not valid json\n").unwrap();
2446
2447 let outcomes = compute_outcomes(root).unwrap();
2448 assert_eq!(outcomes.autonomy_ratio.closed_missions, 1);
2449 }
2450
2451 #[test]
2454 fn divergence_event_fleet_ledger_sums_only_pool_missions() {
2455 let candidate = |run_id: &str| crate::types::DivergenceCandidate {
2456 run_id: run_id.into(),
2457 branch: format!("kranz/pool/m-pool/f-1-1-{run_id}"),
2458 backend: "claude".into(),
2459 tree: "aaa".into(),
2460 };
2461 let tmp = TempDir::new().unwrap();
2462 let root = tmp.path();
2463 seed_mission(
2464 root,
2465 "m-pool",
2466 vec![
2467 created("pool"),
2468 EventKind::DivergenceNoted {
2469 unit: "f-1-1".into(),
2470 candidates: vec![candidate("r-c0"), candidate("r-c1")],
2471 diverged: true,
2472 },
2473 EventKind::DivergenceResolved {
2474 unit: "f-1-1".into(),
2475 selected: Some(0),
2476 reason: "kept".into(),
2477 decided_by: "operator".into(),
2478 },
2479 ],
2480 );
2481 seed_mission(root, "m-quiet", vec![created("quiet")]);
2482
2483 let outcomes = compute_outcomes(root).unwrap();
2484 let ledger = outcomes
2485 .divergences
2486 .expect("a repo with a pool mission reports a fleet ledger");
2487 assert_eq!(ledger.noted, 1);
2488 assert_eq!(ledger.diverged, 1);
2489 assert_eq!(ledger.agreed, 0);
2490 assert_eq!(ledger.resolved_selected, 1);
2491 assert_eq!(ledger.resolved_none, 0);
2492
2493 let tmp2 = TempDir::new().unwrap();
2496 seed_mission(tmp2.path(), "m-quiet", vec![created("quiet")]);
2497 let outcomes = compute_outcomes(tmp2.path()).unwrap();
2498 assert_eq!(outcomes.divergences, None);
2499 let value = serde_json::to_value(&outcomes).unwrap();
2500 assert!(
2501 !value.as_object().unwrap().contains_key("divergences"),
2502 "no divergences key on the wire without pools: {value}"
2503 );
2504 }
2505 }
2506
2507 mod outcomes_report_tests {
2511 use super::*;
2512 use crate::paths::MissionPaths;
2513 use crate::types::{GrantKind, MissionConfig, Role, RunResult, TokenUsage};
2514 use tempfile::TempDir;
2515
2516 fn ev_ms(seq: u64, mission_id: &str, ts_ms: i64, kind: EventKind) -> Event {
2517 Event {
2518 seq,
2519 ts: DateTime::from_timestamp_millis(ts_ms).unwrap(),
2520 mission_id: mission_id.to_string(),
2521 kind,
2522 }
2523 }
2524
2525 fn write_log(repo_root: &std::path::Path, id: &str, events: Vec<Event>) {
2526 let paths = MissionPaths::new(repo_root, id);
2527 std::fs::create_dir_all(paths.mission_dir()).unwrap();
2528 let lines: Vec<String> = events
2529 .iter()
2530 .map(|e| serde_json::to_string(e).unwrap())
2531 .collect();
2532 std::fs::write(paths.events_file(), lines.join("\n") + "\n").unwrap();
2533 }
2534
2535 fn sample_plan() -> crate::types::Plan {
2536 crate::types::Plan {
2537 goal: "g".into(),
2538 validation_contract: vec![],
2539 milestones: vec![],
2540 considered_alternatives: None,
2541 command_grants: vec![],
2542 touch_set: vec![],
2543 standards_manifest: None,
2544 reviewer_independence: None,
2545 }
2546 }
2547
2548 fn created_with_class(mission_branch: &str, task_class: Option<&str>) -> EventKind {
2551 let goal = match task_class {
2552 Some(class) => format!("do the thing\n\n## Task class\n{class}\n"),
2553 None => "do the thing".to_string(),
2554 };
2555 created_with_config(mission_branch, goal, MissionConfig::default())
2556 }
2557
2558 fn created_with_config(
2559 mission_branch: &str,
2560 goal: String,
2561 config: MissionConfig,
2562 ) -> EventKind {
2563 EventKind::MissionCreated {
2564 goal,
2565 base_branch: "main".into(),
2566 mission_branch: mission_branch.into(),
2567 config,
2568 }
2569 }
2570
2571 fn worker_spawned(run_id: &str) -> EventKind {
2572 EventKind::WorkerSpawned {
2573 backend: None,
2574 run_id: run_id.into(),
2575 role: Role::Worker,
2576 feature_id: Some("f-1-1".into()),
2577 milestone_id: Some("ms-1".into()),
2578 candidate: None,
2579 executor_route: None,
2580 sdk_session_id: "s".into(),
2581 model: "sonnet".into(),
2582 quant: "n/a".into(),
2583 weight_hash: None,
2584 prompt_hash: "h".into(),
2585 transcript_path: "t".into(),
2586 }
2587 }
2588
2589 fn worker_completed(run_id: &str, tokens: TokenUsage, cost_usd: f64) -> EventKind {
2590 EventKind::WorkerCompleted {
2591 run_id: run_id.into(),
2592 result: RunResult::Pass,
2593 tokens,
2594 cost_usd: Some(cost_usd),
2595 report: None,
2596 }
2597 }
2598
2599 fn grant_req(seq: u64, mission_id: &str, ts_ms: i64, command: &str) -> Event {
2600 ev_ms(
2601 seq,
2602 mission_id,
2603 ts_ms,
2604 EventKind::GrantRequested {
2605 milestone_id: "ms-1".into(),
2606 kind: GrantKind::Command,
2607 command: command.into(),
2608 },
2609 )
2610 }
2611
2612 fn grant_yes(seq: u64, mission_id: &str, ts_ms: i64, command: &str) -> Event {
2613 ev_ms(
2614 seq,
2615 mission_id,
2616 ts_ms,
2617 EventKind::GrantApproved {
2618 kind: GrantKind::Command,
2619 command: command.into(),
2620 },
2621 )
2622 }
2623
2624 #[test]
2625 fn outcomes_report_task_class_rows_group_and_unclassified_last() {
2626 let tmp = TempDir::new().unwrap();
2627 let root = tmp.path();
2628
2629 write_log(
2632 root,
2633 "m-a",
2634 vec![
2635 ev_ms(
2636 1,
2637 "m-a",
2638 0,
2639 created_with_class("kranz/m-a", Some("execution-class")),
2640 ),
2641 ev_ms(2, "m-a", 1_000, worker_spawned("r-a")),
2642 ev_ms(
2643 3,
2644 "m-a",
2645 2_000,
2646 worker_completed(
2647 "r-a",
2648 TokenUsage {
2649 input: 1,
2650 output: 1,
2651 cache_read: 0,
2652 cache_write: 0,
2653 },
2654 10.0,
2655 ),
2656 ),
2657 ev_ms(
2658 4,
2659 "m-a",
2660 3_000,
2661 EventKind::FeatureCompleted {
2662 feature_id: "f-1-1".into(),
2663 commits: vec!["aaa [f-1-1] add the thing".to_string()],
2664 },
2665 ),
2666 grant_req(5, "m-a", 4_000, "cargo test"),
2667 grant_yes(6, "m-a", 64_000, "cargo test"),
2668 ev_ms(7, "m-a", 100_000, EventKind::MissionCompleted {}),
2669 ],
2670 );
2671 write_log(
2674 root,
2675 "m-b",
2676 vec![
2677 ev_ms(
2678 1,
2679 "m-b",
2680 0,
2681 created_with_class("kranz/m-b", Some("execution-class")),
2682 ),
2683 ev_ms(2, "m-b", 1_000, worker_spawned("r-b")),
2684 ev_ms(
2685 3,
2686 "m-b",
2687 2_000,
2688 worker_completed(
2689 "r-b",
2690 TokenUsage {
2691 input: 1,
2692 output: 1,
2693 cache_read: 0,
2694 cache_write: 0,
2695 },
2696 5.0,
2697 ),
2698 ),
2699 grant_req(4, "m-b", 3_000, "cargo clippy"),
2700 ],
2701 );
2702 write_log(
2705 root,
2706 "m-c",
2707 vec![
2708 ev_ms(1, "m-c", 0, created_with_class("kranz/m-c", None)),
2709 ev_ms(2, "m-c", 50_000, EventKind::MissionCompleted {}),
2710 ],
2711 );
2712
2713 let outcomes =
2714 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
2715 assert_eq!(outcomes.task_classes.len(), 2);
2716 let exec = &outcomes.task_classes[0];
2717 assert_eq!(exec.task_class, "execution-class");
2718 assert_eq!(exec.missions, 2);
2719 assert_eq!(exec.closed_missions, 1);
2720 assert_eq!(exec.total_cost_usd, 15.0);
2721 assert_eq!(exec.non_meta_commits, 1);
2722 assert_eq!(exec.usd_per_commit, Some(15.0));
2723 assert_eq!(exec.escalations, 2);
2724 assert_eq!(exec.advisor_invocations, 2);
2725 assert_eq!(exec.escalations_per_mission, 1.0);
2726 assert_eq!(exec.cycle_mean_ms, Some(100_000.0));
2727
2728 let unclassified = &outcomes.task_classes[1];
2729 assert_eq!(unclassified.task_class, UNCLASSIFIED_TASK_CLASS);
2730 assert_eq!(unclassified.missions, 1);
2731 assert_eq!(unclassified.closed_missions, 1);
2732 assert_eq!(unclassified.non_meta_commits, 0);
2733 assert_eq!(unclassified.usd_per_commit, None);
2735 assert_eq!(unclassified.escalations, 0);
2736 assert_eq!(unclassified.advisor_invocations, 0);
2737 assert_eq!(unclassified.escalations_per_mission, 0.0);
2738 assert_eq!(unclassified.cycle_mean_ms, Some(50_000.0));
2739 }
2740
2741 #[test]
2742 fn outcomes_report_context_reuse_split_absent_for_unreporting_backends() {
2743 let tmp = TempDir::new().unwrap();
2744 let root = tmp.path();
2745
2746 write_log(
2748 root,
2749 "m-claude",
2750 vec![
2751 ev_ms(1, "m-claude", 0, created_with_class("kranz/m-c", None)),
2752 ev_ms(2, "m-claude", 1_000, worker_spawned("r-1")),
2753 ev_ms(
2754 3,
2755 "m-claude",
2756 2_000,
2757 worker_completed(
2758 "r-1",
2759 TokenUsage {
2760 input: 500,
2761 output: 10,
2762 cache_read: 800,
2763 cache_write: 200,
2764 },
2765 1.0,
2766 ),
2767 ),
2768 ev_ms(4, "m-claude", 3_000, EventKind::MissionCompleted {}),
2769 ],
2770 );
2771 let mut local_cfg = MissionConfig::default();
2775 local_cfg.worker.backend = Some("local".into());
2776 write_log(
2777 root,
2778 "m-local",
2779 vec![
2780 ev_ms(
2781 1,
2782 "m-local",
2783 0,
2784 created_with_config("kranz/m-l", "g".into(), local_cfg),
2785 ),
2786 ev_ms(2, "m-local", 1_000, worker_spawned("r-2")),
2787 ev_ms(
2788 3,
2789 "m-local",
2790 2_000,
2791 worker_completed(
2792 "r-2",
2793 TokenUsage {
2794 input: 100,
2795 output: 10,
2796 cache_read: 0,
2797 cache_write: 0,
2798 },
2799 0.0,
2800 ),
2801 ),
2802 ev_ms(4, "m-local", 3_000, EventKind::MissionCompleted {}),
2803 ],
2804 );
2805
2806 let outcomes =
2807 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
2808 assert_eq!(outcomes.context_reuse.len(), 1);
2809 let row = &outcomes.context_reuse[0];
2810 assert_eq!(row.backend, "claude");
2811 assert_eq!(row.missions, 1);
2812 assert_eq!(row.runs, 1);
2813 assert_eq!(row.fresh_input, 500);
2814 assert_eq!(row.cache_read, 800);
2815 assert_eq!(row.cache_write, Some(200));
2816 assert_eq!(row.reuse_share, Some(1_000.0 / 1_500.0));
2817 }
2818
2819 #[test]
2820 fn outcomes_report_context_reuse_codex_cache_write_is_absent() {
2821 let tmp = TempDir::new().unwrap();
2822 let root = tmp.path();
2823
2824 let mut codex_cfg = MissionConfig::default();
2828 codex_cfg.worker.backend = Some("codex".into());
2829 write_log(
2830 root,
2831 "m-codex",
2832 vec![
2833 ev_ms(
2834 1,
2835 "m-codex",
2836 0,
2837 created_with_config("kranz/m-x", "g".into(), codex_cfg),
2838 ),
2839 ev_ms(2, "m-codex", 1_000, worker_spawned("r-1")),
2840 ev_ms(
2841 3,
2842 "m-codex",
2843 2_000,
2844 worker_completed(
2845 "r-1",
2846 TokenUsage {
2847 input: 900,
2848 output: 10,
2849 cache_read: 100,
2850 cache_write: 0,
2851 },
2852 1.0,
2853 ),
2854 ),
2855 ev_ms(4, "m-codex", 3_000, EventKind::MissionCompleted {}),
2856 ],
2857 );
2858
2859 let outcomes =
2860 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
2861 assert_eq!(outcomes.context_reuse.len(), 1);
2862 let row = &outcomes.context_reuse[0];
2863 assert_eq!(row.backend, "codex");
2864 assert_eq!(row.cache_read, 100);
2865 assert_eq!(row.cache_write, None);
2866 assert_eq!(row.reuse_share, Some(100.0 / 1_000.0));
2867 }
2868
2869 #[test]
2870 fn outcomes_report_rubber_stamp_boundary_at_threshold() {
2871 let tmp = TempDir::new().unwrap();
2872 let root = tmp.path();
2873
2874 write_log(
2882 root,
2883 "m-1",
2884 vec![
2885 ev_ms(1, "m-1", 0, created_with_class("kranz/m-1", None)),
2886 ev_ms(
2887 2,
2888 "m-1",
2889 1_000,
2890 EventKind::PlanApproved {
2891 plan: sample_plan(),
2892 base_sha: None,
2893 },
2894 ),
2895 grant_req(3, "m-1", 2_000, "under"),
2896 grant_yes(4, "m-1", 11_999, "under"),
2897 grant_req(5, "m-1", 20_000, "at"),
2898 grant_yes(6, "m-1", 30_000, "at"),
2899 grant_req(7, "m-1", 40_000, "over"),
2900 grant_yes(8, "m-1", 55_000, "over"),
2901 grant_req(9, "m-1", 60_000, "denied-fast"),
2902 ev_ms(
2903 10,
2904 "m-1",
2905 65_000,
2906 EventKind::GrantDenied {
2907 kind: GrantKind::Command,
2908 command: "denied-fast".into(),
2909 reason: "no".into(),
2910 },
2911 ),
2912 grant_req(11, "m-1", 70_000, "pending"),
2913 ev_ms(12, "m-1", 80_000, EventKind::MissionCompleted {}),
2914 ],
2915 );
2916
2917 let outcomes =
2918 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
2919 let stamp = &outcomes.rubber_stamp;
2920 assert_eq!(
2921 stamp.threshold_ms,
2922 crate::types::DEFAULT_RUBBER_STAMP_THRESHOLD_MS
2923 );
2924 assert_eq!(stamp.approved_decisions, 3);
2925 assert_eq!(stamp.flagged, 1);
2926 assert_eq!(stamp.share, Some(1.0 / 3.0));
2927
2928 let marker = |summary: &str| {
2929 outcomes
2930 .escalations
2931 .iter()
2932 .find(|r| r.summary == summary)
2933 .unwrap()
2934 .rubber_stamp
2935 };
2936 assert_eq!(marker("under"), Some(true));
2937 assert_eq!(
2938 marker("at"),
2939 Some(false),
2940 "at the threshold is not under it"
2941 );
2942 assert_eq!(marker("over"), Some(false));
2943 assert_eq!(marker("denied-fast"), None);
2944 assert_eq!(marker("pending"), None);
2945 }
2946
2947 #[test]
2948 fn outcomes_report_rubber_stamp_threshold_resolves_from_config() {
2949 let tmp = TempDir::new().unwrap();
2950 let root = tmp.path();
2951
2952 std::fs::create_dir_all(root.join(".kranz")).unwrap();
2955 std::fs::write(
2956 root.join(".kranz").join("config.json"),
2957 "{\"rubberStampThresholdMs\": 60000}",
2958 )
2959 .unwrap();
2960 assert_eq!(
2961 OutcomesOptions::resolve(root).rubber_stamp_threshold_ms,
2962 60_000
2963 );
2964
2965 write_log(
2966 root,
2967 "m-1",
2968 vec![
2969 ev_ms(1, "m-1", 0, created_with_class("kranz/m-1", None)),
2970 ev_ms(
2971 2,
2972 "m-1",
2973 1_000,
2974 EventKind::PlanApproved {
2975 plan: sample_plan(),
2976 base_sha: None,
2977 },
2978 ),
2979 grant_req(3, "m-1", 2_000, "fifteen seconds"),
2980 grant_yes(4, "m-1", 17_000, "fifteen seconds"),
2981 ev_ms(5, "m-1", 20_000, EventKind::MissionCompleted {}),
2982 ],
2983 );
2984
2985 let outcomes = compute_outcomes(root).unwrap();
2988 assert_eq!(outcomes.rubber_stamp.threshold_ms, 60_000);
2989 assert_eq!(outcomes.rubber_stamp.flagged, 1);
2990 assert_eq!(outcomes.rubber_stamp.share, Some(1.0));
2991 assert_eq!(outcomes.escalations[0].rubber_stamp, Some(true));
2992 }
2993
2994 #[test]
2995 fn outcomes_report_rubber_stamp_absent_without_approvals() {
2996 let tmp = TempDir::new().unwrap();
2997 let root = tmp.path();
2998
2999 write_log(
3000 root,
3001 "m-1",
3002 vec![
3003 ev_ms(1, "m-1", 0, created_with_class("kranz/m-1", None)),
3004 ev_ms(
3005 2,
3006 "m-1",
3007 1_000,
3008 EventKind::PlanApproved {
3009 plan: sample_plan(),
3010 base_sha: None,
3011 },
3012 ),
3013 grant_req(3, "m-1", 2_000, "only-denied"),
3014 ev_ms(
3015 4,
3016 "m-1",
3017 3_000,
3018 EventKind::GrantDenied {
3019 kind: GrantKind::Command,
3020 command: "only-denied".into(),
3021 reason: "no".into(),
3022 },
3023 ),
3024 ev_ms(5, "m-1", 4_000, EventKind::MissionCompleted {}),
3025 ],
3026 );
3027
3028 let outcomes =
3029 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
3030 assert_eq!(outcomes.rubber_stamp.approved_decisions, 0);
3031 assert_eq!(outcomes.rubber_stamp.flagged, 0);
3032 assert_eq!(outcomes.rubber_stamp.share, None);
3033 assert_eq!(outcomes.escalations[0].rubber_stamp, None);
3034 }
3035
3036 #[test]
3037 fn outcomes_report_fold_is_byte_identical_across_repeated_computes() {
3038 let tmp = TempDir::new().unwrap();
3039 let root = tmp.path();
3040
3041 write_log(
3042 root,
3043 "m-1",
3044 vec![
3045 ev_ms(
3046 1,
3047 "m-1",
3048 0,
3049 created_with_class("kranz/m-1", Some("execution-class")),
3050 ),
3051 ev_ms(2, "m-1", 1_000, worker_spawned("r-1")),
3052 ev_ms(
3053 3,
3054 "m-1",
3055 2_000,
3056 worker_completed(
3057 "r-1",
3058 TokenUsage {
3059 input: 500,
3060 output: 10,
3061 cache_read: 800,
3062 cache_write: 200,
3063 },
3064 3.0,
3065 ),
3066 ),
3067 grant_req(4, "m-1", 3_000, "cargo test"),
3068 grant_yes(5, "m-1", 6_000, "cargo test"),
3069 ev_ms(6, "m-1", 10_000, EventKind::MissionCompleted {}),
3070 ],
3071 );
3072
3073 let options = OutcomesOptions::default();
3074 let first = compute_outcomes_with_options(root, &options).unwrap();
3075 let second = compute_outcomes_with_options(root, &options).unwrap();
3076 assert_eq!(first, second);
3077 assert_eq!(
3078 serde_json::to_string(&first).unwrap(),
3079 serde_json::to_string(&second).unwrap(),
3080 "the same log plus the same options yields byte-identical data"
3081 );
3082 }
3083
3084 fn gate_scored(
3088 seq: u64,
3089 mission_id: &str,
3090 ts_ms: i64,
3091 gate: &str,
3092 score: Option<(f64, f64)>,
3093 ) -> Event {
3094 ev_ms(
3095 seq,
3096 mission_id,
3097 ts_ms,
3098 EventKind::GateResult {
3099 gate: gate.into(),
3100 surface: crate::gate::GateSurface::Approval,
3101 kind: crate::gate::GateKind::Deterministic,
3102 index: 0,
3103 verdict: crate::gate::GateVerdict::Pass,
3104 artefact_ref: format!("contract gate {gate}"),
3105 artefact_detail: None,
3106 score: score.map(|(score, _)| score),
3107 threshold: score.map(|(_, threshold)| threshold),
3108 rule_ids: Vec::new(),
3109 },
3110 )
3111 }
3112
3113 #[test]
3120 fn score_distribution_flag_outcomes_fold_flags_beside_rubber_stamp() {
3121 let tmp = TempDir::new().unwrap();
3122 let root = tmp.path();
3123
3124 let mut m1 = vec![
3125 ev_ms(1, "m-1", 0, created_with_class("kranz/m-1", None)),
3126 ev_ms(
3127 2,
3128 "m-1",
3129 1_000,
3130 EventKind::PlanApproved {
3131 plan: sample_plan(),
3132 base_sha: None,
3133 },
3134 ),
3135 grant_req(3, "m-1", 2_000, "cargo test"),
3136 grant_yes(4, "m-1", 4_000, "cargo test"),
3137 ];
3138 for i in 0..5 {
3139 m1.push(gate_scored(
3140 5 + i,
3141 "m-1",
3142 5_000 + i as i64,
3143 "vacuous-filter",
3144 Some((0.5, 1.0)),
3145 ));
3146 }
3147 m1.push(ev_ms(10, "m-1", 10_000, EventKind::MissionCompleted {}));
3148 write_log(root, "m-1", m1);
3149
3150 let mut m2 = vec![ev_ms(1, "m-2", 0, created_with_class("kranz/m-2", None))];
3151 for i in 0..5 {
3152 m2.push(gate_scored(
3153 2 + i,
3154 "m-2",
3155 5_000 + i as i64,
3156 "vacuous-filter",
3157 Some((0.5, 1.0)),
3158 ));
3159 }
3160 m2.push(ev_ms(7, "m-2", 10_000, EventKind::MissionCompleted {}));
3161 write_log(root, "m-2", m2);
3162
3163 let outcomes =
3164 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
3165
3166 assert_eq!(outcomes.rubber_stamp.flagged, 1);
3168 assert_eq!(outcomes.escalations[0].rubber_stamp, Some(true));
3169
3170 let report = &outcomes.gate_score_flags;
3172 assert_eq!(report.scored_gates, 1);
3173 assert_eq!(report.assessed_gates, 1);
3174 assert_eq!(report.flags.len(), 2);
3175 assert!(
3176 report.flags.iter().all(|f| f.gate == "vacuous-filter"),
3177 "the flag names the gate: {report:?}"
3178 );
3179 let kinds: Vec<_> = report.flags.iter().map(|f| f.kind).collect();
3180 assert_eq!(
3181 kinds,
3182 [
3183 crate::gate_score_flags::GateScoreFlagKind::NeverApproachesThreshold,
3184 crate::gate_score_flags::GateScoreFlagKind::NearConstant,
3185 ]
3186 );
3187 let d = &report.flags[0].distribution;
3190 assert_eq!(d.samples, 10);
3191 assert_eq!(d.closest_approach, 0.5);
3192 assert_eq!(d.variance, 0.0);
3193 assert_eq!(
3195 report.min_samples,
3196 crate::gate_score_flags::MIN_SAMPLE_COUNT
3197 );
3198 }
3199
3200 #[test]
3205 fn score_distribution_flag_outcomes_fold_sub_minimum_and_unscored_absent() {
3206 let tmp = TempDir::new().unwrap();
3207 let root = tmp.path();
3208
3209 let mut m1 = vec![ev_ms(1, "m-1", 0, created_with_class("kranz/m-1", None))];
3211 for i in 0..3 {
3212 m1.push(gate_scored(
3213 2 + i,
3214 "m-1",
3215 5_000 + i as i64,
3216 "vacuous-filter",
3217 Some((0.5, 1.0)),
3218 ));
3219 }
3220 m1.push(ev_ms(5, "m-1", 10_000, EventKind::MissionCompleted {}));
3221 write_log(root, "m-1", m1);
3222
3223 write_log(
3225 root,
3226 "m-2",
3227 vec![
3228 ev_ms(1, "m-2", 0, created_with_class("kranz/m-2", None)),
3229 gate_scored(2, "m-2", 5_000, "env-sensitive", None),
3230 gate_scored(3, "m-2", 6_000, "env-sensitive", None),
3231 ev_ms(4, "m-2", 10_000, EventKind::MissionCompleted {}),
3232 ],
3233 );
3234
3235 let outcomes =
3236 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
3237 let report = &outcomes.gate_score_flags;
3238 assert_eq!(
3239 report.scored_gates, 1,
3240 "the unscored gate adds no population: {report:?}"
3241 );
3242 assert_eq!(report.assessed_gates, 0, "under the minimum: unassessed");
3243 assert!(
3244 report.flags.is_empty(),
3245 "absent, never a zero-filled row: {report:?}"
3246 );
3247 }
3248 }
3249}