1use crate::events::{Event, EventKind};
15use chrono::{DateTime, Utc};
16use serde::{Deserialize, Serialize};
17
18#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
19#[serde(rename_all = "camelCase")]
20pub struct AutonomyRatio {
21 pub closed_missions: u64,
22 pub total_interventions: u64,
23 pub interventions_per_closed_mission: f64,
24 pub zero_intervention_missions: u64,
25 pub zero_intervention_share: f64,
26}
27
28#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
29#[serde(rename_all = "camelCase")]
30pub struct LatencyBucket {
31 pub label: String,
32 pub count: u64,
33}
34
35#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
36#[serde(rename_all = "camelCase")]
37pub struct GrantLatency {
38 pub buckets: Vec<LatencyBucket>,
39 pub total_decided: u64,
40}
41
42#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
43#[serde(rename_all = "lowercase")]
44pub enum EscalationKind {
45 Block,
46 Grant,
47 Revision,
48}
49
50impl EscalationKind {
51 pub fn as_str(&self) -> &'static str {
53 match self {
54 Self::Block => "block",
55 Self::Grant => "grant",
56 Self::Revision => "revision",
57 }
58 }
59}
60
61#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
62#[serde(rename_all = "camelCase")]
63pub struct EscalationRow {
64 pub ts: DateTime<Utc>,
65 pub mission_id: String,
66 pub kind: EscalationKind,
67 pub summary: String,
68 pub decision: String,
69 pub latency_ms: Option<u64>,
70 #[serde(default)]
77 pub rubber_stamp: Option<bool>,
78}
79
80#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
90#[serde(rename_all = "camelCase")]
91pub struct DivergenceOutcomes {
92 pub noted: u64,
95 pub diverged: u64,
97 pub agreed: u64,
100 pub resolved_selected: u64,
104 pub resolved_none: u64,
107}
108
109#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
110#[serde(rename_all = "camelCase")]
111pub struct Outcomes {
112 pub autonomy_ratio: AutonomyRatio,
113 pub grant_latency: GrantLatency,
114 pub escalations: Vec<EscalationRow>,
115 pub cost_per_change: CostPerChange,
117 pub cycle_time: CycleTime,
119 #[serde(default)]
125 pub task_classes: Vec<TaskClassRow>,
126 #[serde(default)]
131 pub context_reuse: Vec<ContextReuseRow>,
132 #[serde(default)]
135 pub rubber_stamp: RubberStampReport,
136 #[serde(default)]
147 pub gate_score_flags: crate::gate_score_flags::GateScoreFlagsReport,
148 #[serde(default, skip_serializing_if = "Option::is_none")]
153 pub divergences: Option<DivergenceOutcomes>,
154 #[serde(default, skip_serializing_if = "Option::is_none")]
163 pub comparison: Option<crate::comparison_metrics::ComparisonReport>,
164}
165
166#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
168#[serde(rename_all = "camelCase")]
169pub struct TaskClassRow {
170 pub task_class: String,
173 pub missions: u64,
174 pub closed_missions: u64,
175 pub total_cost_usd: f64,
178 pub non_meta_commits: u64,
179 pub usd_per_commit: Option<f64>,
182 pub escalations: u64,
184 pub advisor_invocations: u64,
186 pub escalations_per_mission: f64,
188 pub cycle_mean_ms: Option<f64>,
191}
192
193pub const UNCLASSIFIED_TASK_CLASS: &str = "unclassified";
196
197#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
203#[serde(rename_all = "camelCase")]
204pub struct ContextReuseRow {
205 pub backend: String,
208 pub missions: u64,
210 pub runs: u64,
212 pub fresh_input: u64,
214 pub cache_read: u64,
216 pub cache_write: Option<u64>,
219 pub reuse_share: Option<f64>,
222}
223
224#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
228#[serde(rename_all = "camelCase", default)]
229pub struct RubberStampReport {
230 pub threshold_ms: u64,
233 pub approved_decisions: u64,
235 pub flagged: u64,
238 pub share: Option<f64>,
240}
241
242impl Default for RubberStampReport {
243 fn default() -> Self {
246 Self {
247 threshold_ms: crate::types::DEFAULT_RUBBER_STAMP_THRESHOLD_MS,
248 approved_decisions: 0,
249 flagged: 0,
250 share: None,
251 }
252 }
253}
254
255#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
256#[serde(rename_all = "camelCase")]
257pub struct CostPerChange {
258 pub total_cost_usd: f64,
259 pub non_meta_commits: u64,
263 pub usd_per_commit: Option<f64>,
266}
267
268#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
269#[serde(rename_all = "camelCase")]
270pub struct CycleTime {
271 pub closed_missions: u64,
273 pub total_ms: u64,
274 pub mean_ms: Option<f64>,
276}
277
278#[derive(Debug, Clone, PartialEq)]
281pub struct MissionOutcomes {
282 pub interventions: u64,
283 pub is_closed: bool,
284 pub latencies_ms: Vec<u64>,
285 pub escalations: Vec<EscalationRow>,
286 pub cost_usd: f64,
289 pub non_meta_commits: u64,
290 pub cycle_time_ms: Option<u64>,
292 pub task_class: Option<String>,
296 pub token_sums: Vec<BackendTokenSum>,
299 pub divergences: Option<DivergenceOutcomes>,
303 pub gate_score_samples: Vec<crate::gate_score_flags::GateScoreSample>,
308 pub comparison: ComparisonInputs,
316}
317
318#[derive(Debug, Clone, PartialEq)]
322pub struct ComparisonInputs {
323 pub terminal_ts: Option<DateTime<Utc>>,
327 pub base_branch: Option<String>,
332 pub folded: Option<FoldedMissionRefs>,
339}
340
341#[derive(Debug, Clone, PartialEq)]
344pub struct FoldedMissionRefs {
345 pub status: crate::types::MissionStatus,
346 pub base_branch: String,
347 pub mission_branch: String,
348}
349
350#[derive(Debug, Clone, Copy, PartialEq)]
353pub struct BackendTokenSum {
354 pub backend: crate::types::BackendKind,
355 pub runs: u64,
356 pub fresh_input: u64,
358 pub cache_read: u64,
359 pub cache_write: u64,
360}
361
362const BUCKET_LABELS: [&str; 4] = ["<10s", "<60s", "<10m", ">=10m"];
364
365#[derive(Clone)]
377struct CachedMission {
378 len: u64,
379 mtime: std::time::SystemTime,
380 outcomes: MissionOutcomes,
381 computes: u64,
382 hits: u64,
383}
384
385static MISSION_CACHE: std::sync::OnceLock<
386 std::sync::Mutex<std::collections::HashMap<std::path::PathBuf, CachedMission>>,
387> = std::sync::OnceLock::new();
388
389#[cfg(test)]
391fn cache_entry_stats(events_path: &std::path::Path) -> Option<(u64, u64)> {
392 MISSION_CACHE
393 .get()?
394 .lock()
395 .ok()?
396 .get(events_path)
397 .map(|c| (c.computes, c.hits))
398}
399
400pub(crate) fn cached_mission_outcomes(
407 mission_id: &str,
408 events_path: &std::path::Path,
409) -> Option<MissionOutcomes> {
410 let meta = std::fs::metadata(events_path).ok()?;
411 let (len, mtime) = (meta.len(), meta.modified().ok()?);
412 let cache =
413 MISSION_CACHE.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()));
414 {
415 let mut guard = cache.lock().ok()?;
416 if let Some(hit) = guard.get_mut(events_path) {
417 if hit.len == len && hit.mtime == mtime {
418 hit.hits += 1;
419 return Some(hit.outcomes.clone());
420 }
421 }
422 }
423 let events = crate::event_log::EventLog::read_events(events_path).ok()?;
424 let outcomes = mission_outcomes(mission_id, &events);
425 if let Ok(mut guard) = cache.lock() {
426 guard
427 .entry(events_path.to_path_buf())
428 .and_modify(|entry| {
429 entry.len = len;
430 entry.mtime = mtime;
431 entry.outcomes = outcomes.clone();
432 entry.computes += 1;
433 })
434 .or_insert_with(|| CachedMission {
435 len,
436 mtime,
437 outcomes: outcomes.clone(),
438 computes: 1,
439 hits: 0,
440 });
441 }
442 Some(outcomes)
443}
444
445pub fn mission_outcomes(mission_id: &str, events: &[Event]) -> MissionOutcomes {
450 let mission_events: Vec<&Event> = events
451 .iter()
452 .filter(|e| e.mission_id == mission_id)
453 .collect();
454
455 let is_closed = mission_events.iter().any(|e| {
456 matches!(
457 e.kind,
458 EventKind::MissionCompleted {}
459 | EventKind::MissionFailed { .. }
460 | EventKind::MissionAbandoned { .. }
461 )
462 });
463
464 let plan_approved_ts = mission_events
465 .iter()
466 .find(|e| matches!(e.kind, EventKind::PlanApproved { .. }))
467 .map(|e| e.ts);
468
469 let mut interventions: u64 = 0;
470 for e in &mission_events {
471 match &e.kind {
472 EventKind::UserMessage { .. } => {
473 if let Some(approved_ts) = plan_approved_ts {
474 if e.ts >= approved_ts {
475 interventions += 1;
476 }
477 }
478 }
479 EventKind::GrantApproved { .. }
480 | EventKind::GrantDenied { .. }
481 | EventKind::PlanRevised { .. }
482 | EventKind::PlanRevisionRejected { .. } => {
483 interventions += 1;
484 }
485 _ => {}
486 }
487 }
488
489 let mut latencies_ms = Vec::new();
490 let mut escalations = Vec::new();
491
492 let mut used_decisions = vec![false; mission_events.len()];
496 for (req_idx, req) in mission_events.iter().enumerate() {
497 let EventKind::GrantRequested { command, .. } = &req.kind else {
498 continue;
499 };
500
501 let mut matched: Option<usize> = None;
502 for (i, cand) in mission_events.iter().enumerate() {
503 if i <= req_idx || used_decisions[i] {
504 continue;
505 }
506 let cand_command = match &cand.kind {
507 EventKind::GrantApproved { command, .. }
508 | EventKind::GrantDenied { command, .. } => command,
509 _ => continue,
510 };
511 if cand_command == command {
512 matched = Some(i);
513 break;
514 }
515 }
516 if matched.is_none() {
517 for (i, cand) in mission_events.iter().enumerate() {
522 if i <= req_idx || used_decisions[i] {
523 continue;
524 }
525 if matches!(
526 cand.kind,
527 EventKind::GrantApproved { .. } | EventKind::GrantDenied { .. }
528 ) {
529 matched = Some(i);
530 break;
531 }
532 }
533 }
534
535 let (decision, latency_ms) = match matched {
536 Some(i) => {
537 used_decisions[i] = true;
538 let decided = mission_events[i];
539 let latency = (decided.ts - req.ts).num_milliseconds();
540 let latency_ms = if latency >= 0 {
541 Some(latency as u64)
542 } else {
543 None
544 };
545 if let Some(l) = latency_ms {
546 latencies_ms.push(l);
547 }
548 let decision = match &decided.kind {
549 EventKind::GrantApproved { .. } => "approved".to_string(),
550 EventKind::GrantDenied { reason, .. } => format!("denied: {reason}"),
551 _ => unreachable!(),
552 };
553 (decision, latency_ms)
554 }
555 None => ("pending".to_string(), None),
556 };
557
558 escalations.push(EscalationRow {
559 ts: req.ts,
560 mission_id: mission_id.to_string(),
561 kind: EscalationKind::Grant,
562 summary: command.clone(),
563 decision,
564 latency_ms,
565 rubber_stamp: None,
567 });
568 }
569
570 let mut used_unblocks = vec![false; mission_events.len()];
573 for (idx, e) in mission_events.iter().enumerate() {
574 let EventKind::MilestoneBlocked {
575 milestone_id,
576 reason,
577 ..
578 } = &e.kind
579 else {
580 continue;
581 };
582 let mut matched: Option<usize> = None;
583 for (i, cand) in mission_events.iter().enumerate() {
584 if i <= idx || used_unblocks[i] {
585 continue;
586 }
587 if let EventKind::MilestoneUnblocked {
588 milestone_id: mid, ..
589 } = &cand.kind
590 {
591 if mid == milestone_id {
592 matched = Some(i);
593 break;
594 }
595 }
596 }
597 let decision = match matched {
598 Some(i) => {
599 used_unblocks[i] = true;
600 let EventKind::MilestoneUnblocked {
601 reason: unblock_reason,
602 ..
603 } = &mission_events[i].kind
604 else {
605 unreachable!()
606 };
607 format!("unblocked: {unblock_reason}")
608 }
609 None => "open".to_string(),
610 };
611 escalations.push(EscalationRow {
612 ts: e.ts,
613 mission_id: mission_id.to_string(),
614 kind: EscalationKind::Block,
615 summary: reason.clone(),
616 decision,
617 latency_ms: None,
618 rubber_stamp: None,
619 });
620 }
621
622 for (idx, e) in mission_events.iter().enumerate() {
625 let EventKind::PlanRevisionProposed {
626 revision,
627 instructions,
628 ..
629 } = &e.kind
630 else {
631 continue;
632 };
633 let mut decision = "pending".to_string();
634 for cand in mission_events.iter().skip(idx + 1) {
635 match &cand.kind {
636 EventKind::PlanRevised { revision: rev, .. } if rev == revision => {
637 decision = format!("accepted (rev {rev})");
638 break;
639 }
640 EventKind::PlanRevisionRejected {
641 revision: rev,
642 reason,
643 } if rev == revision => {
644 decision = format!("rejected: {reason}");
645 break;
646 }
647 _ => {}
648 }
649 }
650 escalations.push(EscalationRow {
651 ts: e.ts,
652 mission_id: mission_id.to_string(),
653 kind: EscalationKind::Revision,
654 summary: instructions.clone(),
655 decision,
656 latency_ms: None,
657 rubber_stamp: None,
658 });
659 }
660
661 let mut non_meta_commits: u64 = 0;
665 for e in &mission_events {
666 if let EventKind::FeatureCompleted { commits, .. } = &e.kind {
667 for commit in commits {
668 let subject = commit.split_once(' ').map(|(_, s)| s).unwrap_or("");
669 if !crate::contract_sweep::is_meta_commit(subject) {
670 non_meta_commits += 1;
671 }
672 }
673 }
674 }
675
676 let config = mission_events.iter().find_map(|e| match &e.kind {
679 EventKind::MissionCreated { config, .. } => Some(config),
680 _ => None,
681 });
682 let task_class = mission_events.iter().find_map(|e| match &e.kind {
685 EventKind::MissionCreated { goal, .. } => crate::ticket::parse_task_class_from_goal(goal),
686 _ => None,
687 });
688 let mut run_models = std::collections::HashMap::new();
689 for e in &mission_events {
690 if let EventKind::WorkerSpawned {
691 run_id,
692 role,
693 model,
694 backend,
695 ..
696 } = &e.kind
697 {
698 run_models.insert(run_id.as_str(), (model.as_str(), *role, *backend));
699 }
700 }
701 let mut cost_usd = 0.0;
702 let mut token_sums: std::collections::BTreeMap<&'static str, BackendTokenSum> =
706 std::collections::BTreeMap::new();
707 for e in &mission_events {
708 if let EventKind::WorkerCompleted {
709 run_id,
710 tokens,
711 cost_usd: recorded,
712 ..
713 } = &e.kind
714 {
715 let (model, role, recorded_backend) = run_models
716 .get(run_id.as_str())
717 .copied()
718 .unwrap_or(("", crate::types::Role::Worker, None));
719 let backend = crate::cost::resolved_run_backend(recorded_backend, role, config);
720 cost_usd += crate::cost::resolved_run_cost(*recorded, tokens, model, backend);
721 let sum = token_sums
722 .entry(backend.as_str())
723 .or_insert(BackendTokenSum {
724 backend,
725 runs: 0,
726 fresh_input: 0,
727 cache_read: 0,
728 cache_write: 0,
729 });
730 sum.runs += 1;
731 sum.fresh_input += tokens.input;
732 sum.cache_read += tokens.cache_read;
733 sum.cache_write += tokens.cache_write;
734 }
735 }
736
737 let created_ts = mission_events
740 .iter()
741 .find(|e| matches!(e.kind, EventKind::MissionCreated { .. }))
742 .map(|e| e.ts);
743 let terminal_ts = mission_events.iter().find_map(|e| {
744 matches!(
745 e.kind,
746 EventKind::MissionCompleted {}
747 | EventKind::MissionFailed { .. }
748 | EventKind::MissionAbandoned { .. }
749 )
750 .then_some(e.ts)
751 });
752 let cycle_time_ms = match (created_ts, terminal_ts) {
753 (Some(start), Some(end)) => {
754 let mut paused_ms: i64 = 0;
755 let mut pause_start: Option<DateTime<Utc>> = None;
756 for e in &mission_events {
757 match &e.kind {
758 EventKind::MissionPaused {} => pause_start = Some(e.ts),
759 EventKind::MissionResumed {} => {
760 if let Some(p) = pause_start.take() {
761 paused_ms += (e.ts - p).num_milliseconds().max(0);
762 }
763 }
764 _ => {}
765 }
766 }
767 if let Some(p) = pause_start {
768 paused_ms += (end - p).num_milliseconds().max(0);
769 }
770 Some(((end - start).num_milliseconds() - paused_ms).max(0) as u64)
771 }
772 _ => None,
773 };
774
775 let mut noted: u64 = 0;
782 let mut diverged: u64 = 0;
783 let mut resolved_units: std::collections::HashSet<&str> = std::collections::HashSet::new();
784 let mut resolved_selected: u64 = 0;
785 let mut resolved_none: u64 = 0;
786 for e in &mission_events {
787 match &e.kind {
788 EventKind::DivergenceNoted { diverged: d, .. } => {
789 noted += 1;
790 if *d {
791 diverged += 1;
792 }
793 }
794 EventKind::DivergenceResolved { unit, selected, .. } => {
795 let first_for_unit = resolved_units.insert(unit.as_str());
796 match (first_for_unit, selected) {
797 (true, Some(_)) => resolved_selected += 1,
798 (true, None) => resolved_none += 1,
799 (false, _) => {}
800 }
801 }
802 _ => {}
803 }
804 }
805 let divergences = (noted > 0 || !resolved_units.is_empty()).then(|| DivergenceOutcomes {
806 noted,
807 diverged,
808 agreed: noted - diverged,
809 resolved_selected,
810 resolved_none,
811 });
812
813 let gate_score_samples = crate::gate_score_flags::collect_scored_samples(&mission_events);
818
819 let comparison = ComparisonInputs {
828 terminal_ts,
829 base_branch: mission_events.iter().find_map(|e| match &e.kind {
830 EventKind::MissionCreated { base_branch, .. } => Some(base_branch.clone()),
831 _ => None,
832 }),
833 folded: crate::reducer::fold(events)
834 .ok()
835 .map(|state| FoldedMissionRefs {
836 status: state.mission.status,
837 base_branch: state.mission.base_branch,
838 mission_branch: state.mission.mission_branch,
839 }),
840 };
841
842 MissionOutcomes {
843 interventions,
844 is_closed,
845 latencies_ms,
846 escalations,
847 cost_usd,
848 non_meta_commits,
849 cycle_time_ms,
850 task_class,
851 token_sums: token_sums.into_values().collect(),
852 divergences,
853 gate_score_samples,
854 comparison,
855 }
856}
857
858#[derive(Default)]
860struct TaskClassAcc {
861 missions: u64,
862 closed_missions: u64,
863 total_cost_usd: f64,
864 non_meta_commits: u64,
865 escalations: u64,
866 advisor_invocations: u64,
867 cycle_count: u64,
868 cycle_total_ms: u64,
869}
870
871struct ReuseAcc {
873 backend: crate::types::BackendKind,
874 missions: u64,
875 runs: u64,
876 fresh_input: u64,
877 cache_read: u64,
878 cache_write: u64,
879}
880
881#[derive(Debug, Clone, Copy, PartialEq, Eq)]
885pub struct OutcomesOptions {
886 pub rubber_stamp_threshold_ms: u64,
889 pub comparison_window: Option<(u64, DateTime<Utc>)>,
897}
898
899impl Default for OutcomesOptions {
900 fn default() -> Self {
901 Self {
902 rubber_stamp_threshold_ms: crate::types::DEFAULT_RUBBER_STAMP_THRESHOLD_MS,
903 comparison_window: None,
904 }
905 }
906}
907
908impl OutcomesOptions {
909 pub fn resolve(repo_root: &std::path::Path) -> Self {
914 match crate::config::load(repo_root) {
915 Ok(cfg) => Self {
916 rubber_stamp_threshold_ms: cfg.rubber_stamp_threshold_ms,
917 ..Self::default()
918 },
919 Err(_) => Self::default(),
920 }
921 .with_comparison_window()
922 }
923
924 fn with_comparison_window(mut self) -> Self {
928 self.comparison_window = Some((DEFAULT_MERGED_CHANGE_WINDOW_DAYS, Utc::now()));
929 self
930 }
931}
932
933pub fn compute_outcomes(repo_root: &std::path::Path) -> anyhow::Result<Outcomes> {
940 compute_outcomes_with_options(repo_root, &OutcomesOptions::resolve(repo_root))
941}
942
943pub fn compute_outcomes_with_options(
946 repo_root: &std::path::Path,
947 options: &OutcomesOptions,
948) -> anyhow::Result<Outcomes> {
949 let index_contents = std::fs::read_to_string(
950 crate::paths::MissionPaths::new(repo_root, "_")
951 .missions_dir()
952 .join("index.md"),
953 )
954 .unwrap_or_default();
955
956 let mut ids = crate::paths::MissionPaths::list_missions(repo_root);
957 for id in crate::mission_catalog::mission_index_ids(&index_contents) {
958 if !ids.contains(&id) {
959 ids.push(id);
960 }
961 }
962 ids.sort();
963
964 let mut closed_missions: u64 = 0;
965 let mut total_interventions: u64 = 0;
966 let mut zero_intervention_missions: u64 = 0;
967 let mut all_latencies_ms = Vec::new();
968 let mut escalations = Vec::new();
969 let mut total_cost_usd = 0.0;
970 let mut total_non_meta_commits: u64 = 0;
971 let mut cycle_closed: u64 = 0;
972 let mut cycle_total_ms: u64 = 0;
973 let mut class_accs: std::collections::BTreeMap<String, TaskClassAcc> =
976 std::collections::BTreeMap::new();
977 let mut reuse_accs: std::collections::BTreeMap<&'static str, ReuseAcc> =
979 std::collections::BTreeMap::new();
980 let mut divergence_acc: Option<DivergenceOutcomes> = None;
983 let mut all_score_samples: Vec<crate::gate_score_flags::GateScoreSample> = Vec::new();
986 let mut comparison_inputs: Vec<(String, ComparisonInputs)> = Vec::new();
990
991 for id in ids {
992 let paths = crate::paths::MissionPaths::new(repo_root, &id);
993 let events_path = paths.events_file();
994 if !events_path.is_file() {
995 continue;
996 }
997 if paths.require_no_follow().is_err() {
1000 continue;
1001 }
1002 let Some(out) = cached_mission_outcomes(&id, &events_path) else {
1006 continue;
1007 };
1008 comparison_inputs.push((id.clone(), out.comparison.clone()));
1009 if out.is_closed {
1010 closed_missions += 1;
1011 total_interventions += out.interventions;
1012 if out.interventions == 0 {
1013 zero_intervention_missions += 1;
1014 }
1015 }
1016 all_latencies_ms.extend(out.latencies_ms);
1017 total_cost_usd += out.cost_usd;
1018 total_non_meta_commits += out.non_meta_commits;
1019 if let Some(ms) = out.cycle_time_ms {
1020 cycle_closed += 1;
1021 cycle_total_ms += ms;
1022 }
1023
1024 let class_key = out
1026 .task_class
1027 .clone()
1028 .unwrap_or_else(|| UNCLASSIFIED_TASK_CLASS.to_string());
1029 let acc = class_accs.entry(class_key).or_default();
1030 acc.missions += 1;
1031 if out.is_closed {
1032 acc.closed_missions += 1;
1033 }
1034 acc.total_cost_usd += out.cost_usd;
1035 acc.non_meta_commits += out.non_meta_commits;
1036 if let Some(ms) = out.cycle_time_ms {
1037 acc.cycle_count += 1;
1038 acc.cycle_total_ms += ms;
1039 }
1040 acc.escalations += out.escalations.len() as u64;
1041 acc.advisor_invocations += out
1042 .escalations
1043 .iter()
1044 .filter(|r| r.kind == EscalationKind::Grant)
1045 .count() as u64;
1046
1047 for sum in &out.token_sums {
1049 let acc = reuse_accs
1050 .entry(sum.backend.as_str())
1051 .or_insert_with(|| ReuseAcc {
1052 backend: sum.backend,
1053 missions: 0,
1054 runs: 0,
1055 fresh_input: 0,
1056 cache_read: 0,
1057 cache_write: 0,
1058 });
1059 acc.missions += 1;
1060 acc.runs += sum.runs;
1061 acc.fresh_input += sum.fresh_input;
1062 acc.cache_read += sum.cache_read;
1063 acc.cache_write += sum.cache_write;
1064 }
1065
1066 if let Some(d) = &out.divergences {
1068 let acc = divergence_acc.get_or_insert_with(DivergenceOutcomes::default);
1069 acc.noted += d.noted;
1070 acc.diverged += d.diverged;
1071 acc.agreed += d.agreed;
1072 acc.resolved_selected += d.resolved_selected;
1073 acc.resolved_none += d.resolved_none;
1074 }
1075
1076 all_score_samples.extend(out.gate_score_samples);
1077
1078 escalations.extend(out.escalations);
1079 }
1080
1081 let interventions_per_closed_mission = if closed_missions > 0 {
1082 total_interventions as f64 / closed_missions as f64
1083 } else {
1084 0.0
1085 };
1086 let zero_intervention_share = if closed_missions > 0 {
1087 zero_intervention_missions as f64 / closed_missions as f64
1088 } else {
1089 0.0
1090 };
1091
1092 escalations.sort_by_key(|e| std::cmp::Reverse(e.ts));
1093
1094 let mut approved_decisions: u64 = 0;
1099 let mut flagged: u64 = 0;
1100 for row in &mut escalations {
1101 if row.kind != EscalationKind::Grant || row.decision != "approved" {
1102 continue;
1103 }
1104 let Some(latency) = row.latency_ms else {
1105 continue;
1106 };
1107 approved_decisions += 1;
1108 let is_flagged = latency < options.rubber_stamp_threshold_ms;
1109 if is_flagged {
1110 flagged += 1;
1111 }
1112 row.rubber_stamp = Some(is_flagged);
1113 }
1114
1115 let mut task_classes: Vec<TaskClassRow> = class_accs
1118 .into_iter()
1119 .map(|(task_class, acc)| TaskClassRow {
1120 escalations_per_mission: acc.escalations as f64 / acc.missions as f64,
1121 usd_per_commit: (acc.non_meta_commits > 0)
1122 .then(|| acc.total_cost_usd / acc.non_meta_commits as f64),
1123 cycle_mean_ms: (acc.cycle_count > 0)
1124 .then(|| acc.cycle_total_ms as f64 / acc.cycle_count as f64),
1125 task_class,
1126 missions: acc.missions,
1127 closed_missions: acc.closed_missions,
1128 total_cost_usd: acc.total_cost_usd,
1129 non_meta_commits: acc.non_meta_commits,
1130 escalations: acc.escalations,
1131 advisor_invocations: acc.advisor_invocations,
1132 })
1133 .collect();
1134 task_classes.sort_by_key(|row| {
1135 (
1136 row.task_class == UNCLASSIFIED_TASK_CLASS,
1137 row.task_class.clone(),
1138 )
1139 });
1140
1141 let context_reuse: Vec<ContextReuseRow> = reuse_accs
1145 .into_values()
1146 .filter(|acc| acc.backend.reports_cache_read_tokens())
1147 .map(|acc| {
1148 let cache_write = acc
1149 .backend
1150 .reports_cache_write_tokens()
1151 .then_some(acc.cache_write);
1152 let cached = acc.cache_read + cache_write.unwrap_or(0);
1153 let total = acc.fresh_input + cached;
1154 ContextReuseRow {
1155 backend: acc.backend.as_str().to_string(),
1156 missions: acc.missions,
1157 runs: acc.runs,
1158 fresh_input: acc.fresh_input,
1159 cache_read: acc.cache_read,
1160 cache_write,
1161 reuse_share: (total > 0).then(|| cached as f64 / total as f64),
1162 }
1163 })
1164 .collect();
1165
1166 let comparison = options
1174 .comparison_window
1175 .map(|(window_days, now)| {
1176 crate::comparison_metrics::comparison_report_from_inputs(
1177 repo_root,
1178 &comparison_inputs,
1179 window_days,
1180 now,
1181 )
1182 })
1183 .transpose()?;
1184
1185 Ok(Outcomes {
1186 autonomy_ratio: AutonomyRatio {
1187 closed_missions,
1188 total_interventions,
1189 interventions_per_closed_mission,
1190 zero_intervention_missions,
1191 zero_intervention_share,
1192 },
1193 grant_latency: bucketize(&all_latencies_ms),
1194 escalations,
1195 cost_per_change: CostPerChange {
1196 total_cost_usd,
1197 non_meta_commits: total_non_meta_commits,
1198 usd_per_commit: (total_non_meta_commits > 0)
1199 .then(|| total_cost_usd / total_non_meta_commits as f64),
1200 },
1201 cycle_time: CycleTime {
1202 closed_missions: cycle_closed,
1203 total_ms: cycle_total_ms,
1204 mean_ms: (cycle_closed > 0).then(|| cycle_total_ms as f64 / cycle_closed as f64),
1205 },
1206 task_classes,
1207 context_reuse,
1208 rubber_stamp: RubberStampReport {
1209 threshold_ms: options.rubber_stamp_threshold_ms,
1210 approved_decisions,
1211 flagged,
1212 share: (approved_decisions > 0).then(|| flagged as f64 / approved_decisions as f64),
1213 },
1214 gate_score_flags: crate::gate_score_flags::score_distribution_report(&all_score_samples),
1215 divergences: divergence_acc,
1216 comparison,
1217 })
1218}
1219
1220pub const DEFAULT_MERGED_CHANGE_WINDOW_DAYS: u64 = 30;
1224
1225pub const MAX_MERGED_CHANGE_WINDOW_DAYS: u64 = 36_525;
1232
1233#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1239#[serde(rename_all = "camelCase")]
1240pub struct CostPerMergedChange {
1241 pub window_days: u64,
1243 pub closed_in_window: u64,
1246 pub total_cost_usd: f64,
1249 pub merged_changes: u64,
1251 pub usd_per_merged_change: Option<f64>,
1254 pub zero_intervention_share: Option<f64>,
1257}
1258
1259pub fn compute_cost_per_merged_change(
1268 repo_root: &std::path::Path,
1269 window_days: u64,
1270 now: DateTime<Utc>,
1271) -> anyhow::Result<CostPerMergedChange> {
1272 let index_contents = std::fs::read_to_string(
1273 crate::paths::MissionPaths::new(repo_root, "_")
1274 .missions_dir()
1275 .join("index.md"),
1276 )
1277 .unwrap_or_default();
1278
1279 let mut ids = crate::paths::MissionPaths::list_missions(repo_root);
1280 for id in crate::mission_catalog::mission_index_ids(&index_contents) {
1281 if !ids.contains(&id) {
1282 ids.push(id);
1283 }
1284 }
1285 ids.sort();
1286
1287 if window_days > MAX_MERGED_CHANGE_WINDOW_DAYS {
1294 return Err(crate::error::EngineError::InvalidState(format!(
1295 "window_days {window_days} exceeds the maximum {MAX_MERGED_CHANGE_WINDOW_DAYS} days"
1296 ))
1297 .into());
1298 }
1299 let days = i64::try_from(window_days).map_err(|_| {
1300 crate::error::EngineError::InvalidState(format!(
1301 "window_days {window_days} is out of range"
1302 ))
1303 })?;
1304 let window = chrono::Duration::try_days(days).ok_or_else(|| {
1305 crate::error::EngineError::InvalidState(format!(
1306 "window_days {window_days} is out of range"
1307 ))
1308 })?;
1309 let cutoff = now - window;
1310 let repo = crate::git_ops::GitRepo::open(repo_root).ok();
1311
1312 let mut closed_in_window: u64 = 0;
1313 let mut zero_intervention: u64 = 0;
1314 let mut total_cost_usd = 0.0;
1315 let mut merged_changes: u64 = 0;
1316
1317 for id in ids {
1318 let paths = crate::paths::MissionPaths::new(repo_root, &id);
1319 let events_path = paths.events_file();
1320 if !events_path.is_file() {
1321 continue;
1322 }
1323 if paths.require_no_follow().is_err() {
1324 continue;
1325 }
1326 let events = match crate::event_log::EventLog::read_events(&events_path) {
1327 Ok(events) => events,
1328 Err(_) => continue, };
1330 let Some(terminal_ts) = events.iter().find_map(|e| {
1333 matches!(
1334 e.kind,
1335 EventKind::MissionCompleted {}
1336 | EventKind::MissionFailed { .. }
1337 | EventKind::MissionAbandoned { .. }
1338 )
1339 .then_some(e.ts)
1340 }) else {
1341 continue; };
1343 if terminal_ts < cutoff || terminal_ts > now {
1344 continue;
1345 }
1346 let out = mission_outcomes(&id, &events);
1347 closed_in_window += 1;
1348 if out.interventions == 0 {
1349 zero_intervention += 1;
1350 }
1351 total_cost_usd += out.cost_usd;
1352
1353 if let (Some(repo), Some(folded)) = (repo.as_ref(), out.comparison.folded.as_ref()) {
1359 if folded.status == crate::types::MissionStatus::Complete
1360 && crate::merged::merged_bit_for_branches(
1361 repo,
1362 &folded.mission_branch,
1363 &folded.base_branch,
1364 ) == Some(true)
1365 {
1366 merged_changes += 1;
1367 }
1368 }
1369 }
1370
1371 Ok(CostPerMergedChange {
1372 window_days,
1373 closed_in_window,
1374 total_cost_usd,
1375 merged_changes,
1376 usd_per_merged_change: (merged_changes > 0).then(|| total_cost_usd / merged_changes as f64),
1377 zero_intervention_share: (closed_in_window > 0)
1378 .then(|| zero_intervention as f64 / closed_in_window as f64),
1379 })
1380}
1381
1382pub fn bucketize(latencies_ms: &[u64]) -> GrantLatency {
1385 let mut counts = [0u64; 4];
1386 for &latency in latencies_ms {
1387 let idx = if latency < 10_000 {
1388 0
1389 } else if latency < 60_000 {
1390 1
1391 } else if latency < 600_000 {
1392 2
1393 } else {
1394 3
1395 };
1396 counts[idx] += 1;
1397 }
1398 let buckets = BUCKET_LABELS
1399 .iter()
1400 .zip(counts)
1401 .map(|(label, count)| LatencyBucket {
1402 label: label.to_string(),
1403 count,
1404 })
1405 .collect();
1406 GrantLatency {
1407 buckets,
1408 total_decided: latencies_ms.len() as u64,
1409 }
1410}
1411
1412#[cfg(test)]
1413mod tests {
1414 use super::*;
1415 use crate::types::GrantKind;
1416
1417 fn ev(seq: u64, mission_id: &str, ts_secs: i64, kind: EventKind) -> Event {
1418 Event {
1419 seq,
1420 ts: DateTime::from_timestamp(ts_secs, 0).unwrap(),
1421 mission_id: mission_id.to_string(),
1422 kind,
1423 }
1424 }
1425
1426 fn ev_ms(seq: u64, mission_id: &str, ts_ms: i64, kind: EventKind) -> Event {
1427 Event {
1428 seq,
1429 ts: DateTime::from_timestamp_millis(ts_ms).unwrap(),
1430 mission_id: mission_id.to_string(),
1431 kind,
1432 }
1433 }
1434
1435 #[test]
1436 fn outcomes_latency_bucket_boundaries() {
1437 let latencies = vec![9_999, 10_000, 59_999, 60_000, 599_999, 600_000];
1438 let result = bucketize(&latencies);
1439 assert_eq!(result.total_decided, 6);
1440 assert_eq!(result.buckets.len(), 4);
1441 assert_eq!(result.buckets[0].label, "<10s");
1442 assert_eq!(result.buckets[0].count, 1); assert_eq!(result.buckets[1].label, "<60s");
1444 assert_eq!(result.buckets[1].count, 2); assert_eq!(result.buckets[2].label, "<10m");
1446 assert_eq!(result.buckets[2].count, 2); assert_eq!(result.buckets[3].label, ">=10m");
1448 assert_eq!(result.buckets[3].count, 1); }
1450
1451 #[test]
1452 fn outcomes_latency_empty_fills_all_zero_buckets() {
1453 let result = bucketize(&[]);
1454 assert_eq!(result.total_decided, 0);
1455 assert_eq!(result.buckets.len(), 4);
1456 assert!(result.buckets.iter().all(|b| b.count == 0));
1457 }
1458
1459 #[test]
1460 fn interventions_ignore_pre_approval_messages_but_count_post_approval() {
1461 let events = vec![
1462 ev(
1463 1,
1464 "m-1",
1465 100,
1466 EventKind::UserMessage {
1467 text: "before".into(),
1468 interrupt: false,
1469 },
1470 ),
1471 ev(
1472 2,
1473 "m-1",
1474 200,
1475 EventKind::PlanApproved {
1476 plan: sample_plan(),
1477 base_sha: None,
1478 },
1479 ),
1480 ev(
1481 3,
1482 "m-1",
1483 300,
1484 EventKind::UserMessage {
1485 text: "after".into(),
1486 interrupt: false,
1487 },
1488 ),
1489 ];
1490 let out = mission_outcomes("m-1", &events);
1491 assert_eq!(out.interventions, 1);
1492 }
1493
1494 #[test]
1495 fn interventions_count_each_decision_kind() {
1496 let events = vec![
1497 ev(
1498 1,
1499 "m-1",
1500 100,
1501 EventKind::PlanApproved {
1502 plan: sample_plan(),
1503 base_sha: None,
1504 },
1505 ),
1506 ev(
1507 2,
1508 "m-1",
1509 200,
1510 EventKind::GrantApproved {
1511 kind: GrantKind::Command,
1512 command: "cargo test".into(),
1513 },
1514 ),
1515 ev(
1516 3,
1517 "m-1",
1518 300,
1519 EventKind::GrantDenied {
1520 kind: GrantKind::Command,
1521 command: "rm -rf".into(),
1522 reason: "no".into(),
1523 },
1524 ),
1525 ev(
1526 4,
1527 "m-1",
1528 400,
1529 EventKind::PlanRevised {
1530 revision: 1,
1531 plan: sample_plan(),
1532 },
1533 ),
1534 ev(
1535 5,
1536 "m-1",
1537 500,
1538 EventKind::PlanRevisionRejected {
1539 revision: 2,
1540 reason: "bad".into(),
1541 },
1542 ),
1543 ];
1544 let out = mission_outcomes("m-1", &events);
1545 assert_eq!(out.interventions, 4);
1546 }
1547
1548 #[test]
1549 fn interventions_no_plan_approved_counts_zero_user_messages() {
1550 let events = vec![ev(
1551 1,
1552 "m-1",
1553 100,
1554 EventKind::UserMessage {
1555 text: "hi".into(),
1556 interrupt: false,
1557 },
1558 )];
1559 let out = mission_outcomes("m-1", &events);
1560 assert_eq!(out.interventions, 0);
1561 }
1562
1563 #[test]
1564 fn is_closed_true_for_each_terminal_event() {
1565 for kind in [
1566 EventKind::MissionCompleted {},
1567 EventKind::MissionFailed { reason: "x".into() },
1568 EventKind::MissionAbandoned { reason: "x".into() },
1569 ] {
1570 let events = vec![ev(1, "m-1", 100, kind)];
1571 let out = mission_outcomes("m-1", &events);
1572 assert!(out.is_closed);
1573 }
1574 }
1575
1576 #[test]
1577 fn is_closed_false_without_terminal_event() {
1578 let events = vec![ev(
1579 1,
1580 "m-1",
1581 100,
1582 EventKind::PlanApproved {
1583 plan: sample_plan(),
1584 base_sha: None,
1585 },
1586 )];
1587 let out = mission_outcomes("m-1", &events);
1588 assert!(!out.is_closed);
1589 }
1590
1591 #[test]
1592 fn escalation_grant_row_approved_denied_pending() {
1593 let events = vec![
1594 ev_ms(
1595 1,
1596 "m-1",
1597 0,
1598 EventKind::GrantRequested {
1599 milestone_id: "ms-1".into(),
1600 kind: GrantKind::Command,
1601 command: "cargo test".into(),
1602 },
1603 ),
1604 ev_ms(
1605 2,
1606 "m-1",
1607 5_000,
1608 EventKind::GrantApproved {
1609 kind: GrantKind::Command,
1610 command: "cargo test".into(),
1611 },
1612 ),
1613 ev_ms(
1614 3,
1615 "m-1",
1616 10_000,
1617 EventKind::GrantRequested {
1618 milestone_id: "ms-1".into(),
1619 kind: GrantKind::Command,
1620 command: "rm -rf".into(),
1621 },
1622 ),
1623 ev_ms(
1624 4,
1625 "m-1",
1626 15_000,
1627 EventKind::GrantDenied {
1628 kind: GrantKind::Command,
1629 command: "rm -rf".into(),
1630 reason: "unsafe".into(),
1631 },
1632 ),
1633 ev_ms(
1634 5,
1635 "m-1",
1636 20_000,
1637 EventKind::GrantRequested {
1638 milestone_id: "ms-1".into(),
1639 kind: GrantKind::Command,
1640 command: "still pending".into(),
1641 },
1642 ),
1643 ];
1644 let out = mission_outcomes("m-1", &events);
1645 let grants: Vec<_> = out
1646 .escalations
1647 .iter()
1648 .filter(|r| r.kind == EscalationKind::Grant)
1649 .collect();
1650 assert_eq!(grants.len(), 3);
1651 assert_eq!(grants[0].summary, "cargo test");
1652 assert_eq!(grants[0].decision, "approved");
1653 assert_eq!(grants[0].latency_ms, Some(5_000));
1654 assert_eq!(grants[1].summary, "rm -rf");
1655 assert_eq!(grants[1].decision, "denied: unsafe");
1656 assert_eq!(grants[1].latency_ms, Some(5_000));
1657 assert_eq!(grants[2].summary, "still pending");
1658 assert_eq!(grants[2].decision, "pending");
1659 assert_eq!(grants[2].latency_ms, None);
1660 }
1661
1662 #[test]
1663 fn escalation_block_row_open_and_unblocked() {
1664 let events = vec![
1665 ev(
1666 1,
1667 "m-1",
1668 0,
1669 EventKind::MilestoneBlocked {
1670 block_context: None,
1671 milestone_id: "ms-1".into(),
1672 reason: "waiting".into(),
1673 },
1674 ),
1675 ev(
1676 2,
1677 "m-1",
1678 10,
1679 EventKind::MilestoneUnblocked {
1680 block_context: None,
1681 milestone_id: "ms-1".into(),
1682 reason: "cap raised".into(),
1683 validator_guidance: None,
1684 },
1685 ),
1686 ev(
1687 3,
1688 "m-1",
1689 20,
1690 EventKind::MilestoneBlocked {
1691 block_context: None,
1692 milestone_id: "ms-2".into(),
1693 reason: "still stuck".into(),
1694 },
1695 ),
1696 ];
1697 let out = mission_outcomes("m-1", &events);
1698 let blocks: Vec<_> = out
1699 .escalations
1700 .iter()
1701 .filter(|r| r.kind == EscalationKind::Block)
1702 .collect();
1703 assert_eq!(blocks.len(), 2);
1704 assert_eq!(blocks[0].summary, "waiting");
1705 assert_eq!(blocks[0].decision, "unblocked: cap raised");
1706 assert_eq!(blocks[1].summary, "still stuck");
1707 assert_eq!(blocks[1].decision, "open");
1708 }
1709
1710 #[test]
1711 fn escalation_revision_row_accepted_rejected_pending() {
1712 let events = vec![
1713 ev(
1714 1,
1715 "m-1",
1716 0,
1717 EventKind::PlanRevisionProposed {
1718 revision: 1,
1719 plan: sample_plan(),
1720 instructions: "add tests".into(),
1721 },
1722 ),
1723 ev(
1724 2,
1725 "m-1",
1726 10,
1727 EventKind::PlanRevised {
1728 revision: 1,
1729 plan: sample_plan(),
1730 },
1731 ),
1732 ev(
1733 3,
1734 "m-1",
1735 20,
1736 EventKind::PlanRevisionProposed {
1737 revision: 2,
1738 plan: sample_plan(),
1739 instructions: "drop scope".into(),
1740 },
1741 ),
1742 ev(
1743 4,
1744 "m-1",
1745 30,
1746 EventKind::PlanRevisionRejected {
1747 revision: 2,
1748 reason: "too risky".into(),
1749 },
1750 ),
1751 ev(
1752 5,
1753 "m-1",
1754 40,
1755 EventKind::PlanRevisionProposed {
1756 revision: 3,
1757 plan: sample_plan(),
1758 instructions: "pending one".into(),
1759 },
1760 ),
1761 ];
1762 let out = mission_outcomes("m-1", &events);
1763 let revisions: Vec<_> = out
1764 .escalations
1765 .iter()
1766 .filter(|r| r.kind == EscalationKind::Revision)
1767 .collect();
1768 assert_eq!(revisions.len(), 3);
1769 assert_eq!(revisions[0].summary, "add tests");
1770 assert_eq!(revisions[0].decision, "accepted (rev 1)");
1771 assert_eq!(revisions[1].summary, "drop scope");
1772 assert_eq!(revisions[1].decision, "rejected: too risky");
1773 assert_eq!(revisions[2].summary, "pending one");
1774 assert_eq!(revisions[2].decision, "pending");
1775 }
1776
1777 fn sample_plan() -> crate::types::Plan {
1778 crate::types::Plan {
1779 goal: "g".into(),
1780 validation_contract: vec![],
1781 milestones: vec![],
1782 considered_alternatives: None,
1783 command_grants: vec![],
1784 touch_set: vec![],
1785 standards_manifest: None,
1786 reviewer_independence: None,
1787 }
1788 }
1789
1790 #[test]
1795 fn divergence_event_outcomes_fold_surfaces_count_and_resolution_kind() {
1796 let candidate = |run_id: &str, tree: &str| crate::types::DivergenceCandidate {
1797 run_id: run_id.into(),
1798 branch: format!("kranz/pool/m-1/f-1-1-{run_id}"),
1799 backend: "claude".into(),
1800 tree: tree.into(),
1801 };
1802 let noted = |seq: u64, unit: &str, diverged: bool| {
1803 ev(
1804 seq,
1805 "m-1",
1806 seq as i64,
1807 EventKind::DivergenceNoted {
1808 unit: unit.into(),
1809 candidates: vec![candidate("r-c0", "aaa"), candidate("r-c1", "bbb")],
1810 diverged,
1811 },
1812 )
1813 };
1814 let resolved = |seq: u64, unit: &str, selected: Option<u32>| {
1815 ev(
1816 seq,
1817 "m-1",
1818 seq as i64,
1819 EventKind::DivergenceResolved {
1820 unit: unit.into(),
1821 selected,
1822 reason: "r".into(),
1823 decided_by: "operator".into(),
1824 },
1825 )
1826 };
1827 let events = vec![
1828 noted(1, "f-1-1", true), noted(2, "f-1-2", false), resolved(3, "f-1-1", Some(1)), resolved(4, "f-1-2", None), resolved(5, "f-1-1", Some(0)), ];
1834 let out = mission_outcomes("m-1", &events);
1835 let ledger = out.divergences.expect("a pool mission has a ledger");
1836 assert_eq!(ledger.noted, 2, "two units compared");
1837 assert_eq!(ledger.diverged, 1);
1838 assert_eq!(
1839 ledger.agreed, 1,
1840 "the agreement record counts — logged, never trusted"
1841 );
1842 assert_eq!(ledger.resolved_selected, 1, "first-wins dedupes the repeat");
1843 assert_eq!(ledger.resolved_none, 1);
1844
1845 let quiet = mission_outcomes(
1847 "m-1",
1848 &[ev(
1849 1,
1850 "m-1",
1851 1,
1852 EventKind::UserMessage {
1853 text: "hi".into(),
1854 interrupt: false,
1855 },
1856 )],
1857 );
1858 assert_eq!(
1859 quiet.divergences, None,
1860 "absent for missions without pools — never a zeroed row"
1861 );
1862 }
1863
1864 mod compute_outcomes_tests {
1865 use super::*;
1866 use crate::event_log::{EventLog, LockForce};
1867 use crate::paths::MissionPaths;
1868 use crate::types::{GrantKind, MissionConfig};
1869 use std::time::Duration;
1870 use tempfile::TempDir;
1871
1872 fn seed_mission(repo_root: &std::path::Path, id: &str, kinds: Vec<EventKind>) {
1874 let paths = MissionPaths::new(repo_root, id);
1875 let mut log = EventLog::acquire(&paths, id, Duration::ZERO, LockForce::No).unwrap();
1876 for kind in kinds {
1877 log.append(kind).unwrap();
1878 }
1879 }
1880
1881 fn write_timed_log(repo_root: &std::path::Path, id: &str, events: Vec<Event>) {
1884 let paths = MissionPaths::new(repo_root, id);
1885 std::fs::create_dir_all(paths.mission_dir()).unwrap();
1886 let lines: Vec<String> = events
1887 .iter()
1888 .map(|e| serde_json::to_string(e).unwrap())
1889 .collect();
1890 std::fs::write(paths.events_file(), lines.join("\n") + "\n").unwrap();
1891 }
1892
1893 fn timed(seq: u64, secs: i64, kind: EventKind) -> Event {
1894 Event {
1895 seq,
1896 ts: chrono::DateTime::from_timestamp(secs, 0).unwrap(),
1897 mission_id: "m-cost".into(),
1898 kind,
1899 }
1900 }
1901
1902 #[test]
1903 fn cost_per_change_and_cycle_time_fold_from_events() {
1904 use crate::types::{Role, RunResult, TokenUsage};
1905
1906 let tmp = TempDir::new().unwrap();
1907 let root = tmp.path();
1908 write_timed_log(
1909 root,
1910 "m-cost",
1911 vec![
1912 timed(1, 0, created("g")),
1913 timed(
1914 2,
1915 10,
1916 EventKind::PlanApproved {
1917 plan: sample_plan(),
1918 base_sha: None,
1919 },
1920 ),
1921 timed(
1922 3,
1923 20,
1924 EventKind::WorkerSpawned {
1925 backend: None,
1926 run_id: "r-1".into(),
1927 role: Role::Worker,
1928 feature_id: Some("f-1-1".into()),
1929 milestone_id: Some("ms-1".into()),
1930 candidate: None,
1931 executor_route: None,
1932 sdk_session_id: "s".into(),
1933 model: "sonnet".into(),
1934 quant: "n/a".into(),
1935 weight_hash: None,
1936 prompt_hash: "h".into(),
1937 transcript_path: "t".into(),
1938 },
1939 ),
1940 timed(
1941 4,
1942 100,
1943 EventKind::WorkerCompleted {
1944 run_id: "r-1".into(),
1945 result: RunResult::Pass,
1946 tokens: TokenUsage {
1947 input: 1,
1948 output: 1,
1949 cache_read: 0,
1950 cache_write: 0,
1951 },
1952 cost_usd: Some(12.50),
1953 report: None,
1954 },
1955 ),
1956 timed(
1957 5,
1958 110,
1959 EventKind::FeatureCompleted {
1960 feature_id: "f-1-1".into(),
1961 commits: vec![
1962 "aaa [f-1-1] add the thing".to_string(),
1963 "bbb [kranz] mission report for m-cost".to_string(),
1964 ],
1965 },
1966 ),
1967 timed(6, 120, EventKind::MissionPaused {}),
1968 timed(7, 130, EventKind::MissionResumed {}),
1969 timed(8, 160, EventKind::MissionCompleted {}),
1970 ],
1971 );
1972
1973 let outcomes = compute_outcomes(root).unwrap();
1974 let cost = &outcomes.cost_per_change;
1975 assert_eq!(cost.total_cost_usd, 12.50);
1976 assert_eq!(cost.non_meta_commits, 1);
1978 assert_eq!(cost.usd_per_commit, Some(12.50));
1979
1980 let cycle = &outcomes.cycle_time;
1981 assert_eq!(cycle.closed_missions, 1);
1982 assert_eq!(cycle.total_ms, 150_000);
1984 assert_eq!(cycle.mean_ms, Some(150_000.0));
1985 }
1986
1987 #[test]
1988 fn cost_fallback_prices_tokens_with_spawned_model_and_local_is_zero() {
1989 use crate::types::{Role, RunResult, TokenUsage};
1990
1991 let tmp = TempDir::new().unwrap();
1992 let root = tmp.path();
1993 let mut local_cfg = MissionConfig::default();
1996 local_cfg.worker.backend = Some("local".into());
1997 let tokens = TokenUsage {
1998 input: 2_000_000,
1999 output: 100_000,
2000 cache_read: 0,
2001 cache_write: 0,
2002 };
2003 write_timed_log(
2004 root,
2005 "m-cost",
2006 vec![
2007 timed(
2008 1,
2009 0,
2010 EventKind::MissionCreated {
2011 goal: "g".into(),
2012 base_branch: "main".into(),
2013 mission_branch: "kranz/mission-x".into(),
2014 config: local_cfg,
2015 },
2016 ),
2017 timed(
2018 2,
2019 10,
2020 EventKind::WorkerSpawned {
2021 backend: None,
2022 run_id: "r-1".into(),
2023 role: Role::Worker,
2024 feature_id: None,
2025 milestone_id: Some("ms-1".into()),
2026 candidate: None,
2027 executor_route: None,
2028 sdk_session_id: "s".into(),
2029 model: "my-local-model".into(),
2030 quant: "n/a".into(),
2031 weight_hash: None,
2032 prompt_hash: "h".into(),
2033 transcript_path: "t".into(),
2034 },
2035 ),
2036 timed(
2037 3,
2038 20,
2039 EventKind::WorkerCompleted {
2040 run_id: "r-1".into(),
2041 result: RunResult::Pass,
2042 tokens,
2043 cost_usd: None,
2044 report: None,
2045 },
2046 ),
2047 timed(4, 30, EventKind::MissionCompleted {}),
2048 ],
2049 );
2050
2051 let outcomes = compute_outcomes(root).unwrap();
2052 assert_eq!(
2053 outcomes.cost_per_change.total_cost_usd, 0.0,
2054 "the local tier must price at $0, never the frontier fallback"
2055 );
2056 }
2057
2058 #[test]
2059 fn memoized_fold_skips_unchanged_logs_and_invalidates_on_new_events() {
2060 let tmp = TempDir::new().unwrap();
2061 let root = tmp.path();
2062 let events_path = MissionPaths::new(root, "m-cache").events_file();
2063
2064 seed_mission(
2065 root,
2066 "m-cache",
2067 vec![
2068 created("g"),
2069 EventKind::PlanApproved {
2070 plan: sample_plan(),
2071 base_sha: None,
2072 },
2073 EventKind::MissionCompleted {},
2074 ],
2075 );
2076 let first = compute_outcomes(root).unwrap();
2077 assert_eq!(
2078 cache_entry_stats(&events_path),
2079 Some((1, 0)),
2080 "first fold computes once, no hits"
2081 );
2082
2083 let second = compute_outcomes(root).unwrap();
2084 assert_eq!(
2085 cache_entry_stats(&events_path),
2086 Some((1, 1)),
2087 "an unchanged log is served from the cache — no re-parse"
2088 );
2089 assert_eq!(first, second);
2090
2091 seed_mission(
2094 root,
2095 "m-cache",
2096 vec![
2097 EventKind::GrantRequested {
2098 milestone_id: "ms-1".into(),
2099 kind: GrantKind::Command,
2100 command: "cargo test".into(),
2101 },
2102 EventKind::GrantApproved {
2103 kind: GrantKind::Command,
2104 command: "cargo test".into(),
2105 },
2106 ],
2107 );
2108 let third = compute_outcomes(root).unwrap();
2109 assert_eq!(
2110 cache_entry_stats(&events_path),
2111 Some((2, 1)),
2112 "appended events invalidate the memo entry"
2113 );
2114 assert_ne!(third, second);
2115 }
2116
2117 #[test]
2124 fn comparison_fold_reuses_the_memoized_native_fold_scan() {
2125 let tmp = TempDir::new().unwrap();
2126 let root = tmp.path();
2127 let events_path = MissionPaths::new(root, "m-cmp").events_file();
2128 seed_mission(
2129 root,
2130 "m-cmp",
2131 vec![
2132 created("g"),
2133 EventKind::PlanApproved {
2134 plan: sample_plan(),
2135 base_sha: None,
2136 },
2137 EventKind::MissionCompleted {},
2138 ],
2139 );
2140
2141 let outcomes = compute_outcomes(root).unwrap();
2142 assert_eq!(
2143 cache_entry_stats(&events_path),
2144 Some((1, 0)),
2145 "the native fold computes once"
2146 );
2147 let attached = outcomes
2151 .comparison
2152 .as_ref()
2153 .expect("resolve() pins the comparison window");
2154 assert_eq!(
2155 attached.assisted_change_share.base_branch.as_deref(),
2156 Some("main")
2157 );
2158 assert_eq!(attached.window_days, DEFAULT_MERGED_CHANGE_WINDOW_DAYS);
2159
2160 let report = crate::comparison_metrics::compute_comparison_report(
2162 root,
2163 DEFAULT_MERGED_CHANGE_WINDOW_DAYS,
2164 chrono::Utc::now(),
2165 )
2166 .unwrap();
2167 assert_eq!(
2168 cache_entry_stats(&events_path),
2169 Some((1, 1)),
2170 "the comparison fold must ride the memoized scan, not re-read the log"
2171 );
2172 assert_eq!(&report, attached, "same inputs, same report");
2173 }
2174
2175 fn created(goal: &str) -> EventKind {
2176 EventKind::MissionCreated {
2177 goal: goal.into(),
2178 base_branch: "main".into(),
2179 mission_branch: "kranz/mission-x".into(),
2180 config: MissionConfig::default(),
2181 }
2182 }
2183
2184 #[test]
2185 fn outcomes_ratio_denominator_is_closed_missions() {
2186 let tmp = TempDir::new().unwrap();
2187 let root = tmp.path();
2188
2189 seed_mission(
2191 root,
2192 "m-closed",
2193 vec![
2194 created("closed one"),
2195 EventKind::PlanApproved {
2196 plan: sample_plan(),
2197 base_sha: None,
2198 },
2199 EventKind::GrantApproved {
2200 kind: GrantKind::Command,
2201 command: "cargo test".into(),
2202 },
2203 EventKind::GrantDenied {
2204 kind: GrantKind::Command,
2205 command: "rm -rf".into(),
2206 reason: "no".into(),
2207 },
2208 EventKind::MissionCompleted {},
2209 ],
2210 );
2211
2212 seed_mission(
2215 root,
2216 "m-open",
2217 vec![
2218 created("still running"),
2219 EventKind::PlanApproved {
2220 plan: sample_plan(),
2221 base_sha: None,
2222 },
2223 EventKind::GrantApproved {
2224 kind: GrantKind::Command,
2225 command: "echo hi".into(),
2226 },
2227 ],
2228 );
2229
2230 let outcomes = compute_outcomes(root).unwrap();
2231 let ratio = outcomes.autonomy_ratio;
2232 assert_eq!(ratio.closed_missions, 1);
2233 assert_eq!(ratio.total_interventions, 2);
2234 assert_eq!(ratio.interventions_per_closed_mission, 2.0);
2235 assert_eq!(ratio.zero_intervention_missions, 0);
2236 assert_eq!(ratio.zero_intervention_share, 0.0);
2237 }
2238
2239 #[test]
2240 fn outcomes_ratio_zero_intervention_share_counts_clean_closed_missions() {
2241 let tmp = TempDir::new().unwrap();
2242 let root = tmp.path();
2243
2244 seed_mission(
2245 root,
2246 "m-clean",
2247 vec![
2248 created("clean"),
2249 EventKind::PlanApproved {
2250 plan: sample_plan(),
2251 base_sha: None,
2252 },
2253 EventKind::MissionCompleted {},
2254 ],
2255 );
2256 seed_mission(
2257 root,
2258 "m-dirty",
2259 vec![
2260 created("dirty"),
2261 EventKind::PlanApproved {
2262 plan: sample_plan(),
2263 base_sha: None,
2264 },
2265 EventKind::GrantApproved {
2266 kind: GrantKind::Command,
2267 command: "cargo test".into(),
2268 },
2269 EventKind::MissionCompleted {},
2270 ],
2271 );
2272
2273 let outcomes = compute_outcomes(root).unwrap();
2274 let ratio = outcomes.autonomy_ratio;
2275 assert_eq!(ratio.closed_missions, 2);
2276 assert_eq!(ratio.zero_intervention_missions, 1);
2277 assert_eq!(ratio.zero_intervention_share, 0.5);
2278 }
2279
2280 #[test]
2281 fn outcomes_ledger_newest_first() {
2282 let tmp = TempDir::new().unwrap();
2283 let root = tmp.path();
2284
2285 seed_mission(
2287 root,
2288 "m-a",
2289 vec![
2290 created("a"),
2291 EventKind::GrantRequested {
2292 milestone_id: "ms-1".into(),
2293 kind: GrantKind::Command,
2294 command: "cargo test".into(),
2295 },
2296 EventKind::GrantApproved {
2297 kind: GrantKind::Command,
2298 command: "cargo test".into(),
2299 },
2300 ],
2301 );
2302 seed_mission(
2303 root,
2304 "m-b",
2305 vec![
2306 created("b"),
2307 EventKind::GrantRequested {
2308 milestone_id: "ms-1".into(),
2309 kind: GrantKind::Command,
2310 command: "npm test".into(),
2311 },
2312 EventKind::GrantDenied {
2313 kind: GrantKind::Command,
2314 command: "npm test".into(),
2315 reason: "no".into(),
2316 },
2317 ],
2318 );
2319
2320 let outcomes = compute_outcomes(root).unwrap();
2321 assert!(outcomes.escalations.len() >= 2);
2322 for pair in outcomes.escalations.windows(2) {
2323 assert!(pair[0].ts >= pair[1].ts);
2324 }
2325 let mission_order: Vec<&str> = outcomes
2328 .escalations
2329 .iter()
2330 .map(|r| r.mission_id.as_str())
2331 .collect();
2332 assert_eq!(mission_order[0], "m-b");
2333 }
2334
2335 #[test]
2336 fn outcomes_empty_repo_yields_all_zero_defaults() {
2337 let tmp = TempDir::new().unwrap();
2338 let outcomes = compute_outcomes(tmp.path()).unwrap();
2339
2340 let ratio = outcomes.autonomy_ratio;
2341 assert_eq!(ratio.closed_missions, 0);
2342 assert_eq!(ratio.interventions_per_closed_mission, 0.0);
2343 assert_eq!(ratio.zero_intervention_share, 0.0);
2344
2345 assert_eq!(outcomes.grant_latency.buckets.len(), 4);
2346 assert!(outcomes.grant_latency.buckets.iter().all(|b| b.count == 0));
2347 assert_eq!(outcomes.grant_latency.total_decided, 0);
2348
2349 assert!(outcomes.escalations.is_empty());
2350 }
2351
2352 #[test]
2358 fn window_days_bound_over_max_errors_instead_of_panicking() {
2359 let tmp = TempDir::new().unwrap();
2360 let now = Utc::now();
2361 let err = compute_cost_per_merged_change(tmp.path(), u64::MAX, now)
2362 .expect_err("u64::MAX must error, never wrap or panic");
2363 assert!(err.to_string().contains("exceeds the maximum"), "{err}");
2364 let err =
2365 compute_cost_per_merged_change(tmp.path(), MAX_MERGED_CHANGE_WINDOW_DAYS + 1, now)
2366 .expect_err("just over the bound errors");
2367 assert!(err.to_string().contains("exceeds the maximum"), "{err}");
2368 let report =
2369 compute_cost_per_merged_change(tmp.path(), MAX_MERGED_CHANGE_WINDOW_DAYS, now)
2370 .expect("the documented maximum computes");
2371 assert_eq!(report.window_days, MAX_MERGED_CHANGE_WINDOW_DAYS);
2372 assert_eq!(report.closed_in_window, 0);
2373 }
2374
2375 #[test]
2376 fn outcomes_skips_mission_with_corrupt_event_log() {
2377 let tmp = TempDir::new().unwrap();
2378 let root = tmp.path();
2379
2380 seed_mission(
2381 root,
2382 "m-good",
2383 vec![
2384 created("good"),
2385 EventKind::PlanApproved {
2386 plan: sample_plan(),
2387 base_sha: None,
2388 },
2389 EventKind::MissionCompleted {},
2390 ],
2391 );
2392
2393 let bad_paths = MissionPaths::new(root, "m-bad");
2395 std::fs::create_dir_all(bad_paths.mission_dir()).unwrap();
2396 std::fs::write(bad_paths.events_file(), "not valid json\n").unwrap();
2397
2398 let outcomes = compute_outcomes(root).unwrap();
2399 assert_eq!(outcomes.autonomy_ratio.closed_missions, 1);
2400 }
2401
2402 #[test]
2405 fn divergence_event_fleet_ledger_sums_only_pool_missions() {
2406 let candidate = |run_id: &str| crate::types::DivergenceCandidate {
2407 run_id: run_id.into(),
2408 branch: format!("kranz/pool/m-pool/f-1-1-{run_id}"),
2409 backend: "claude".into(),
2410 tree: "aaa".into(),
2411 };
2412 let tmp = TempDir::new().unwrap();
2413 let root = tmp.path();
2414 seed_mission(
2415 root,
2416 "m-pool",
2417 vec![
2418 created("pool"),
2419 EventKind::DivergenceNoted {
2420 unit: "f-1-1".into(),
2421 candidates: vec![candidate("r-c0"), candidate("r-c1")],
2422 diverged: true,
2423 },
2424 EventKind::DivergenceResolved {
2425 unit: "f-1-1".into(),
2426 selected: Some(0),
2427 reason: "kept".into(),
2428 decided_by: "operator".into(),
2429 },
2430 ],
2431 );
2432 seed_mission(root, "m-quiet", vec![created("quiet")]);
2433
2434 let outcomes = compute_outcomes(root).unwrap();
2435 let ledger = outcomes
2436 .divergences
2437 .expect("a repo with a pool mission reports a fleet ledger");
2438 assert_eq!(ledger.noted, 1);
2439 assert_eq!(ledger.diverged, 1);
2440 assert_eq!(ledger.agreed, 0);
2441 assert_eq!(ledger.resolved_selected, 1);
2442 assert_eq!(ledger.resolved_none, 0);
2443
2444 let tmp2 = TempDir::new().unwrap();
2447 seed_mission(tmp2.path(), "m-quiet", vec![created("quiet")]);
2448 let outcomes = compute_outcomes(tmp2.path()).unwrap();
2449 assert_eq!(outcomes.divergences, None);
2450 let value = serde_json::to_value(&outcomes).unwrap();
2451 assert!(
2452 !value.as_object().unwrap().contains_key("divergences"),
2453 "no divergences key on the wire without pools: {value}"
2454 );
2455 }
2456 }
2457
2458 mod outcomes_report_tests {
2462 use super::*;
2463 use crate::paths::MissionPaths;
2464 use crate::types::{GrantKind, MissionConfig, Role, RunResult, TokenUsage};
2465 use tempfile::TempDir;
2466
2467 fn ev_ms(seq: u64, mission_id: &str, ts_ms: i64, kind: EventKind) -> Event {
2468 Event {
2469 seq,
2470 ts: DateTime::from_timestamp_millis(ts_ms).unwrap(),
2471 mission_id: mission_id.to_string(),
2472 kind,
2473 }
2474 }
2475
2476 fn write_log(repo_root: &std::path::Path, id: &str, events: Vec<Event>) {
2477 let paths = MissionPaths::new(repo_root, id);
2478 std::fs::create_dir_all(paths.mission_dir()).unwrap();
2479 let lines: Vec<String> = events
2480 .iter()
2481 .map(|e| serde_json::to_string(e).unwrap())
2482 .collect();
2483 std::fs::write(paths.events_file(), lines.join("\n") + "\n").unwrap();
2484 }
2485
2486 fn sample_plan() -> crate::types::Plan {
2487 crate::types::Plan {
2488 goal: "g".into(),
2489 validation_contract: vec![],
2490 milestones: vec![],
2491 considered_alternatives: None,
2492 command_grants: vec![],
2493 touch_set: vec![],
2494 standards_manifest: None,
2495 reviewer_independence: None,
2496 }
2497 }
2498
2499 fn created_with_class(mission_branch: &str, task_class: Option<&str>) -> EventKind {
2502 let goal = match task_class {
2503 Some(class) => format!("do the thing\n\n## Task class\n{class}\n"),
2504 None => "do the thing".to_string(),
2505 };
2506 created_with_config(mission_branch, goal, MissionConfig::default())
2507 }
2508
2509 fn created_with_config(
2510 mission_branch: &str,
2511 goal: String,
2512 config: MissionConfig,
2513 ) -> EventKind {
2514 EventKind::MissionCreated {
2515 goal,
2516 base_branch: "main".into(),
2517 mission_branch: mission_branch.into(),
2518 config,
2519 }
2520 }
2521
2522 fn worker_spawned(run_id: &str) -> EventKind {
2523 EventKind::WorkerSpawned {
2524 backend: None,
2525 run_id: run_id.into(),
2526 role: Role::Worker,
2527 feature_id: Some("f-1-1".into()),
2528 milestone_id: Some("ms-1".into()),
2529 candidate: None,
2530 executor_route: None,
2531 sdk_session_id: "s".into(),
2532 model: "sonnet".into(),
2533 quant: "n/a".into(),
2534 weight_hash: None,
2535 prompt_hash: "h".into(),
2536 transcript_path: "t".into(),
2537 }
2538 }
2539
2540 fn worker_completed(run_id: &str, tokens: TokenUsage, cost_usd: f64) -> EventKind {
2541 EventKind::WorkerCompleted {
2542 run_id: run_id.into(),
2543 result: RunResult::Pass,
2544 tokens,
2545 cost_usd: Some(cost_usd),
2546 report: None,
2547 }
2548 }
2549
2550 fn grant_req(seq: u64, mission_id: &str, ts_ms: i64, command: &str) -> Event {
2551 ev_ms(
2552 seq,
2553 mission_id,
2554 ts_ms,
2555 EventKind::GrantRequested {
2556 milestone_id: "ms-1".into(),
2557 kind: GrantKind::Command,
2558 command: command.into(),
2559 },
2560 )
2561 }
2562
2563 fn grant_yes(seq: u64, mission_id: &str, ts_ms: i64, command: &str) -> Event {
2564 ev_ms(
2565 seq,
2566 mission_id,
2567 ts_ms,
2568 EventKind::GrantApproved {
2569 kind: GrantKind::Command,
2570 command: command.into(),
2571 },
2572 )
2573 }
2574
2575 #[test]
2576 fn outcomes_report_task_class_rows_group_and_unclassified_last() {
2577 let tmp = TempDir::new().unwrap();
2578 let root = tmp.path();
2579
2580 write_log(
2583 root,
2584 "m-a",
2585 vec![
2586 ev_ms(
2587 1,
2588 "m-a",
2589 0,
2590 created_with_class("kranz/m-a", Some("execution-class")),
2591 ),
2592 ev_ms(2, "m-a", 1_000, worker_spawned("r-a")),
2593 ev_ms(
2594 3,
2595 "m-a",
2596 2_000,
2597 worker_completed(
2598 "r-a",
2599 TokenUsage {
2600 input: 1,
2601 output: 1,
2602 cache_read: 0,
2603 cache_write: 0,
2604 },
2605 10.0,
2606 ),
2607 ),
2608 ev_ms(
2609 4,
2610 "m-a",
2611 3_000,
2612 EventKind::FeatureCompleted {
2613 feature_id: "f-1-1".into(),
2614 commits: vec!["aaa [f-1-1] add the thing".to_string()],
2615 },
2616 ),
2617 grant_req(5, "m-a", 4_000, "cargo test"),
2618 grant_yes(6, "m-a", 64_000, "cargo test"),
2619 ev_ms(7, "m-a", 100_000, EventKind::MissionCompleted {}),
2620 ],
2621 );
2622 write_log(
2625 root,
2626 "m-b",
2627 vec![
2628 ev_ms(
2629 1,
2630 "m-b",
2631 0,
2632 created_with_class("kranz/m-b", Some("execution-class")),
2633 ),
2634 ev_ms(2, "m-b", 1_000, worker_spawned("r-b")),
2635 ev_ms(
2636 3,
2637 "m-b",
2638 2_000,
2639 worker_completed(
2640 "r-b",
2641 TokenUsage {
2642 input: 1,
2643 output: 1,
2644 cache_read: 0,
2645 cache_write: 0,
2646 },
2647 5.0,
2648 ),
2649 ),
2650 grant_req(4, "m-b", 3_000, "cargo clippy"),
2651 ],
2652 );
2653 write_log(
2656 root,
2657 "m-c",
2658 vec![
2659 ev_ms(1, "m-c", 0, created_with_class("kranz/m-c", None)),
2660 ev_ms(2, "m-c", 50_000, EventKind::MissionCompleted {}),
2661 ],
2662 );
2663
2664 let outcomes =
2665 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
2666 assert_eq!(outcomes.task_classes.len(), 2);
2667 let exec = &outcomes.task_classes[0];
2668 assert_eq!(exec.task_class, "execution-class");
2669 assert_eq!(exec.missions, 2);
2670 assert_eq!(exec.closed_missions, 1);
2671 assert_eq!(exec.total_cost_usd, 15.0);
2672 assert_eq!(exec.non_meta_commits, 1);
2673 assert_eq!(exec.usd_per_commit, Some(15.0));
2674 assert_eq!(exec.escalations, 2);
2675 assert_eq!(exec.advisor_invocations, 2);
2676 assert_eq!(exec.escalations_per_mission, 1.0);
2677 assert_eq!(exec.cycle_mean_ms, Some(100_000.0));
2678
2679 let unclassified = &outcomes.task_classes[1];
2680 assert_eq!(unclassified.task_class, UNCLASSIFIED_TASK_CLASS);
2681 assert_eq!(unclassified.missions, 1);
2682 assert_eq!(unclassified.closed_missions, 1);
2683 assert_eq!(unclassified.non_meta_commits, 0);
2684 assert_eq!(unclassified.usd_per_commit, None);
2686 assert_eq!(unclassified.escalations, 0);
2687 assert_eq!(unclassified.advisor_invocations, 0);
2688 assert_eq!(unclassified.escalations_per_mission, 0.0);
2689 assert_eq!(unclassified.cycle_mean_ms, Some(50_000.0));
2690 }
2691
2692 #[test]
2693 fn outcomes_report_context_reuse_split_absent_for_unreporting_backends() {
2694 let tmp = TempDir::new().unwrap();
2695 let root = tmp.path();
2696
2697 write_log(
2699 root,
2700 "m-claude",
2701 vec![
2702 ev_ms(1, "m-claude", 0, created_with_class("kranz/m-c", None)),
2703 ev_ms(2, "m-claude", 1_000, worker_spawned("r-1")),
2704 ev_ms(
2705 3,
2706 "m-claude",
2707 2_000,
2708 worker_completed(
2709 "r-1",
2710 TokenUsage {
2711 input: 500,
2712 output: 10,
2713 cache_read: 800,
2714 cache_write: 200,
2715 },
2716 1.0,
2717 ),
2718 ),
2719 ev_ms(4, "m-claude", 3_000, EventKind::MissionCompleted {}),
2720 ],
2721 );
2722 let mut local_cfg = MissionConfig::default();
2726 local_cfg.worker.backend = Some("local".into());
2727 write_log(
2728 root,
2729 "m-local",
2730 vec![
2731 ev_ms(
2732 1,
2733 "m-local",
2734 0,
2735 created_with_config("kranz/m-l", "g".into(), local_cfg),
2736 ),
2737 ev_ms(2, "m-local", 1_000, worker_spawned("r-2")),
2738 ev_ms(
2739 3,
2740 "m-local",
2741 2_000,
2742 worker_completed(
2743 "r-2",
2744 TokenUsage {
2745 input: 100,
2746 output: 10,
2747 cache_read: 0,
2748 cache_write: 0,
2749 },
2750 0.0,
2751 ),
2752 ),
2753 ev_ms(4, "m-local", 3_000, EventKind::MissionCompleted {}),
2754 ],
2755 );
2756
2757 let outcomes =
2758 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
2759 assert_eq!(outcomes.context_reuse.len(), 1);
2760 let row = &outcomes.context_reuse[0];
2761 assert_eq!(row.backend, "claude");
2762 assert_eq!(row.missions, 1);
2763 assert_eq!(row.runs, 1);
2764 assert_eq!(row.fresh_input, 500);
2765 assert_eq!(row.cache_read, 800);
2766 assert_eq!(row.cache_write, Some(200));
2767 assert_eq!(row.reuse_share, Some(1_000.0 / 1_500.0));
2768 }
2769
2770 #[test]
2771 fn outcomes_report_context_reuse_codex_cache_write_is_absent() {
2772 let tmp = TempDir::new().unwrap();
2773 let root = tmp.path();
2774
2775 let mut codex_cfg = MissionConfig::default();
2779 codex_cfg.worker.backend = Some("codex".into());
2780 write_log(
2781 root,
2782 "m-codex",
2783 vec![
2784 ev_ms(
2785 1,
2786 "m-codex",
2787 0,
2788 created_with_config("kranz/m-x", "g".into(), codex_cfg),
2789 ),
2790 ev_ms(2, "m-codex", 1_000, worker_spawned("r-1")),
2791 ev_ms(
2792 3,
2793 "m-codex",
2794 2_000,
2795 worker_completed(
2796 "r-1",
2797 TokenUsage {
2798 input: 900,
2799 output: 10,
2800 cache_read: 100,
2801 cache_write: 0,
2802 },
2803 1.0,
2804 ),
2805 ),
2806 ev_ms(4, "m-codex", 3_000, EventKind::MissionCompleted {}),
2807 ],
2808 );
2809
2810 let outcomes =
2811 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
2812 assert_eq!(outcomes.context_reuse.len(), 1);
2813 let row = &outcomes.context_reuse[0];
2814 assert_eq!(row.backend, "codex");
2815 assert_eq!(row.cache_read, 100);
2816 assert_eq!(row.cache_write, None);
2817 assert_eq!(row.reuse_share, Some(100.0 / 1_000.0));
2818 }
2819
2820 #[test]
2821 fn outcomes_report_rubber_stamp_boundary_at_threshold() {
2822 let tmp = TempDir::new().unwrap();
2823 let root = tmp.path();
2824
2825 write_log(
2833 root,
2834 "m-1",
2835 vec![
2836 ev_ms(1, "m-1", 0, created_with_class("kranz/m-1", None)),
2837 ev_ms(
2838 2,
2839 "m-1",
2840 1_000,
2841 EventKind::PlanApproved {
2842 plan: sample_plan(),
2843 base_sha: None,
2844 },
2845 ),
2846 grant_req(3, "m-1", 2_000, "under"),
2847 grant_yes(4, "m-1", 11_999, "under"),
2848 grant_req(5, "m-1", 20_000, "at"),
2849 grant_yes(6, "m-1", 30_000, "at"),
2850 grant_req(7, "m-1", 40_000, "over"),
2851 grant_yes(8, "m-1", 55_000, "over"),
2852 grant_req(9, "m-1", 60_000, "denied-fast"),
2853 ev_ms(
2854 10,
2855 "m-1",
2856 65_000,
2857 EventKind::GrantDenied {
2858 kind: GrantKind::Command,
2859 command: "denied-fast".into(),
2860 reason: "no".into(),
2861 },
2862 ),
2863 grant_req(11, "m-1", 70_000, "pending"),
2864 ev_ms(12, "m-1", 80_000, EventKind::MissionCompleted {}),
2865 ],
2866 );
2867
2868 let outcomes =
2869 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
2870 let stamp = &outcomes.rubber_stamp;
2871 assert_eq!(
2872 stamp.threshold_ms,
2873 crate::types::DEFAULT_RUBBER_STAMP_THRESHOLD_MS
2874 );
2875 assert_eq!(stamp.approved_decisions, 3);
2876 assert_eq!(stamp.flagged, 1);
2877 assert_eq!(stamp.share, Some(1.0 / 3.0));
2878
2879 let marker = |summary: &str| {
2880 outcomes
2881 .escalations
2882 .iter()
2883 .find(|r| r.summary == summary)
2884 .unwrap()
2885 .rubber_stamp
2886 };
2887 assert_eq!(marker("under"), Some(true));
2888 assert_eq!(
2889 marker("at"),
2890 Some(false),
2891 "at the threshold is not under it"
2892 );
2893 assert_eq!(marker("over"), Some(false));
2894 assert_eq!(marker("denied-fast"), None);
2895 assert_eq!(marker("pending"), None);
2896 }
2897
2898 #[test]
2899 fn outcomes_report_rubber_stamp_threshold_resolves_from_config() {
2900 let tmp = TempDir::new().unwrap();
2901 let root = tmp.path();
2902
2903 std::fs::create_dir_all(root.join(".kranz")).unwrap();
2906 std::fs::write(
2907 root.join(".kranz").join("config.json"),
2908 "{\"rubberStampThresholdMs\": 60000}",
2909 )
2910 .unwrap();
2911 assert_eq!(
2912 OutcomesOptions::resolve(root).rubber_stamp_threshold_ms,
2913 60_000
2914 );
2915
2916 write_log(
2917 root,
2918 "m-1",
2919 vec![
2920 ev_ms(1, "m-1", 0, created_with_class("kranz/m-1", None)),
2921 ev_ms(
2922 2,
2923 "m-1",
2924 1_000,
2925 EventKind::PlanApproved {
2926 plan: sample_plan(),
2927 base_sha: None,
2928 },
2929 ),
2930 grant_req(3, "m-1", 2_000, "fifteen seconds"),
2931 grant_yes(4, "m-1", 17_000, "fifteen seconds"),
2932 ev_ms(5, "m-1", 20_000, EventKind::MissionCompleted {}),
2933 ],
2934 );
2935
2936 let outcomes = compute_outcomes(root).unwrap();
2939 assert_eq!(outcomes.rubber_stamp.threshold_ms, 60_000);
2940 assert_eq!(outcomes.rubber_stamp.flagged, 1);
2941 assert_eq!(outcomes.rubber_stamp.share, Some(1.0));
2942 assert_eq!(outcomes.escalations[0].rubber_stamp, Some(true));
2943 }
2944
2945 #[test]
2946 fn outcomes_report_rubber_stamp_absent_without_approvals() {
2947 let tmp = TempDir::new().unwrap();
2948 let root = tmp.path();
2949
2950 write_log(
2951 root,
2952 "m-1",
2953 vec![
2954 ev_ms(1, "m-1", 0, created_with_class("kranz/m-1", None)),
2955 ev_ms(
2956 2,
2957 "m-1",
2958 1_000,
2959 EventKind::PlanApproved {
2960 plan: sample_plan(),
2961 base_sha: None,
2962 },
2963 ),
2964 grant_req(3, "m-1", 2_000, "only-denied"),
2965 ev_ms(
2966 4,
2967 "m-1",
2968 3_000,
2969 EventKind::GrantDenied {
2970 kind: GrantKind::Command,
2971 command: "only-denied".into(),
2972 reason: "no".into(),
2973 },
2974 ),
2975 ev_ms(5, "m-1", 4_000, EventKind::MissionCompleted {}),
2976 ],
2977 );
2978
2979 let outcomes =
2980 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
2981 assert_eq!(outcomes.rubber_stamp.approved_decisions, 0);
2982 assert_eq!(outcomes.rubber_stamp.flagged, 0);
2983 assert_eq!(outcomes.rubber_stamp.share, None);
2984 assert_eq!(outcomes.escalations[0].rubber_stamp, None);
2985 }
2986
2987 #[test]
2988 fn outcomes_report_fold_is_byte_identical_across_repeated_computes() {
2989 let tmp = TempDir::new().unwrap();
2990 let root = tmp.path();
2991
2992 write_log(
2993 root,
2994 "m-1",
2995 vec![
2996 ev_ms(
2997 1,
2998 "m-1",
2999 0,
3000 created_with_class("kranz/m-1", Some("execution-class")),
3001 ),
3002 ev_ms(2, "m-1", 1_000, worker_spawned("r-1")),
3003 ev_ms(
3004 3,
3005 "m-1",
3006 2_000,
3007 worker_completed(
3008 "r-1",
3009 TokenUsage {
3010 input: 500,
3011 output: 10,
3012 cache_read: 800,
3013 cache_write: 200,
3014 },
3015 3.0,
3016 ),
3017 ),
3018 grant_req(4, "m-1", 3_000, "cargo test"),
3019 grant_yes(5, "m-1", 6_000, "cargo test"),
3020 ev_ms(6, "m-1", 10_000, EventKind::MissionCompleted {}),
3021 ],
3022 );
3023
3024 let options = OutcomesOptions::default();
3025 let first = compute_outcomes_with_options(root, &options).unwrap();
3026 let second = compute_outcomes_with_options(root, &options).unwrap();
3027 assert_eq!(first, second);
3028 assert_eq!(
3029 serde_json::to_string(&first).unwrap(),
3030 serde_json::to_string(&second).unwrap(),
3031 "the same log plus the same options yields byte-identical data"
3032 );
3033 }
3034
3035 fn gate_scored(
3039 seq: u64,
3040 mission_id: &str,
3041 ts_ms: i64,
3042 gate: &str,
3043 score: Option<(f64, f64)>,
3044 ) -> Event {
3045 ev_ms(
3046 seq,
3047 mission_id,
3048 ts_ms,
3049 EventKind::GateResult {
3050 gate: gate.into(),
3051 surface: crate::gate::GateSurface::Approval,
3052 kind: crate::gate::GateKind::Deterministic,
3053 index: 0,
3054 verdict: crate::gate::GateVerdict::Pass,
3055 artefact_ref: format!("contract gate {gate}"),
3056 artefact_detail: None,
3057 score: score.map(|(score, _)| score),
3058 threshold: score.map(|(_, threshold)| threshold),
3059 rule_ids: Vec::new(),
3060 },
3061 )
3062 }
3063
3064 #[test]
3071 fn score_distribution_flag_outcomes_fold_flags_beside_rubber_stamp() {
3072 let tmp = TempDir::new().unwrap();
3073 let root = tmp.path();
3074
3075 let mut m1 = vec![
3076 ev_ms(1, "m-1", 0, created_with_class("kranz/m-1", None)),
3077 ev_ms(
3078 2,
3079 "m-1",
3080 1_000,
3081 EventKind::PlanApproved {
3082 plan: sample_plan(),
3083 base_sha: None,
3084 },
3085 ),
3086 grant_req(3, "m-1", 2_000, "cargo test"),
3087 grant_yes(4, "m-1", 4_000, "cargo test"),
3088 ];
3089 for i in 0..5 {
3090 m1.push(gate_scored(
3091 5 + i,
3092 "m-1",
3093 5_000 + i as i64,
3094 "vacuous-filter",
3095 Some((0.5, 1.0)),
3096 ));
3097 }
3098 m1.push(ev_ms(10, "m-1", 10_000, EventKind::MissionCompleted {}));
3099 write_log(root, "m-1", m1);
3100
3101 let mut m2 = vec![ev_ms(1, "m-2", 0, created_with_class("kranz/m-2", None))];
3102 for i in 0..5 {
3103 m2.push(gate_scored(
3104 2 + i,
3105 "m-2",
3106 5_000 + i as i64,
3107 "vacuous-filter",
3108 Some((0.5, 1.0)),
3109 ));
3110 }
3111 m2.push(ev_ms(7, "m-2", 10_000, EventKind::MissionCompleted {}));
3112 write_log(root, "m-2", m2);
3113
3114 let outcomes =
3115 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
3116
3117 assert_eq!(outcomes.rubber_stamp.flagged, 1);
3119 assert_eq!(outcomes.escalations[0].rubber_stamp, Some(true));
3120
3121 let report = &outcomes.gate_score_flags;
3123 assert_eq!(report.scored_gates, 1);
3124 assert_eq!(report.assessed_gates, 1);
3125 assert_eq!(report.flags.len(), 2);
3126 assert!(
3127 report.flags.iter().all(|f| f.gate == "vacuous-filter"),
3128 "the flag names the gate: {report:?}"
3129 );
3130 let kinds: Vec<_> = report.flags.iter().map(|f| f.kind).collect();
3131 assert_eq!(
3132 kinds,
3133 [
3134 crate::gate_score_flags::GateScoreFlagKind::NeverApproachesThreshold,
3135 crate::gate_score_flags::GateScoreFlagKind::NearConstant,
3136 ]
3137 );
3138 let d = &report.flags[0].distribution;
3141 assert_eq!(d.samples, 10);
3142 assert_eq!(d.closest_approach, 0.5);
3143 assert_eq!(d.variance, 0.0);
3144 assert_eq!(
3146 report.min_samples,
3147 crate::gate_score_flags::MIN_SAMPLE_COUNT
3148 );
3149 }
3150
3151 #[test]
3156 fn score_distribution_flag_outcomes_fold_sub_minimum_and_unscored_absent() {
3157 let tmp = TempDir::new().unwrap();
3158 let root = tmp.path();
3159
3160 let mut m1 = vec![ev_ms(1, "m-1", 0, created_with_class("kranz/m-1", None))];
3162 for i in 0..3 {
3163 m1.push(gate_scored(
3164 2 + i,
3165 "m-1",
3166 5_000 + i as i64,
3167 "vacuous-filter",
3168 Some((0.5, 1.0)),
3169 ));
3170 }
3171 m1.push(ev_ms(5, "m-1", 10_000, EventKind::MissionCompleted {}));
3172 write_log(root, "m-1", m1);
3173
3174 write_log(
3176 root,
3177 "m-2",
3178 vec![
3179 ev_ms(1, "m-2", 0, created_with_class("kranz/m-2", None)),
3180 gate_scored(2, "m-2", 5_000, "env-sensitive", None),
3181 gate_scored(3, "m-2", 6_000, "env-sensitive", None),
3182 ev_ms(4, "m-2", 10_000, EventKind::MissionCompleted {}),
3183 ],
3184 );
3185
3186 let outcomes =
3187 compute_outcomes_with_options(root, &OutcomesOptions::default()).unwrap();
3188 let report = &outcomes.gate_score_flags;
3189 assert_eq!(
3190 report.scored_gates, 1,
3191 "the unscored gate adds no population: {report:?}"
3192 );
3193 assert_eq!(report.assessed_gates, 0, "under the minimum: unassessed");
3194 assert!(
3195 report.flags.is_empty(),
3196 "absent, never a zero-filled row: {report:?}"
3197 );
3198 }
3199 }
3200}