1#![forbid(unsafe_code)]
2
3use crate::benchmark_gate::GateResult;
39use crate::shadow_run::{ShadowRunResult, ShadowVerdict};
40use ftui_runtime::effect_system::QueueTelemetry;
41
42#[must_use]
43fn normalized_ratio(ratio: f64) -> f64 {
44 if ratio.is_finite() {
45 ratio.clamp(0.0, 1.0)
46 } else {
47 0.0
48 }
49}
50
51#[must_use]
52fn json_string(value: &str) -> String {
53 let mut out = String::with_capacity(value.len() + 2);
54 out.push('"');
55 for ch in value.chars() {
56 match ch {
57 '"' => out.push_str("\\\""),
58 '\\' => out.push_str("\\\\"),
59 '\n' => out.push_str("\\n"),
60 '\r' => out.push_str("\\r"),
61 '\t' => out.push_str("\\t"),
62 c if c.is_control() => {
63 out.push_str(&format!("\\u{:04x}", c as u32));
64 }
65 c => out.push(c),
66 }
67 }
68 out.push('"');
69 out
70}
71
72#[must_use]
73fn json_string_array(values: &[String]) -> String {
74 values
75 .iter()
76 .map(|value| json_string(value))
77 .collect::<Vec<_>>()
78 .join(",")
79}
80
81#[derive(Debug, Clone)]
87pub struct RolloutScorecardConfig {
88 pub min_shadow_scenarios: usize,
91 pub min_match_ratio: f64,
94 pub require_benchmark_pass: bool,
97}
98
99impl Default for RolloutScorecardConfig {
100 fn default() -> Self {
101 Self {
102 min_shadow_scenarios: 1,
103 min_match_ratio: 1.0,
104 require_benchmark_pass: false,
105 }
106 }
107}
108
109impl RolloutScorecardConfig {
110 #[must_use]
112 pub fn min_shadow_scenarios(mut self, n: usize) -> Self {
113 self.min_shadow_scenarios = n;
114 self
115 }
116
117 #[must_use]
119 pub fn min_match_ratio(mut self, ratio: f64) -> Self {
120 self.min_match_ratio = ratio.clamp(0.0, 1.0);
121 self
122 }
123
124 #[must_use]
126 pub fn require_benchmark_pass(mut self, required: bool) -> Self {
127 self.require_benchmark_pass = required;
128 self
129 }
130}
131
132#[derive(Debug, Clone, Copy, PartialEq, Eq)]
138pub enum RolloutVerdict {
139 Go,
141 NoGo,
143 Inconclusive,
145}
146
147impl RolloutVerdict {
148 #[must_use]
150 pub fn is_go(self) -> bool {
151 matches!(self, Self::Go)
152 }
153
154 #[must_use]
156 pub fn label(self) -> &'static str {
157 match self {
158 Self::Go => "GO",
159 Self::NoGo => "NO-GO",
160 Self::Inconclusive => "INCONCLUSIVE",
161 }
162 }
163}
164
165impl std::fmt::Display for RolloutVerdict {
166 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
167 f.write_str(self.label())
168 }
169}
170
171#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
177pub enum MigrationRolloutStage {
178 Alpha,
180 Beta,
182 Ga,
184}
185
186impl MigrationRolloutStage {
187 #[must_use]
189 pub const fn label(self) -> &'static str {
190 match self {
191 Self::Alpha => "alpha",
192 Self::Beta => "beta",
193 Self::Ga => "ga",
194 }
195 }
196
197 pub const ALL: &'static [Self] = &[Self::Alpha, Self::Beta, Self::Ga];
199}
200
201#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
203pub enum OperatorAuthority {
204 Automation,
206 OnCall,
208 ReleaseOwner,
210 MaintainerQuorum,
212}
213
214impl OperatorAuthority {
215 #[must_use]
217 pub const fn label(self) -> &'static str {
218 match self {
219 Self::Automation => "automation",
220 Self::OnCall => "on-call",
221 Self::ReleaseOwner => "release-owner",
222 Self::MaintainerQuorum => "maintainer-quorum",
223 }
224 }
225}
226
227#[derive(Debug, Clone, Copy, PartialEq, Eq)]
229pub enum EmergencyHoldReason {
230 CertificationRegression,
232 DeterminismDivergence,
234 SecurityIncident,
236 ReliabilityBreach,
238 MissingEvidence,
240 OperatorOverride,
242}
243
244impl EmergencyHoldReason {
245 #[must_use]
247 pub const fn label(self) -> &'static str {
248 match self {
249 Self::CertificationRegression => "certification-regression",
250 Self::DeterminismDivergence => "determinism-divergence",
251 Self::SecurityIncident => "security-incident",
252 Self::ReliabilityBreach => "reliability-breach",
253 Self::MissingEvidence => "missing-evidence",
254 Self::OperatorOverride => "operator-override",
255 }
256 }
257}
258
259#[derive(Debug, Clone, Copy, PartialEq, Eq)]
261pub struct EmergencyHold {
262 pub reason: EmergencyHoldReason,
264 pub authority: OperatorAuthority,
266}
267
268impl EmergencyHold {
269 #[must_use]
271 pub const fn new(reason: EmergencyHoldReason, authority: OperatorAuthority) -> Self {
272 Self { reason, authority }
273 }
274}
275
276#[derive(Debug, Clone)]
278pub struct MigrationReadinessEvidence {
279 pub certification_pass_ratio: f64,
281 pub corpus_coverage_ratio: f64,
283 pub reliability_pass_ratio: f64,
285 pub deterministic_artifact_count: usize,
287 pub benchmark_gate_passed: bool,
289 pub open_blocker_count: usize,
291 pub operator_authority: OperatorAuthority,
293 pub emergency_hold: Option<EmergencyHold>,
295}
296
297impl MigrationReadinessEvidence {
298 #[must_use]
300 pub const fn new(operator_authority: OperatorAuthority) -> Self {
301 Self {
302 certification_pass_ratio: 0.0,
303 corpus_coverage_ratio: 0.0,
304 reliability_pass_ratio: 0.0,
305 deterministic_artifact_count: 0,
306 benchmark_gate_passed: false,
307 open_blocker_count: 0,
308 operator_authority,
309 emergency_hold: None,
310 }
311 }
312
313 #[must_use]
315 pub fn certification_pass_ratio(mut self, ratio: f64) -> Self {
316 self.certification_pass_ratio = normalized_ratio(ratio);
317 self
318 }
319
320 #[must_use]
322 pub fn corpus_coverage_ratio(mut self, ratio: f64) -> Self {
323 self.corpus_coverage_ratio = normalized_ratio(ratio);
324 self
325 }
326
327 #[must_use]
329 pub fn reliability_pass_ratio(mut self, ratio: f64) -> Self {
330 self.reliability_pass_ratio = normalized_ratio(ratio);
331 self
332 }
333
334 #[must_use]
336 pub const fn deterministic_artifact_count(mut self, count: usize) -> Self {
337 self.deterministic_artifact_count = count;
338 self
339 }
340
341 #[must_use]
343 pub const fn benchmark_gate_passed(mut self, passed: bool) -> Self {
344 self.benchmark_gate_passed = passed;
345 self
346 }
347
348 #[must_use]
350 pub const fn open_blocker_count(mut self, count: usize) -> Self {
351 self.open_blocker_count = count;
352 self
353 }
354
355 #[must_use]
357 pub const fn emergency_hold(mut self, hold: EmergencyHold) -> Self {
358 self.emergency_hold = Some(hold);
359 self
360 }
361}
362
363#[derive(Debug, Clone)]
365pub struct MigrationStageGate {
366 pub stage: MigrationRolloutStage,
368 pub min_certification_pass_ratio: f64,
370 pub min_corpus_coverage_ratio: f64,
372 pub min_reliability_pass_ratio: f64,
374 pub min_deterministic_artifacts: usize,
376 pub require_benchmark_gate: bool,
378 pub max_open_blockers: usize,
380 pub required_authority: OperatorAuthority,
382}
383
384impl MigrationStageGate {
385 #[allow(clippy::too_many_arguments)]
387 #[must_use]
388 pub const fn new(
389 stage: MigrationRolloutStage,
390 min_certification_pass_ratio: f64,
391 min_corpus_coverage_ratio: f64,
392 min_reliability_pass_ratio: f64,
393 min_deterministic_artifacts: usize,
394 require_benchmark_gate: bool,
395 max_open_blockers: usize,
396 required_authority: OperatorAuthority,
397 ) -> Self {
398 Self {
399 stage,
400 min_certification_pass_ratio,
401 min_corpus_coverage_ratio,
402 min_reliability_pass_ratio,
403 min_deterministic_artifacts,
404 require_benchmark_gate,
405 max_open_blockers,
406 required_authority,
407 }
408 }
409}
410
411#[derive(Debug, Clone, Copy, PartialEq, Eq)]
413pub enum MigrationReadinessVerdict {
414 Advance,
416 Hold,
418 EmergencyHold,
420}
421
422impl MigrationReadinessVerdict {
423 #[must_use]
425 pub const fn label(self) -> &'static str {
426 match self {
427 Self::Advance => "advance",
428 Self::Hold => "hold",
429 Self::EmergencyHold => "emergency-hold",
430 }
431 }
432}
433
434#[derive(Debug, Clone)]
436pub struct MigrationReadinessDecision {
437 pub stage: MigrationRolloutStage,
439 pub verdict: MigrationReadinessVerdict,
441 pub reasons: Vec<&'static str>,
443}
444
445impl MigrationReadinessDecision {
446 #[must_use]
448 pub fn may_advance(&self) -> bool {
449 self.verdict == MigrationReadinessVerdict::Advance
450 }
451
452 #[must_use]
454 pub fn to_json(&self) -> String {
455 let reasons = self
456 .reasons
457 .iter()
458 .map(|reason| format!("\"{reason}\""))
459 .collect::<Vec<_>>()
460 .join(",");
461 format!(
462 concat!(
463 "{{",
464 "\"stage\":\"{stage}\",",
465 "\"verdict\":\"{verdict}\",",
466 "\"reasons\":[{reasons}]",
467 "}}"
468 ),
469 stage = self.stage.label(),
470 verdict = self.verdict.label(),
471 reasons = reasons,
472 )
473 }
474}
475
476#[derive(Debug, Clone)]
478pub struct MigrationReadinessRubric {
479 gates: Vec<MigrationStageGate>,
480}
481
482impl MigrationReadinessRubric {
483 #[must_use]
485 pub fn new(gates: Vec<MigrationStageGate>) -> Self {
486 Self { gates }
487 }
488
489 #[must_use]
491 pub fn opentui_default() -> Self {
492 Self::new(vec![
493 MigrationStageGate::new(
494 MigrationRolloutStage::Alpha,
495 0.90,
496 0.25,
497 0.95,
498 3,
499 true,
500 2,
501 OperatorAuthority::ReleaseOwner,
502 ),
503 MigrationStageGate::new(
504 MigrationRolloutStage::Beta,
505 0.97,
506 0.60,
507 0.98,
508 5,
509 true,
510 0,
511 OperatorAuthority::ReleaseOwner,
512 ),
513 MigrationStageGate::new(
514 MigrationRolloutStage::Ga,
515 1.00,
516 0.90,
517 0.995,
518 7,
519 true,
520 0,
521 OperatorAuthority::MaintainerQuorum,
522 ),
523 ])
524 }
525
526 #[must_use]
528 pub fn gate(&self, stage: MigrationRolloutStage) -> Option<&MigrationStageGate> {
529 self.gates.iter().find(|gate| gate.stage == stage)
530 }
531
532 #[must_use]
534 pub fn evaluate(
535 &self,
536 stage: MigrationRolloutStage,
537 evidence: &MigrationReadinessEvidence,
538 ) -> MigrationReadinessDecision {
539 let Some(gate) = self.gate(stage) else {
540 return MigrationReadinessDecision {
541 stage,
542 verdict: MigrationReadinessVerdict::Hold,
543 reasons: vec!["stage-gate-missing"],
544 };
545 };
546
547 if let Some(hold) = evidence.emergency_hold {
548 return MigrationReadinessDecision {
549 stage,
550 verdict: MigrationReadinessVerdict::EmergencyHold,
551 reasons: vec![hold.reason.label()],
552 };
553 }
554
555 fn fails_min(value: f64, min: f64) -> bool {
559 !value.is_finite() || value < min
560 }
561
562 let mut reasons = Vec::new();
563 if fails_min(
564 evidence.certification_pass_ratio,
565 gate.min_certification_pass_ratio,
566 ) {
567 reasons.push("certification-threshold");
568 }
569 if fails_min(
570 evidence.corpus_coverage_ratio,
571 gate.min_corpus_coverage_ratio,
572 ) {
573 reasons.push("corpus-coverage-threshold");
574 }
575 if fails_min(
576 evidence.reliability_pass_ratio,
577 gate.min_reliability_pass_ratio,
578 ) {
579 reasons.push("operational-reliability-threshold");
580 }
581 if evidence.deterministic_artifact_count < gate.min_deterministic_artifacts {
582 reasons.push("deterministic-artifact-threshold");
583 }
584 if gate.require_benchmark_gate && !evidence.benchmark_gate_passed {
585 reasons.push("benchmark-gate");
586 }
587 if evidence.open_blocker_count > gate.max_open_blockers {
588 reasons.push("release-blockers");
589 }
590 if evidence.operator_authority < gate.required_authority {
591 reasons.push("operator-authority");
592 }
593
594 let verdict = if reasons.is_empty() {
595 MigrationReadinessVerdict::Advance
596 } else {
597 MigrationReadinessVerdict::Hold
598 };
599 MigrationReadinessDecision {
600 stage,
601 verdict,
602 reasons,
603 }
604 }
605
606 #[must_use]
608 pub fn recommended_stage(
609 &self,
610 evidence: &MigrationReadinessEvidence,
611 ) -> Option<MigrationRolloutStage> {
612 MigrationRolloutStage::ALL
613 .iter()
614 .rev()
615 .copied()
616 .find(|stage| self.evaluate(*stage, evidence).may_advance())
617 }
618}
619
620#[derive(Debug, Clone, Copy, PartialEq, Eq)]
626pub enum MigrationReleaseGateMode {
627 DryRun,
629 Enforce,
631}
632
633impl MigrationReleaseGateMode {
634 #[must_use]
636 pub const fn label(self) -> &'static str {
637 match self {
638 Self::DryRun => "dry-run",
639 Self::Enforce => "enforce",
640 }
641 }
642}
643
644#[derive(Debug, Clone, Copy, PartialEq, Eq)]
646pub enum MigrationReleaseGateVerdict {
647 Pass,
649 Fail,
651}
652
653impl MigrationReleaseGateVerdict {
654 #[must_use]
656 pub const fn label(self) -> &'static str {
657 match self {
658 Self::Pass => "pass",
659 Self::Fail => "fail",
660 }
661 }
662}
663
664#[derive(Debug, Clone, PartialEq, Eq)]
666pub struct MigrationReleaseGateArtifact {
667 pub artifact_id: String,
669 pub kind: String,
671 pub digest: String,
673 pub uri: String,
675}
676
677impl MigrationReleaseGateArtifact {
678 #[must_use]
680 pub fn new(artifact_id: &str, kind: &str, digest: &str, uri: &str) -> Self {
681 Self {
682 artifact_id: artifact_id.to_string(),
683 kind: kind.to_string(),
684 digest: digest.to_string(),
685 uri: uri.to_string(),
686 }
687 }
688
689 #[must_use]
691 pub fn is_immutable(&self) -> bool {
692 let Some(hex_digest) = self.digest.strip_prefix("sha256:") else {
693 return false;
694 };
695 !self.artifact_id.is_empty()
696 && !self.kind.is_empty()
697 && !self.uri.is_empty()
698 && hex_digest.len() == 64
699 && hex_digest.bytes().all(|byte| byte.is_ascii_hexdigit())
700 }
701}
702
703#[derive(Debug, Clone)]
705pub struct MigrationReleaseGateEvidence {
706 pub migration_version: String,
708 pub readiness: MigrationReadinessEvidence,
710 pub determinism_pass_ratio: f64,
712 pub performance_budget_passed: bool,
714 pub unresolved_critical_gap_count: usize,
716 pub artifacts: Vec<MigrationReleaseGateArtifact>,
718}
719
720impl MigrationReleaseGateEvidence {
721 #[must_use]
723 pub fn new(migration_version: &str, readiness: MigrationReadinessEvidence) -> Self {
724 Self {
725 migration_version: migration_version.to_string(),
726 readiness,
727 determinism_pass_ratio: 0.0,
728 performance_budget_passed: false,
729 unresolved_critical_gap_count: 0,
730 artifacts: Vec::new(),
731 }
732 }
733
734 #[must_use]
736 pub fn determinism_pass_ratio(mut self, ratio: f64) -> Self {
737 self.determinism_pass_ratio = normalized_ratio(ratio);
738 self
739 }
740
741 #[must_use]
743 pub const fn performance_budget_passed(mut self, passed: bool) -> Self {
744 self.performance_budget_passed = passed;
745 self
746 }
747
748 #[must_use]
750 pub const fn unresolved_critical_gap_count(mut self, count: usize) -> Self {
751 self.unresolved_critical_gap_count = count;
752 self
753 }
754
755 #[must_use]
757 pub fn artifact(mut self, artifact: MigrationReleaseGateArtifact) -> Self {
758 self.artifacts.push(artifact);
759 self
760 }
761}
762
763#[derive(Debug, Clone)]
765pub struct MigrationReleaseGatePolicy {
766 pub mode: MigrationReleaseGateMode,
768 pub target_stage: MigrationRolloutStage,
770 pub min_certification_pass_ratio: f64,
772 pub min_determinism_pass_ratio: f64,
774 pub require_performance_budget_pass: bool,
776 pub max_unresolved_critical_gaps: usize,
778 pub min_traceable_artifacts: usize,
780 pub required_artifact_kinds: Vec<&'static str>,
782}
783
784impl MigrationReleaseGatePolicy {
785 #[must_use]
787 pub fn for_stage(target_stage: MigrationRolloutStage, mode: MigrationReleaseGateMode) -> Self {
788 let (
789 min_certification_pass_ratio,
790 min_determinism_pass_ratio,
791 max_unresolved_critical_gaps,
792 min_traceable_artifacts,
793 ) = match target_stage {
794 MigrationRolloutStage::Alpha => (0.90, 0.99, 2, 5),
795 MigrationRolloutStage::Beta => (0.97, 1.00, 0, 6),
796 MigrationRolloutStage::Ga => (1.00, 1.00, 0, 8),
797 };
798 Self {
799 mode,
800 target_stage,
801 min_certification_pass_ratio,
802 min_determinism_pass_ratio,
803 require_performance_budget_pass: true,
804 max_unresolved_critical_gaps,
805 min_traceable_artifacts,
806 required_artifact_kinds: vec![
807 "certification",
808 "critical-gaps",
809 "determinism",
810 "performance",
811 "readiness",
812 ],
813 }
814 }
815
816 #[must_use]
818 pub const fn mode(mut self, mode: MigrationReleaseGateMode) -> Self {
819 self.mode = mode;
820 self
821 }
822}
823
824#[derive(Debug, Clone)]
826pub struct MigrationReleaseGateClauseResult {
827 pub clause: &'static str,
829 pub passed: bool,
831 pub reason: String,
833 pub artifact_ids: Vec<String>,
835}
836
837impl MigrationReleaseGateClauseResult {
838 #[must_use]
839 fn pass(clause: &'static str, reason: String, artifact_ids: Vec<String>) -> Self {
840 Self {
841 clause,
842 passed: true,
843 reason,
844 artifact_ids,
845 }
846 }
847
848 #[must_use]
849 fn fail(clause: &'static str, reason: String, artifact_ids: Vec<String>) -> Self {
850 Self {
851 clause,
852 passed: false,
853 reason,
854 artifact_ids,
855 }
856 }
857
858 #[must_use]
859 fn to_json(&self) -> String {
860 format!(
861 concat!(
862 "{{",
863 "\"clause\":{clause},",
864 "\"passed\":{passed},",
865 "\"reason\":{reason},",
866 "\"artifact_ids\":[{artifact_ids}]",
867 "}}"
868 ),
869 clause = json_string(self.clause),
870 passed = self.passed,
871 reason = json_string(&self.reason),
872 artifact_ids = json_string_array(&self.artifact_ids),
873 )
874 }
875}
876
877#[derive(Debug, Clone)]
879pub struct MigrationReleaseGateDecision {
880 pub migration_version: String,
882 pub mode: MigrationReleaseGateMode,
884 pub target_stage: MigrationRolloutStage,
886 pub verdict: MigrationReleaseGateVerdict,
888 pub clauses: Vec<MigrationReleaseGateClauseResult>,
890}
891
892impl MigrationReleaseGateDecision {
893 #[must_use]
895 pub fn passed(&self) -> bool {
896 self.verdict == MigrationReleaseGateVerdict::Pass
897 }
898
899 #[must_use]
901 pub fn blocks_release(&self) -> bool {
902 self.mode == MigrationReleaseGateMode::Enforce && !self.passed()
903 }
904
905 #[must_use]
907 pub fn failed_clauses(&self) -> Vec<&'static str> {
908 self.clauses
909 .iter()
910 .filter(|clause| !clause.passed)
911 .map(|clause| clause.clause)
912 .collect()
913 }
914
915 #[must_use]
917 pub fn to_json(&self) -> String {
918 let clauses = self
919 .clauses
920 .iter()
921 .map(MigrationReleaseGateClauseResult::to_json)
922 .collect::<Vec<_>>()
923 .join(",");
924 format!(
925 concat!(
926 "{{",
927 "\"schema_version\":\"1.0.0\",",
928 "\"migration_version\":{version},",
929 "\"mode\":\"{mode}\",",
930 "\"target_stage\":\"{stage}\",",
931 "\"verdict\":\"{verdict}\",",
932 "\"blocks_release\":{blocks_release},",
933 "\"clauses\":[{clauses}]",
934 "}}"
935 ),
936 version = json_string(&self.migration_version),
937 mode = self.mode.label(),
938 stage = self.target_stage.label(),
939 verdict = self.verdict.label(),
940 blocks_release = self.blocks_release(),
941 clauses = clauses,
942 )
943 }
944}
945
946#[derive(Debug, Clone)]
949pub struct MigrationReleaseGateEvaluator {
950 policy: MigrationReleaseGatePolicy,
951 readiness_rubric: MigrationReadinessRubric,
952}
953
954impl MigrationReleaseGateEvaluator {
955 #[must_use]
957 pub fn new(policy: MigrationReleaseGatePolicy) -> Self {
958 Self {
959 policy,
960 readiness_rubric: MigrationReadinessRubric::opentui_default(),
961 }
962 }
963
964 #[must_use]
966 pub fn with_readiness_rubric(mut self, rubric: MigrationReadinessRubric) -> Self {
967 self.readiness_rubric = rubric;
968 self
969 }
970
971 #[must_use]
973 pub fn evaluate(
974 &self,
975 evidence: &MigrationReleaseGateEvidence,
976 ) -> MigrationReleaseGateDecision {
977 let mut clauses = Vec::new();
978 let readiness_decision = self
979 .readiness_rubric
980 .evaluate(self.policy.target_stage, &evidence.readiness);
981 let readiness_artifact_ids = artifact_ids_for_kind(evidence, "readiness");
982 if readiness_decision.may_advance() {
983 clauses.push(MigrationReleaseGateClauseResult::pass(
984 "readiness",
985 format!(
986 "target stage {} readiness advanced",
987 self.policy.target_stage.label()
988 ),
989 readiness_artifact_ids,
990 ));
991 } else {
992 clauses.push(MigrationReleaseGateClauseResult::fail(
993 "readiness",
994 format!("readiness held: {}", readiness_decision.reasons.join(",")),
995 readiness_artifact_ids,
996 ));
997 }
998
999 clauses.push(compare_ratio_clause(
1000 "certification",
1001 evidence.readiness.certification_pass_ratio,
1002 self.policy.min_certification_pass_ratio,
1003 artifact_ids_for_kind(evidence, "certification"),
1004 ));
1005 clauses.push(compare_ratio_clause(
1006 "determinism",
1007 evidence.determinism_pass_ratio,
1008 self.policy.min_determinism_pass_ratio,
1009 artifact_ids_for_kind(evidence, "determinism"),
1010 ));
1011
1012 let performance_ids = artifact_ids_for_kind(evidence, "performance");
1013 if !self.policy.require_performance_budget_pass || evidence.performance_budget_passed {
1014 clauses.push(MigrationReleaseGateClauseResult::pass(
1015 "performance-budget",
1016 "performance budget gate passed".to_string(),
1017 performance_ids,
1018 ));
1019 } else {
1020 clauses.push(MigrationReleaseGateClauseResult::fail(
1021 "performance-budget",
1022 "performance budget gate failed".to_string(),
1023 performance_ids,
1024 ));
1025 }
1026
1027 let critical_gap_ids = artifact_ids_for_kind(evidence, "critical-gaps");
1028 if evidence.unresolved_critical_gap_count <= self.policy.max_unresolved_critical_gaps {
1029 clauses.push(MigrationReleaseGateClauseResult::pass(
1030 "critical-gaps",
1031 format!(
1032 "{} critical gaps <= {} allowed",
1033 evidence.unresolved_critical_gap_count,
1034 self.policy.max_unresolved_critical_gaps
1035 ),
1036 critical_gap_ids,
1037 ));
1038 } else {
1039 clauses.push(MigrationReleaseGateClauseResult::fail(
1040 "critical-gaps",
1041 format!(
1042 "{} critical gaps > {} allowed",
1043 evidence.unresolved_critical_gap_count,
1044 self.policy.max_unresolved_critical_gaps
1045 ),
1046 critical_gap_ids,
1047 ));
1048 }
1049
1050 clauses.push(self.evaluate_artifact_traceability(evidence));
1051
1052 let verdict = if clauses.iter().all(|clause| clause.passed) {
1053 MigrationReleaseGateVerdict::Pass
1054 } else {
1055 MigrationReleaseGateVerdict::Fail
1056 };
1057 MigrationReleaseGateDecision {
1058 migration_version: evidence.migration_version.clone(),
1059 mode: self.policy.mode,
1060 target_stage: self.policy.target_stage,
1061 verdict,
1062 clauses,
1063 }
1064 }
1065
1066 #[must_use]
1067 fn evaluate_artifact_traceability(
1068 &self,
1069 evidence: &MigrationReleaseGateEvidence,
1070 ) -> MigrationReleaseGateClauseResult {
1071 let traceable_ids = evidence
1072 .artifacts
1073 .iter()
1074 .filter(|artifact| artifact.is_immutable())
1075 .map(|artifact| artifact.artifact_id.clone())
1076 .collect::<Vec<_>>();
1077 let invalid_ids = evidence
1078 .artifacts
1079 .iter()
1080 .filter(|artifact| !artifact.is_immutable())
1081 .map(|artifact| artifact.artifact_id.clone())
1082 .collect::<Vec<_>>();
1083 let missing_kinds = self
1084 .policy
1085 .required_artifact_kinds
1086 .iter()
1087 .copied()
1088 .filter(|kind| {
1089 !evidence
1090 .artifacts
1091 .iter()
1092 .any(|artifact| artifact.kind == *kind && artifact.is_immutable())
1093 })
1094 .collect::<Vec<_>>();
1095
1096 if traceable_ids.len() >= self.policy.min_traceable_artifacts
1097 && invalid_ids.is_empty()
1098 && missing_kinds.is_empty()
1099 {
1100 return MigrationReleaseGateClauseResult::pass(
1101 "artifact-traceability",
1102 format!(
1103 "{} immutable artifacts cover required input kinds",
1104 traceable_ids.len()
1105 ),
1106 traceable_ids,
1107 );
1108 }
1109
1110 let mut reasons = Vec::new();
1111 if traceable_ids.len() < self.policy.min_traceable_artifacts {
1112 reasons.push(format!(
1113 "{} immutable artifacts < {} required",
1114 traceable_ids.len(),
1115 self.policy.min_traceable_artifacts
1116 ));
1117 }
1118 if !invalid_ids.is_empty() {
1119 reasons.push(format!(
1120 "invalid immutable references: {}",
1121 invalid_ids.join(",")
1122 ));
1123 }
1124 if !missing_kinds.is_empty() {
1125 reasons.push(format!(
1126 "missing artifact kinds: {}",
1127 missing_kinds.join(",")
1128 ));
1129 }
1130 MigrationReleaseGateClauseResult::fail(
1131 "artifact-traceability",
1132 reasons.join("; "),
1133 traceable_ids,
1134 )
1135 }
1136}
1137
1138#[must_use]
1139fn artifact_ids_for_kind(evidence: &MigrationReleaseGateEvidence, kind: &str) -> Vec<String> {
1140 evidence
1141 .artifacts
1142 .iter()
1143 .filter(|artifact| artifact.kind == kind && artifact.is_immutable())
1144 .map(|artifact| artifact.artifact_id.clone())
1145 .collect()
1146}
1147
1148#[must_use]
1149fn compare_ratio_clause(
1150 clause: &'static str,
1151 actual: f64,
1152 required: f64,
1153 artifact_ids: Vec<String>,
1154) -> MigrationReleaseGateClauseResult {
1155 if actual >= required {
1156 MigrationReleaseGateClauseResult::pass(
1157 clause,
1158 format!("{actual:.6} >= {required:.6} required"),
1159 artifact_ids,
1160 )
1161 } else {
1162 MigrationReleaseGateClauseResult::fail(
1163 clause,
1164 format!("{actual:.6} < {required:.6} required"),
1165 artifact_ids,
1166 )
1167 }
1168}
1169
1170#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
1180pub enum MigrationIncidentClass {
1181 SemanticRegression,
1183 DeterminismDivergence,
1185 PerformanceRegression,
1187 CapabilityGapEscape,
1189 SecurityBreach,
1191 CertificationFalsePass,
1193 RollbackFailure,
1195}
1196
1197impl MigrationIncidentClass {
1198 #[must_use]
1200 pub const fn label(self) -> &'static str {
1201 match self {
1202 Self::SemanticRegression => "semantic-regression",
1203 Self::DeterminismDivergence => "determinism-divergence",
1204 Self::PerformanceRegression => "performance-regression",
1205 Self::CapabilityGapEscape => "capability-gap-escape",
1206 Self::SecurityBreach => "security-breach",
1207 Self::CertificationFalsePass => "certification-false-pass",
1208 Self::RollbackFailure => "rollback-failure",
1209 }
1210 }
1211
1212 pub const ALL: &'static [Self] = &[
1214 Self::SemanticRegression,
1215 Self::DeterminismDivergence,
1216 Self::PerformanceRegression,
1217 Self::CapabilityGapEscape,
1218 Self::SecurityBreach,
1219 Self::CertificationFalsePass,
1220 Self::RollbackFailure,
1221 ];
1222
1223 #[must_use]
1225 pub const fn default_severity(self) -> MigrationIncidentSeverity {
1226 match self {
1227 Self::DeterminismDivergence
1228 | Self::SecurityBreach
1229 | Self::CertificationFalsePass
1230 | Self::RollbackFailure => MigrationIncidentSeverity::Sev1,
1231 Self::SemanticRegression | Self::CapabilityGapEscape => MigrationIncidentSeverity::Sev2,
1232 Self::PerformanceRegression => MigrationIncidentSeverity::Sev3,
1233 }
1234 }
1235
1236 #[must_use]
1238 pub const fn emergency_hold_reason(self) -> EmergencyHoldReason {
1239 match self {
1240 Self::SemanticRegression | Self::CapabilityGapEscape | Self::CertificationFalsePass => {
1241 EmergencyHoldReason::CertificationRegression
1242 }
1243 Self::DeterminismDivergence => EmergencyHoldReason::DeterminismDivergence,
1244 Self::SecurityBreach => EmergencyHoldReason::SecurityIncident,
1245 Self::PerformanceRegression | Self::RollbackFailure => {
1246 EmergencyHoldReason::ReliabilityBreach
1247 }
1248 }
1249 }
1250}
1251
1252#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1254pub enum MigrationIncidentSeverity {
1255 Sev1,
1257 Sev2,
1259 Sev3,
1261 Sev4,
1263}
1264
1265impl MigrationIncidentSeverity {
1266 #[must_use]
1268 pub const fn label(self) -> &'static str {
1269 match self {
1270 Self::Sev1 => "sev1",
1271 Self::Sev2 => "sev2",
1272 Self::Sev3 => "sev3",
1273 Self::Sev4 => "sev4",
1274 }
1275 }
1276
1277 #[must_use]
1279 pub const fn rank(self) -> u8 {
1280 match self {
1281 Self::Sev1 => 4,
1282 Self::Sev2 => 3,
1283 Self::Sev3 => 2,
1284 Self::Sev4 => 1,
1285 }
1286 }
1287
1288 #[must_use]
1290 pub const fn ack_deadline_minutes(self) -> u32 {
1291 match self {
1292 Self::Sev1 => 15,
1293 Self::Sev2 => 60,
1294 Self::Sev3 => 240,
1295 Self::Sev4 => 1440,
1296 }
1297 }
1298
1299 #[must_use]
1301 pub const fn requires_immediate_rollback(self) -> bool {
1302 matches!(self, Self::Sev1 | Self::Sev2)
1303 }
1304
1305 #[must_use]
1307 pub const fn rollback_authority(self) -> OperatorAuthority {
1308 match self {
1309 Self::Sev1 | Self::Sev2 => OperatorAuthority::OnCall,
1311 Self::Sev3 | Self::Sev4 => OperatorAuthority::ReleaseOwner,
1312 }
1313 }
1314
1315 #[must_use]
1317 pub const fn escalate(self, other: Self) -> Self {
1318 if other.rank() > self.rank() {
1319 other
1320 } else {
1321 self
1322 }
1323 }
1324}
1325
1326#[derive(Debug, Clone)]
1328pub struct MigrationIncidentSignal {
1329 pub code: String,
1331 pub detail: String,
1333 pub escalates_to: Option<MigrationIncidentSeverity>,
1335}
1336
1337impl MigrationIncidentSignal {
1338 #[must_use]
1340 pub fn new(code: &str, detail: &str) -> Self {
1341 Self {
1342 code: code.to_string(),
1343 detail: detail.to_string(),
1344 escalates_to: None,
1345 }
1346 }
1347
1348 #[must_use]
1350 pub const fn escalates_to(mut self, severity: MigrationIncidentSeverity) -> Self {
1351 self.escalates_to = Some(severity);
1352 self
1353 }
1354
1355 #[must_use]
1356 fn to_json(&self) -> String {
1357 let escalates = match self.escalates_to {
1358 Some(severity) => format!("\"{}\"", severity.label()),
1359 None => "null".to_string(),
1360 };
1361 format!(
1362 concat!(
1363 "{{",
1364 "\"code\":{code},",
1365 "\"detail\":{detail},",
1366 "\"escalates_to\":{escalates}",
1367 "}}"
1368 ),
1369 code = json_string(&self.code),
1370 detail = json_string(&self.detail),
1371 escalates = escalates,
1372 )
1373 }
1374}
1375
1376#[derive(Debug, Clone)]
1378pub struct MigrationIncidentReport {
1379 pub incident_id: String,
1381 pub class: MigrationIncidentClass,
1383 pub detected_stage: MigrationRolloutStage,
1385 pub comparator_id: String,
1387 pub claim_id: String,
1389 pub signals: Vec<MigrationIncidentSignal>,
1391 pub available_artifacts: Vec<MigrationReleaseGateArtifact>,
1393}
1394
1395impl MigrationIncidentReport {
1396 #[must_use]
1398 pub fn new(
1399 incident_id: &str,
1400 class: MigrationIncidentClass,
1401 detected_stage: MigrationRolloutStage,
1402 ) -> Self {
1403 Self {
1404 incident_id: incident_id.to_string(),
1405 class,
1406 detected_stage,
1407 comparator_id: "n/a".to_string(),
1408 claim_id: "n/a".to_string(),
1409 signals: Vec::new(),
1410 available_artifacts: Vec::new(),
1411 }
1412 }
1413
1414 #[must_use]
1416 pub fn comparator_id(mut self, comparator_id: &str) -> Self {
1417 self.comparator_id = comparator_id.to_string();
1418 self
1419 }
1420
1421 #[must_use]
1423 pub fn claim_id(mut self, claim_id: &str) -> Self {
1424 self.claim_id = claim_id.to_string();
1425 self
1426 }
1427
1428 #[must_use]
1430 pub fn signal(mut self, signal: MigrationIncidentSignal) -> Self {
1431 self.signals.push(signal);
1432 self
1433 }
1434
1435 #[must_use]
1437 pub fn artifact(mut self, artifact: MigrationReleaseGateArtifact) -> Self {
1438 self.available_artifacts.push(artifact);
1439 self
1440 }
1441
1442 #[must_use]
1444 pub fn effective_severity(&self) -> MigrationIncidentSeverity {
1445 self.signals
1446 .iter()
1447 .filter_map(|signal| signal.escalates_to)
1448 .fold(self.class.default_severity(), |acc, escalated| {
1449 acc.escalate(escalated)
1450 })
1451 }
1452}
1453
1454#[derive(Debug, Clone, PartialEq, Eq)]
1456pub struct MigrationRollbackStep {
1457 pub order: usize,
1459 pub action: &'static str,
1461 pub description: &'static str,
1463 pub required_artifact_kind: &'static str,
1465 pub artifact_id: Option<String>,
1467}
1468
1469impl MigrationRollbackStep {
1470 #[must_use]
1471 const fn template(
1472 order: usize,
1473 action: &'static str,
1474 description: &'static str,
1475 required_artifact_kind: &'static str,
1476 ) -> Self {
1477 Self {
1478 order,
1479 action,
1480 description,
1481 required_artifact_kind,
1482 artifact_id: None,
1483 }
1484 }
1485
1486 #[must_use]
1488 pub fn is_resolved(&self) -> bool {
1489 self.required_artifact_kind.is_empty() || self.artifact_id.is_some()
1490 }
1491
1492 #[must_use]
1493 fn to_json(&self) -> String {
1494 let artifact_id = match &self.artifact_id {
1495 Some(id) => json_string(id),
1496 None => "null".to_string(),
1497 };
1498 format!(
1499 concat!(
1500 "{{",
1501 "\"order\":{order},",
1502 "\"action\":{action},",
1503 "\"description\":{description},",
1504 "\"required_artifact_kind\":{kind},",
1505 "\"artifact_id\":{artifact_id},",
1506 "\"resolved\":{resolved}",
1507 "}}"
1508 ),
1509 order = self.order,
1510 action = json_string(self.action),
1511 description = json_string(self.description),
1512 kind = json_string(self.required_artifact_kind),
1513 artifact_id = artifact_id,
1514 resolved = self.is_resolved(),
1515 )
1516 }
1517}
1518
1519#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1521pub enum MigrationRollbackReadinessVerdict {
1522 Ready,
1524 Blocked,
1526}
1527
1528impl MigrationRollbackReadinessVerdict {
1529 #[must_use]
1531 pub const fn label(self) -> &'static str {
1532 match self {
1533 Self::Ready => "ready",
1534 Self::Blocked => "blocked",
1535 }
1536 }
1537}
1538
1539#[derive(Debug, Clone)]
1541pub struct MigrationRollbackReadiness {
1542 pub verdict: MigrationRollbackReadinessVerdict,
1544 pub total_steps: usize,
1546 pub resolved_steps: usize,
1548 pub missing_artifact_kinds: Vec<&'static str>,
1550}
1551
1552impl MigrationRollbackReadiness {
1553 #[must_use]
1555 pub fn is_ready(&self) -> bool {
1556 self.verdict == MigrationRollbackReadinessVerdict::Ready
1557 }
1558
1559 #[must_use]
1560 fn to_json(&self) -> String {
1561 let missing = self
1562 .missing_artifact_kinds
1563 .iter()
1564 .map(|kind| json_string(kind))
1565 .collect::<Vec<_>>()
1566 .join(",");
1567 format!(
1568 concat!(
1569 "{{",
1570 "\"verdict\":\"{verdict}\",",
1571 "\"total_steps\":{total},",
1572 "\"resolved_steps\":{resolved},",
1573 "\"missing_artifact_kinds\":[{missing}]",
1574 "}}"
1575 ),
1576 verdict = self.verdict.label(),
1577 total = self.total_steps,
1578 resolved = self.resolved_steps,
1579 missing = missing,
1580 )
1581 }
1582}
1583
1584#[derive(Debug, Clone)]
1586pub struct MigrationRollbackPlan {
1587 pub incident_class: MigrationIncidentClass,
1589 pub target_stage: MigrationRolloutStage,
1591 pub steps: Vec<MigrationRollbackStep>,
1593}
1594
1595impl MigrationRollbackPlan {
1596 #[must_use]
1602 pub fn default_for(
1603 incident_class: MigrationIncidentClass,
1604 target_stage: MigrationRolloutStage,
1605 ) -> Self {
1606 let mut steps = vec![
1607 MigrationRollbackStep::template(
1608 1,
1609 "halt-promotion",
1610 "Freeze promotion and mark the affected migration version non-deployable.",
1611 "release-gate-decision",
1612 ),
1613 MigrationRollbackStep::template(
1614 2,
1615 "quarantine-version",
1616 "Quarantine the failing version using its locked source snapshot.",
1617 "source-snapshot",
1618 ),
1619 MigrationRollbackStep::template(
1620 3,
1621 "restore-last-good",
1622 "Restore the last-good migration release artifact.",
1623 "last-good-release",
1624 ),
1625 MigrationRollbackStep::template(
1626 4,
1627 "verify-determinism",
1628 "Re-verify determinism hashes against the recorded baseline.",
1629 "determinism-baseline",
1630 ),
1631 MigrationRollbackStep::template(
1632 5,
1633 "reverify-certification",
1634 "Re-run the certification smoke against the restored release.",
1635 "certification-report",
1636 ),
1637 ];
1638 match incident_class {
1639 MigrationIncidentClass::SecurityBreach => {
1640 steps.push(MigrationRollbackStep::template(
1641 6,
1642 "rotate-exposed-credentials",
1643 "Rotate any credentials or tokens exposed by the breach.",
1644 "secret-rotation-runbook",
1645 ));
1646 }
1647 MigrationIncidentClass::RollbackFailure => {
1648 steps.push(MigrationRollbackStep::template(
1649 6,
1650 "escalate-manual-recovery",
1651 "Escalate to maintainer-led manual recovery using the recovery runbook.",
1652 "manual-recovery-runbook",
1653 ));
1654 }
1655 _ => {}
1656 }
1657 let order = steps.len() + 1;
1659 steps.push(MigrationRollbackStep::template(
1660 order,
1661 "record-postmortem",
1662 "Open a postmortem record linking the incident to backlog and prevention.",
1663 "",
1664 ));
1665 Self {
1666 incident_class,
1667 target_stage,
1668 steps,
1669 }
1670 }
1671
1672 #[must_use]
1677 pub fn resolve(mut self, artifacts: &[MigrationReleaseGateArtifact]) -> Self {
1678 for step in &mut self.steps {
1679 if step.required_artifact_kind.is_empty() {
1680 continue;
1681 }
1682 let mut candidates = artifacts
1683 .iter()
1684 .filter(|artifact| {
1685 artifact.kind == step.required_artifact_kind && artifact.is_immutable()
1686 })
1687 .map(|artifact| artifact.artifact_id.clone())
1688 .collect::<Vec<_>>();
1689 candidates.sort();
1690 step.artifact_id = candidates.into_iter().next();
1691 }
1692 self
1693 }
1694
1695 #[must_use]
1697 pub fn readiness(&self) -> MigrationRollbackReadiness {
1698 let resolved_steps = self.steps.iter().filter(|step| step.is_resolved()).count();
1699 let mut missing_artifact_kinds = self
1700 .steps
1701 .iter()
1702 .filter(|step| !step.is_resolved())
1703 .map(|step| step.required_artifact_kind)
1704 .collect::<Vec<_>>();
1705 missing_artifact_kinds.sort_unstable();
1706 missing_artifact_kinds.dedup();
1707 let verdict = if missing_artifact_kinds.is_empty() {
1708 MigrationRollbackReadinessVerdict::Ready
1709 } else {
1710 MigrationRollbackReadinessVerdict::Blocked
1711 };
1712 MigrationRollbackReadiness {
1713 verdict,
1714 total_steps: self.steps.len(),
1715 resolved_steps,
1716 missing_artifact_kinds,
1717 }
1718 }
1719
1720 #[must_use]
1721 fn to_json(&self) -> String {
1722 let steps = self
1723 .steps
1724 .iter()
1725 .map(MigrationRollbackStep::to_json)
1726 .collect::<Vec<_>>()
1727 .join(",");
1728 format!(
1729 concat!(
1730 "{{",
1731 "\"incident_class\":\"{class}\",",
1732 "\"target_stage\":\"{stage}\",",
1733 "\"steps\":[{steps}]",
1734 "}}"
1735 ),
1736 class = self.incident_class.label(),
1737 stage = self.target_stage.label(),
1738 steps = steps,
1739 )
1740 }
1741}
1742
1743#[derive(Debug, Clone)]
1745pub struct MigrationPostmortemTemplate {
1746 pub incident_id: String,
1748 pub incident_class: MigrationIncidentClass,
1750 pub severity: MigrationIncidentSeverity,
1752 pub comparator_id: String,
1754 pub claim_id: String,
1756 pub timeline_prompts: Vec<&'static str>,
1758 pub backlog_item_ids: Vec<String>,
1760 pub prevention_actions: Vec<String>,
1762}
1763
1764impl MigrationPostmortemTemplate {
1765 #[must_use]
1769 pub fn for_incident(
1770 report: &MigrationIncidentReport,
1771 severity: MigrationIncidentSeverity,
1772 ) -> Self {
1773 Self {
1774 incident_id: report.incident_id.clone(),
1775 incident_class: report.class,
1776 severity,
1777 comparator_id: report.comparator_id.clone(),
1778 claim_id: report.claim_id.clone(),
1779 timeline_prompts: vec![
1780 "detection: how and when the incident was detected",
1781 "impact: which migrations/versions were affected and for how long",
1782 "root-cause: the verified causal chain, not the first symptom",
1783 "resolution: the rollback executed and how recovery was verified",
1784 ],
1785 backlog_item_ids: Vec::new(),
1786 prevention_actions: default_prevention_actions(report.class),
1787 }
1788 }
1789
1790 #[must_use]
1792 pub fn backlog_item(mut self, backlog_item_id: &str) -> Self {
1793 self.backlog_item_ids.push(backlog_item_id.to_string());
1794 self
1795 }
1796
1797 #[must_use]
1799 pub fn prevention_action(mut self, action: &str) -> Self {
1800 self.prevention_actions.push(action.to_string());
1801 self
1802 }
1803
1804 #[must_use]
1806 pub fn links_backlog(&self) -> bool {
1807 !self.backlog_item_ids.is_empty()
1808 }
1809
1810 #[must_use]
1811 fn to_json(&self) -> String {
1812 let prompts = self
1813 .timeline_prompts
1814 .iter()
1815 .map(|prompt| json_string(prompt))
1816 .collect::<Vec<_>>()
1817 .join(",");
1818 format!(
1819 concat!(
1820 "{{",
1821 "\"incident_id\":{incident_id},",
1822 "\"incident_class\":\"{class}\",",
1823 "\"severity\":\"{severity}\",",
1824 "\"comparator_id\":{comparator_id},",
1825 "\"claim_id\":{claim_id},",
1826 "\"timeline_prompts\":[{prompts}],",
1827 "\"backlog_item_ids\":[{backlog}],",
1828 "\"prevention_actions\":[{prevention}],",
1829 "\"links_backlog\":{links_backlog}",
1830 "}}"
1831 ),
1832 incident_id = json_string(&self.incident_id),
1833 class = self.incident_class.label(),
1834 severity = self.severity.label(),
1835 comparator_id = json_string(&self.comparator_id),
1836 claim_id = json_string(&self.claim_id),
1837 prompts = prompts,
1838 backlog = json_string_array(&self.backlog_item_ids),
1839 prevention = json_string_array(&self.prevention_actions),
1840 links_backlog = self.links_backlog(),
1841 )
1842 }
1843}
1844
1845#[derive(Debug, Clone)]
1847pub struct MigrationIncidentResponse {
1848 pub incident_id: String,
1850 pub class: MigrationIncidentClass,
1852 pub severity: MigrationIncidentSeverity,
1854 pub emergency_hold_reason: EmergencyHoldReason,
1856 pub rollback_authority: OperatorAuthority,
1858 pub requires_immediate_rollback: bool,
1860 pub signals: Vec<MigrationIncidentSignal>,
1862 pub rollback: MigrationRollbackPlan,
1864 pub readiness: MigrationRollbackReadiness,
1866 pub postmortem: MigrationPostmortemTemplate,
1868}
1869
1870impl MigrationIncidentResponse {
1871 #[must_use]
1875 pub fn blocks_promotion(&self) -> bool {
1876 self.severity.rank() >= MigrationIncidentSeverity::Sev3.rank()
1877 }
1878
1879 #[must_use]
1881 pub fn to_json(&self) -> String {
1882 let signals = self
1883 .signals
1884 .iter()
1885 .map(MigrationIncidentSignal::to_json)
1886 .collect::<Vec<_>>()
1887 .join(",");
1888 format!(
1889 concat!(
1890 "{{",
1891 "\"schema_version\":\"1.0.0\",",
1892 "\"incident_id\":{incident_id},",
1893 "\"class\":\"{class}\",",
1894 "\"severity\":\"{severity}\",",
1895 "\"ack_deadline_minutes\":{ack_deadline},",
1896 "\"emergency_hold_reason\":\"{hold}\",",
1897 "\"rollback_authority\":\"{authority}\",",
1898 "\"requires_immediate_rollback\":{immediate},",
1899 "\"blocks_promotion\":{blocks},",
1900 "\"signals\":[{signals}],",
1901 "\"rollback\":{rollback},",
1902 "\"readiness\":{readiness},",
1903 "\"postmortem\":{postmortem}",
1904 "}}"
1905 ),
1906 incident_id = json_string(&self.incident_id),
1907 class = self.class.label(),
1908 severity = self.severity.label(),
1909 ack_deadline = self.severity.ack_deadline_minutes(),
1910 hold = self.emergency_hold_reason.label(),
1911 authority = self.rollback_authority.label(),
1912 immediate = self.requires_immediate_rollback,
1913 blocks = self.blocks_promotion(),
1914 signals = signals,
1915 rollback = self.rollback.to_json(),
1916 readiness = self.readiness.to_json(),
1917 postmortem = self.postmortem.to_json(),
1918 )
1919 }
1920}
1921
1922#[derive(Debug, Clone, Copy, Default)]
1924pub struct MigrationIncidentResponsePlaybook;
1925
1926impl MigrationIncidentResponsePlaybook {
1927 #[must_use]
1929 pub const fn new() -> Self {
1930 Self
1931 }
1932
1933 #[must_use]
1935 pub fn resolve(&self, report: &MigrationIncidentReport) -> MigrationIncidentResponse {
1936 let severity = report.effective_severity();
1937 let rollback = MigrationRollbackPlan::default_for(report.class, report.detected_stage)
1938 .resolve(&report.available_artifacts);
1939 let readiness = rollback.readiness();
1940 let postmortem = MigrationPostmortemTemplate::for_incident(report, severity);
1941 MigrationIncidentResponse {
1942 incident_id: report.incident_id.clone(),
1943 class: report.class,
1944 severity,
1945 emergency_hold_reason: report.class.emergency_hold_reason(),
1946 rollback_authority: severity.rollback_authority(),
1947 requires_immediate_rollback: severity.requires_immediate_rollback(),
1948 signals: report.signals.clone(),
1949 rollback,
1950 readiness,
1951 postmortem,
1952 }
1953 }
1954}
1955
1956#[must_use]
1958fn default_prevention_actions(class: MigrationIncidentClass) -> Vec<String> {
1959 let actions: &[&str] = match class {
1960 MigrationIncidentClass::SemanticRegression => &[
1961 "add-metamorphic-oracle-case",
1962 "expand-golden-isomorphism-corpus",
1963 ],
1964 MigrationIncidentClass::DeterminismDivergence => &[
1965 "add-determinism-soak-fixture",
1966 "pin-nondeterministic-input-source",
1967 ],
1968 MigrationIncidentClass::PerformanceRegression => &[
1969 "add-tail-latency-budget-gate",
1970 "capture-baseline-profile-artifact",
1971 ],
1972 MigrationIncidentClass::CapabilityGapEscape => &[
1973 "add-capability-gap-guard",
1974 "fail-closed-on-unsupported-construct",
1975 ],
1976 MigrationIncidentClass::SecurityBreach => {
1977 &["tighten-sandbox-policy", "add-secret-redaction-fixture"]
1978 }
1979 MigrationIncidentClass::CertificationFalsePass => &[
1980 "strengthen-certification-oracle",
1981 "add-field-evidence-cross-check",
1982 ],
1983 MigrationIncidentClass::RollbackFailure => {
1984 &["add-rollback-readiness-precheck", "rehearse-rollback-drill"]
1985 }
1986 };
1987 actions.iter().map(|action| (*action).to_string()).collect()
1988}
1989
1990#[derive(Debug)]
1999pub struct RolloutScorecard {
2000 config: RolloutScorecardConfig,
2001 shadow_results: Vec<ShadowRunResult>,
2002 benchmark_gate: Option<GateResult>,
2003}
2004
2005impl RolloutScorecard {
2006 pub fn new(config: RolloutScorecardConfig) -> Self {
2008 Self {
2009 config,
2010 shadow_results: Vec::new(),
2011 benchmark_gate: None,
2012 }
2013 }
2014
2015 pub fn add_shadow_result(&mut self, result: ShadowRunResult) {
2017 self.shadow_results.push(result);
2018 }
2019
2020 pub fn set_benchmark_gate(&mut self, result: GateResult) {
2022 self.benchmark_gate = Some(result);
2023 }
2024
2025 #[must_use]
2027 pub fn shadow_scenario_count(&self) -> usize {
2028 self.shadow_results.len()
2029 }
2030
2031 #[must_use]
2033 pub fn shadow_match_count(&self) -> usize {
2034 self.shadow_results
2035 .iter()
2036 .filter(|r| r.verdict == ShadowVerdict::Match)
2037 .count()
2038 }
2039
2040 #[must_use]
2042 pub fn aggregate_match_ratio(&self) -> f64 {
2043 if self.shadow_results.is_empty() {
2044 return 0.0;
2045 }
2046 let total_frames: usize = self.shadow_results.iter().map(|r| r.frames_compared).sum();
2047 if total_frames == 0 {
2048 return 0.0;
2051 }
2052 let matched_frames: usize = self
2053 .shadow_results
2054 .iter()
2055 .flat_map(|r| r.frame_comparisons.iter())
2056 .filter(|c| c.matched)
2057 .count();
2058 matched_frames as f64 / total_frames as f64
2059 }
2060
2061 #[must_use]
2063 pub fn evaluate(&self) -> RolloutVerdict {
2064 if self.shadow_results.len() < self.config.min_shadow_scenarios {
2066 return RolloutVerdict::Inconclusive;
2067 }
2068
2069 let total_frames: usize = self.shadow_results.iter().map(|r| r.frames_compared).sum();
2072 if total_frames == 0 {
2073 return RolloutVerdict::Inconclusive;
2074 }
2075
2076 let match_ratio = self.aggregate_match_ratio();
2078 if match_ratio < self.config.min_match_ratio {
2079 return RolloutVerdict::NoGo;
2080 }
2081
2082 if self
2084 .shadow_results
2085 .iter()
2086 .any(|r| r.verdict == ShadowVerdict::Diverged)
2087 {
2088 return RolloutVerdict::NoGo;
2089 }
2090
2091 if self.config.require_benchmark_pass {
2093 match &self.benchmark_gate {
2094 None => return RolloutVerdict::Inconclusive,
2095 Some(gate) if !gate.passed() => return RolloutVerdict::NoGo,
2096 _ => {}
2097 }
2098 }
2099
2100 RolloutVerdict::Go
2101 }
2102
2103 #[must_use]
2105 pub fn summary(&self) -> RolloutSummary {
2106 let verdict = self.evaluate();
2107 RolloutSummary {
2108 verdict,
2109 shadow_scenarios: self.shadow_results.len(),
2110 shadow_matches: self.shadow_match_count(),
2111 aggregate_match_ratio: self.aggregate_match_ratio(),
2112 total_frames_compared: self.shadow_results.iter().map(|r| r.frames_compared).sum(),
2113 benchmark_passed: self.benchmark_gate.as_ref().map(|g| g.passed()),
2114 min_shadow_scenarios_required: self.config.min_shadow_scenarios,
2115 min_match_ratio_required: self.config.min_match_ratio,
2116 benchmark_required: self.config.require_benchmark_pass,
2117 }
2118 }
2119}
2120
2121#[derive(Debug, Clone)]
2123pub struct RolloutSummary {
2124 pub verdict: RolloutVerdict,
2126 pub shadow_scenarios: usize,
2128 pub shadow_matches: usize,
2130 pub aggregate_match_ratio: f64,
2132 pub total_frames_compared: usize,
2134 pub benchmark_passed: Option<bool>,
2136 pub min_shadow_scenarios_required: usize,
2138 pub min_match_ratio_required: f64,
2140 pub benchmark_required: bool,
2142}
2143
2144impl RolloutSummary {
2145 #[must_use]
2150 pub fn to_json(&self) -> String {
2151 let benchmark_str = match self.benchmark_passed {
2152 Some(true) => "\"pass\"",
2153 Some(false) => "\"fail\"",
2154 None => "null",
2155 };
2156 format!(
2157 concat!(
2158 "{{",
2159 "\"verdict\":\"{verdict}\",",
2160 "\"shadow_scenarios\":{scenarios},",
2161 "\"shadow_matches\":{matches},",
2162 "\"aggregate_match_ratio\":{ratio},",
2163 "\"total_frames_compared\":{frames},",
2164 "\"benchmark_passed\":{bench},",
2165 "\"config\":{{",
2166 "\"min_shadow_scenarios\":{min_scenarios},",
2167 "\"min_match_ratio\":{min_ratio},",
2168 "\"benchmark_required\":{bench_required}",
2169 "}}",
2170 "}}"
2171 ),
2172 verdict = self.verdict.label(),
2173 scenarios = self.shadow_scenarios,
2174 matches = self.shadow_matches,
2175 ratio = self.aggregate_match_ratio,
2176 frames = self.total_frames_compared,
2177 bench = benchmark_str,
2178 min_scenarios = self.min_shadow_scenarios_required,
2179 min_ratio = self.min_match_ratio_required,
2180 bench_required = self.benchmark_required,
2181 )
2182 }
2183}
2184
2185impl std::fmt::Display for RolloutSummary {
2186 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2187 writeln!(f, "=== Rollout Scorecard ===")?;
2188 writeln!(f, "Verdict: {}", self.verdict)?;
2189 writeln!(
2190 f,
2191 "Shadow: {}/{} scenarios matched ({} required)",
2192 self.shadow_matches, self.shadow_scenarios, self.min_shadow_scenarios_required,
2193 )?;
2194 writeln!(
2195 f,
2196 "Match ratio: {:.1}% (>= {:.1}% required)",
2197 self.aggregate_match_ratio * 100.0,
2198 self.min_match_ratio_required * 100.0,
2199 )?;
2200 writeln!(f, "Frames compared: {}", self.total_frames_compared)?;
2201 match self.benchmark_passed {
2202 Some(true) => writeln!(f, "Benchmark: PASS")?,
2203 Some(false) => writeln!(f, "Benchmark: FAIL")?,
2204 None if self.benchmark_required => writeln!(f, "Benchmark: MISSING (required)")?,
2205 None => writeln!(f, "Benchmark: not provided")?,
2206 }
2207 Ok(())
2208 }
2209}
2210
2211#[derive(Debug, Clone)]
2221pub struct RolloutEvidenceBundle {
2222 pub scorecard: RolloutSummary,
2224 pub queue_telemetry: Option<QueueTelemetry>,
2226 pub requested_lane: String,
2228 pub resolved_lane: String,
2230 pub rollout_policy: String,
2232}
2233
2234impl RolloutEvidenceBundle {
2235 #[must_use]
2237 pub fn to_json(&self) -> String {
2238 let qt_json = match &self.queue_telemetry {
2239 Some(qt) => format!(
2240 concat!(
2241 "{{",
2242 "\"enqueued\":{e},",
2243 "\"processed\":{p},",
2244 "\"dropped\":{d},",
2245 "\"high_water\":{hw},",
2246 "\"in_flight\":{inf}",
2247 "}}"
2248 ),
2249 e = qt.enqueued,
2250 p = qt.processed,
2251 d = qt.dropped,
2252 hw = qt.high_water,
2253 inf = qt.in_flight,
2254 ),
2255 None => "null".to_string(),
2256 };
2257 format!(
2258 concat!(
2259 "{{",
2260 "\"schema_version\":\"1.0.0\",",
2261 "\"scorecard\":{sc},",
2262 "\"queue_telemetry\":{qt},",
2263 "\"runtime\":{{",
2264 "\"requested_lane\":\"{rl}\",",
2265 "\"resolved_lane\":\"{rsl}\",",
2266 "\"rollout_policy\":\"{rp}\"",
2267 "}}",
2268 "}}"
2269 ),
2270 sc = self.scorecard.to_json(),
2271 qt = qt_json,
2272 rl = self.requested_lane,
2273 rsl = self.resolved_lane,
2274 rp = self.rollout_policy,
2275 )
2276 }
2277}
2278
2279impl std::fmt::Display for RolloutEvidenceBundle {
2280 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2281 writeln!(f, "=== Rollout Evidence Bundle ===")?;
2282 writeln!(
2283 f,
2284 "Lane: {} (resolved: {})",
2285 self.requested_lane, self.resolved_lane
2286 )?;
2287 writeln!(f, "Policy: {}", self.rollout_policy)?;
2288 write!(f, "{}", self.scorecard)?;
2289 if let Some(qt) = &self.queue_telemetry {
2290 writeln!(
2291 f,
2292 "Queue: enqueued={}, processed={}, dropped={}, high_water={}, in_flight={}",
2293 qt.enqueued, qt.processed, qt.dropped, qt.high_water, qt.in_flight
2294 )?;
2295 }
2296 Ok(())
2297 }
2298}
2299
2300#[cfg(test)]
2301mod tests {
2302 use super::*;
2303 use crate::shadow_run::{FrameComparison, ShadowRunResult, ShadowVerdict};
2304
2305 use crate::lab_integration::LabOutput;
2306
2307 fn empty_lab_output() -> LabOutput {
2308 LabOutput {
2309 frame_count: 0,
2310 frame_records: vec![],
2311 event_count: 0,
2312 event_log: vec![],
2313 tick_count: 0,
2314 anomaly_count: 0,
2315 }
2316 }
2317
2318 fn make_shadow_result(verdict: ShadowVerdict, frames: usize) -> ShadowRunResult {
2319 let frame_comparisons: Vec<FrameComparison> = (0..frames)
2320 .map(|i| FrameComparison {
2321 index: i,
2322 baseline_checksum: 0xDEAD_BEEF,
2323 candidate_checksum: if verdict == ShadowVerdict::Match {
2324 0xDEAD_BEEF
2325 } else {
2326 0xCAFE_BABE
2327 },
2328 matched: verdict == ShadowVerdict::Match,
2329 })
2330 .collect();
2331
2332 ShadowRunResult {
2333 verdict,
2334 scenario_name: "test".to_string(),
2335 seed: 42,
2336 frame_comparisons,
2337 first_divergence: if verdict == ShadowVerdict::Diverged {
2338 Some(0)
2339 } else {
2340 None
2341 },
2342 frames_compared: frames,
2343 baseline: empty_lab_output(),
2344 candidate: empty_lab_output(),
2345 baseline_label: "baseline".to_string(),
2346 candidate_label: "candidate".to_string(),
2347 run_total: 1,
2348 }
2349 }
2350
2351 fn release_gate_artifact(kind: &str) -> MigrationReleaseGateArtifact {
2352 let artifact_id = format!("{kind}-artifact");
2353 let uri = format!("artifacts/{kind}.json");
2354 MigrationReleaseGateArtifact::new(
2355 &artifact_id,
2356 kind,
2357 "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
2358 &uri,
2359 )
2360 }
2361
2362 fn beta_release_gate_evidence() -> MigrationReleaseGateEvidence {
2363 let readiness = MigrationReadinessEvidence::new(OperatorAuthority::ReleaseOwner)
2364 .certification_pass_ratio(0.98)
2365 .corpus_coverage_ratio(0.70)
2366 .reliability_pass_ratio(0.99)
2367 .deterministic_artifact_count(5)
2368 .benchmark_gate_passed(true)
2369 .open_blocker_count(0);
2370
2371 MigrationReleaseGateEvidence::new("migration-service-2026.05.08", readiness)
2372 .determinism_pass_ratio(1.0)
2373 .performance_budget_passed(true)
2374 .unresolved_critical_gap_count(0)
2375 .artifact(release_gate_artifact("certification"))
2376 .artifact(release_gate_artifact("critical-gaps"))
2377 .artifact(release_gate_artifact("determinism"))
2378 .artifact(release_gate_artifact("performance"))
2379 .artifact(release_gate_artifact("readiness"))
2380 .artifact(release_gate_artifact("corpus"))
2381 }
2382
2383 #[test]
2384 fn scorecard_go_with_matching_shadows() {
2385 let config = RolloutScorecardConfig::default().min_shadow_scenarios(2);
2386 let mut sc = RolloutScorecard::new(config);
2387 sc.add_shadow_result(make_shadow_result(ShadowVerdict::Match, 10));
2388 sc.add_shadow_result(make_shadow_result(ShadowVerdict::Match, 15));
2389
2390 let verdict = sc.evaluate();
2391 assert_eq!(verdict, RolloutVerdict::Go);
2392 assert!(verdict.is_go());
2393 assert_eq!(sc.aggregate_match_ratio(), 1.0);
2394 }
2395
2396 #[test]
2397 fn scorecard_nogo_with_diverged_shadow() {
2398 let config = RolloutScorecardConfig::default();
2399 let mut sc = RolloutScorecard::new(config);
2400 sc.add_shadow_result(make_shadow_result(ShadowVerdict::Diverged, 10));
2401
2402 assert_eq!(sc.evaluate(), RolloutVerdict::NoGo);
2403 }
2404
2405 #[test]
2406 fn scorecard_inconclusive_without_enough_scenarios() {
2407 let config = RolloutScorecardConfig::default().min_shadow_scenarios(3);
2408 let mut sc = RolloutScorecard::new(config);
2409 sc.add_shadow_result(make_shadow_result(ShadowVerdict::Match, 10));
2410 sc.add_shadow_result(make_shadow_result(ShadowVerdict::Match, 10));
2411
2412 assert_eq!(sc.evaluate(), RolloutVerdict::Inconclusive);
2413 }
2414
2415 #[test]
2416 fn scorecard_inconclusive_when_benchmark_required_but_missing() {
2417 let config = RolloutScorecardConfig::default().require_benchmark_pass(true);
2418 let mut sc = RolloutScorecard::new(config);
2419 sc.add_shadow_result(make_shadow_result(ShadowVerdict::Match, 10));
2420
2421 assert_eq!(sc.evaluate(), RolloutVerdict::Inconclusive);
2422 }
2423
2424 #[test]
2425 fn scorecard_summary_display() {
2426 let config = RolloutScorecardConfig::default().min_shadow_scenarios(1);
2427 let mut sc = RolloutScorecard::new(config);
2428 sc.add_shadow_result(make_shadow_result(ShadowVerdict::Match, 10));
2429
2430 let summary = sc.summary();
2431 let text = summary.to_string();
2432 assert!(text.contains("GO"));
2433 assert!(text.contains("100.0%"));
2434 assert!(text.contains("10"));
2435 }
2436
2437 #[test]
2438 fn verdict_labels() {
2439 assert_eq!(RolloutVerdict::Go.label(), "GO");
2440 assert_eq!(RolloutVerdict::NoGo.label(), "NO-GO");
2441 assert_eq!(RolloutVerdict::Inconclusive.label(), "INCONCLUSIVE");
2442 assert_eq!(format!("{}", RolloutVerdict::Go), "GO");
2443 }
2444
2445 #[test]
2446 fn readiness_rubric_allows_alpha_at_thresholds() {
2447 let rubric = MigrationReadinessRubric::opentui_default();
2448 let evidence = MigrationReadinessEvidence::new(OperatorAuthority::ReleaseOwner)
2449 .certification_pass_ratio(0.90)
2450 .corpus_coverage_ratio(0.25)
2451 .reliability_pass_ratio(0.95)
2452 .deterministic_artifact_count(3)
2453 .benchmark_gate_passed(true)
2454 .open_blocker_count(2);
2455
2456 let decision = rubric.evaluate(MigrationRolloutStage::Alpha, &evidence);
2457 assert_eq!(decision.verdict, MigrationReadinessVerdict::Advance);
2458 assert!(
2459 decision.reasons.is_empty(),
2460 "passing evidence must not carry hold reasons"
2461 );
2462 }
2463
2464 #[test]
2465 fn readiness_rubric_fails_closed_on_non_finite_evidence() {
2466 let rubric = MigrationReadinessRubric::opentui_default();
2467 let mut evidence = MigrationReadinessEvidence::new(OperatorAuthority::ReleaseOwner)
2468 .certification_pass_ratio(0.99)
2469 .corpus_coverage_ratio(0.80)
2470 .reliability_pass_ratio(0.99)
2471 .deterministic_artifact_count(4)
2472 .benchmark_gate_passed(true);
2473 evidence.certification_pass_ratio = f64::NAN;
2476 let decision = rubric.evaluate(MigrationRolloutStage::Alpha, &evidence);
2477 assert_eq!(decision.verdict, MigrationReadinessVerdict::Hold);
2478 assert!(decision.reasons.contains(&"certification-threshold"));
2479 }
2480
2481 #[test]
2482 fn scorecard_with_zero_compared_frames_is_not_go() {
2483 let mut scorecard =
2484 RolloutScorecard::new(RolloutScorecardConfig::default().min_shadow_scenarios(1));
2485 scorecard.add_shadow_result(make_shadow_result(ShadowVerdict::Match, 0));
2486 assert_eq!(scorecard.aggregate_match_ratio(), 0.0);
2488 assert_eq!(scorecard.evaluate(), RolloutVerdict::Inconclusive);
2489 }
2490
2491 #[test]
2492 fn readiness_rubric_blocks_beta_without_artifacts_and_benchmark() {
2493 let rubric = MigrationReadinessRubric::opentui_default();
2494 let evidence = MigrationReadinessEvidence::new(OperatorAuthority::ReleaseOwner)
2495 .certification_pass_ratio(0.99)
2496 .corpus_coverage_ratio(0.80)
2497 .reliability_pass_ratio(0.99)
2498 .deterministic_artifact_count(4)
2499 .benchmark_gate_passed(false);
2500
2501 let decision = rubric.evaluate(MigrationRolloutStage::Beta, &evidence);
2502 assert_eq!(decision.verdict, MigrationReadinessVerdict::Hold);
2503 assert!(
2504 decision
2505 .reasons
2506 .contains(&"deterministic-artifact-threshold")
2507 );
2508 assert!(decision.reasons.contains(&"benchmark-gate"));
2509 }
2510
2511 #[test]
2512 fn readiness_rubric_requires_maintainer_quorum_for_ga() {
2513 let rubric = MigrationReadinessRubric::opentui_default();
2514 let release_owner_evidence =
2515 MigrationReadinessEvidence::new(OperatorAuthority::ReleaseOwner)
2516 .certification_pass_ratio(1.0)
2517 .corpus_coverage_ratio(0.95)
2518 .reliability_pass_ratio(0.999)
2519 .deterministic_artifact_count(7)
2520 .benchmark_gate_passed(true);
2521
2522 let decision = rubric.evaluate(MigrationRolloutStage::Ga, &release_owner_evidence);
2523 assert_eq!(decision.verdict, MigrationReadinessVerdict::Hold);
2524 assert_eq!(decision.reasons, vec!["operator-authority"]);
2525
2526 let quorum_evidence = MigrationReadinessEvidence {
2527 operator_authority: OperatorAuthority::MaintainerQuorum,
2528 ..release_owner_evidence
2529 };
2530 assert!(
2531 rubric
2532 .evaluate(MigrationRolloutStage::Ga, &quorum_evidence)
2533 .may_advance()
2534 );
2535 }
2536
2537 #[test]
2538 fn readiness_rubric_emergency_hold_overrides_clean_evidence() {
2539 let rubric = MigrationReadinessRubric::opentui_default();
2540 let evidence = MigrationReadinessEvidence::new(OperatorAuthority::MaintainerQuorum)
2541 .certification_pass_ratio(1.0)
2542 .corpus_coverage_ratio(1.0)
2543 .reliability_pass_ratio(1.0)
2544 .deterministic_artifact_count(8)
2545 .benchmark_gate_passed(true)
2546 .emergency_hold(EmergencyHold::new(
2547 EmergencyHoldReason::DeterminismDivergence,
2548 OperatorAuthority::OnCall,
2549 ));
2550
2551 let decision = rubric.evaluate(MigrationRolloutStage::Ga, &evidence);
2552 assert_eq!(decision.verdict, MigrationReadinessVerdict::EmergencyHold);
2553 assert_eq!(decision.reasons, vec!["determinism-divergence"]);
2554 }
2555
2556 #[test]
2557 fn readiness_rubric_recommends_highest_passing_stage() {
2558 let rubric = MigrationReadinessRubric::opentui_default();
2559 let beta_ready = MigrationReadinessEvidence::new(OperatorAuthority::ReleaseOwner)
2560 .certification_pass_ratio(0.98)
2561 .corpus_coverage_ratio(0.75)
2562 .reliability_pass_ratio(0.99)
2563 .deterministic_artifact_count(5)
2564 .benchmark_gate_passed(true);
2565
2566 assert_eq!(
2567 rubric.recommended_stage(&beta_ready),
2568 Some(MigrationRolloutStage::Beta)
2569 );
2570 }
2571
2572 #[test]
2573 fn readiness_decision_json_is_machine_readable() {
2574 let rubric = MigrationReadinessRubric::opentui_default();
2575 let evidence = MigrationReadinessEvidence::new(OperatorAuthority::Automation);
2576 let json = rubric
2577 .evaluate(MigrationRolloutStage::Alpha, &evidence)
2578 .to_json();
2579
2580 assert!(json.contains("\"stage\":\"alpha\""));
2581 assert!(json.contains("\"verdict\":\"hold\""));
2582 assert!(json.contains("\"operator-authority\""));
2583 }
2584
2585 #[test]
2586 fn readiness_rubric_rejects_non_finite_ratios() {
2587 let rubric = MigrationReadinessRubric::opentui_default();
2588 let evidence = MigrationReadinessEvidence::new(OperatorAuthority::MaintainerQuorum)
2589 .certification_pass_ratio(f64::NAN)
2590 .corpus_coverage_ratio(f64::INFINITY)
2591 .reliability_pass_ratio(f64::NEG_INFINITY)
2592 .deterministic_artifact_count(8)
2593 .benchmark_gate_passed(true);
2594
2595 let decision = rubric.evaluate(MigrationRolloutStage::Ga, &evidence);
2596 assert_eq!(decision.verdict, MigrationReadinessVerdict::Hold);
2597 assert!(decision.reasons.contains(&"certification-threshold"));
2598 assert!(decision.reasons.contains(&"corpus-coverage-threshold"));
2599 assert!(
2600 decision
2601 .reasons
2602 .contains(&"operational-reliability-threshold")
2603 );
2604 }
2605
2606 #[test]
2607 fn release_gate_enforce_passes_with_traceable_artifacts() {
2608 let policy = MigrationReleaseGatePolicy::for_stage(
2609 MigrationRolloutStage::Beta,
2610 MigrationReleaseGateMode::Enforce,
2611 );
2612 let evaluator = MigrationReleaseGateEvaluator::new(policy);
2613 let decision = evaluator.evaluate(&beta_release_gate_evidence());
2614
2615 assert_eq!(decision.verdict, MigrationReleaseGateVerdict::Pass);
2616 assert!(decision.passed());
2617 assert!(!decision.blocks_release());
2618 assert!(decision.failed_clauses().is_empty());
2619 assert_eq!(
2620 decision
2621 .clauses
2622 .iter()
2623 .map(|clause| clause.clause)
2624 .collect::<Vec<_>>(),
2625 vec![
2626 "readiness",
2627 "certification",
2628 "determinism",
2629 "performance-budget",
2630 "critical-gaps",
2631 "artifact-traceability",
2632 ]
2633 );
2634 let critical_gap_artifacts = vec!["critical-gaps-artifact".to_string()];
2635 assert!(decision.clauses.iter().any(|clause| {
2636 clause.clause == "critical-gaps" && clause.artifact_ids == critical_gap_artifacts
2637 }));
2638 }
2639
2640 #[test]
2641 fn release_gate_enforce_fails_with_clause_reasons() {
2642 let readiness = MigrationReadinessEvidence::new(OperatorAuthority::ReleaseOwner)
2643 .certification_pass_ratio(0.96)
2644 .corpus_coverage_ratio(0.70)
2645 .reliability_pass_ratio(0.99)
2646 .deterministic_artifact_count(5)
2647 .benchmark_gate_passed(true);
2648 let evidence = MigrationReleaseGateEvidence::new("migration-service-bad", readiness)
2649 .determinism_pass_ratio(0.95)
2650 .performance_budget_passed(false)
2651 .unresolved_critical_gap_count(2)
2652 .artifact(release_gate_artifact("certification"))
2653 .artifact(release_gate_artifact("critical-gaps"))
2654 .artifact(release_gate_artifact("determinism"))
2655 .artifact(release_gate_artifact("performance"))
2656 .artifact(release_gate_artifact("readiness"))
2657 .artifact(release_gate_artifact("corpus"));
2658 let policy = MigrationReleaseGatePolicy::for_stage(
2659 MigrationRolloutStage::Beta,
2660 MigrationReleaseGateMode::Enforce,
2661 );
2662 let decision = MigrationReleaseGateEvaluator::new(policy).evaluate(&evidence);
2663
2664 assert_eq!(decision.verdict, MigrationReleaseGateVerdict::Fail);
2665 assert!(decision.blocks_release());
2666 let failed = decision.failed_clauses();
2667 assert!(failed.contains(&"readiness"));
2668 assert!(failed.contains(&"certification"));
2669 assert!(failed.contains(&"determinism"));
2670 assert!(failed.contains(&"performance-budget"));
2671 assert!(failed.contains(&"critical-gaps"));
2672 assert!(
2673 decision
2674 .clauses
2675 .iter()
2676 .any(|clause| clause.reason.contains("0.960000 < 0.970000"))
2677 );
2678 }
2679
2680 #[test]
2681 fn release_gate_dry_run_failure_does_not_block_release() {
2682 let mut evidence = beta_release_gate_evidence();
2683 evidence.performance_budget_passed = false;
2684 let policy = MigrationReleaseGatePolicy::for_stage(
2685 MigrationRolloutStage::Beta,
2686 MigrationReleaseGateMode::DryRun,
2687 );
2688 let decision = MigrationReleaseGateEvaluator::new(policy).evaluate(&evidence);
2689
2690 assert_eq!(decision.verdict, MigrationReleaseGateVerdict::Fail);
2691 assert!(!decision.blocks_release());
2692 assert!(decision.failed_clauses().contains(&"performance-budget"));
2693 }
2694
2695 #[test]
2696 fn release_gate_rejects_mutable_artifact_evidence() {
2697 let readiness = MigrationReadinessEvidence::new(OperatorAuthority::ReleaseOwner)
2698 .certification_pass_ratio(0.98)
2699 .corpus_coverage_ratio(0.70)
2700 .reliability_pass_ratio(0.99)
2701 .deterministic_artifact_count(5)
2702 .benchmark_gate_passed(true);
2703 let mutable_performance = MigrationReleaseGateArtifact::new(
2704 "performance-artifact",
2705 "performance",
2706 "latest",
2707 "artifacts/performance.json",
2708 );
2709 let evidence = MigrationReleaseGateEvidence::new("migration-service-mutable", readiness)
2710 .determinism_pass_ratio(1.0)
2711 .performance_budget_passed(true)
2712 .artifact(release_gate_artifact("certification"))
2713 .artifact(release_gate_artifact("critical-gaps"))
2714 .artifact(release_gate_artifact("determinism"))
2715 .artifact(mutable_performance)
2716 .artifact(release_gate_artifact("readiness"))
2717 .artifact(release_gate_artifact("corpus"));
2718 let policy = MigrationReleaseGatePolicy::for_stage(
2719 MigrationRolloutStage::Beta,
2720 MigrationReleaseGateMode::Enforce,
2721 );
2722 let decision = MigrationReleaseGateEvaluator::new(policy).evaluate(&evidence);
2723
2724 assert_eq!(decision.verdict, MigrationReleaseGateVerdict::Fail);
2725 assert!(decision.blocks_release());
2726 assert!(decision.clauses.iter().any(|clause| {
2727 clause.clause == "artifact-traceability"
2728 && !clause.passed
2729 && clause.reason.contains("invalid immutable references")
2730 && clause
2731 .reason
2732 .contains("missing artifact kinds: performance")
2733 }));
2734 }
2735
2736 #[test]
2737 fn release_gate_json_is_machine_readable() -> serde_json::Result<()> {
2738 let policy = MigrationReleaseGatePolicy::for_stage(
2739 MigrationRolloutStage::Beta,
2740 MigrationReleaseGateMode::Enforce,
2741 );
2742 let mut evidence = beta_release_gate_evidence();
2743 evidence.migration_version = "migration-service-\"beta\"".to_string();
2744 let decision = MigrationReleaseGateEvaluator::new(policy).evaluate(&evidence);
2745 let json = decision.to_json();
2746 let parsed: serde_json::Value = serde_json::from_str(&json)?;
2747
2748 assert_eq!(parsed["schema_version"], "1.0.0");
2749 assert_eq!(parsed["migration_version"], "migration-service-\"beta\"");
2750 assert_eq!(parsed["mode"], "enforce");
2751 assert_eq!(parsed["target_stage"], "beta");
2752 assert_eq!(parsed["verdict"], "pass");
2753 assert_eq!(parsed["blocks_release"], false);
2754 assert_eq!(parsed["clauses"].as_array().map(Vec::len), Some(6));
2755 Ok(())
2756 }
2757
2758 #[test]
2759 fn scorecard_summary_json_go() {
2760 let config = RolloutScorecardConfig::default().min_shadow_scenarios(1);
2761 let mut sc = RolloutScorecard::new(config);
2762 sc.add_shadow_result(make_shadow_result(ShadowVerdict::Match, 10));
2763
2764 let json = sc.summary().to_json();
2765 assert!(json.contains("\"verdict\":\"GO\""));
2766 assert!(json.contains("\"shadow_scenarios\":1"));
2767 assert!(json.contains("\"shadow_matches\":1"));
2768 assert!(json.contains("\"total_frames_compared\":10"));
2769 assert!(json.contains("\"aggregate_match_ratio\":1"));
2770 assert!(json.contains("\"benchmark_passed\":null"));
2771 }
2772
2773 #[test]
2774 fn scorecard_summary_json_nogo() {
2775 let config = RolloutScorecardConfig::default();
2776 let mut sc = RolloutScorecard::new(config);
2777 sc.add_shadow_result(make_shadow_result(ShadowVerdict::Diverged, 5));
2778
2779 let json = sc.summary().to_json();
2780 assert!(json.contains("\"verdict\":\"NO-GO\""));
2781 assert!(json.contains("\"shadow_matches\":0"));
2782 }
2783
2784 #[test]
2785 fn scorecard_e2e_with_real_shadow_run() {
2786 use crate::shadow_run::{ShadowRun, ShadowRunConfig};
2787 use ftui_core::event::Event;
2788 use ftui_core::geometry::Rect;
2789 use ftui_render::frame::Frame;
2790 use ftui_runtime::program::{Cmd, Model};
2791 use ftui_widgets::Widget;
2792 use ftui_widgets::paragraph::Paragraph;
2793
2794 struct RolloutModel {
2795 ticks: u64,
2796 }
2797
2798 #[derive(Debug, Clone)]
2799 enum RolloutMsg {
2800 Tick,
2801 Quit,
2802 }
2803
2804 impl From<Event> for RolloutMsg {
2805 fn from(e: Event) -> Self {
2806 match e {
2807 Event::Tick => RolloutMsg::Tick,
2808 _ => RolloutMsg::Quit,
2809 }
2810 }
2811 }
2812
2813 impl Model for RolloutModel {
2814 type Message = RolloutMsg;
2815
2816 fn update(&mut self, msg: RolloutMsg) -> Cmd<RolloutMsg> {
2817 match msg {
2818 RolloutMsg::Tick => {
2819 self.ticks += 1;
2820 Cmd::none()
2821 }
2822 RolloutMsg::Quit => Cmd::quit(),
2823 }
2824 }
2825
2826 fn view(&self, frame: &mut Frame) {
2827 let text = format!("Ticks: {}", self.ticks);
2828 let area = Rect::new(0, 0, frame.width(), 1);
2829 Paragraph::new(text).render(area, frame);
2830 }
2831 }
2832
2833 let mut scorecard =
2835 RolloutScorecard::new(RolloutScorecardConfig::default().min_shadow_scenarios(3));
2836
2837 for seed in [42, 99, 7] {
2838 let config = ShadowRunConfig::new("rollout_e2e", "tick_counter", seed).viewport(40, 10);
2839 let result = ShadowRun::compare(
2840 config,
2841 || RolloutModel { ticks: 0 },
2842 |session| {
2843 session.init();
2844 for _ in 0..5 {
2845 session.tick();
2846 session.capture_frame();
2847 }
2848 },
2849 );
2850 scorecard.add_shadow_result(result);
2851 }
2852
2853 let verdict = scorecard.evaluate();
2855 assert_eq!(verdict, RolloutVerdict::Go);
2856
2857 let summary = scorecard.summary();
2858 assert_eq!(summary.shadow_scenarios, 3);
2859 assert_eq!(summary.shadow_matches, 3);
2860 assert_eq!(summary.total_frames_compared, 15); assert!((summary.aggregate_match_ratio - 1.0).abs() < f64::EPSILON);
2862 assert!(summary.to_string().contains("GO"));
2863 }
2864
2865 #[test]
2866 fn evidence_bundle_json_contains_all_sections() {
2867 let config = RolloutScorecardConfig::default().min_shadow_scenarios(1);
2868 let mut sc = RolloutScorecard::new(config);
2869 sc.add_shadow_result(make_shadow_result(ShadowVerdict::Match, 5));
2870
2871 let bundle = RolloutEvidenceBundle {
2872 scorecard: sc.summary(),
2873 queue_telemetry: Some(QueueTelemetry {
2874 enqueued: 10,
2875 processed: 8,
2876 dropped: 1,
2877 high_water: 4,
2878 in_flight: 1,
2879 }),
2880 requested_lane: "structured".to_string(),
2881 resolved_lane: "structured".to_string(),
2882 rollout_policy: "shadow".to_string(),
2883 };
2884
2885 let json = bundle.to_json();
2886 assert!(json.contains("\"schema_version\":\"1.0.0\""));
2887 assert!(json.contains("\"scorecard\":{"));
2888 assert!(json.contains("\"verdict\":\"GO\""));
2889 assert!(json.contains("\"queue_telemetry\":{"));
2890 assert!(json.contains("\"enqueued\":10"));
2891 assert!(json.contains("\"dropped\":1"));
2892 assert!(json.contains("\"runtime\":{"));
2893 assert!(json.contains("\"requested_lane\":\"structured\""));
2894 assert!(json.contains("\"rollout_policy\":\"shadow\""));
2895 }
2896
2897 #[test]
2898 fn evidence_bundle_display_readable() {
2899 let config = RolloutScorecardConfig::default().min_shadow_scenarios(1);
2900 let mut sc = RolloutScorecard::new(config);
2901 sc.add_shadow_result(make_shadow_result(ShadowVerdict::Match, 5));
2902
2903 let bundle = RolloutEvidenceBundle {
2904 scorecard: sc.summary(),
2905 queue_telemetry: None,
2906 requested_lane: "asupersync".to_string(),
2907 resolved_lane: "structured".to_string(),
2908 rollout_policy: "off".to_string(),
2909 };
2910
2911 let text = bundle.to_string();
2912 assert!(text.contains("Rollout Evidence Bundle"));
2913 assert!(text.contains("asupersync"));
2914 assert!(text.contains("structured"));
2915 assert!(text.contains("GO"));
2916 }
2917
2918 fn rollback_artifact_set() -> Vec<MigrationReleaseGateArtifact> {
2924 [
2925 "release-gate-decision",
2926 "source-snapshot",
2927 "last-good-release",
2928 "determinism-baseline",
2929 "certification-report",
2930 "secret-rotation-runbook",
2931 "manual-recovery-runbook",
2932 ]
2933 .iter()
2934 .map(|kind| release_gate_artifact(kind))
2935 .collect()
2936 }
2937
2938 #[test]
2939 fn incident_class_maps_to_severity_and_hold_reason() {
2940 assert_eq!(MigrationIncidentClass::ALL.len(), 7);
2941 let mut labels: Vec<&str> = MigrationIncidentClass::ALL
2943 .iter()
2944 .map(|class| class.label())
2945 .collect();
2946 let count = labels.len();
2947 labels.sort_unstable();
2948 labels.dedup();
2949 assert_eq!(labels.len(), count, "labels must be unique");
2950
2951 assert_eq!(
2952 MigrationIncidentClass::DeterminismDivergence.default_severity(),
2953 MigrationIncidentSeverity::Sev1
2954 );
2955 assert_eq!(
2956 MigrationIncidentClass::PerformanceRegression.default_severity(),
2957 MigrationIncidentSeverity::Sev3
2958 );
2959 assert_eq!(
2960 MigrationIncidentClass::SecurityBreach.emergency_hold_reason(),
2961 EmergencyHoldReason::SecurityIncident
2962 );
2963 assert_eq!(
2964 MigrationIncidentClass::DeterminismDivergence.emergency_hold_reason(),
2965 EmergencyHoldReason::DeterminismDivergence
2966 );
2967 }
2968
2969 #[test]
2970 fn severity_rank_escalation_and_response_metadata() {
2971 assert!(MigrationIncidentSeverity::Sev1.rank() > MigrationIncidentSeverity::Sev2.rank());
2973 assert!(MigrationIncidentSeverity::Sev2.rank() > MigrationIncidentSeverity::Sev3.rank());
2974 assert!(MigrationIncidentSeverity::Sev3.rank() > MigrationIncidentSeverity::Sev4.rank());
2975 assert_eq!(
2977 MigrationIncidentSeverity::Sev3.escalate(MigrationIncidentSeverity::Sev1),
2978 MigrationIncidentSeverity::Sev1
2979 );
2980 assert_eq!(
2981 MigrationIncidentSeverity::Sev1.escalate(MigrationIncidentSeverity::Sev4),
2982 MigrationIncidentSeverity::Sev1
2983 );
2984 assert!(
2986 MigrationIncidentSeverity::Sev1.ack_deadline_minutes()
2987 < MigrationIncidentSeverity::Sev4.ack_deadline_minutes()
2988 );
2989 assert!(MigrationIncidentSeverity::Sev1.requires_immediate_rollback());
2990 assert!(!MigrationIncidentSeverity::Sev3.requires_immediate_rollback());
2991 assert_eq!(
2992 MigrationIncidentSeverity::Sev1.rollback_authority(),
2993 OperatorAuthority::OnCall
2994 );
2995 assert_eq!(
2996 MigrationIncidentSeverity::Sev4.rollback_authority(),
2997 OperatorAuthority::ReleaseOwner
2998 );
2999 }
3000
3001 #[test]
3002 fn signals_escalate_effective_severity() {
3003 let report = MigrationIncidentReport::new(
3005 "INC-1",
3006 MigrationIncidentClass::PerformanceRegression,
3007 MigrationRolloutStage::Beta,
3008 )
3009 .signal(MigrationIncidentSignal::new(
3010 "p95-breach",
3011 "p95 over budget",
3012 ))
3013 .signal(
3014 MigrationIncidentSignal::new("data-corruption", "rows lost")
3015 .escalates_to(MigrationIncidentSeverity::Sev1),
3016 );
3017 assert_eq!(report.effective_severity(), MigrationIncidentSeverity::Sev1);
3018
3019 let plain = MigrationIncidentReport::new(
3021 "INC-2",
3022 MigrationIncidentClass::PerformanceRegression,
3023 MigrationRolloutStage::Beta,
3024 )
3025 .signal(MigrationIncidentSignal::new(
3026 "p95-breach",
3027 "p95 over budget",
3028 ));
3029 assert_eq!(plain.effective_severity(), MigrationIncidentSeverity::Sev3);
3030 }
3031
3032 #[test]
3033 fn rollback_plan_has_spine_and_class_specific_steps() {
3034 let plain = MigrationRollbackPlan::default_for(
3036 MigrationIncidentClass::SemanticRegression,
3037 MigrationRolloutStage::Beta,
3038 );
3039 assert_eq!(plain.steps.len(), 6);
3040 let last = plain.steps.last().expect("non-empty plan");
3041 assert_eq!(last.action, "record-postmortem");
3042 assert!(last.required_artifact_kind.is_empty());
3043 assert!(last.is_resolved());
3045
3046 let security = MigrationRollbackPlan::default_for(
3048 MigrationIncidentClass::SecurityBreach,
3049 MigrationRolloutStage::Ga,
3050 );
3051 assert_eq!(security.steps.len(), 7);
3052 assert!(
3053 security
3054 .steps
3055 .iter()
3056 .any(|step| step.action == "rotate-exposed-credentials")
3057 );
3058
3059 let recovery = MigrationRollbackPlan::default_for(
3061 MigrationIncidentClass::RollbackFailure,
3062 MigrationRolloutStage::Ga,
3063 );
3064 assert!(
3065 recovery
3066 .steps
3067 .iter()
3068 .any(|step| step.action == "escalate-manual-recovery")
3069 );
3070
3071 for (index, step) in plain.steps.iter().enumerate() {
3073 assert_eq!(step.order, index + 1);
3074 }
3075 }
3076
3077 #[test]
3078 fn rollback_ready_when_all_artifacts_present() {
3079 let artifacts = rollback_artifact_set();
3080 let plan = MigrationRollbackPlan::default_for(
3081 MigrationIncidentClass::SecurityBreach,
3082 MigrationRolloutStage::Ga,
3083 )
3084 .resolve(&artifacts);
3085 let readiness = plan.readiness();
3086 assert!(readiness.is_ready());
3087 assert_eq!(readiness.verdict, MigrationRollbackReadinessVerdict::Ready);
3088 assert!(readiness.missing_artifact_kinds.is_empty());
3089 assert_eq!(readiness.resolved_steps, readiness.total_steps);
3090 }
3091
3092 #[test]
3093 fn rollback_blocked_when_required_artifact_missing() {
3094 let artifacts: Vec<MigrationReleaseGateArtifact> = rollback_artifact_set()
3096 .into_iter()
3097 .filter(|artifact| artifact.kind != "determinism-baseline")
3098 .collect();
3099 let plan = MigrationRollbackPlan::default_for(
3100 MigrationIncidentClass::DeterminismDivergence,
3101 MigrationRolloutStage::Beta,
3102 )
3103 .resolve(&artifacts);
3104 let readiness = plan.readiness();
3105 assert!(!readiness.is_ready());
3106 assert_eq!(
3107 readiness.verdict,
3108 MigrationRollbackReadinessVerdict::Blocked
3109 );
3110 assert_eq!(
3111 readiness.missing_artifact_kinds,
3112 vec!["determinism-baseline"]
3113 );
3114 }
3115
3116 #[test]
3117 fn rollback_ignores_mutable_artifacts() {
3118 let mut artifacts = rollback_artifact_set();
3120 artifacts.retain(|artifact| artifact.kind != "last-good-release");
3121 artifacts.push(MigrationReleaseGateArtifact::new(
3122 "last-good-release-mutable",
3123 "last-good-release",
3124 "md5:not-a-real-sha",
3125 "artifacts/last-good.json",
3126 ));
3127 let plan = MigrationRollbackPlan::default_for(
3128 MigrationIncidentClass::SemanticRegression,
3129 MigrationRolloutStage::Beta,
3130 )
3131 .resolve(&artifacts);
3132 let readiness = plan.readiness();
3133 assert!(!readiness.is_ready());
3134 assert!(
3135 readiness
3136 .missing_artifact_kinds
3137 .contains(&"last-good-release")
3138 );
3139 }
3140
3141 #[test]
3142 fn rollback_resolution_is_deterministic() {
3143 let mut artifacts = rollback_artifact_set();
3145 artifacts.push(MigrationReleaseGateArtifact::new(
3146 "aaa-source-snapshot",
3147 "source-snapshot",
3148 "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
3149 "artifacts/aaa.json",
3150 ));
3151 let plan = MigrationRollbackPlan::default_for(
3152 MigrationIncidentClass::SemanticRegression,
3153 MigrationRolloutStage::Beta,
3154 )
3155 .resolve(&artifacts);
3156 let step = plan
3157 .steps
3158 .iter()
3159 .find(|step| step.required_artifact_kind == "source-snapshot")
3160 .expect("source-snapshot step present");
3161 assert_eq!(step.artifact_id.as_deref(), Some("aaa-source-snapshot"));
3162 }
3163
3164 #[test]
3165 fn postmortem_links_backlog_and_seeds_prevention() {
3166 let report = MigrationIncidentReport::new(
3167 "INC-9",
3168 MigrationIncidentClass::CertificationFalsePass,
3169 MigrationRolloutStage::Ga,
3170 )
3171 .comparator_id("cmp-cert-7")
3172 .claim_id("claim-42");
3173 let postmortem =
3174 MigrationPostmortemTemplate::for_incident(&report, MigrationIncidentSeverity::Sev1);
3175 assert!(!postmortem.links_backlog());
3176 assert!(!postmortem.prevention_actions.is_empty());
3177 assert!(!postmortem.timeline_prompts.is_empty());
3178
3179 let linked = postmortem
3180 .backlog_item("br-555")
3181 .prevention_action("add-cross-validator");
3182 assert!(linked.links_backlog());
3183 assert!(
3184 linked
3185 .prevention_actions
3186 .iter()
3187 .any(|action| action == "add-cross-validator")
3188 );
3189 let json = linked.to_json();
3190 assert!(json.contains("\"claim_id\":\"claim-42\""));
3191 assert!(json.contains("\"comparator_id\":\"cmp-cert-7\""));
3192 assert!(json.contains("br-555"));
3193 assert!(json.contains("\"links_backlog\":true"));
3194 }
3195
3196 #[test]
3197 fn playbook_resolves_complete_incident_response() {
3198 let report = MigrationIncidentReport::new(
3199 "INC-100",
3200 MigrationIncidentClass::SecurityBreach,
3201 MigrationRolloutStage::Ga,
3202 )
3203 .comparator_id("cmp-sec-1")
3204 .claim_id("claim-sec")
3205 .signal(MigrationIncidentSignal::new("sandbox-escape", "fs write"));
3206 let report = rollback_artifact_set()
3208 .into_iter()
3209 .fold(report, MigrationIncidentReport::artifact);
3210
3211 let playbook = MigrationIncidentResponsePlaybook::new();
3212 let response = playbook.resolve(&report);
3213
3214 assert_eq!(response.class, MigrationIncidentClass::SecurityBreach);
3215 assert_eq!(response.severity, MigrationIncidentSeverity::Sev1);
3216 assert_eq!(
3217 response.emergency_hold_reason,
3218 EmergencyHoldReason::SecurityIncident
3219 );
3220 assert_eq!(response.rollback_authority, OperatorAuthority::OnCall);
3221 assert!(response.requires_immediate_rollback);
3222 assert!(response.blocks_promotion());
3223 assert!(response.readiness.is_ready());
3224
3225 let json = response.to_json();
3226 assert!(json.contains("\"schema_version\":\"1.0.0\""));
3227 assert!(json.contains("\"class\":\"security-breach\""));
3228 assert!(json.contains("\"severity\":\"sev1\""));
3229 assert!(json.contains("\"emergency_hold_reason\":\"security-incident\""));
3230 assert!(json.contains("\"rollback_authority\":\"on-call\""));
3231 assert!(json.contains("\"requires_immediate_rollback\":true"));
3232 assert!(json.contains("rotate-exposed-credentials"));
3233 assert!(json.contains("sandbox-escape"));
3235 }
3236
3237 #[test]
3238 fn incident_response_is_deterministic() {
3239 let build = || {
3240 let report = MigrationIncidentReport::new(
3241 "INC-200",
3242 MigrationIncidentClass::DeterminismDivergence,
3243 MigrationRolloutStage::Beta,
3244 )
3245 .comparator_id("cmp-det")
3246 .claim_id("claim-det");
3247 let report = rollback_artifact_set()
3248 .into_iter()
3249 .fold(report, MigrationIncidentReport::artifact);
3250 MigrationIncidentResponsePlaybook::new()
3251 .resolve(&report)
3252 .to_json()
3253 };
3254 assert_eq!(build(), build());
3255 }
3256
3257 #[test]
3258 fn low_severity_incident_does_not_force_immediate_rollback() {
3259 let report = MigrationIncidentReport::new(
3260 "INC-300",
3261 MigrationIncidentClass::PerformanceRegression,
3262 MigrationRolloutStage::Alpha,
3263 );
3264 let response = MigrationIncidentResponsePlaybook::new().resolve(&report);
3265 assert_eq!(response.severity, MigrationIncidentSeverity::Sev3);
3266 assert!(!response.requires_immediate_rollback);
3267 assert!(response.blocks_promotion());
3269 assert_eq!(response.rollback_authority, OperatorAuthority::ReleaseOwner);
3270 }
3271}