glass/
reliability.rs

1//! Browser-free reliability scenario contracts.
2//!
3//! A scenario is an input to the reliability laboratory, not a browser
4//! command. Validation is deliberately independent from fixture execution so
5//! malformed expectations fail before any browser is started.
6
7use serde::{Deserialize, Serialize};
8use serde_json::Value;
9use sha2::{Digest, Sha256};
10use std::collections::{BTreeMap, BTreeSet};
11use std::fmt;
12
13pub const RELIABILITY_SCENARIO_SCHEMA_VERSION: u32 = 1;
14pub const RELIABILITY_FIXTURE_SCHEMA_VERSION: u32 = 1;
15pub const RELIABILITY_REPLAY_SCHEMA_VERSION: u32 = 1;
16const MAX_SCENARIO_ID_BYTES: usize = 128;
17const MAX_CATEGORY_BYTES: usize = 128;
18const MAX_FIXTURE_BYTES: usize = 256;
19const MAX_CAPABILITIES: usize = 32;
20const MAX_STEPS: usize = 64;
21const MAX_FORBIDDEN_OUTCOMES: usize = 32;
22const MAX_SIDE_EFFECT_COUNTERS: usize = 32;
23const MAX_DURATION_MS: u64 = 15 * 60 * 1_000;
24const MAX_BROWSER_ACTIONS: u32 = 1_024;
25const MAX_REPLAY_EVENTS: usize = 1_024;
26
27/// Supported platform labels for deterministic reliability evidence.
28#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)]
29#[serde(rename_all = "kebab-case")]
30pub enum ReliabilityPlatform {
31    LinuxX86_64,
32    LinuxArm64,
33    MacosX86_64,
34    MacosArm64,
35}
36
37/// Release-blocking outcomes recognized by the reliability laboratory.
38#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)]
39#[serde(rename_all = "camelCase")]
40pub enum ReliabilityForbiddenOutcome {
41    WrongTargetExecuted,
42    StaleRevisionExecuted,
43    AmbiguousTargetSilentlyExecuted,
44    NonIdempotentMutationDuplicated,
45    FalseWorkflowCompletion,
46    UnsafeResumeReplay,
47    SecretLeaked,
48    CrossProfileKnowledgeLeak,
49    UnboundedLoopEscapedBudget,
50    PolicyBypassed,
51    CheckpointAcceptedAfterIncompatibleDefinitionChange,
52    SemanticCacheHidCriticalUnexpectedState,
53}
54
55/// Faults that the reliability lab can inject without relying on timing races.
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)]
57#[serde(rename_all = "camelCase")]
58pub enum ReliabilityFaultKind {
59    LoseResponse,
60    RendererDisconnect,
61    BrowserDisconnect,
62    DelayedEffect,
63    DropEvent,
64}
65
66impl ReliabilityFaultKind {
67    /// Return the stable fixture argument used by a runner.
68    pub const fn fixture_name(self) -> &'static str {
69        match self {
70            Self::LoseResponse => "loseResponse",
71            Self::RendererDisconnect => "rendererDisconnect",
72            Self::BrowserDisconnect => "browserDisconnect",
73            Self::DelayedEffect => "delayedEffect",
74            Self::DropEvent => "dropEvent",
75        }
76    }
77}
78
79/// Deterministic controls exposed by the checked-in adversarial fixture.
80#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)]
81#[serde(rename_all = "camelCase")]
82pub enum ReliabilityFixtureControl {
83    Reset,
84    ReplaceTarget,
85    RenameTarget,
86    DuplicateTarget,
87    ReorderTargets,
88    MoveTargetToOtherRegion,
89    ShowOverlay,
90    MoveTarget,
91    DetachFrame,
92    ScheduleEffectMarker,
93    CommitSubmit,
94}
95
96impl ReliabilityFixtureControl {
97    /// Return the allowlisted fixture method for this control.
98    pub const fn javascript_method(self) -> &'static str {
99        match self {
100            Self::Reset => "reset",
101            Self::ReplaceTarget => "replaceTarget",
102            Self::RenameTarget => "renameTarget",
103            Self::DuplicateTarget => "duplicateTarget",
104            Self::ReorderTargets => "reorderTargets",
105            Self::MoveTargetToOtherRegion => "moveTargetToOtherRegion",
106            Self::ShowOverlay => "showOverlay",
107            Self::MoveTarget => "moveTarget",
108            Self::DetachFrame => "detachFrame",
109            Self::ScheduleEffectMarker => "scheduleEffectMarker",
110            Self::CommitSubmit => "commitSubmit",
111        }
112    }
113}
114
115/// Independent oracle exposed by a reliability fixture.
116#[derive(Debug, Clone, Copy, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)]
117#[serde(rename_all = "camelCase")]
118pub enum ReliabilityFixtureOracle {
119    Snapshot,
120    SubmitSideEffectCount,
121}
122
123/// Versioned manifest for a deterministic browser fixture.
124#[derive(Debug, Clone, Serialize, Deserialize)]
125#[serde(rename_all = "camelCase", deny_unknown_fields)]
126pub struct ReliabilityFixtureManifest {
127    pub schema_version: u32,
128    pub id: String,
129    pub entrypoint: String,
130    pub controls: Vec<ReliabilityFixtureControl>,
131    pub faults: Vec<ReliabilityFaultKind>,
132    pub oracles: Vec<ReliabilityFixtureOracle>,
133}
134
135impl ReliabilityFixtureManifest {
136    /// Parse and validate one fixture manifest.
137    pub fn from_json(input: &str) -> Result<Self, ReliabilityScenarioError> {
138        let manifest: Self = serde_json::from_str(input).map_err(|error| {
139            ReliabilityScenarioError::new("$", format!("invalid fixture JSON: {error}"))
140        })?;
141        manifest.validate()?;
142        Ok(manifest)
143    }
144
145    /// Validate the fixture identity, controls, faults, and independent oracles.
146    pub fn validate(&self) -> Result<(), ReliabilityScenarioError> {
147        if self.schema_version != RELIABILITY_FIXTURE_SCHEMA_VERSION {
148            return Err(ReliabilityScenarioError::new(
149                "schemaVersion",
150                format!(
151                    "unsupported fixture schema {}; expected {}",
152                    self.schema_version, RELIABILITY_FIXTURE_SCHEMA_VERSION
153                ),
154            ));
155        }
156        validate_text("id", &self.id, MAX_SCENARIO_ID_BYTES)?;
157        validate_text("entrypoint", &self.entrypoint, MAX_FIXTURE_BYTES)?;
158        validate_unique("controls", &self.controls)?;
159        validate_unique("faults", &self.faults)?;
160        validate_unique("oracles", &self.oracles)?;
161        if self.controls.is_empty() {
162            return Err(ReliabilityScenarioError::new(
163                "controls",
164                "must expose at least one deterministic control",
165            ));
166        }
167        if self.faults.is_empty() {
168            return Err(ReliabilityScenarioError::new(
169                "faults",
170                "must expose at least one deterministic fault",
171            ));
172        }
173        if self.oracles.is_empty() {
174            return Err(ReliabilityScenarioError::new(
175                "oracles",
176                "must expose at least one independent oracle",
177            ));
178        }
179        Ok(())
180    }
181
182    /// Return the stable content hash used to bind fixture evidence.
183    pub fn content_hash(&self) -> Result<String, ReliabilityScenarioError> {
184        self.validate()?;
185        let canonical = serde_json::to_string(self).map_err(|error| {
186            ReliabilityScenarioError::new("$", format!("cannot serialize fixture: {error}"))
187        })?;
188        let digest = Sha256::digest(canonical.as_bytes());
189        Ok(format!("sha256:{digest:x}"))
190    }
191
192    /// Check that a scenario only uses controls and faults exposed by this fixture.
193    pub fn validate_scenario(
194        &self,
195        scenario: &ReliabilityScenario,
196    ) -> Result<(), ReliabilityScenarioError> {
197        self.validate()?;
198        scenario.validate()?;
199        if scenario.fixture != self.id {
200            return Err(ReliabilityScenarioError::new(
201                "fixture",
202                "scenario references a different fixture manifest",
203            ));
204        }
205        for (index, step) in scenario.steps.iter().enumerate() {
206            if let Some(control) = step.apply_control
207                && !self.controls.contains(&control)
208            {
209                return Err(ReliabilityScenarioError::new(
210                    format!("steps[{index}].applyControl"),
211                    "fixture does not expose this control",
212                ));
213            }
214            if let Some(injection) = &step.inject
215                && !self.faults.contains(&injection.fault)
216            {
217                return Err(ReliabilityScenarioError::new(
218                    format!("steps[{index}].inject.fault"),
219                    "fixture does not expose this fault",
220                ));
221            }
222        }
223        Ok(())
224    }
225}
226
227/// Host and browser metadata attached to a replay bundle.
228#[derive(Debug, Clone, Serialize, Deserialize)]
229#[serde(rename_all = "camelCase", deny_unknown_fields)]
230pub struct ReliabilityRunMetadata {
231    pub platform: ReliabilityPlatform,
232    pub browser: String,
233    pub browser_version: String,
234    pub duration_ms: u64,
235    pub browser_actions: u32,
236}
237
238impl ReliabilityRunMetadata {
239    fn validate(
240        &self,
241        path: &str,
242        budgets: &ReliabilityScenarioBudgets,
243    ) -> Result<(), ReliabilityScenarioError> {
244        validate_text(
245            &format!("{path}.browser"),
246            &self.browser,
247            MAX_CATEGORY_BYTES,
248        )?;
249        validate_text(
250            &format!("{path}.browserVersion"),
251            &self.browser_version,
252            MAX_CATEGORY_BYTES,
253        )?;
254        if self.duration_ms == 0 || self.duration_ms > budgets.max_duration_ms {
255            return Err(ReliabilityScenarioError::new(
256                format!("{path}.durationMs"),
257                "must be positive and within the scenario duration budget",
258            ));
259        }
260        if self.browser_actions == 0 || self.browser_actions > budgets.max_browser_actions {
261            return Err(ReliabilityScenarioError::new(
262                format!("{path}.browserActions"),
263                "must be positive and within the scenario action budget",
264            ));
265        }
266        Ok(())
267    }
268}
269
270/// Redacted replay event. Values and page content are intentionally absent.
271#[derive(Debug, Clone, Serialize, Deserialize)]
272#[serde(rename_all = "camelCase", deny_unknown_fields)]
273pub struct ReliabilityReplayEvent {
274    pub sequence: u32,
275    pub operation: String,
276    pub result: String,
277}
278
279/// Stable comparison result for two replay runs of the same scenario.
280#[derive(Debug, Clone, Serialize, Deserialize)]
281#[serde(rename_all = "camelCase", deny_unknown_fields)]
282pub struct ReliabilityReplayComparison {
283    pub scenario_id: String,
284    pub equivalent: bool,
285    pub changed_fields: Vec<String>,
286}
287
288/// Versioned, redacted evidence bundle for replay and regression comparison.
289#[derive(Debug, Clone, Serialize, Deserialize)]
290#[serde(rename_all = "camelCase", deny_unknown_fields)]
291pub struct ReliabilityReplayBundle {
292    pub schema_version: u32,
293    pub scenario_id: String,
294    pub scenario_hash: String,
295    pub fixture_id: String,
296    pub fixture_hash: String,
297    pub events: Vec<ReliabilityReplayEvent>,
298    pub observation: ReliabilityScenarioObservation,
299}
300
301impl ReliabilityReplayBundle {
302    /// Parse and validate one redacted replay bundle against its scenario.
303    pub fn from_json(
304        input: &str,
305        scenario: &ReliabilityScenario,
306    ) -> Result<Self, ReliabilityScenarioError> {
307        let value = serde_json::from_str(input).map_err(|error| {
308            ReliabilityScenarioError::new("$", format!("invalid replay JSON: {error}"))
309        })?;
310        Self::from_value(value, scenario)
311    }
312
313    /// Parse and validate one replay JSON value against its scenario.
314    pub fn from_value(
315        value: Value,
316        scenario: &ReliabilityScenario,
317    ) -> Result<Self, ReliabilityScenarioError> {
318        let bundle: Self = serde_json::from_value(value).map_err(|error| {
319            ReliabilityScenarioError::new("$", format!("invalid replay shape: {error}"))
320        })?;
321        bundle.validate(scenario)?;
322        Ok(bundle)
323    }
324
325    /// Validate binding, budgets, event ordering, and redacted evidence shape.
326    pub fn validate(&self, scenario: &ReliabilityScenario) -> Result<(), ReliabilityScenarioError> {
327        if self.schema_version != RELIABILITY_REPLAY_SCHEMA_VERSION {
328            return Err(ReliabilityScenarioError::new(
329                "schemaVersion",
330                format!(
331                    "unsupported replay schema {}; expected {}",
332                    self.schema_version, RELIABILITY_REPLAY_SCHEMA_VERSION
333                ),
334            ));
335        }
336        validate_text("scenarioId", &self.scenario_id, MAX_SCENARIO_ID_BYTES)?;
337        validate_text("fixtureId", &self.fixture_id, MAX_SCENARIO_ID_BYTES)?;
338        validate_digest("scenarioHash", &self.scenario_hash)?;
339        validate_digest("fixtureHash", &self.fixture_hash)?;
340        if self.scenario_id != scenario.id {
341            return Err(ReliabilityScenarioError::new(
342                "scenarioId",
343                "replay bundle is bound to a different scenario",
344            ));
345        }
346        if self.scenario_hash != scenario.content_hash()? {
347            return Err(ReliabilityScenarioError::new(
348                "scenarioHash",
349                "replay bundle is not bound to the exact scenario content",
350            ));
351        }
352        self.observation
353            .metadata
354            .validate("observation.metadata", &scenario.budgets)?;
355        if self.events.is_empty() || self.events.len() > MAX_REPLAY_EVENTS {
356            return Err(ReliabilityScenarioError::new(
357                "events",
358                format!("must contain 1..={MAX_REPLAY_EVENTS} entries"),
359            ));
360        }
361        for (index, event) in self.events.iter().enumerate() {
362            if event.sequence != index as u32 {
363                return Err(ReliabilityScenarioError::new(
364                    format!("events[{index}].sequence"),
365                    "must be contiguous and start at zero",
366                ));
367            }
368            validate_text(
369                &format!("events[{index}].operation"),
370                &event.operation,
371                MAX_CATEGORY_BYTES,
372            )?;
373            validate_redacted_text(&format!("events[{index}].operation"), &event.operation)?;
374            validate_text(
375                &format!("events[{index}].result"),
376                &event.result,
377                MAX_CATEGORY_BYTES,
378            )?;
379            validate_redacted_text(&format!("events[{index}].result"), &event.result)?;
380        }
381        if self.observation.scenario_id != self.scenario_id
382            || self.observation.scenario_hash != self.scenario_hash
383        {
384            return Err(ReliabilityScenarioError::new(
385                "observation",
386                "observation identity does not match the replay bundle",
387            ));
388        }
389        Ok(())
390    }
391
392    /// Return stable JSON suitable for replay storage and comparison.
393    pub fn to_canonical_json(
394        &self,
395        scenario: &ReliabilityScenario,
396    ) -> Result<String, ReliabilityScenarioError> {
397        self.validate(scenario)?;
398        serde_json::to_string(self).map_err(|error| {
399            ReliabilityScenarioError::new("$", format!("cannot serialize replay: {error}"))
400        })
401    }
402
403    /// Return a stable hash for this validated replay bundle.
404    pub fn content_hash(
405        &self,
406        scenario: &ReliabilityScenario,
407    ) -> Result<String, ReliabilityScenarioError> {
408        let canonical = self.to_canonical_json(scenario)?;
409        let digest = Sha256::digest(canonical.as_bytes());
410        Ok(format!("sha256:{digest:x}"))
411    }
412
413    /// Compare stable replay fields after validating both bundles.
414    pub fn compare(
415        &self,
416        other: &Self,
417        scenario: &ReliabilityScenario,
418    ) -> Result<ReliabilityReplayComparison, ReliabilityScenarioError> {
419        self.validate(scenario)?;
420        other.validate(scenario)?;
421        let left = serde_json::to_value(self).map_err(|error| {
422            ReliabilityScenarioError::new("$", format!("cannot serialize replay: {error}"))
423        })?;
424        let right = serde_json::to_value(other).map_err(|error| {
425            ReliabilityScenarioError::new("$", format!("cannot serialize replay: {error}"))
426        })?;
427        let mut changed_fields = Vec::new();
428        for field in ["fixtureId", "fixtureHash", "events", "observation"] {
429            if left[field] != right[field] {
430                changed_fields.push(field.to_string());
431            }
432        }
433        Ok(ReliabilityReplayComparison {
434            scenario_id: self.scenario_id.clone(),
435            equivalent: changed_fields.is_empty(),
436            changed_fields,
437        })
438    }
439}
440
441/// Browser and policy setup for a scenario.
442#[derive(Debug, Clone, Serialize, Deserialize)]
443#[serde(rename_all = "camelCase", deny_unknown_fields)]
444pub struct ReliabilityScenarioSetup {
445    pub browser: String,
446    pub policy: String,
447}
448
449/// One operation handed to a reliability scenario runner.
450#[derive(Debug, Clone, Serialize, Deserialize)]
451#[serde(rename_all = "camelCase", deny_unknown_fields)]
452pub enum ReliabilityExecutionOperation {
453    RunWorkflow {
454        source: String,
455    },
456    ApplyControl {
457        control: ReliabilityFixtureControl,
458    },
459    InjectFault {
460        injection: ReliabilityFaultInjection,
461    },
462    ResumeFromCheckpoint {
463        checkpoint: String,
464    },
465}
466
467/// Ordered, manifest-bound execution plan for one scenario.
468#[derive(Debug, Clone, Serialize, Deserialize)]
469#[serde(rename_all = "camelCase", deny_unknown_fields)]
470pub struct ReliabilityExecutionPlan {
471    pub scenario_id: String,
472    pub fixture_id: String,
473    pub operations: Vec<ReliabilityExecutionOperation>,
474    pub budgets: ReliabilityScenarioBudgets,
475}
476
477/// A controlled fault injected during a scenario step.
478#[derive(Debug, Clone, Serialize, Deserialize)]
479#[serde(rename_all = "camelCase", deny_unknown_fields)]
480pub struct ReliabilityFaultInjection {
481    pub after_dispatch: String,
482    pub fault: ReliabilityFaultKind,
483}
484
485/// One declarative scenario operation.
486#[derive(Debug, Clone, Serialize, Deserialize)]
487#[serde(rename_all = "camelCase", deny_unknown_fields)]
488pub struct ReliabilityScenarioStep {
489    #[serde(default, skip_serializing_if = "Option::is_none")]
490    pub run_workflow: Option<String>,
491    #[serde(default, skip_serializing_if = "Option::is_none")]
492    pub apply_control: Option<ReliabilityFixtureControl>,
493    #[serde(default, skip_serializing_if = "Option::is_none")]
494    pub inject: Option<ReliabilityFaultInjection>,
495    #[serde(default, skip_serializing_if = "Option::is_none")]
496    pub resume_from_checkpoint: Option<String>,
497}
498
499/// Expected typed outcome and independent side-effect oracle.
500#[derive(Debug, Clone, Serialize, Deserialize)]
501#[serde(rename_all = "camelCase", deny_unknown_fields)]
502pub struct ReliabilityScenarioExpectation {
503    pub terminal_state: String,
504    #[serde(default)]
505    pub side_effect_count: BTreeMap<String, u64>,
506}
507
508/// Resource limits for one scenario execution.
509#[derive(Debug, Clone, Serialize, Deserialize)]
510#[serde(rename_all = "camelCase", deny_unknown_fields)]
511pub struct ReliabilityScenarioBudgets {
512    pub max_duration_ms: u64,
513    pub max_browser_actions: u32,
514}
515
516/// Classification reported by one executed scenario.
517#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
518#[serde(rename_all = "snake_case")]
519pub enum ReliabilityRunClassification {
520    Passed,
521    Failed,
522    SafeRefusal,
523    Indeterminate,
524    Unsupported,
525}
526
527/// Bounded evidence submitted to the forbidden-outcome evaluator.
528#[derive(Debug, Clone, Serialize, Deserialize)]
529#[serde(rename_all = "camelCase", deny_unknown_fields)]
530pub struct ReliabilityScenarioObservation {
531    pub scenario_id: String,
532    pub scenario_hash: String,
533    pub metadata: ReliabilityRunMetadata,
534    pub classification: ReliabilityRunClassification,
535    #[serde(default, skip_serializing_if = "Option::is_none")]
536    pub terminal_state: Option<String>,
537    #[serde(default)]
538    pub side_effect_count: BTreeMap<String, u64>,
539    #[serde(default)]
540    pub forbidden_outcomes: Vec<ReliabilityForbiddenOutcome>,
541    pub oracle_evidence: bool,
542    pub artifacts_complete: bool,
543}
544
545/// One reason certification cannot be granted.
546#[derive(Debug, Clone, Serialize, Deserialize)]
547#[serde(rename_all = "camelCase", deny_unknown_fields)]
548pub struct ReliabilityGateFailure {
549    pub scenario_id: String,
550    pub code: String,
551    pub detail: String,
552}
553
554/// Release-blocking result over a complete scenario set.
555#[derive(Debug, Clone, Serialize, Deserialize)]
556#[serde(rename_all = "camelCase", deny_unknown_fields)]
557pub struct ReliabilityGateReport {
558    pub schema_version: u32,
559    pub certified: bool,
560    pub scenario_count: usize,
561    pub passed: usize,
562    pub safe_refusals: usize,
563    pub indeterminate: usize,
564    pub unsupported: usize,
565    pub forbidden_outcomes: BTreeMap<ReliabilityForbiddenOutcome, u64>,
566    pub failures: Vec<ReliabilityGateFailure>,
567}
568
569/// Category-level release summary derived from the exact gate report.
570#[derive(Debug, Clone, Serialize, Deserialize)]
571#[serde(rename_all = "camelCase", deny_unknown_fields)]
572pub struct ReliabilityScorecardCategory {
573    pub category: String,
574    pub scenario_count: usize,
575    pub certified_count: usize,
576    pub blocked_count: usize,
577}
578
579/// Inspectable release scorecard. It never replaces the detailed gate report.
580#[derive(Debug, Clone, Serialize, Deserialize)]
581#[serde(rename_all = "camelCase", deny_unknown_fields)]
582pub struct ReliabilityScorecard {
583    pub schema_version: u32,
584    pub certified: bool,
585    pub categories: Vec<ReliabilityScorecardCategory>,
586    pub gate: ReliabilityGateReport,
587}
588
589/// Versioned, browser-free reliability scenario.
590#[derive(Debug, Clone, Serialize, Deserialize)]
591#[serde(rename_all = "camelCase", deny_unknown_fields)]
592pub struct ReliabilityScenario {
593    pub schema_version: u32,
594    pub id: String,
595    pub category: String,
596    pub fixture: String,
597    #[serde(default = "default_platforms")]
598    pub platforms: Vec<ReliabilityPlatform>,
599    #[serde(default)]
600    pub capabilities: Vec<String>,
601    pub setup: ReliabilityScenarioSetup,
602    pub steps: Vec<ReliabilityScenarioStep>,
603    pub expect: ReliabilityScenarioExpectation,
604    #[serde(default)]
605    pub forbid: Vec<ReliabilityForbiddenOutcome>,
606    pub budgets: ReliabilityScenarioBudgets,
607}
608
609impl ReliabilityScenario {
610    /// Parse and validate one JSON scenario.
611    pub fn from_json(input: &str) -> Result<Self, ReliabilityScenarioError> {
612        let scenario: Self = serde_json::from_str(input).map_err(|error| {
613            ReliabilityScenarioError::new("$", format!("invalid scenario JSON: {error}"))
614        })?;
615        scenario.validate()?;
616        Ok(scenario)
617    }
618
619    /// Parse and validate one JSON value.
620    pub fn from_value(value: Value) -> Result<Self, ReliabilityScenarioError> {
621        let scenario: Self = serde_json::from_value(value).map_err(|error| {
622            ReliabilityScenarioError::new("$", format!("invalid scenario shape: {error}"))
623        })?;
624        scenario.validate()?;
625        Ok(scenario)
626    }
627
628    /// Expand validated steps into an ordered plan for a browser runner.
629    pub fn execution_plan(
630        &self,
631        fixture: &ReliabilityFixtureManifest,
632    ) -> Result<ReliabilityExecutionPlan, ReliabilityScenarioError> {
633        fixture.validate_scenario(self)?;
634        let operations = self
635            .steps
636            .iter()
637            .map(|step| {
638                Ok(if let Some(source) = &step.run_workflow {
639                    ReliabilityExecutionOperation::RunWorkflow {
640                        source: source.clone(),
641                    }
642                } else if let Some(control) = step.apply_control {
643                    ReliabilityExecutionOperation::ApplyControl { control }
644                } else if let Some(injection) = &step.inject {
645                    ReliabilityExecutionOperation::InjectFault {
646                        injection: injection.clone(),
647                    }
648                } else if let Some(checkpoint) = &step.resume_from_checkpoint {
649                    ReliabilityExecutionOperation::ResumeFromCheckpoint {
650                        checkpoint: checkpoint.clone(),
651                    }
652                } else {
653                    return Err(ReliabilityScenarioError::new(
654                        "steps",
655                        "validated scenario step has no executable operation",
656                    ));
657                })
658            })
659            .collect::<Result<Vec<_>, _>>()?;
660        Ok(ReliabilityExecutionPlan {
661            scenario_id: self.id.clone(),
662            fixture_id: self.fixture.clone(),
663            operations,
664            budgets: self.budgets.clone(),
665        })
666    }
667
668    /// Validate bounds, supported platforms, operations, and outcome oracles.
669    pub fn validate(&self) -> Result<(), ReliabilityScenarioError> {
670        if self.schema_version != RELIABILITY_SCENARIO_SCHEMA_VERSION {
671            return Err(ReliabilityScenarioError::new(
672                "schemaVersion",
673                format!(
674                    "unsupported scenario schema {}; expected {}",
675                    self.schema_version, RELIABILITY_SCENARIO_SCHEMA_VERSION
676                ),
677            ));
678        }
679        validate_text("id", &self.id, MAX_SCENARIO_ID_BYTES)?;
680        validate_text("category", &self.category, MAX_CATEGORY_BYTES)?;
681        validate_text("fixture", &self.fixture, MAX_FIXTURE_BYTES)?;
682        if self.platforms.is_empty() {
683            return Err(ReliabilityScenarioError::new(
684                "platforms",
685                "must list at least one supported platform",
686            ));
687        }
688        validate_unique("platforms", &self.platforms)?;
689        if self.capabilities.len() > MAX_CAPABILITIES {
690            return Err(ReliabilityScenarioError::new(
691                "capabilities",
692                format!("must contain at most {MAX_CAPABILITIES} entries"),
693            ));
694        }
695        for (index, capability) in self.capabilities.iter().enumerate() {
696            validate_text(
697                &format!("capabilities[{index}]"),
698                capability,
699                MAX_CATEGORY_BYTES,
700            )?;
701        }
702        validate_text("setup.browser", &self.setup.browser, MAX_CATEGORY_BYTES)?;
703        validate_text("setup.policy", &self.setup.policy, MAX_CATEGORY_BYTES)?;
704        if self.steps.is_empty() || self.steps.len() > MAX_STEPS {
705            return Err(ReliabilityScenarioError::new(
706                "steps",
707                format!("must contain 1..={MAX_STEPS} entries"),
708            ));
709        }
710        for (index, step) in self.steps.iter().enumerate() {
711            let path = format!("steps[{index}]");
712            let operations = step.run_workflow.is_some() as u8
713                + step.apply_control.is_some() as u8
714                + step.inject.is_some() as u8
715                + step.resume_from_checkpoint.is_some() as u8;
716            if operations != 1 {
717                return Err(ReliabilityScenarioError::new(
718                    path,
719                    "provide exactly one of runWorkflow, applyControl, inject, or resumeFromCheckpoint",
720                ));
721            }
722            if let Some(workflow) = &step.run_workflow {
723                validate_text(&format!("{path}.runWorkflow"), workflow, MAX_FIXTURE_BYTES)?;
724            }
725            if let Some(checkpoint) = &step.resume_from_checkpoint {
726                validate_text(
727                    &format!("{path}.resumeFromCheckpoint"),
728                    checkpoint,
729                    MAX_FIXTURE_BYTES,
730                )?;
731            }
732            if let Some(injection) = &step.inject {
733                validate_text(
734                    &format!("{path}.inject.afterDispatch"),
735                    &injection.after_dispatch,
736                    MAX_SCENARIO_ID_BYTES,
737                )?;
738            }
739        }
740        validate_text(
741            "expect.terminalState",
742            &self.expect.terminal_state,
743            MAX_CATEGORY_BYTES,
744        )?;
745        if self.expect.side_effect_count.len() > MAX_SIDE_EFFECT_COUNTERS {
746            return Err(ReliabilityScenarioError::new(
747                "expect.sideEffectCount",
748                format!("must contain at most {MAX_SIDE_EFFECT_COUNTERS} counters"),
749            ));
750        }
751        for (name, count) in &self.expect.side_effect_count {
752            validate_text("expect.sideEffectCount", name, MAX_SCENARIO_ID_BYTES)?;
753            if *count > MAX_BROWSER_ACTIONS as u64 {
754                return Err(ReliabilityScenarioError::new(
755                    "expect.sideEffectCount",
756                    format!("counter {name:?} exceeds the action bound"),
757                ));
758            }
759        }
760        if self.forbid.len() > MAX_FORBIDDEN_OUTCOMES {
761            return Err(ReliabilityScenarioError::new(
762                "forbid",
763                format!("must contain at most {MAX_FORBIDDEN_OUTCOMES} entries"),
764            ));
765        }
766        if self.forbid.is_empty() {
767            return Err(ReliabilityScenarioError::new(
768                "forbid",
769                "must declare at least one release-blocking outcome",
770            ));
771        }
772        let mut forbidden = BTreeSet::new();
773        for outcome in &self.forbid {
774            if !forbidden.insert(*outcome) {
775                return Err(ReliabilityScenarioError::new(
776                    "forbid",
777                    "duplicate forbidden outcome",
778                ));
779            }
780        }
781        if self.budgets.max_duration_ms == 0 || self.budgets.max_duration_ms > MAX_DURATION_MS {
782            return Err(ReliabilityScenarioError::new(
783                "budgets.maxDurationMs",
784                format!("must be 1..={MAX_DURATION_MS}"),
785            ));
786        }
787        if self.budgets.max_browser_actions == 0
788            || self.budgets.max_browser_actions > MAX_BROWSER_ACTIONS
789        {
790            return Err(ReliabilityScenarioError::new(
791                "budgets.maxBrowserActions",
792                format!("must be 1..={MAX_BROWSER_ACTIONS}"),
793            ));
794        }
795        Ok(())
796    }
797
798    /// Return stable JSON suitable for fixture hashes and report metadata.
799    pub fn to_canonical_json(&self) -> Result<String, ReliabilityScenarioError> {
800        self.validate()?;
801        serde_json::to_string(self).map_err(|error| {
802            ReliabilityScenarioError::new("$", format!("cannot serialize scenario: {error}"))
803        })
804    }
805
806    /// Return the content hash used to bind observations to this scenario.
807    pub fn content_hash(&self) -> Result<String, ReliabilityScenarioError> {
808        let canonical = self.to_canonical_json()?;
809        let digest = Sha256::digest(canonical.as_bytes());
810        Ok(format!("sha256:{digest:x}"))
811    }
812}
813
814/// Evaluate one complete scenario corpus using independent oracle evidence.
815pub fn evaluate_reliability_gate(
816    scenarios: &[ReliabilityScenario],
817    observations: &[ReliabilityScenarioObservation],
818) -> Result<ReliabilityGateReport, ReliabilityScenarioError> {
819    let mut scenario_map = BTreeMap::new();
820    for scenario in scenarios {
821        scenario.validate()?;
822        if scenario_map
823            .insert(scenario.id.as_str(), scenario)
824            .is_some()
825        {
826            return Err(ReliabilityScenarioError::new(
827                "scenarios",
828                format!("duplicate scenario ID {:?}", scenario.id),
829            ));
830        }
831    }
832    let mut observation_map = BTreeMap::new();
833    for observation in observations {
834        validate_text(
835            "observations.scenarioId",
836            &observation.scenario_id,
837            MAX_SCENARIO_ID_BYTES,
838        )?;
839        validate_unique(
840            "observations.forbiddenOutcomes",
841            &observation.forbidden_outcomes,
842        )?;
843        if observation_map
844            .insert(observation.scenario_id.as_str(), observation)
845            .is_some()
846        {
847            return Err(ReliabilityScenarioError::new(
848                "observations",
849                format!(
850                    "duplicate observation for scenario {:?}",
851                    observation.scenario_id
852                ),
853            ));
854        }
855    }
856    let mut forbidden_outcomes = BTreeMap::new();
857    let mut failures = Vec::new();
858    let mut passed = 0;
859    let mut safe_refusals = 0;
860    let mut indeterminate = 0;
861    let mut unsupported = 0;
862
863    for scenario in scenarios {
864        let Some(observation) = observation_map.get(scenario.id.as_str()) else {
865            failures.push(gate_failure(
866                &scenario.id,
867                "missing_evidence",
868                "no observation was provided for the required scenario",
869            ));
870            continue;
871        };
872        let expected_hash = scenario.content_hash()?;
873        if observation.scenario_hash != expected_hash {
874            failures.push(gate_failure(
875                &scenario.id,
876                "scenario_hash_mismatch",
877                "observation is not bound to the exact scenario content",
878            ));
879        }
880        if !observation.oracle_evidence {
881            failures.push(gate_failure(
882                &scenario.id,
883                "missing_oracle_evidence",
884                "required independent oracle evidence was not collected",
885            ));
886        }
887        if !observation.artifacts_complete {
888            failures.push(gate_failure(
889                &scenario.id,
890                "incomplete_artifacts",
891                "required release evidence artifacts are incomplete",
892            ));
893        }
894        if !scenario.platforms.contains(&observation.metadata.platform) {
895            failures.push(gate_failure(
896                &scenario.id,
897                "unsupported_platform",
898                "observation platform is not listed by the scenario",
899            ));
900        }
901        if let Err(error) = observation.metadata.validate("metadata", &scenario.budgets) {
902            failures.push(gate_failure(
903                &scenario.id,
904                "invalid_run_metadata",
905                &error.to_string(),
906            ));
907        }
908        match observation.classification {
909            ReliabilityRunClassification::Passed => passed += 1,
910            ReliabilityRunClassification::SafeRefusal => safe_refusals += 1,
911            ReliabilityRunClassification::Indeterminate => indeterminate += 1,
912            ReliabilityRunClassification::Unsupported => unsupported += 1,
913            ReliabilityRunClassification::Failed => failures.push(gate_failure(
914                &scenario.id,
915                "scenario_failed",
916                "scenario classification is failed",
917            )),
918        }
919        if matches!(
920            observation.classification,
921            ReliabilityRunClassification::Indeterminate | ReliabilityRunClassification::Unsupported
922        ) {
923            failures.push(gate_failure(
924                &scenario.id,
925                "non_certifying_classification",
926                "indeterminate and unsupported runs cannot certify a release",
927            ));
928        }
929        if observation.terminal_state.as_deref() != Some(scenario.expect.terminal_state.as_str()) {
930            failures.push(gate_failure(
931                &scenario.id,
932                "terminal_state_mismatch",
933                "observed terminal state does not match the declared expectation",
934            ));
935        }
936        if observation.side_effect_count != scenario.expect.side_effect_count {
937            failures.push(gate_failure(
938                &scenario.id,
939                "side_effect_oracle_mismatch",
940                "observed side-effect counters do not match the declared expectation",
941            ));
942        }
943        for outcome in &observation.forbidden_outcomes {
944            *forbidden_outcomes.entry(*outcome).or_default() += 1;
945        }
946    }
947    for observation in observations {
948        if !scenario_map.contains_key(observation.scenario_id.as_str()) {
949            failures.push(gate_failure(
950                &observation.scenario_id,
951                "unexpected_observation",
952                "observation does not identify a required scenario",
953            ));
954        }
955    }
956    if !forbidden_outcomes.is_empty() {
957        failures.push(gate_failure(
958            "*",
959            "forbidden_outcome",
960            "one or more release-blocking forbidden outcomes were observed",
961        ));
962    }
963    let report = ReliabilityGateReport {
964        schema_version: RELIABILITY_SCENARIO_SCHEMA_VERSION,
965        certified: failures.is_empty(),
966        scenario_count: scenarios.len(),
967        passed,
968        safe_refusals,
969        indeterminate,
970        unsupported,
971        forbidden_outcomes,
972        failures,
973    };
974    Ok(report)
975}
976
977/// Build a category summary from the same evidence used by certification.
978pub fn build_reliability_scorecard(
979    scenarios: &[ReliabilityScenario],
980    observations: &[ReliabilityScenarioObservation],
981) -> Result<ReliabilityScorecard, ReliabilityScenarioError> {
982    let gate = evaluate_reliability_gate(scenarios, observations)?;
983    let mut categories: BTreeMap<&str, ReliabilityScorecardCategory> = BTreeMap::new();
984    for scenario in scenarios {
985        let category = categories
986            .entry(scenario.category.as_str())
987            .or_insert_with(|| ReliabilityScorecardCategory {
988                category: scenario.category.clone(),
989                scenario_count: 0,
990                certified_count: 0,
991                blocked_count: 0,
992            });
993        category.scenario_count += 1;
994        let blocked = gate
995            .failures
996            .iter()
997            .any(|failure| failure.scenario_id == scenario.id || failure.scenario_id == "*");
998        if blocked {
999            category.blocked_count += 1;
1000        } else {
1001            category.certified_count += 1;
1002        }
1003    }
1004    Ok(ReliabilityScorecard {
1005        schema_version: RELIABILITY_SCENARIO_SCHEMA_VERSION,
1006        certified: gate.certified,
1007        categories: categories.into_values().collect(),
1008        gate,
1009    })
1010}
1011
1012fn gate_failure(scenario_id: &str, code: &str, detail: &str) -> ReliabilityGateFailure {
1013    ReliabilityGateFailure {
1014        scenario_id: scenario_id.into(),
1015        code: code.into(),
1016        detail: detail.into(),
1017    }
1018}
1019
1020fn default_platforms() -> Vec<ReliabilityPlatform> {
1021    vec![
1022        ReliabilityPlatform::LinuxX86_64,
1023        ReliabilityPlatform::LinuxArm64,
1024        ReliabilityPlatform::MacosX86_64,
1025        ReliabilityPlatform::MacosArm64,
1026    ]
1027}
1028
1029fn validate_unique<T: Ord>(path: &str, values: &[T]) -> Result<(), ReliabilityScenarioError> {
1030    let mut unique = BTreeSet::new();
1031    for value in values {
1032        if !unique.insert(value) {
1033            return Err(ReliabilityScenarioError::new(
1034                path,
1035                "must not contain duplicates",
1036            ));
1037        }
1038    }
1039    Ok(())
1040}
1041
1042fn validate_text(path: &str, value: &str, maximum: usize) -> Result<(), ReliabilityScenarioError> {
1043    if value.is_empty() || value.len() > maximum || value.chars().any(char::is_control) {
1044        return Err(ReliabilityScenarioError::new(
1045            path,
1046            format!("must contain 1..={maximum} non-control bytes"),
1047        ));
1048    }
1049    Ok(())
1050}
1051
1052fn validate_digest(path: &str, value: &str) -> Result<(), ReliabilityScenarioError> {
1053    if !value.starts_with("sha256:") || value.len() != "sha256:".len() + 64 {
1054        return Err(ReliabilityScenarioError::new(
1055            path,
1056            "must be a sha256 digest",
1057        ));
1058    }
1059    if value["sha256:".len()..]
1060        .chars()
1061        .any(|character| !character.is_ascii_hexdigit())
1062    {
1063        return Err(ReliabilityScenarioError::new(
1064            path,
1065            "must contain only hexadecimal digest characters",
1066        ));
1067    }
1068    Ok(())
1069}
1070
1071fn validate_redacted_text(path: &str, value: &str) -> Result<(), ReliabilityScenarioError> {
1072    let normalized = value.to_ascii_lowercase();
1073    for marker in [
1074        "password",
1075        "secret",
1076        "cookie",
1077        "authorization",
1078        "bearer ",
1079        "token=",
1080    ] {
1081        if normalized.contains(marker) {
1082            return Err(ReliabilityScenarioError::new(
1083                path,
1084                "replay text contains a sensitive-value marker",
1085            ));
1086        }
1087    }
1088    Ok(())
1089}
1090
1091/// A path-aware scenario contract failure.
1092#[derive(Debug, Clone, PartialEq, Eq)]
1093pub struct ReliabilityScenarioError {
1094    pub path: String,
1095    pub reason: String,
1096}
1097
1098impl ReliabilityScenarioError {
1099    fn new(path: impl Into<String>, reason: impl Into<String>) -> Self {
1100        Self {
1101            path: path.into(),
1102            reason: reason.into(),
1103        }
1104    }
1105}
1106
1107impl fmt::Display for ReliabilityScenarioError {
1108    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1109        write!(formatter, "{}: {}", self.path, self.reason)
1110    }
1111}
1112
1113impl std::error::Error for ReliabilityScenarioError {}
1114
1115#[cfg(test)]
1116mod tests {
1117    use super::*;
1118    use serde_json::json;
1119
1120    fn scenario() -> ReliabilityScenario {
1121        ReliabilityScenario {
1122            schema_version: RELIABILITY_SCENARIO_SCHEMA_VERSION,
1123            id: "duplicate-submit".into(),
1124            category: "transactional-workflow".into(),
1125            fixture: "checkout-submit".into(),
1126            platforms: vec![ReliabilityPlatform::LinuxX86_64],
1127            capabilities: vec!["workflow".into(), "idempotency".into()],
1128            setup: ReliabilityScenarioSetup {
1129                browser: "chromium".into(),
1130                policy: "hardened".into(),
1131            },
1132            steps: vec![
1133                ReliabilityScenarioStep {
1134                    run_workflow: Some("submit-request.json".into()),
1135                    apply_control: None,
1136                    inject: None,
1137                    resume_from_checkpoint: None,
1138                },
1139                ReliabilityScenarioStep {
1140                    run_workflow: None,
1141                    apply_control: None,
1142                    inject: Some(ReliabilityFaultInjection {
1143                        after_dispatch: "submit".into(),
1144                        fault: ReliabilityFaultKind::LoseResponse,
1145                    }),
1146                    resume_from_checkpoint: None,
1147                },
1148                ReliabilityScenarioStep {
1149                    run_workflow: None,
1150                    apply_control: None,
1151                    inject: None,
1152                    resume_from_checkpoint: Some("latest".into()),
1153                },
1154            ],
1155            expect: ReliabilityScenarioExpectation {
1156                terminal_state: "completed".into(),
1157                side_effect_count: BTreeMap::from([(String::from("submit"), 1)]),
1158            },
1159            forbid: vec![
1160                ReliabilityForbiddenOutcome::NonIdempotentMutationDuplicated,
1161                ReliabilityForbiddenOutcome::FalseWorkflowCompletion,
1162            ],
1163            budgets: ReliabilityScenarioBudgets {
1164                max_duration_ms: 30_000,
1165                max_browser_actions: 20,
1166            },
1167        }
1168    }
1169
1170    #[test]
1171    fn scenario_round_trip_is_canonical_and_bounded() {
1172        let scenario = scenario();
1173        let canonical = scenario.to_canonical_json().unwrap();
1174        let parsed = ReliabilityScenario::from_json(&canonical).unwrap();
1175        assert_eq!(parsed.id, "duplicate-submit");
1176        assert!(canonical.contains("nonIdempotentMutationDuplicated"));
1177    }
1178
1179    #[test]
1180    fn replay_bundle_is_redacted_bound_and_ordered() {
1181        let scenario = scenario();
1182        let observation = observation(&scenario, ReliabilityRunClassification::Passed, Vec::new());
1183        let bundle = ReliabilityReplayBundle {
1184            schema_version: RELIABILITY_REPLAY_SCHEMA_VERSION,
1185            scenario_id: scenario.id.clone(),
1186            scenario_hash: scenario.content_hash().unwrap(),
1187            fixture_id: scenario.fixture.clone(),
1188            fixture_hash: format!("sha256:{}", "a".repeat(64)),
1189            events: vec![ReliabilityReplayEvent {
1190                sequence: 0,
1191                operation: "runWorkflow".into(),
1192                result: "committed".into(),
1193            }],
1194            observation,
1195        };
1196        let canonical = bundle.to_canonical_json(&scenario).unwrap();
1197        let parsed = ReliabilityReplayBundle::from_json(&canonical, &scenario).unwrap();
1198        assert_eq!(parsed.events[0].sequence, 0);
1199        assert!(!canonical.contains("secret"));
1200
1201        let mut changed = parsed.clone();
1202        changed.events[0].result = "refused".into();
1203        let comparison = parsed.compare(&changed, &scenario).unwrap();
1204        assert!(!comparison.equivalent);
1205        assert_eq!(comparison.changed_fields, vec!["events"]);
1206    }
1207
1208    #[test]
1209    fn replay_bundle_rejects_non_contiguous_events() {
1210        let scenario = scenario();
1211        let mut observation =
1212            observation(&scenario, ReliabilityRunClassification::Passed, Vec::new());
1213        observation.scenario_hash = scenario.content_hash().unwrap();
1214        let bundle = ReliabilityReplayBundle {
1215            schema_version: RELIABILITY_REPLAY_SCHEMA_VERSION,
1216            scenario_id: scenario.id.clone(),
1217            scenario_hash: scenario.content_hash().unwrap(),
1218            fixture_id: scenario.fixture.clone(),
1219            fixture_hash: format!("sha256:{}", "b".repeat(64)),
1220            events: vec![ReliabilityReplayEvent {
1221                sequence: 1,
1222                operation: "runWorkflow".into(),
1223                result: "committed".into(),
1224            }],
1225            observation,
1226        };
1227        let error = bundle.validate(&scenario).unwrap_err();
1228        assert_eq!(error.path, "events[0].sequence");
1229    }
1230
1231    #[test]
1232    fn replay_bundle_rejects_sensitive_event_markers() {
1233        let scenario = scenario();
1234        let observation = observation(&scenario, ReliabilityRunClassification::Passed, Vec::new());
1235        let bundle = ReliabilityReplayBundle {
1236            schema_version: RELIABILITY_REPLAY_SCHEMA_VERSION,
1237            scenario_id: scenario.id.clone(),
1238            scenario_hash: scenario.content_hash().unwrap(),
1239            fixture_id: scenario.fixture.clone(),
1240            fixture_hash: format!("sha256:{}", "c".repeat(64)),
1241            events: vec![ReliabilityReplayEvent {
1242                sequence: 0,
1243                operation: "observe".into(),
1244                result: "secret=redacted".into(),
1245            }],
1246            observation,
1247        };
1248        let error = bundle.validate(&scenario).unwrap_err();
1249        assert_eq!(error.path, "events[0].result");
1250    }
1251
1252    #[test]
1253    fn scenario_rejects_multiple_operations_in_one_step() {
1254        let mut value = serde_json::to_value(scenario()).unwrap();
1255        value["steps"][0]["inject"] = json!({
1256            "afterDispatch": "submit",
1257            "fault": "loseResponse"
1258        });
1259        let error = ReliabilityScenario::from_value(value).unwrap_err();
1260        assert_eq!(error.path, "steps[0]");
1261    }
1262
1263    #[test]
1264    fn scenario_rejects_windows_and_duplicate_forbidden_outcomes() {
1265        let mut value = serde_json::to_value(scenario()).unwrap();
1266        value["platforms"] = json!(["windows-x86-64"]);
1267        assert!(ReliabilityScenario::from_value(value).is_err());
1268
1269        let mut value = serde_json::to_value(scenario()).unwrap();
1270        value["forbid"] = json!(["secretLeaked", "secretLeaked"]);
1271        let error = ReliabilityScenario::from_value(value).unwrap_err();
1272        assert_eq!(error.path, "forbid");
1273
1274        let mut value = serde_json::to_value(scenario()).unwrap();
1275        value["forbid"] = json!([]);
1276        let error = ReliabilityScenario::from_value(value).unwrap_err();
1277        assert_eq!(error.path, "forbid");
1278    }
1279
1280    fn observation(
1281        scenario: &ReliabilityScenario,
1282        classification: ReliabilityRunClassification,
1283        forbidden_outcomes: Vec<ReliabilityForbiddenOutcome>,
1284    ) -> ReliabilityScenarioObservation {
1285        ReliabilityScenarioObservation {
1286            scenario_id: scenario.id.clone(),
1287            scenario_hash: scenario.content_hash().unwrap(),
1288            metadata: ReliabilityRunMetadata {
1289                platform: ReliabilityPlatform::LinuxX86_64,
1290                browser: "chromium".into(),
1291                browser_version: "stable".into(),
1292                duration_ms: 100,
1293                browser_actions: 3,
1294            },
1295            classification,
1296            terminal_state: Some(scenario.expect.terminal_state.clone()),
1297            side_effect_count: scenario.expect.side_effect_count.clone(),
1298            forbidden_outcomes,
1299            oracle_evidence: true,
1300            artifacts_complete: true,
1301        }
1302    }
1303
1304    #[test]
1305    fn reliability_gate_certifies_complete_clean_evidence() {
1306        let scenario = scenario();
1307        let report = evaluate_reliability_gate(
1308            std::slice::from_ref(&scenario),
1309            &[observation(
1310                &scenario,
1311                ReliabilityRunClassification::Passed,
1312                Vec::new(),
1313            )],
1314        )
1315        .unwrap();
1316        assert!(report.certified);
1317        assert_eq!(report.passed, 1);
1318        assert!(report.failures.is_empty());
1319    }
1320
1321    #[test]
1322    fn reliability_scorecard_preserves_gate_and_groups_categories() {
1323        let scenario = scenario();
1324        let scorecard = build_reliability_scorecard(
1325            std::slice::from_ref(&scenario),
1326            &[observation(
1327                &scenario,
1328                ReliabilityRunClassification::Passed,
1329                Vec::new(),
1330            )],
1331        )
1332        .unwrap();
1333        assert!(scorecard.certified);
1334        assert_eq!(scorecard.gate.passed, 1);
1335        assert_eq!(scorecard.categories[0].category, "transactional-workflow");
1336        assert_eq!(scorecard.categories[0].certified_count, 1);
1337        assert_eq!(scorecard.categories[0].blocked_count, 0);
1338    }
1339
1340    #[test]
1341    fn reliability_gate_blocks_forbidden_outcomes_and_missing_evidence() {
1342        let scenario = scenario();
1343        let mut failed = observation(
1344            &scenario,
1345            ReliabilityRunClassification::SafeRefusal,
1346            vec![ReliabilityForbiddenOutcome::SecretLeaked],
1347        );
1348        failed.artifacts_complete = false;
1349        let report = evaluate_reliability_gate(std::slice::from_ref(&scenario), &[failed]).unwrap();
1350        assert!(!report.certified);
1351        assert_eq!(
1352            report.forbidden_outcomes[&ReliabilityForbiddenOutcome::SecretLeaked],
1353            1
1354        );
1355        assert!(
1356            report
1357                .failures
1358                .iter()
1359                .any(|failure| failure.code == "incomplete_artifacts")
1360        );
1361    }
1362
1363    #[test]
1364    fn reliability_gate_blocks_non_certifying_platform_and_classification() {
1365        let scenario = scenario();
1366        let mut unsupported = observation(
1367            &scenario,
1368            ReliabilityRunClassification::Indeterminate,
1369            Vec::new(),
1370        );
1371        unsupported.metadata.platform = ReliabilityPlatform::MacosArm64;
1372        unsupported.metadata.duration_ms = scenario.budgets.max_duration_ms + 1;
1373        let report =
1374            evaluate_reliability_gate(std::slice::from_ref(&scenario), &[unsupported]).unwrap();
1375        assert!(!report.certified);
1376        assert!(
1377            report
1378                .failures
1379                .iter()
1380                .any(|failure| failure.code == "unsupported_platform")
1381        );
1382        assert!(
1383            report
1384                .failures
1385                .iter()
1386                .any(|failure| failure.code == "non_certifying_classification")
1387        );
1388        assert!(
1389            report
1390                .failures
1391                .iter()
1392                .any(|failure| failure.code == "invalid_run_metadata")
1393        );
1394    }
1395
1396    #[test]
1397    fn reliability_gate_rejects_duplicate_observed_forbidden_outcomes() {
1398        let scenario = scenario();
1399        let mut observation = observation(
1400            &scenario,
1401            ReliabilityRunClassification::Passed,
1402            vec![ReliabilityForbiddenOutcome::SecretLeaked],
1403        );
1404        observation
1405            .forbidden_outcomes
1406            .push(ReliabilityForbiddenOutcome::SecretLeaked);
1407        let error = evaluate_reliability_gate(&[scenario], &[observation]).unwrap_err();
1408        assert_eq!(error.path, "observations.forbiddenOutcomes");
1409    }
1410}
glass/reliability.rs

glass/
reliability.rs
