Skip to main content

stateknot_testkit/
run.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4use crate::{
5    environment::QualificationEnvironment,
6    fault::{FaultCaseId, FaultError, FaultPlan, FaultResult},
7    report::{
8        FairnessEvidence, LatencyBucket, LatencyDistribution, LatencySignal, PhaseDurations,
9        QualificationCounter, QualificationCounters, QualificationProfile, QualificationReport,
10        QualificationScenario, ReportError,
11    },
12};
13use hdrhistogram::Histogram;
14use serde::{Deserialize, Serialize};
15use std::{
16    collections::BTreeMap,
17    sync::{Arc, Mutex, MutexGuard},
18    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
19};
20use thiserror::Error;
21
22const MIN_PHASE_MICROS: u64 = 1_000;
23const MAX_PHASE_MICROS: u64 = 86_400_000_000;
24const MAX_LATENCY_MICROS: u64 = 3_600_000_000;
25const MAX_IN_FLIGHT: u64 = 1_000_000;
26const MAX_BUCKETS: usize = 16_384;
27const RELEASE_WARMUP_MICROS: u64 = 600_000_000;
28const RELEASE_MEASUREMENT_MICROS: u64 = 1_800_000_000;
29const RELEASE_DRAIN_MICROS: u64 = 300_000_000;
30
31/// Minimum monotonic duration for each qualification phase.
32#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
33#[serde(deny_unknown_fields)]
34#[allow(clippy::struct_field_names)] // Units belong in the serialized field names.
35pub struct QualificationWindow {
36    warmup_micros: u64,
37    measurement_micros: u64,
38    drain_micros: u64,
39}
40
41impl QualificationWindow {
42    /// Creates bounded phase windows with microsecond-exact durations.
43    pub fn new(warmup: Duration, measurement: Duration, drain: Duration) -> Result<Self, RunError> {
44        let value = Self {
45            warmup_micros: duration_micros(warmup)?,
46            measurement_micros: duration_micros(measurement)?,
47            drain_micros: duration_micros(drain)?,
48        };
49        value.validate(QualificationProfile::CiReduced)?;
50        Ok(value)
51    }
52
53    /// A practical reduced window for local or CI correctness qualification.
54    pub const fn ci_default() -> Self {
55        Self {
56            warmup_micros: 1_000_000,
57            measurement_micros: 5_000_000,
58            drain_micros: 1_000_000,
59        }
60    }
61
62    /// The minimum release window: 10 minute warm-up, 30 minute measurement and 5 minute drain.
63    pub const fn release_default() -> Self {
64        Self {
65            warmup_micros: RELEASE_WARMUP_MICROS,
66            measurement_micros: RELEASE_MEASUREMENT_MICROS,
67            drain_micros: RELEASE_DRAIN_MICROS,
68        }
69    }
70
71    /// Returns the warm-up minimum.
72    pub const fn warmup(&self) -> Duration {
73        Duration::from_micros(self.warmup_micros)
74    }
75
76    /// Returns the measurement minimum.
77    pub const fn measurement(&self) -> Duration {
78        Duration::from_micros(self.measurement_micros)
79    }
80
81    /// Returns the drain minimum.
82    pub const fn drain(&self) -> Duration {
83        Duration::from_micros(self.drain_micros)
84    }
85
86    pub(crate) fn validate(&self, profile: QualificationProfile) -> Result<(), RunError> {
87        for value in [
88            self.warmup_micros,
89            self.measurement_micros,
90            self.drain_micros,
91        ] {
92            if !(MIN_PHASE_MICROS..=MAX_PHASE_MICROS).contains(&value) {
93                return Err(RunError::InvalidWindow);
94            }
95        }
96        if profile == QualificationProfile::ReleaseCandidate
97            && (self.warmup_micros < RELEASE_WARMUP_MICROS
98                || self.measurement_micros < RELEASE_MEASUREMENT_MICROS
99                || self.drain_micros < RELEASE_DRAIN_MICROS)
100        {
101            return Err(RunError::ReleaseWindowTooShort);
102        }
103        Ok(())
104    }
105
106    pub(crate) const fn minimum_micros(&self) -> PhaseDurations {
107        PhaseDurations {
108            warmup_micros: self.warmup_micros,
109            measurement_micros: self.measurement_micros,
110            drain_micros: self.drain_micros,
111        }
112    }
113}
114
115/// Monotonic qualification phase.
116#[derive(Clone, Copy, Debug, Eq, PartialEq)]
117pub enum RunPhase {
118    /// Target warm-up; observations are deliberately not evaluated.
119    Warmup,
120    /// Evaluated measurement window.
121    Measurement,
122    /// Bounded drain; no new evaluated operations are admitted.
123    Drain,
124    /// Immutable report was successfully produced.
125    Finished,
126}
127
128/// Final classification for an offered operation.
129#[derive(Clone, Copy, Debug, Eq, PartialEq)]
130pub enum OperationOutcome {
131    /// The target accepted and completed the operation.
132    Completed,
133    /// The target rejected the operation without accepting it.
134    Rejected,
135    /// The accepted operation ended in an intentionally injected failure.
136    ExpectedInjectedFailure,
137    /// The accepted operation ended in an unexpected failure.
138    UnexpectedFailure,
139}
140
141/// A qualification run with recorder-owned monotonic phase transitions.
142pub struct QualificationRun {
143    profile: QualificationProfile,
144    scenario: QualificationScenario,
145    environment: QualificationEnvironment,
146    window: QualificationWindow,
147    release_fault_plan_complete: bool,
148    started_at_unix_millis: u64,
149    shared: Arc<Mutex<RunState>>,
150}
151
152impl QualificationRun {
153    /// Starts in Warmup after validating profile, environment and fault plan.
154    pub fn start(
155        profile: QualificationProfile,
156        scenario: QualificationScenario,
157        window: QualificationWindow,
158        environment: QualificationEnvironment,
159        fault_plan: &FaultPlan,
160    ) -> Result<Self, RunError> {
161        environment.validate()?;
162        window.validate(profile)?;
163        fault_plan.validate()?;
164        if profile == QualificationProfile::ReleaseCandidate {
165            if !environment.reference_violations().is_empty() {
166                return Err(RunError::NonReferenceReleaseEnvironment);
167            }
168            if !fault_plan.is_release_complete() {
169                return Err(RunError::IncompleteReleaseFaultPlan);
170            }
171        }
172        let now = Instant::now();
173        let started_at_unix_millis = unix_millis(SystemTime::now())?;
174        let measurements = Measurements::new()?;
175        let faults = fault_plan
176            .requirements()
177            .iter()
178            .map(FaultResult::planned)
179            .collect();
180        Ok(Self {
181            profile,
182            scenario,
183            environment,
184            window,
185            release_fault_plan_complete: fault_plan.is_release_complete(),
186            started_at_unix_millis,
187            shared: Arc::new(Mutex::new(RunState {
188                phase: RunPhase::Warmup,
189                phase_started: now,
190                warmup_micros: None,
191                measurement_micros: None,
192                measurements,
193                faults,
194            })),
195        })
196    }
197
198    /// Returns a cloneable thread-safe recorder bound to this run.
199    pub fn recorder(&self) -> QualificationRecorder {
200        QualificationRecorder {
201            shared: self.shared.clone(),
202        }
203    }
204
205    /// Returns the current phase through the same synchronization boundary as recording.
206    pub fn phase(&self) -> Result<RunPhase, RunError> {
207        Ok(self.lock()?.phase)
208    }
209
210    /// Ends Warmup after its recorder-owned minimum and starts Measurement.
211    pub fn begin_measurement(&self) -> Result<(), RunError> {
212        let mut state = self.lock()?;
213        require_phase(&state, RunPhase::Warmup)?;
214        let elapsed = state.phase_started.elapsed();
215        require_elapsed(elapsed, self.window.warmup())?;
216        state.warmup_micros = Some(elapsed_micros(elapsed)?);
217        state.phase = RunPhase::Measurement;
218        state.phase_started = Instant::now();
219        Ok(())
220    }
221
222    /// Ends Measurement after its recorder-owned minimum and starts Drain.
223    pub fn begin_drain(&self) -> Result<(), RunError> {
224        let mut state = self.lock()?;
225        require_phase(&state, RunPhase::Measurement)?;
226        let elapsed = state.phase_started.elapsed();
227        require_elapsed(elapsed, self.window.measurement())?;
228        state.measurement_micros = Some(elapsed_micros(elapsed)?);
229        state.phase = RunPhase::Drain;
230        state.phase_started = Instant::now();
231        Ok(())
232    }
233
234    /// Finishes Drain with no in-flight operation and returns immutable evidence.
235    pub fn finish(&self) -> Result<QualificationReport, RunError> {
236        let mut state = self.lock()?;
237        require_phase(&state, RunPhase::Drain)?;
238        let drain = state.phase_started.elapsed();
239        require_elapsed(drain, self.window.drain())?;
240        if state.measurements.in_flight != 0 {
241            return Err(RunError::OperationsInFlight(state.measurements.in_flight));
242        }
243        let saturation = state.measurements.max_saturation_basis_points;
244        let fairness = state.measurements.fairness.clone();
245        let latencies = state.measurements.distributions()?;
246        let durations = PhaseDurations {
247            warmup_micros: state
248                .warmup_micros
249                .ok_or(RunError::MissingEvidence("warmup duration"))?,
250            measurement_micros: state
251                .measurement_micros
252                .ok_or(RunError::MissingEvidence("measurement duration"))?,
253            drain_micros: elapsed_micros(drain)?,
254        };
255        let report = QualificationReport::build(
256            self.profile,
257            self.scenario,
258            self.environment.clone(),
259            self.window.clone(),
260            durations,
261            self.started_at_unix_millis,
262            unix_millis(SystemTime::now())?,
263            latencies,
264            state.measurements.counters.clone(),
265            saturation,
266            fairness,
267            state.faults.clone(),
268            self.release_fault_plan_complete,
269        )?;
270        state.phase = RunPhase::Finished;
271        Ok(report)
272    }
273
274    fn lock(&self) -> Result<MutexGuard<'_, RunState>, RunError> {
275        self.shared.lock().map_err(|_| RunError::Poisoned)
276    }
277}
278
279/// Thread-safe measurement handle. Every mutating method uses the phase lock.
280#[derive(Clone)]
281pub struct QualificationRecorder {
282    shared: Arc<Mutex<RunState>>,
283}
284
285impl QualificationRecorder {
286    /// Begins one offered operation. Dropping the returned guard records an unexpected failure.
287    pub fn begin_operation(&self) -> Result<Operation, RunError> {
288        let mut state = self.lock()?;
289        if state.phase != RunPhase::Measurement {
290            return Ok(Operation {
291                recorder: self.clone(),
292                measured: false,
293                finished: false,
294            });
295        }
296        checked_increment(&mut state.measurements.counters.offered, 1)?;
297        if state.measurements.in_flight >= MAX_IN_FLIGHT {
298            checked_increment(&mut state.measurements.counters.rejected, 1)?;
299            checked_increment(
300                &mut state.measurements.counters.harness_capacity_rejections,
301                1,
302            )?;
303            return Err(RunError::HarnessCapacityRejected);
304        }
305        checked_increment(&mut state.measurements.in_flight, 1)?;
306        Ok(Operation {
307            recorder: self.clone(),
308            measured: true,
309            finished: false,
310        })
311    }
312
313    /// Records a bounded latency, optionally correcting a fixed-rate interval.
314    pub fn record_latency(
315        &self,
316        signal: LatencySignal,
317        elapsed: Duration,
318        expected_interval: Option<Duration>,
319    ) -> Result<(), RunError> {
320        let mut state = self.lock()?;
321        require_phase(&state, RunPhase::Measurement)?;
322        let value = latency_micros(elapsed)?;
323        let expected = expected_interval.map(latency_micros).transpose()?;
324        let histogram = state
325            .measurements
326            .latencies
327            .get_mut(&signal)
328            .ok_or(RunError::MissingEvidence("latency histogram"))?;
329        if let Some(expected) = expected {
330            histogram
331                .histogram
332                .record_correct(value, expected)
333                .map_err(|_| RunError::LatencyOutOfRange)?;
334        } else {
335            histogram
336                .histogram
337                .record(value)
338                .map_err(|_| RunError::LatencyOutOfRange)?;
339        }
340        checked_increment(&mut histogram.observed_samples, 1)
341    }
342
343    /// Records a correctness or security failure count during Measurement.
344    pub fn record_counter(
345        &self,
346        counter: QualificationCounter,
347        amount: u64,
348    ) -> Result<(), RunError> {
349        let mut state = self.lock()?;
350        require_phase(&state, RunPhase::Measurement)?;
351        state.measurements.counters.checked_add(counter, amount)?;
352        Ok(())
353    }
354
355    /// Records current shared-bottleneck saturation and retains the maximum.
356    pub fn record_saturation_basis_points(&self, value: u16) -> Result<(), RunError> {
357        if value > 10_000 {
358            return Err(RunError::InvalidSaturation);
359        }
360        let mut state = self.lock()?;
361        require_phase(&state, RunPhase::Measurement)?;
362        state.measurements.max_saturation_basis_points = Some(
363            state
364                .measurements
365                .max_saturation_basis_points
366                .map_or(value, |current| current.max(value)),
367        );
368        Ok(())
369    }
370
371    /// Records the single driver-computed fairness evidence set.
372    pub fn record_fairness(&self, fairness: FairnessEvidence) -> Result<(), RunError> {
373        let mut state = self.lock()?;
374        require_phase(&state, RunPhase::Measurement)?;
375        if state.measurements.fairness.replace(fairness).is_some() {
376            return Err(RunError::DuplicateEvidence("fairness"));
377        }
378        Ok(())
379    }
380
381    /// Marks one planned fault injection.
382    pub fn fault_injected(&self, case: FaultCaseId) -> Result<(), RunError> {
383        self.update_fault(case, FaultResult::injected)
384    }
385
386    /// Marks one observed recovery for a planned fault.
387    pub fn fault_recovered(&self, case: FaultCaseId) -> Result<(), RunError> {
388        self.update_fault(case, FaultResult::recovered)
389    }
390
391    /// Marks one checked invariant for a planned fault.
392    pub fn fault_invariant_checked(&self, case: FaultCaseId) -> Result<(), RunError> {
393        self.update_fault(case, FaultResult::invariant_checked)
394    }
395
396    fn update_fault(
397        &self,
398        case: FaultCaseId,
399        update: fn(&mut FaultResult) -> Result<(), FaultError>,
400    ) -> Result<(), RunError> {
401        let mut state = self.lock()?;
402        require_phase(&state, RunPhase::Measurement)?;
403        let result = state
404            .faults
405            .iter_mut()
406            .find(|result| result.case() == case)
407            .ok_or(FaultError::UnplannedCase(case))?;
408        update(result)?;
409        Ok(())
410    }
411
412    fn finish_operation(&self, outcome: OperationOutcome) -> Result<(), RunError> {
413        let mut state = self.lock()?;
414        if state.measurements.in_flight == 0 {
415            return Err(RunError::OperationAccounting);
416        }
417        state.measurements.in_flight -= 1;
418        match outcome {
419            OperationOutcome::Completed => {
420                checked_increment(&mut state.measurements.counters.accepted, 1)?;
421                checked_increment(&mut state.measurements.counters.completed, 1)?;
422            }
423            OperationOutcome::Rejected => {
424                checked_increment(&mut state.measurements.counters.rejected, 1)?;
425            }
426            OperationOutcome::ExpectedInjectedFailure => {
427                checked_increment(&mut state.measurements.counters.accepted, 1)?;
428                checked_increment(
429                    &mut state.measurements.counters.expected_injected_failures,
430                    1,
431                )?;
432            }
433            OperationOutcome::UnexpectedFailure => {
434                checked_increment(&mut state.measurements.counters.accepted, 1)?;
435                checked_increment(&mut state.measurements.counters.unexpected_failures, 1)?;
436            }
437        }
438        Ok(())
439    }
440
441    fn abandoned_operation(&self) {
442        let Ok(mut state) = self.shared.lock() else {
443            return;
444        };
445        if state.measurements.in_flight == 0 {
446            return;
447        }
448        state.measurements.in_flight -= 1;
449        let accepted = state.measurements.counters.accepted.checked_add(1);
450        let failures = state
451            .measurements
452            .counters
453            .unexpected_failures
454            .checked_add(1);
455        if let (Some(accepted), Some(failures)) = (accepted, failures) {
456            state.measurements.counters.accepted = accepted;
457            state.measurements.counters.unexpected_failures = failures;
458        }
459    }
460
461    fn lock(&self) -> Result<MutexGuard<'_, RunState>, RunError> {
462        self.shared.lock().map_err(|_| RunError::Poisoned)
463    }
464}
465
466/// RAII accounting guard for one offered operation.
467pub struct Operation {
468    recorder: QualificationRecorder,
469    measured: bool,
470    finished: bool,
471}
472
473impl Operation {
474    /// Returns whether this operation began during Measurement.
475    pub const fn is_measured(&self) -> bool {
476        self.measured
477    }
478
479    /// Completes accounting exactly once.
480    pub fn finish(mut self, outcome: OperationOutcome) -> Result<(), RunError> {
481        if self.measured {
482            self.recorder.finish_operation(outcome)?;
483        }
484        self.finished = true;
485        Ok(())
486    }
487}
488
489impl Drop for Operation {
490    fn drop(&mut self) {
491        if self.measured && !self.finished {
492            self.recorder.abandoned_operation();
493        }
494    }
495}
496
497/// Qualification recording or lifecycle failure.
498#[derive(Clone, Debug, Eq, Error, PartialEq)]
499pub enum RunError {
500    /// Environment metadata is invalid.
501    #[error(transparent)]
502    Environment(#[from] crate::EnvironmentError),
503    /// Fault plan or transition is invalid.
504    #[error(transparent)]
505    Fault(#[from] FaultError),
506    /// Report construction failed.
507    #[error(transparent)]
508    Report(#[from] ReportError),
509    /// A phase duration is outside the supported bound.
510    #[error("qualification window is outside the supported bound")]
511    InvalidWindow,
512    /// The release profile is shorter than 10/30/5 minutes.
513    #[error("release qualification requires at least 10/30/5 minute phases")]
514    ReleaseWindowTooShort,
515    /// The release profile environment is not the documented reference environment.
516    #[error("release qualification requires the documented reference environment")]
517    NonReferenceReleaseEnvironment,
518    /// The release profile does not plan every version-one fault.
519    #[error("release qualification requires the complete fault matrix")]
520    IncompleteReleaseFaultPlan,
521    /// The requested transition or recording call occurs in the wrong phase.
522    #[error("expected qualification phase {expected:?}, found {actual:?}")]
523    WrongPhase {
524        /// Required phase.
525        expected: RunPhase,
526        /// Observed phase.
527        actual: RunPhase,
528    },
529    /// The recorder-owned minimum has not elapsed.
530    #[error("qualification phase minimum has not elapsed")]
531    PhaseTooShort,
532    /// An elapsed or wall-clock value cannot be represented.
533    #[error("qualification clock value is not representable")]
534    Clock,
535    /// Shared recorder state was poisoned by a panic.
536    #[error("qualification recorder state is poisoned")]
537    Poisoned,
538    /// A measurement exceeds the bounded HDR range.
539    #[error("latency is outside the one-microsecond through one-hour range")]
540    LatencyOutOfRange,
541    /// A numeric counter overflowed.
542    #[error("qualification counter overflowed")]
543    CounterOverflow,
544    /// The explicit harness capacity ceiling rejected an offered operation.
545    #[error("qualification harness in-flight ceiling reached")]
546    HarnessCapacityRejected,
547    /// Drain cannot finish while operations remain in flight.
548    #[error("{0} qualification operations remain in flight")]
549    OperationsInFlight(u64),
550    /// Operation accounting is internally inconsistent.
551    #[error("qualification operation accounting is inconsistent")]
552    OperationAccounting,
553    /// Saturation is outside zero through 10,000 basis points.
554    #[error("saturation basis points exceed 10,000")]
555    InvalidSaturation,
556    /// A single-valued evidence set was recorded more than once.
557    #[error("duplicate qualification evidence: {0}")]
558    DuplicateEvidence(&'static str),
559    /// Required evidence was not recorded.
560    #[error("missing qualification evidence: {0}")]
561    MissingEvidence(&'static str),
562}
563
564struct RunState {
565    phase: RunPhase,
566    phase_started: Instant,
567    warmup_micros: Option<u64>,
568    measurement_micros: Option<u64>,
569    measurements: Measurements,
570    faults: Vec<FaultResult>,
571}
572
573struct Measurements {
574    latencies: BTreeMap<LatencySignal, LatencyHistogram>,
575    counters: QualificationCounters,
576    in_flight: u64,
577    max_saturation_basis_points: Option<u16>,
578    fairness: Option<FairnessEvidence>,
579}
580
581impl Measurements {
582    fn new() -> Result<Self, RunError> {
583        let mut latencies = BTreeMap::new();
584        for signal in LatencySignal::ALL {
585            let histogram = Histogram::<u64>::new_with_bounds(1, MAX_LATENCY_MICROS, 3)
586                .map_err(|_| RunError::LatencyOutOfRange)?;
587            latencies.insert(
588                signal,
589                LatencyHistogram {
590                    histogram,
591                    observed_samples: 0,
592                },
593            );
594        }
595        Ok(Self {
596            latencies,
597            counters: QualificationCounters::default(),
598            in_flight: 0,
599            max_saturation_basis_points: None,
600            fairness: None,
601        })
602    }
603
604    fn distributions(&self) -> Result<Vec<LatencyDistribution>, RunError> {
605        self.latencies
606            .iter()
607            .map(|(signal, latency)| {
608                if latency.observed_samples == 0 {
609                    return Err(RunError::MissingEvidence("latency samples"));
610                }
611                let buckets = latency
612                    .histogram
613                    .iter_recorded()
614                    .map(|item| LatencyBucket {
615                        upper_bound_micros: item.value_iterated_to(),
616                        count: item.count_since_last_iteration(),
617                    })
618                    .collect::<Vec<_>>();
619                if buckets.len() > MAX_BUCKETS {
620                    return Err(RunError::Report(ReportError::InvalidReport(
621                        "too many latency buckets",
622                    )));
623                }
624                Ok(LatencyDistribution {
625                    signal: *signal,
626                    observed_samples: latency.observed_samples,
627                    histogram_samples: latency.histogram.len(),
628                    p50_micros: latency.histogram.value_at_quantile(0.50),
629                    p95_micros: latency.histogram.value_at_quantile(0.95),
630                    p99_micros: latency.histogram.value_at_quantile(0.99),
631                    max_micros: latency.histogram.max(),
632                    buckets,
633                })
634            })
635            .collect()
636    }
637}
638
639struct LatencyHistogram {
640    histogram: Histogram<u64>,
641    observed_samples: u64,
642}
643
644fn duration_micros(value: Duration) -> Result<u64, RunError> {
645    let micros = u64::try_from(value.as_micros()).map_err(|_| RunError::InvalidWindow)?;
646    if !(MIN_PHASE_MICROS..=MAX_PHASE_MICROS).contains(&micros)
647        || Duration::from_micros(micros) != value
648    {
649        return Err(RunError::InvalidWindow);
650    }
651    Ok(micros)
652}
653
654fn latency_micros(value: Duration) -> Result<u64, RunError> {
655    let micros =
656        u64::try_from(value.as_nanos().div_ceil(1_000)).map_err(|_| RunError::LatencyOutOfRange)?;
657    if !(1..=MAX_LATENCY_MICROS).contains(&micros) {
658        return Err(RunError::LatencyOutOfRange);
659    }
660    Ok(micros)
661}
662
663fn elapsed_micros(value: Duration) -> Result<u64, RunError> {
664    u64::try_from(value.as_micros()).map_err(|_| RunError::Clock)
665}
666
667fn unix_millis(value: SystemTime) -> Result<u64, RunError> {
668    let elapsed = value
669        .duration_since(UNIX_EPOCH)
670        .map_err(|_| RunError::Clock)?;
671    u64::try_from(elapsed.as_millis()).map_err(|_| RunError::Clock)
672}
673
674fn require_phase(state: &RunState, expected: RunPhase) -> Result<(), RunError> {
675    if state.phase != expected {
676        return Err(RunError::WrongPhase {
677            expected,
678            actual: state.phase,
679        });
680    }
681    Ok(())
682}
683
684fn require_elapsed(actual: Duration, minimum: Duration) -> Result<(), RunError> {
685    if actual < minimum {
686        return Err(RunError::PhaseTooShort);
687    }
688    Ok(())
689}
690
691fn checked_increment(value: &mut u64, amount: u64) -> Result<(), RunError> {
692    *value = value.checked_add(amount).ok_or(RunError::CounterOverflow)?;
693    Ok(())
694}
695
696#[cfg(test)]
697mod tests {
698    use super::*;
699    use crate::environment::test_environment;
700    use std::thread::sleep;
701
702    fn small_window() -> QualificationWindow {
703        QualificationWindow::new(
704            Duration::from_millis(1),
705            Duration::from_millis(1),
706            Duration::from_millis(1),
707        )
708        .unwrap()
709    }
710
711    fn begin_run() -> QualificationRun {
712        let run = QualificationRun::start(
713            QualificationProfile::CiReduced,
714            QualificationScenario::AgentHostControlPlaneV1,
715            small_window(),
716            test_environment(false),
717            &FaultPlan::ci_default(),
718        )
719        .unwrap();
720        sleep(Duration::from_millis(2));
721        run.begin_measurement().unwrap();
722        run
723    }
724
725    fn complete_evidence(run: &QualificationRun) {
726        let recorder = run.recorder();
727        for signal in LatencySignal::ALL {
728            recorder
729                .record_latency(
730                    signal,
731                    Duration::from_micros(100),
732                    Some(Duration::from_micros(50)),
733                )
734                .unwrap();
735        }
736        recorder.record_saturation_basis_points(1_000).unwrap();
737        recorder
738            .record_fairness(FairnessEvidence::new(100, 150, 200).unwrap())
739            .unwrap();
740        for requirement in FaultPlan::ci_default().requirements() {
741            recorder.fault_injected(requirement.case()).unwrap();
742            recorder.fault_recovered(requirement.case()).unwrap();
743            for _ in 0..requirement.required_invariant_checks() {
744                recorder
745                    .fault_invariant_checked(requirement.case())
746                    .unwrap();
747            }
748        }
749        recorder
750            .begin_operation()
751            .unwrap()
752            .finish(OperationOutcome::Completed)
753            .unwrap();
754    }
755
756    #[test]
757    fn reduced_report_is_valid_but_never_release_qualified() {
758        let run = begin_run();
759        complete_evidence(&run);
760        sleep(Duration::from_millis(2));
761        run.begin_drain().unwrap();
762        sleep(Duration::from_millis(2));
763        let report = run.finish().unwrap();
764        assert!(report.evidence_valid());
765        assert!(!report.release_qualified());
766        let bytes = report
767            .to_integrity_envelope()
768            .unwrap()
769            .canonical_bytes()
770            .unwrap();
771        let parsed = crate::IntegrityEnvelope::from_json(&bytes).unwrap();
772        assert_eq!(parsed.report(), &report);
773    }
774
775    #[test]
776    fn abandoned_operation_is_an_explicit_failure() {
777        let run = begin_run();
778        complete_evidence(&run);
779        drop(run.recorder().begin_operation().unwrap());
780        sleep(Duration::from_millis(2));
781        run.begin_drain().unwrap();
782        sleep(Duration::from_millis(2));
783        assert!(!run.finish().unwrap().evidence_valid());
784    }
785
786    #[test]
787    fn release_profile_cannot_use_reduced_inputs() {
788        assert!(matches!(
789            QualificationRun::start(
790                QualificationProfile::ReleaseCandidate,
791                QualificationScenario::AgentHostControlPlaneV1,
792                small_window(),
793                test_environment(false),
794                &FaultPlan::ci_default(),
795            ),
796            Err(RunError::ReleaseWindowTooShort)
797        ));
798    }
799}