ferrum_testkit/op_diff/
required.rs

1//! Strict checks for an explicitly required accelerator. Compilation and CPU
2//! reference execution cannot substitute for the requested backend's output.
3//!
4//! The operator adapter remains responsible for checked device synchronization:
5//! a Vec-returning trait cannot independently attest driver completion. This
6//! module catches unwind failures; process aborts require the caller to reject
7//! missing/incomplete reports. Neither an op label nor this report proves an
8//! unexecuted shape, precision, execution path, model, or performance claim.
9use super::{OpUnderTest, Output};
10pub use ferrum_bench_core::release_regression::numerics::{
11    compare_outputs, NumericalFailure, NumericalMetrics, RawOutput,
12};
13use ferrum_bench_core::release_regression::numerics::{
14    validate_reference as valid_reference, validate_tolerance as valid_tolerance,
15};
16use serde::{Deserialize, Serialize};
17use std::any::Any;
18use std::panic::{catch_unwind, AssertUnwindSafe};
19
20#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
21#[serde(rename_all = "snake_case")]
22pub enum RequiredBackend {
23    Metal,
24    Cuda,
25}
26
27#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
28#[serde(rename_all = "snake_case")]
29pub enum RequiredStatus {
30    Passed,
31    Failed,
32    NotRun,
33}
34
35#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
36#[serde(deny_unknown_fields)]
37pub struct RequiredReport {
38    pub schema_version: u32,
39    pub op: String,
40    pub backend: RequiredBackend,
41    pub seed: u64,
42    /// None only when the submitted tolerance was non-finite. Its original
43    /// bits are always retained below so invalid inputs remain auditable.
44    pub tolerance: Option<f64>,
45    pub tolerance_f64_bits: u64,
46    pub status: RequiredStatus,
47    pub reason: Option<String>,
48    pub reference: Option<RawOutput>,
49    pub actual: Option<RawOutput>,
50    pub metrics: Option<NumericalMetrics>,
51}
52
53impl RequiredReport {
54    /// This convenience describes a freshly generated report. Consumers of
55    /// stored reports must replay the raw data and verify execution provenance;
56    /// deserializing a self-reported status is not a release authorization.
57    pub fn is_passed(&self) -> bool {
58        self.status == RequiredStatus::Passed
59    }
60
61    fn failed(&mut self, reason: impl Into<String>) {
62        self.status = RequiredStatus::Failed;
63        self.reason = Some(reason.into());
64    }
65}
66
67#[derive(Debug)]
68enum Availability {
69    #[cfg_attr(
70        not(any(test, feature = "cuda", all(target_os = "macos", feature = "metal"))),
71        allow(dead_code)
72    )]
73    Available,
74    Unavailable(String),
75}
76
77fn availability(backend: RequiredBackend) -> Result<Availability, String> {
78    match backend {
79        RequiredBackend::Metal => {
80            #[cfg(all(target_os = "macos", feature = "metal"))]
81            {
82                if ferrum_kernels::attention::metal::is_available() {
83                    Ok(Availability::Available)
84                } else {
85                    Ok(Availability::Unavailable(
86                        "no Metal device is available".into(),
87                    ))
88                }
89            }
90            #[cfg(not(all(target_os = "macos", feature = "metal")))]
91            {
92                Ok(Availability::Unavailable(
93                    "Metal backend is not compiled for this platform".into(),
94                ))
95            }
96        }
97        RequiredBackend::Cuda => {
98            #[cfg(feature = "cuda")]
99            {
100                match ferrum_kernels::cuda_device_count()? {
101                    0 => Ok(Availability::Unavailable(
102                        "no CUDA device is available".into(),
103                    )),
104                    _ => Ok(Availability::Available),
105                }
106            }
107            #[cfg(not(feature = "cuda"))]
108            {
109                Ok(Availability::Unavailable(
110                    "CUDA backend is not compiled".into(),
111                ))
112            }
113        }
114    }
115}
116
117fn backend_output(op: &dyn OpUnderTest, backend: RequiredBackend, seed: u64) -> Option<Output> {
118    // The branches deliberately dispatch only the explicitly required backend.
119    // This does not call compare_backends, which runs every compiled accelerator.
120    let _ = (op, seed);
121    match backend {
122        RequiredBackend::Metal => {
123            #[cfg(all(target_os = "macos", feature = "metal"))]
124            {
125                Some(op.run_metal(seed))
126            }
127            #[cfg(not(all(target_os = "macos", feature = "metal")))]
128            {
129                None
130            }
131        }
132        RequiredBackend::Cuda => {
133            #[cfg(feature = "cuda")]
134            {
135                Some(op.run_cuda(seed))
136            }
137            #[cfg(not(feature = "cuda"))]
138            {
139                None
140            }
141        }
142    }
143}
144
145fn panic_text(payload: Box<dyn Any + Send>) -> String {
146    if let Some(message) = payload.downcast_ref::<String>() {
147        message.clone()
148    } else if let Some(message) = payload.downcast_ref::<&str>() {
149        (*message).into()
150    } else {
151        "non-string panic payload".into()
152    }
153}
154
155/// Run the CPU reference and exactly one requested backend. Uncompiled or
156/// unavailable backends return NotRun; failed probes, driver/dispatch panics,
157/// invalid outputs and excessive error return Failed. No fallback is a pass.
158pub fn run_required(
159    op: &dyn OpUnderTest,
160    backend: RequiredBackend,
161    seed: u64,
162    tolerance: f64,
163) -> RequiredReport {
164    run_with(
165        op.name(),
166        backend,
167        seed,
168        tolerance,
169        || availability(backend),
170        || op.run_cpu(seed),
171        || backend_output(op, backend, seed),
172    )
173}
174
175fn run_with(
176    op: &str,
177    backend: RequiredBackend,
178    seed: u64,
179    tolerance: f64,
180    probe: impl FnOnce() -> Result<Availability, String>,
181    reference: impl FnOnce() -> Output,
182    execute: impl FnOnce() -> Option<Output>,
183) -> RequiredReport {
184    let mut report = RequiredReport {
185        schema_version: 1,
186        op: op.into(),
187        backend,
188        seed,
189        tolerance: tolerance.is_finite().then_some(tolerance),
190        tolerance_f64_bits: tolerance.to_bits(),
191        status: RequiredStatus::NotRun,
192        reason: None,
193        reference: None,
194        actual: None,
195        metrics: None,
196    };
197    if let Err(error) = valid_tolerance(tolerance) {
198        report.failed(error.reason);
199        return report;
200    }
201    match catch_unwind(AssertUnwindSafe(probe)) {
202        Ok(Ok(Availability::Available)) => {}
203        Ok(Ok(Availability::Unavailable(reason))) => {
204            report.reason = Some(reason);
205            return report;
206        }
207        Ok(Err(error)) => {
208            report.failed(format!("backend availability query failed: {error}"));
209            return report;
210        }
211        Err(payload) => {
212            report.failed(format!(
213                "backend availability query panicked: {}",
214                panic_text(payload)
215            ));
216            return report;
217        }
218    }
219    let reference = match catch_unwind(AssertUnwindSafe(reference)) {
220        Ok(output) => output,
221        Err(payload) => {
222            report.failed(format!("CPU reference panicked: {}", panic_text(payload)));
223            return report;
224        }
225    };
226    report.reference = Some(RawOutput::from_f32(&reference));
227    if let Err(error) = valid_reference(&reference) {
228        report.failed(error.reason);
229        return report;
230    }
231    let actual = match catch_unwind(AssertUnwindSafe(execute)) {
232        Ok(Some(output)) => output,
233        Ok(None) => {
234            report.reason =
235                Some("required backend was not executed; no output was produced".into());
236            return report;
237        }
238        Err(payload) => {
239            report.failed(format!(
240                "required backend execution panicked: {}",
241                panic_text(payload)
242            ));
243            return report;
244        }
245    };
246    report.actual = Some(RawOutput::from_f32(&actual));
247    match compare_outputs(&reference, &actual, tolerance) {
248        Ok(metrics) => {
249            report.metrics = Some(metrics);
250            report.status = RequiredStatus::Passed;
251        }
252        Err(error) => {
253            report.metrics = error.metrics;
254            report.failed(error.reason);
255        }
256    }
257    report
258}
259
260#[cfg(test)]
261mod tests {
262    use super::*;
263    use std::cell::Cell;
264
265    #[test]
266    fn missing_backend_does_not_even_run_the_reference() {
267        let report = run_with(
268            "fixture",
269            RequiredBackend::Metal,
270            7,
271            1e-7,
272            || Ok(Availability::Unavailable("no device".into())),
273            || panic!("reference must not run"),
274            || panic!("backend must not run"),
275        );
276        assert_eq!(report.status, RequiredStatus::NotRun);
277        assert!(report.reference.is_none());
278        assert!(report.actual.is_none());
279        assert!(!report.is_passed());
280    }
281
282    #[test]
283    fn cpu_reference_alone_cannot_pass_required_backend() {
284        let report = run_with(
285            "fixture",
286            RequiredBackend::Cuda,
287            7,
288            1e-7,
289            || Ok(Availability::Available),
290            || vec![1.0, 2.0],
291            || None,
292        );
293        assert_eq!(report.status, RequiredStatus::NotRun);
294        assert_eq!(report.reference.unwrap().to_f32(), [1.0, 2.0]);
295        assert!(report.actual.is_none());
296        assert!(report.metrics.is_none());
297    }
298
299    #[test]
300    fn failed_probe_and_panicking_execution_are_failed_not_not_run() {
301        let probe = run_with(
302            "fixture",
303            RequiredBackend::Cuda,
304            7,
305            1e-7,
306            || Err("driver query failed".into()),
307            || panic!("no reference"),
308            || panic!("no backend"),
309        );
310        assert_eq!(probe.status, RequiredStatus::Failed);
311        let dispatch = run_with(
312            "fixture",
313            RequiredBackend::Metal,
314            7,
315            1e-7,
316            || Ok(Availability::Available),
317            || vec![1.0],
318            || panic!("checked GPU completion failed"),
319        );
320        assert_eq!(dispatch.status, RequiredStatus::Failed);
321        assert!(dispatch.reference.is_some());
322        assert!(dispatch.actual.is_none());
323        assert!(dispatch
324            .reason
325            .unwrap()
326            .contains("checked GPU completion failed"));
327    }
328
329    #[test]
330    fn runner_and_replay_share_the_same_comparator() {
331        for actual in [vec![1.0, 2.0], vec![2.0, 4.0], vec![], vec![f32::NAN, 2.0]] {
332            let result = run_with(
333                "fixture",
334                RequiredBackend::Metal,
335                7,
336                1e-7,
337                || Ok(Availability::Available),
338                || vec![1.0, 2.0],
339                || Some(actual),
340            );
341            let reference = result.reference.as_ref().unwrap().to_f32();
342            let actual = result.actual.as_ref().unwrap().to_f32();
343            let replay = compare_outputs(
344                &reference,
345                &actual,
346                f64::from_bits(result.tolerance_f64_bits),
347            );
348            assert_eq!(result.is_passed(), replay.is_ok());
349            assert_eq!(
350                result.metrics,
351                match replay {
352                    Ok(metrics) => Some(metrics),
353                    Err(error) => error.metrics,
354                }
355            );
356        }
357    }
358
359    #[test]
360    fn invalid_reference_or_tolerance_prevents_backend_execution() {
361        let dispatched = Cell::new(false);
362        let result = run_with(
363            "fixture",
364            RequiredBackend::Metal,
365            7,
366            1e-7,
367            || Ok(Availability::Available),
368            || vec![],
369            || {
370                dispatched.set(true);
371                Some(vec![])
372            },
373        );
374        assert_eq!(result.status, RequiredStatus::Failed);
375        assert!(!dispatched.get());
376        let result = run_with(
377            "fixture",
378            RequiredBackend::Metal,
379            7,
380            f64::NAN,
381            || panic!("invalid input must not probe"),
382            || panic!("no reference"),
383            || panic!("no backend"),
384        );
385        assert_eq!(result.status, RequiredStatus::Failed);
386        assert_eq!(result.tolerance, None);
387        assert_eq!(result.tolerance_f64_bits, f64::NAN.to_bits());
388    }
389
390    struct MustNotExecute;
391    impl OpUnderTest for MustNotExecute {
392        fn name(&self) -> &str {
393            "fixture"
394        }
395        fn run_cpu(&self, _: u64) -> Output {
396            panic!("CPU reference must not execute for an uncompiled backend")
397        }
398        #[cfg(all(target_os = "macos", feature = "metal"))]
399        fn run_metal(&self, _: u64) -> Output {
400            panic!("unrequested Metal must not execute")
401        }
402        #[cfg(feature = "cuda")]
403        fn run_cuda(&self, _: u64) -> Output {
404            panic!("unrequested CUDA must not execute")
405        }
406    }
407
408    #[cfg(not(all(target_os = "macos", feature = "metal")))]
409    #[test]
410    fn missing_metal_feature_is_not_run_through_public_api() {
411        let result = run_required(&MustNotExecute, RequiredBackend::Metal, 7, 1e-7);
412        assert_eq!(result.status, RequiredStatus::NotRun);
413        assert!(result.actual.is_none());
414    }
415
416    #[cfg(not(feature = "cuda"))]
417    #[test]
418    fn missing_cuda_feature_is_not_run_through_public_api() {
419        let result = run_required(&MustNotExecute, RequiredBackend::Cuda, 7, 1e-7);
420        assert_eq!(result.status, RequiredStatus::NotRun);
421        assert!(result.actual.is_none());
422    }
423}
ferrum_testkit/op_diff/required.rs

ferrum_testkit/op_diff/
required.rs
