sui_spec/
probe.rs

1//! Parity probes — each a typed question of the form
2//! "does sui agree with CppNix on this expression?".
3//!
4//! The sweep harness used to be a bash script enumerating probes
5//! inline.  It's now a Lisp corpus interpreted by a small Rust
6//! runner.  Authoring a new probe is one `(defprobe …)` form;
7//! promoting a probe to a regression guard is one `:tags ("regression")`
8//! edit.  No bash, no JSON munging at the shell level — the runner
9//! reads typed probes, executes both engines, and classifies.
10//!
11//! Two canonical corpora compile into the binary:
12//!
13//! - [`CANONICAL_PROBES_LISP`] — `parity_probes.lisp`, the original
14//!   seven cross-flake parity probes.
15//! - [`CANONICAL_BUILTIN_SMOKE_LISP`] — `builtin_smoke_probes.lisp`,
16//!   one probe per sui builtin module so a regression in any one of
17//!   them shows up at sweep time without needing a real flake.
18//!
19//! Both produce values of the same [`Probe`] type — they differ only
20//! in tag set + corpus location.  The [`BuiltinSmokeProbe`] wrapper
21//! relabels the [`ProbeKind`] from `Eval` to `BuiltinSmoke` for
22//! reporting; everything else is shared.
23//!
24//! ## Authoring surface
25//!
26//! ```lisp
27//! (defprobe
28//!   :name     "getflake-outPath"
29//!   :expr     "(builtins.getFlake \"path:$FLAKE\").outPath"
30//!   :classify JsonEqual
31//!   :tags     ("smoke" "drop-in-replacement"))
32//! ```
33//!
34//! The `$FLAKE` token in `expr` is substituted at run time with
35//! each flake's absolute path.  `classify` determines how the two
36//! engines' outputs are compared.
37
38use std::path::Path;
39use std::process::Command;
40
41use serde::{Deserialize, Serialize};
42use tatara_lisp::DeriveTataraDomain;
43
44use crate::SpecError;
45use crate::cli::{nix_cli, sui_cli};
46use crate::exec::CapturedOutput;
47use crate::parity::{default_classify, ParityCheck, ProbeContext, ProbeKind, Verdict};
48
49/// A single parity probe — an expression to evaluate and a rule
50/// for comparing outputs.
51#[derive(DeriveTataraDomain, Serialize, Deserialize, Debug, Clone)]
52#[tatara(keyword = "defprobe")]
53pub struct Probe {
54    pub name: String,
55    pub expr: String,
56    pub classify: Classify,
57    #[serde(default)]
58    pub tags: Vec<String>,
59}
60
61/// How a probe compares sui's result to CppNix's.  Enum variants
62/// are the typed border — the runner interprets exactly these
63/// cases, and adding a new one is adding a new primitive to the
64/// spec language.
65#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
66pub enum Classify {
67    /// Byte-exact JSON equality after parsing each engine's stdout.
68    JsonEqual,
69    /// Attribute names only, order-insensitive (for attrset probes
70    /// where the value at each key may differ but the keyset should
71    /// match).
72    AttrNamesEqual,
73    /// Both engines must produce a `/nix/store/...` path with the
74    /// store-path format.  Does not require the paths to match
75    /// (useful while the outPath store-copy bug is open).
76    BothAreStorePaths,
77}
78
79// ── ParityCheck impl ───────────────────────────────────────────────
80
81impl ParityCheck for Probe {
82    fn name(&self) -> &str { &self.name }
83    fn tags(&self) -> &[String] { &self.tags }
84    fn kind(&self) -> ProbeKind { ProbeKind::Eval }
85
86    fn sui_invocation(&self, ctx: &ProbeContext, sui_bin: &Path) -> Command {
87        sui_cli::eval_expr(sui_bin, &ctx.substitute(&self.expr))
88    }
89
90    fn nix_invocation(&self, ctx: &ProbeContext, nix_bin: &Path) -> Command {
91        nix_cli::eval_expr(nix_bin, &ctx.substitute(&self.expr))
92    }
93
94    fn classify(&self, sui: &CapturedOutput, nix: &CapturedOutput) -> Verdict {
95        let mode = self.classify;
96        default_classify(sui, nix, |s, n| {
97            classify_outputs(mode, s.stdout.trim(), n.stdout.trim())
98        })
99    }
100}
101
102fn classify_outputs(mode: Classify, sui: &str, nix: &str) -> bool {
103    match mode {
104        Classify::JsonEqual => sui == nix,
105        Classify::AttrNamesEqual => {
106            let sui_v: Option<Vec<String>> = serde_json::from_str(sui).ok();
107            let nix_v: Option<Vec<String>> = serde_json::from_str(nix).ok();
108            match (sui_v, nix_v) {
109                (Some(mut a), Some(mut b)) => {
110                    a.sort();
111                    b.sort();
112                    a == b
113                }
114                _ => false,
115            }
116        }
117        Classify::BothAreStorePaths => {
118            let is_sp = |s: &str| s.trim_matches('"').starts_with("/nix/store/");
119            is_sp(sui) && is_sp(nix)
120        }
121    }
122}
123
124// ── Wrapper for the builtin-smoke corpus ───────────────────────────
125
126/// Newtype wrapper around [`Probe`] that re-labels [`ProbeKind`] for
127/// the builtin-smoke corpus.  Lets the sweep report distinguish a
128/// failing `(defprobe ...)` from `builtin_smoke_probes.lisp` from one
129/// in `parity_probes.lisp` without authoring a parallel typed domain.
130pub struct BuiltinSmokeProbe(pub Probe);
131
132impl ParityCheck for BuiltinSmokeProbe {
133    fn name(&self) -> &str { self.0.name() }
134    fn tags(&self) -> &[String] { self.0.tags() }
135    fn kind(&self) -> ProbeKind { ProbeKind::BuiltinSmoke }
136    fn applies(&self, ctx: &ProbeContext) -> bool { self.0.applies(ctx) }
137    fn sui_invocation(&self, ctx: &ProbeContext, sui_bin: &Path) -> Command {
138        self.0.sui_invocation(ctx, sui_bin)
139    }
140    fn nix_invocation(&self, ctx: &ProbeContext, nix_bin: &Path) -> Command {
141        self.0.nix_invocation(ctx, nix_bin)
142    }
143    fn classify(&self, sui: &CapturedOutput, nix: &CapturedOutput) -> Verdict {
144        self.0.classify(sui, nix)
145    }
146}
147
148// ── Canonical probe corpora ────────────────────────────────────────
149
150pub const CANONICAL_PROBES_LISP: &str = include_str!("../specs/parity_probes.lisp");
151
152pub const CANONICAL_BUILTIN_SMOKE_LISP: &str =
153    include_str!("../specs/builtin_smoke_probes.lisp");
154
155/// Compile the embedded probe corpus.  Returns every `(defprobe …)`
156/// form in document order.
157///
158/// # Errors
159///
160/// Returns an error if the Lisp source fails to parse.
161pub fn load_canonical() -> Result<Vec<Probe>, SpecError> {
162    crate::loader::load_all::<Probe>(CANONICAL_PROBES_LISP)
163}
164
165/// Compile the builtin-smoke corpus — one probe per sui builtin module.
166///
167/// # Errors
168///
169/// Returns an error if the Lisp source fails to parse.
170pub fn load_builtin_smoke() -> Result<Vec<Probe>, SpecError> {
171    crate::loader::load_all::<Probe>(CANONICAL_BUILTIN_SMOKE_LISP)
172}
173
174#[cfg(test)]
175mod tests {
176    use super::*;
177
178    #[test]
179    fn canonical_corpus_parses() {
180        let probes = load_canonical().expect("canonical probes must compile");
181        assert!(!probes.is_empty(), "corpus must contain at least one probe");
182        // Every probe has a non-empty name and either references the
183        // `$FLAKE` placeholder (so the runner can substitute the flake
184        // path) or is explicitly tagged "context-free" (asserting the
185        // probe doesn't depend on a flake — e.g. a builtin or pure
186        // derivation algorithm probe).
187        for p in &probes {
188            assert!(!p.name.is_empty(), "probe must have a name: {p:?}");
189            let context_free =
190                p.tags.iter().any(|t| t == "context-free");
191            assert!(p.expr.contains("$FLAKE") || context_free,
192                "probe {} must contain $FLAKE placeholder OR carry the \"context-free\" tag",
193                p.name);
194        }
195    }
196
197    #[test]
198    fn canonical_corpus_has_the_flake_shape_probes() {
199        let probes = load_canonical().unwrap();
200        let names: Vec<&str> = probes.iter().map(|p| p.name.as_str()).collect();
201        // These are the probes that caught the original bugs the
202        // refactor fixed.  If they disappear the regression guard
203        // is gone too.
204        for required in ["getflake-outPath", "getflake-outputs-keys"] {
205            assert!(names.contains(&required),
206                "canonical corpus must contain {required}: have {names:?}");
207        }
208    }
209
210    #[test]
211    fn builtin_smoke_corpus_parses() {
212        let probes = load_builtin_smoke().expect("builtin-smoke corpus must compile");
213        // The corpus targets sui-eval's 19 builtin modules; require at
214        // least 19 probes so we know the sweep exercises each one.
215        assert!(
216            probes.len() >= 19,
217            "builtin smoke corpus is sparse — got {}, expected ≥19 (one per module)",
218            probes.len(),
219        );
220        for p in &probes {
221            assert!(
222                p.tags.iter().any(|t| t == "builtin-smoke"),
223                "builtin-smoke probe {} missing :tags (\"builtin-smoke\" …)",
224                p.name,
225            );
226        }
227    }
228
229    #[test]
230    fn parity_check_impl_constructs_typed_invocation() {
231        let probe = Probe {
232            name: "test".into(),
233            expr: "1 + 2".into(),
234            classify: Classify::JsonEqual,
235            tags: vec![],
236        };
237        let ctx = ProbeContext::current(std::path::PathBuf::from("/tmp/flake"));
238        let cmd = probe.sui_invocation(&ctx, Path::new("/usr/local/bin/sui"));
239        let argv = crate::exec::command_argv(&cmd);
240        assert_eq!(argv[0], "/usr/local/bin/sui");
241        assert_eq!(argv[1], "eval");
242        assert!(argv.contains(&"1 + 2".to_string()));
243    }
244}
sui_spec/probe.rs

sui_spec/
probe.rs
