batty_cli/team/
equivalence.rs

1//! Equivalence testing harness for comparing synthetic emulator runs.
2
3use std::collections::HashMap;
4use std::path::Path;
5use std::process::{Command, Stdio};
6
7use anyhow::{Context, Result, bail};
8
9use super::parity::{self, VerificationStatus};
10
11#[derive(Debug, Clone, PartialEq, Eq, Default)]
12pub struct InputSequence {
13    pub events: Vec<String>,
14}
15
16impl InputSequence {
17    pub fn new(events: impl IntoIterator<Item = impl Into<String>>) -> Self {
18        Self {
19            events: events.into_iter().map(Into::into).collect(),
20        }
21    }
22}
23
24#[derive(Debug, Clone, PartialEq, Eq, Default)]
25pub struct OutputCapture {
26    pub frames: Vec<String>,
27    pub audio_events: Vec<String>,
28    pub io_events: Vec<String>,
29}
30
31impl OutputCapture {
32    pub fn new(frames: impl IntoIterator<Item = impl Into<String>>) -> Self {
33        Self {
34            frames: frames.into_iter().map(Into::into).collect(),
35            audio_events: Vec::new(),
36            io_events: Vec::new(),
37        }
38    }
39}
40
41#[derive(Debug, Clone, PartialEq, Eq)]
42pub struct TestRun {
43    pub name: String,
44    pub inputs: InputSequence,
45    pub outputs: OutputCapture,
46}
47
48impl TestRun {
49    pub fn frames(&self) -> &[String] {
50        &self.outputs.frames
51    }
52}
53
54#[derive(Debug, Clone, PartialEq, Eq)]
55pub struct DiffReport {
56    pub matching_frames: usize,
57    pub divergent_frames: usize,
58    pub timing_difference: isize,
59    pub divergent_indices: Vec<usize>,
60}
61
62impl DiffReport {
63    pub fn passed(&self) -> bool {
64        self.divergent_frames == 0 && self.timing_difference == 0
65    }
66
67    pub fn summary(&self) -> String {
68        format!(
69            "matching_frames={}, divergent_frames={}, timing_difference={}",
70            self.matching_frames, self.divergent_frames, self.timing_difference
71        )
72    }
73}
74
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub struct Z80Snapshot {
77    pub program_counter: u16,
78    pub stack_pointer: u16,
79    pub memory: Vec<u8>,
80}
81
82pub fn load_z80_snapshot(bytes: &[u8]) -> Result<Z80Snapshot> {
83    const HEADER_LEN: usize = 30;
84    const MEMORY_LEN: usize = 48 * 1024;
85
86    if bytes.len() < HEADER_LEN + MEMORY_LEN {
87        bail!(
88            ".z80 snapshot too short: expected at least {} bytes, got {}",
89            HEADER_LEN + MEMORY_LEN,
90            bytes.len()
91        );
92    }
93
94    let program_counter = u16::from_le_bytes([bytes[6], bytes[7]]);
95    if program_counter == 0 {
96        bail!("version 2/3 .z80 snapshots are not supported by this fixture loader");
97    }
98
99    let flags = bytes[12];
100    if flags & 0x20 != 0 {
101        bail!("compressed version 1 .z80 snapshots are not supported by this fixture loader");
102    }
103
104    let stack_pointer = u16::from_le_bytes([bytes[8], bytes[9]]);
105    let memory = bytes[HEADER_LEN..HEADER_LEN + MEMORY_LEN].to_vec();
106    Ok(Z80Snapshot {
107        program_counter,
108        stack_pointer,
109        memory,
110    })
111}
112
113pub trait EmulatorBackend {
114    fn run(&self, binary: &Path, inputs: &InputSequence) -> Result<OutputCapture>;
115}
116
117#[derive(Debug, Default, Clone)]
118pub struct MockBackend {
119    fixtures: HashMap<String, OutputCapture>,
120}
121
122impl MockBackend {
123    pub fn with_fixture(mut self, binary: impl Into<String>, capture: OutputCapture) -> Self {
124        self.fixtures.insert(binary.into(), capture);
125        self
126    }
127}
128
129impl EmulatorBackend for MockBackend {
130    fn run(&self, binary: &Path, _inputs: &InputSequence) -> Result<OutputCapture> {
131        let key = binary.to_string_lossy().into_owned();
132        self.fixtures
133            .get(&key)
134            .cloned()
135            .with_context(|| format!("no mock fixture registered for `{key}`"))
136    }
137}
138
139#[derive(Debug, Default, Clone, Copy)]
140pub struct CommandBackend;
141
142impl EmulatorBackend for CommandBackend {
143    fn run(&self, binary: &Path, inputs: &InputSequence) -> Result<OutputCapture> {
144        let mut child = Command::new(binary)
145            .stdin(Stdio::piped())
146            .stdout(Stdio::piped())
147            .stderr(Stdio::piped())
148            .spawn()
149            .with_context(|| format!("failed to spawn `{}`", binary.display()))?;
150
151        if let Some(mut stdin) = child.stdin.take() {
152            use std::io::Write;
153            for input in &inputs.events {
154                writeln!(stdin, "{input}")
155                    .with_context(|| format!("failed to write input to `{}`", binary.display()))?;
156            }
157        }
158
159        let output = child
160            .wait_with_output()
161            .with_context(|| format!("failed to read output from `{}`", binary.display()))?;
162        if !output.status.success() {
163            let stderr = String::from_utf8_lossy(&output.stderr);
164            bail!(
165                "backend command `{}` failed: {}",
166                binary.display(),
167                stderr.trim()
168            );
169        }
170
171        let frames = String::from_utf8_lossy(&output.stdout)
172            .lines()
173            .map(str::trim)
174            .filter(|line| !line.is_empty())
175            .map(ToString::to_string)
176            .collect();
177
178        Ok(OutputCapture {
179            frames,
180            audio_events: Vec::new(),
181            io_events: Vec::new(),
182        })
183    }
184}
185
186pub fn execute_test_run(
187    backend: &dyn EmulatorBackend,
188    name: &str,
189    binary: &Path,
190    inputs: InputSequence,
191) -> Result<TestRun> {
192    let outputs = backend.run(binary, &inputs)?;
193    Ok(TestRun {
194        name: name.to_string(),
195        inputs,
196        outputs,
197    })
198}
199
200pub fn compare_outputs(expected: &OutputCapture, actual: &OutputCapture) -> DiffReport {
201    let compared_len = expected.frames.len().min(actual.frames.len());
202    let mut matching_frames = 0;
203    let mut divergent_indices = Vec::new();
204
205    for idx in 0..compared_len {
206        if expected.frames[idx] == actual.frames[idx] {
207            matching_frames += 1;
208        } else {
209            divergent_indices.push(idx);
210        }
211    }
212
213    if expected.frames.len() > compared_len {
214        divergent_indices.extend(compared_len..expected.frames.len());
215    } else if actual.frames.len() > compared_len {
216        divergent_indices.extend(compared_len..actual.frames.len());
217    }
218
219    DiffReport {
220        matching_frames,
221        divergent_frames: divergent_indices.len(),
222        timing_difference: actual.frames.len() as isize - expected.frames.len() as isize,
223        divergent_indices,
224    }
225}
226
227pub fn update_parity_from_diff(
228    project_root: &Path,
229    behavior: &str,
230    report: &DiffReport,
231) -> Result<()> {
232    let verification = if report.passed() {
233        VerificationStatus::Pass
234    } else {
235        VerificationStatus::Fail
236    };
237    parity::update_parity_verification(project_root, behavior, verification, &report.summary())
238}
239
240#[cfg(test)]
241mod tests {
242    use super::*;
243
244    const FIXTURE_PATH: &str = concat!(
245        env!("CARGO_MANIFEST_DIR"),
246        "/tests/fixtures/zx_spectrum/minimal_test_program.z80"
247    );
248    const FIXTURE_BYTES: &[u8] = include_bytes!(concat!(
249        env!("CARGO_MANIFEST_DIR"),
250        "/tests/fixtures/zx_spectrum/minimal_test_program.z80"
251    ));
252
253    fn parity_fixture() -> String {
254        r#"---
255project: trivial
256target: trivial.z80
257source_platform: zx-spectrum-z80
258target_language: rust
259last_verified: 2026-04-05
260overall_parity: 0%
261---
262
263| Behavior | Spec | Test | Implementation | Verified | Notes |
264| --- | --- | --- | --- | --- | --- |
265| Screen fill | complete | complete | complete | -- | pending |
266"#
267        .to_string()
268    }
269
270    #[test]
271    fn compare_outputs_counts_matching_and_divergent_frames() {
272        let expected = OutputCapture::new(["frame-1", "frame-2"]);
273        let actual = OutputCapture::new(["frame-1", "frame-x"]);
274
275        let diff = compare_outputs(&expected, &actual);
276        assert_eq!(diff.matching_frames, 1);
277        assert_eq!(diff.divergent_frames, 1);
278        assert_eq!(diff.timing_difference, 0);
279        assert_eq!(diff.divergent_indices, vec![1]);
280    }
281
282    #[test]
283    fn z80_fixture_loads_with_expected_header_and_program_bytes() {
284        let snapshot = load_z80_snapshot(FIXTURE_BYTES).unwrap();
285
286        assert_eq!(snapshot.program_counter, 0x8000);
287        assert_eq!(snapshot.stack_pointer, 0x5c00);
288        assert_eq!(snapshot.memory.len(), 48 * 1024);
289
290        let program_offset = 0x8000usize - 0x4000usize;
291        assert_eq!(snapshot.memory[program_offset], 0xF3);
292        assert_eq!(
293            &snapshot.memory[program_offset..program_offset + 7],
294            &[0xF3, 0x21, 0x00, 0x40, 0x11, 0x01, 0x40]
295        );
296        assert_eq!(
297            &snapshot.memory[0x9000usize - 0x4000usize..0x9003usize - 0x4000usize],
298            &[0x00, 0x00, 0x00]
299        );
300    }
301
302    #[test]
303    fn z80_fixture_behavior_doc_mentions_observable_outputs() {
304        let behavior =
305            std::fs::read_to_string(Path::new(FIXTURE_PATH).with_file_name("BEHAVIOR.md")).unwrap();
306
307        assert!(behavior.contains("Fills the 6912-byte display file"));
308        assert!(behavior.contains("Toggles port `0xFE` twice"));
309        assert!(behavior.contains("Writes `0x42` to address `0x9000`"));
310    }
311
312    #[test]
313    fn mock_backend_runs_trivial_fixture_and_updates_parity() {
314        let tmp = tempfile::tempdir().unwrap();
315        std::fs::write(tmp.path().join("PARITY.md"), parity_fixture()).unwrap();
316
317        let inputs = InputSequence::new(["fill", "flip"]);
318        let backend = MockBackend::default()
319            .with_fixture("original.bin", OutputCapture::new(["frame-a", "frame-b"]))
320            .with_fixture("reimpl.bin", OutputCapture::new(["frame-a", "frame-b"]));
321
322        let expected = execute_test_run(
323            &backend,
324            "original",
325            Path::new("original.bin"),
326            inputs.clone(),
327        )
328        .unwrap();
329        let actual = execute_test_run(&backend, "reimpl", Path::new("reimpl.bin"), inputs).unwrap();
330
331        assert_eq!(
332            expected.frames(),
333            &["frame-a".to_string(), "frame-b".to_string()]
334        );
335
336        let diff = compare_outputs(&expected.outputs, &actual.outputs);
337        assert!(diff.passed(), "diff should match: {diff:?}");
338
339        update_parity_from_diff(tmp.path(), "Screen fill", &diff).unwrap();
340
341        let updated = std::fs::read_to_string(tmp.path().join("PARITY.md")).unwrap();
342        assert!(updated.contains("| Screen fill | complete | complete | complete | PASS |"));
343        assert!(updated.contains("matching_frames=2"));
344        assert!(updated.contains("overall_parity: 100%"));
345    }
346}
batty_cli/team/equivalence.rs

batty_cli/team/
equivalence.rs
