wasmsh_testkit/
compat.rs

1//! Differential compatibility test harness.
2//!
3//! Defines a format for origin-authored compatibility cases and a
4//! harness that can compare wasmsh behavior against local reference
5//! shells (opt-in, skippable if not available).
6
7/// A single compatibility test case.
8#[derive(Debug, Clone)]
9pub struct CompatCase {
10    /// Descriptive name for the test case.
11    pub name: &'static str,
12    /// Shell input to execute.
13    pub input: &'static str,
14    /// Expected exit code (None = don't check).
15    pub expected_status: Option<i32>,
16    /// Expected stdout content (None = don't check).
17    pub expected_stdout: Option<&'static str>,
18    /// Whether to compare against reference shells.
19    pub compare_with_oracle: bool,
20}
21
22/// Result of running a compatibility case.
23#[derive(Debug)]
24pub struct CompatResult {
25    pub name: String,
26    pub wasmsh_status: Option<i32>,
27    pub wasmsh_stdout: Option<String>,
28    pub oracle_status: Option<i32>,
29    pub oracle_stdout: Option<String>,
30    pub passed: bool,
31    pub notes: Vec<String>,
32}
33
34/// Run a compatibility case against expected values (no oracle).
35pub fn check_case(case: &CompatCase, actual_status: i32, actual_stdout: &str) -> CompatResult {
36    let mut passed = true;
37    let mut notes = Vec::new();
38
39    if let Some(expected) = case.expected_status {
40        if actual_status != expected {
41            passed = false;
42            notes.push(format!(
43                "status mismatch: expected {expected}, got {actual_status}"
44            ));
45        }
46    }
47
48    if let Some(expected) = case.expected_stdout {
49        if actual_stdout != expected {
50            passed = false;
51            notes.push(format!(
52                "stdout mismatch:\n  expected: {expected:?}\n  got:      {actual_stdout:?}"
53            ));
54        }
55    }
56
57    CompatResult {
58        name: case.name.to_string(),
59        wasmsh_status: Some(actual_status),
60        wasmsh_stdout: Some(actual_stdout.to_string()),
61        oracle_status: None,
62        oracle_stdout: None,
63        passed,
64        notes,
65    }
66}
67
68/// Original compatibility test cases authored for this repository.
69pub fn core_cases() -> Vec<CompatCase> {
70    vec![
71        CompatCase {
72            name: "true returns 0",
73            input: "true",
74            expected_status: Some(0),
75            expected_stdout: Some(""),
76            compare_with_oracle: true,
77        },
78        CompatCase {
79            name: "false returns 1",
80            input: "false",
81            expected_status: Some(1),
82            expected_stdout: Some(""),
83            compare_with_oracle: true,
84        },
85        CompatCase {
86            name: "echo hello",
87            input: "echo hello",
88            expected_status: Some(0),
89            expected_stdout: Some("hello\n"),
90            compare_with_oracle: true,
91        },
92        CompatCase {
93            name: "echo multiple args",
94            input: "echo hello world",
95            expected_status: Some(0),
96            expected_stdout: Some("hello world\n"),
97            compare_with_oracle: true,
98        },
99        CompatCase {
100            name: "echo -n suppresses newline",
101            input: "echo -n hello",
102            expected_status: Some(0),
103            expected_stdout: Some("hello"),
104            compare_with_oracle: true,
105        },
106        CompatCase {
107            name: "colon is no-op",
108            input: ":",
109            expected_status: Some(0),
110            expected_stdout: Some(""),
111            compare_with_oracle: true,
112        },
113        CompatCase {
114            name: "variable assignment and echo",
115            input: "X=hello; echo $X",
116            expected_status: Some(0),
117            expected_stdout: Some("hello\n"),
118            compare_with_oracle: true,
119        },
120        CompatCase {
121            name: "pipeline exit status",
122            input: "true | false",
123            expected_status: Some(1),
124            expected_stdout: Some(""),
125            compare_with_oracle: true,
126        },
127    ]
128}
129
130#[cfg(test)]
131mod tests {
132    use super::*;
133
134    #[test]
135    fn check_case_passes() {
136        let case = CompatCase {
137            name: "simple",
138            input: "echo hi",
139            expected_status: Some(0),
140            expected_stdout: Some("hi\n"),
141            compare_with_oracle: false,
142        };
143        let result = check_case(&case, 0, "hi\n");
144        assert!(result.passed);
145    }
146
147    #[test]
148    fn check_case_status_mismatch() {
149        let case = CompatCase {
150            name: "status",
151            input: "false",
152            expected_status: Some(1),
153            expected_stdout: None,
154            compare_with_oracle: false,
155        };
156        let result = check_case(&case, 0, "");
157        assert!(!result.passed);
158    }
159
160    #[test]
161    fn check_case_stdout_mismatch() {
162        let case = CompatCase {
163            name: "output",
164            input: "echo hi",
165            expected_status: None,
166            expected_stdout: Some("hi\n"),
167            compare_with_oracle: false,
168        };
169        let result = check_case(&case, 0, "bye\n");
170        assert!(!result.passed);
171    }
172
173    #[test]
174    fn core_cases_not_empty() {
175        assert!(!core_cases().is_empty());
176    }
177}
wasmsh_testkit/compat.rs

wasmsh_testkit/
compat.rs
