sim/runtime/
testing.rs

1use sim_kernel::{
2    CapabilityName, Cx, Expr, Object, ObjectCompat, Result, Symbol, Test, TestReport, Value,
3};
4#[cfg(any(
5    feature = "codec-lisp",
6    feature = "codec-json",
7    feature = "codec-binary",
8    feature = "codec-binary-base64",
9    feature = "codec-chat",
10    feature = "codec-algol"
11))]
12use sim_kernel::{EncodeOptions, ReadPolicy};
13
14use super::test_runs;
15
16/// Expectation a [`SimTest`] checks against the evaluated expression.
17#[derive(Clone, Debug, PartialEq, Eq)]
18pub enum TestExpected {
19    /// The result must be truthy.
20    Truthy,
21    /// The result must equal this expression.
22    Value(Expr),
23    /// Evaluation must fail with an error containing this substring.
24    ErrorContains(String),
25    /// The expression must round-trip unchanged through these codecs.
26    RoundTrip {
27        /// Codecs applied in sequence for the round-trip check.
28        codecs: Vec<Symbol>,
29    },
30}
31
32impl TestExpected {
33    /// Returns the symbol naming this expectation mode.
34    pub fn mode(&self) -> Symbol {
35        match self {
36            Self::Truthy => Symbol::qualified("test", "truthy"),
37            Self::Value(_) => Symbol::qualified("test", "value"),
38            Self::ErrorContains(_) => Symbol::qualified("test", "error-contains"),
39            Self::RoundTrip { .. } => Symbol::qualified("test", "round-trip"),
40        }
41    }
42
43    fn expected_codec(&self, expr_codec: &Symbol) -> Option<Symbol> {
44        matches!(self, Self::Value(_)).then(|| expr_codec.clone())
45    }
46}
47
48/// Executable test or example: an expression, its expectation, and the codec
49/// and capability context it runs in.
50#[derive(Clone)]
51pub struct SimTest {
52    /// Name the test is registered under.
53    pub name: Symbol,
54    /// Lib the test belongs to.
55    pub lib: Symbol,
56    /// Expression evaluated when the test runs.
57    pub expr: Expr,
58    /// Expectation checked against the result.
59    pub expected: TestExpected,
60    /// Symbols the test exercises, for reporting and discovery.
61    pub subjects: Vec<Symbol>,
62    /// Codec the test expression round-trips through before evaluation.
63    pub expr_codec: Symbol,
64    /// Codec the expected value round-trips through, if any.
65    pub expected_codec: Option<Symbol>,
66    /// Whether the test also serves as a documentation example.
67    pub example: bool,
68    /// Capabilities the test requires to run.
69    pub capabilities: Vec<CapabilityName>,
70}
71
72impl SimTest {
73    /// Creates a truthiness-by-default test from its name, lib, expression,
74    /// expectation, and subjects.
75    pub fn new(
76        name: Symbol,
77        lib: Symbol,
78        expr: Expr,
79        expected: TestExpected,
80        subjects: Vec<Symbol>,
81    ) -> Self {
82        let expr_codec = default_test_codec();
83        let expected_codec = expected.expected_codec(&expr_codec);
84        Self {
85            name,
86            lib,
87            expr,
88            expected,
89            subjects,
90            expr_codec,
91            expected_codec,
92            example: false,
93            capabilities: Vec::new(),
94        }
95    }
96
97    /// Sets the expression codec and returns the updated test.
98    pub fn with_expr_codec(mut self, codec: Symbol) -> Self {
99        self.expr_codec = codec;
100        self
101    }
102
103    /// Sets the expected-value codec and returns the updated test.
104    pub fn with_expected_codec(mut self, codec: Symbol) -> Self {
105        self.expected_codec = Some(codec);
106        self
107    }
108
109    /// Marks the test as a documentation example and returns it.
110    pub fn as_example(mut self) -> Self {
111        self.example = true;
112        self
113    }
114
115    /// Adds a required capability and returns the updated test.
116    pub fn requiring(mut self, capability: CapabilityName) -> Self {
117        self.capabilities.push(capability);
118        self
119    }
120
121    fn run_inner(&self, cx: &mut Cx) -> Result<TestReport> {
122        let expr = match checked_roundtrip(cx, &self.expr, &self.expr_codec, "expr")? {
123            Ok(expr) => expr,
124            Err(detail) => return Ok(failed_report(self.name.clone(), detail)),
125        };
126        match &self.expected {
127            TestExpected::Truthy => {
128                let value = cx.eval_expr(expr)?;
129                let passed = value.object().truth(cx)?;
130                Ok(TestReport::from_result(
131                    self.name.clone(),
132                    passed,
133                    if passed {
134                        None
135                    } else {
136                        Some("test result was not truthy".to_owned())
137                    },
138                ))
139            }
140            TestExpected::Value(expected) => {
141                let expected = match self.checked_expected(cx, expected)? {
142                    Ok(expected) => expected,
143                    Err(detail) => return Ok(failed_report(self.name.clone(), detail)),
144                };
145                match cx.eval_expr(expr) {
146                    Ok(value) => {
147                        let actual = value.object().as_expr(cx)?;
148                        Ok(TestReport::from_result(
149                            self.name.clone(),
150                            actual == expected,
151                            (actual != expected)
152                                .then(|| format!("expected {expected:?}, found {actual:?}")),
153                        ))
154                    }
155                    Err(error) => Ok(TestReport::from_result(
156                        self.name.clone(),
157                        false,
158                        Some(format!("evaluation failed: {error}")),
159                    )),
160                }
161            }
162            TestExpected::ErrorContains(expected) => match cx.eval_expr(expr) {
163                Ok(value) => Ok(TestReport::from_result(
164                    self.name.clone(),
165                    false,
166                    Some(format!(
167                        "expected error containing {expected:?}, found value {:?}",
168                        value.object().as_expr(cx)?
169                    )),
170                )),
171                Err(error) => {
172                    let message = error.to_string();
173                    Ok(TestReport::from_result(
174                        self.name.clone(),
175                        message.contains(expected),
176                        (!message.contains(expected)).then(|| {
177                            format!("expected error containing {expected:?}, found {message}")
178                        }),
179                    ))
180                }
181            },
182            TestExpected::RoundTrip { codecs } => {
183                let passed = roundtrip_expr(cx, &expr, codecs)?;
184                Ok(TestReport::from_result(
185                    self.name.clone(),
186                    passed,
187                    (!passed).then(|| "round-trip value changed".to_owned()),
188                ))
189            }
190        }
191    }
192
193    fn checked_expected(
194        &self,
195        cx: &mut Cx,
196        expected: &Expr,
197    ) -> Result<std::result::Result<Expr, String>> {
198        match &self.expected_codec {
199            Some(codec) => checked_roundtrip(cx, expected, codec, "expected"),
200            None => Ok(Ok(expected.clone())),
201        }
202    }
203}
204
205impl Test for SimTest {
206    fn symbol(&self) -> Symbol {
207        self.name.clone()
208    }
209
210    fn lib(&self) -> Symbol {
211        self.lib.clone()
212    }
213
214    fn describe(&self, cx: &mut Cx) -> Result<Value> {
215        self.as_table(cx)
216    }
217
218    fn run(&self, cx: &mut Cx) -> Result<TestReport> {
219        test_runs::run_effect_backed(
220            cx,
221            self.name.clone(),
222            self.expected.mode(),
223            &self.capabilities,
224            |cx| self.run_inner(cx),
225        )
226    }
227}
228
229impl Object for SimTest {
230    fn display(&self, _cx: &mut Cx) -> Result<String> {
231        Ok(format!("#<test {}>", self.name))
232    }
233
234    fn as_any(&self) -> &dyn std::any::Any {
235        self
236    }
237}
238
239impl sim_kernel::ObjectCompat for SimTest {
240    fn class(&self, cx: &mut Cx) -> Result<sim_kernel::ClassRef> {
241        if let Some(value) = cx
242            .registry()
243            .class_by_symbol(&sim_kernel::Symbol::qualified("core", "Test"))
244        {
245            return Ok(value.clone());
246        }
247        cx.factory().class_stub(
248            sim_kernel::CORE_TEST_CLASS_ID,
249            sim_kernel::Symbol::qualified("core", "Test"),
250        )
251    }
252    fn as_expr(&self, _cx: &mut Cx) -> Result<Expr> {
253        Ok(Expr::Symbol(self.name.clone()))
254    }
255    fn as_table(&self, cx: &mut Cx) -> Result<Value> {
256        let mut entries = vec![
257            (Symbol::new("name"), cx.factory().symbol(self.name.clone())?),
258            (
259                Symbol::new("subjects"),
260                cx.factory().list(
261                    self.subjects
262                        .iter()
263                        .cloned()
264                        .map(|symbol| cx.factory().symbol(symbol))
265                        .collect::<Result<Vec<_>>>()?,
266                )?,
267            ),
268            (Symbol::new("lib"), cx.factory().symbol(self.lib.clone())?),
269            (
270                Symbol::new("mode"),
271                cx.factory().symbol(self.expected.mode())?,
272            ),
273            (Symbol::new("expr"), cx.factory().expr(self.expr.clone())?),
274            (
275                Symbol::new("expr-codec"),
276                cx.factory().symbol(self.expr_codec.clone())?,
277            ),
278        ];
279        match &self.expected {
280            TestExpected::Truthy => {
281                entries.push((Symbol::new("expected"), cx.factory().nil()?));
282                entries.push((Symbol::new("expected-codec"), cx.factory().nil()?));
283                entries.push((Symbol::new("expected-error"), cx.factory().nil()?));
284                entries.push((Symbol::new("codecs"), cx.factory().list(Vec::new())?));
285            }
286            TestExpected::Value(expected) => {
287                entries.push((
288                    Symbol::new("expected"),
289                    cx.factory().expr(expected.clone())?,
290                ));
291                entries.push((
292                    Symbol::new("expected-codec"),
293                    match &self.expected_codec {
294                        Some(codec) => cx.factory().symbol(codec.clone())?,
295                        None => cx.factory().nil()?,
296                    },
297                ));
298                entries.push((Symbol::new("expected-error"), cx.factory().nil()?));
299                entries.push((Symbol::new("codecs"), cx.factory().list(Vec::new())?));
300            }
301            TestExpected::ErrorContains(expected) => {
302                entries.push((Symbol::new("expected"), cx.factory().nil()?));
303                entries.push((Symbol::new("expected-codec"), cx.factory().nil()?));
304                entries.push((
305                    Symbol::new("expected-error"),
306                    cx.factory().string(expected.clone())?,
307                ));
308                entries.push((Symbol::new("codecs"), cx.factory().list(Vec::new())?));
309            }
310            TestExpected::RoundTrip { codecs } => {
311                entries.push((Symbol::new("expected"), cx.factory().nil()?));
312                entries.push((Symbol::new("expected-codec"), cx.factory().nil()?));
313                entries.push((Symbol::new("expected-error"), cx.factory().nil()?));
314                entries.push((
315                    Symbol::new("codecs"),
316                    cx.factory().list(
317                        codecs
318                            .iter()
319                            .cloned()
320                            .map(|symbol| cx.factory().symbol(symbol))
321                            .collect::<Result<Vec<_>>>()?,
322                    )?,
323                ));
324            }
325        }
326        entries.push((Symbol::new("example"), cx.factory().bool(self.example)?));
327        entries.push((
328            Symbol::new("capabilities"),
329            cx.factory().list(
330                self.capabilities
331                    .iter()
332                    .map(|capability| cx.factory().symbol(capability.as_symbol()))
333                    .collect::<Result<Vec<_>>>()?,
334            )?,
335        ));
336        cx.factory().table(entries)
337    }
338}
339
340pub(crate) fn roundtrip_expr(cx: &mut Cx, expr: &Expr, codecs: &[Symbol]) -> Result<bool> {
341    #[cfg(any(
342        feature = "codec-lisp",
343        feature = "codec-json",
344        feature = "codec-binary",
345        feature = "codec-binary-base64",
346        feature = "codec-chat",
347        feature = "codec-algol"
348    ))]
349    {
350        let mut current = expr.clone();
351        for codec in codecs {
352            current = codec_roundtrip(cx, &current, codec)?;
353        }
354        Ok(current == *expr)
355    }
356    #[cfg(not(any(
357        feature = "codec-lisp",
358        feature = "codec-json",
359        feature = "codec-binary",
360        feature = "codec-binary-base64",
361        feature = "codec-chat",
362        feature = "codec-algol"
363    )))]
364    {
365        let _ = (cx, expr, codecs);
366        Err(sim_kernel::Error::HostError(
367            "round-trip tests require at least one codec feature".to_owned(),
368        ))
369    }
370}
371
372fn checked_roundtrip(
373    cx: &mut Cx,
374    expr: &Expr,
375    codec: &Symbol,
376    label: &str,
377) -> Result<std::result::Result<Expr, String>> {
378    let decoded = match codec_roundtrip(cx, expr, codec) {
379        Ok(decoded) => decoded,
380        Err(error) => return Ok(Err(format!("{label} codec {codec} failed: {error}"))),
381    };
382    if decoded == *expr {
383        Ok(Ok(decoded))
384    } else {
385        Ok(Err(format!(
386            "{label} codec {codec} did not preserve expression: expected {expr:?}, decoded {decoded:?}"
387        )))
388    }
389}
390
391fn codec_roundtrip(cx: &mut Cx, expr: &Expr, codec: &Symbol) -> Result<Expr> {
392    #[cfg(any(
393        feature = "codec-lisp",
394        feature = "codec-json",
395        feature = "codec-binary",
396        feature = "codec-binary-base64",
397        feature = "codec-chat",
398        feature = "codec-algol"
399    ))]
400    {
401        let output = sim_codec::encode_with_codec(cx, codec, expr, EncodeOptions::default())?;
402        let input = match output {
403            sim_codec::Output::Text(text) => sim_codec::Input::Text(text),
404            sim_codec::Output::Bytes(bytes) => sim_codec::Input::Bytes(bytes),
405        };
406        sim_codec::decode_with_codec(cx, codec, input, ReadPolicy::default())
407    }
408    #[cfg(not(any(
409        feature = "codec-lisp",
410        feature = "codec-json",
411        feature = "codec-binary",
412        feature = "codec-binary-base64",
413        feature = "codec-chat",
414        feature = "codec-algol"
415    )))]
416    {
417        let _ = (cx, expr, codec);
418        Err(sim_kernel::Error::HostError(
419            "codec-faithful tests require at least one codec feature".to_owned(),
420        ))
421    }
422}
423
424fn failed_report(name: Symbol, detail: String) -> TestReport {
425    TestReport::from_result(name, false, Some(detail))
426}
427
428fn default_test_codec() -> Symbol {
429    Symbol::qualified("codec", "lisp")
430}
sim/runtime/testing.rs

sim/runtime/
testing.rs
