lex_runtime/
examples.rs

1//! Behavioral evaluation of signature-level `examples { ... }` blocks (#369 slice 2).
2//!
3//! Slice 1 (PR #370) shipped the AST + parser + type-checking of example args
4//! and expected values against the function's signature. This pass takes the
5//! next step: it actually *runs* each example through the bytecode VM and
6//! compares the result to the declared `expected` value. A mismatch becomes
7//! a [`TypeError::ExampleMismatch`] with `rule_tag = "example-mismatch"`,
8//! surfaced through the same structured-JSON error envelope as every other
9//! `lex check` diagnostic.
10//!
11//! ## Implementation strategy
12//!
13//! For each pure function with non-empty examples, synthesize a small set of
14//! zero-argument helper functions and append them as new stages alongside
15//! the original program:
16//!
17//! - `__ex_<fn>_<K>_arg_<I>` returning the *I*th argument of case *K*.
18//! - `__ex_<fn>_<K>_expected` returning the declared expected value of case *K*.
19//!
20//! Compile the augmented program to bytecode (the user's program plus the
21//! helpers all see the same global scope), and for each case:
22//!
23//! 1. Call each `__ex_<fn>_<K>_arg_<I>` helper through the VM to get a
24//!    runtime `Value` for the argument.
25//! 2. Call the original function with those values to get the actual `Value`.
26//! 3. Call `__ex_<fn>_<K>_expected` to get the declared `Value`.
27//! 4. Compare the two via [`Value`]'s `PartialEq`. On mismatch, emit
28//!    `ExampleMismatch` with pretty-printed `expected` and `got`.
29//!
30//! ## v1 restrictions
31//!
32//! - Generic functions (with `type_params`) are skipped — the helper
33//!   synthesis would need to monomorphize. Examples on generic functions
34//!   still get the slice-1 *type-level* checks; they just don't get
35//!   *behavioral* checks. Worth a follow-up issue if examples on generics
36//!   become a real need.
37//! - Pure-only (already enforced by `ExamplesOnEffectfulFn` in slice 1).
38
39use lex_ast as a;
40use lex_bytecode::{compile_program, vm::Vm, Value};
41use crate::handler::DefaultHandler;
42use crate::policy::Policy;
43use lex_types::TypeError;
44
45/// Run the behavioral-evaluation pass over `stages` and return any
46/// `ExampleMismatch` errors discovered. Returns the empty vec when every
47/// example case passes (or when there are no eligible cases).
48///
49/// Stages that fail VM execution (panics, step-limit, etc.) surface as
50/// `ExampleMismatch` with a synthetic "got" string describing the failure.
51/// We deliberately do not wrap them in a separate error variant so the
52/// downstream JSON envelope and repair-loop wiring stays uniform.
53pub fn evaluate_examples(stages: &[a::Stage]) -> Vec<TypeError> {
54    let helpers = synthesize_helpers(stages);
55    if helpers.cases.is_empty() {
56        return Vec::new();
57    }
58
59    let mut augmented: Vec<a::Stage> = stages.to_vec();
60    augmented.extend(helpers.helper_stages);
61
62    let bc = compile_program(&augmented);
63    let bc = std::sync::Arc::new(bc);
64
65    let mut out = Vec::new();
66    for case in &helpers.cases {
67        match run_case(&bc, case) {
68            CaseOutcome::Pass => {}
69            CaseOutcome::Mismatch { expected, got } => {
70                out.push(TypeError::ExampleMismatch {
71                    at_node: "n_0".into(),
72                    fn_name: case.fn_name.clone(),
73                    case_index: case.case_index,
74                    expected,
75                    got,
76                });
77            }
78            CaseOutcome::RuntimeError(msg) => {
79                // Surface VM panics as ExampleMismatch so the user sees a
80                // clear "this example failed" diagnostic with the panic
81                // message in the `got` slot. Keeps the error envelope uniform.
82                out.push(TypeError::ExampleMismatch {
83                    at_node: "n_0".into(),
84                    fn_name: case.fn_name.clone(),
85                    case_index: case.case_index,
86                    expected: "(declared value)".into(),
87                    got: format!("runtime error: {msg}"),
88                });
89            }
90        }
91    }
92    out
93}
94
95struct Helpers {
96    helper_stages: Vec<a::Stage>,
97    cases: Vec<Case>,
98}
99
100struct Case {
101    fn_name: String,
102    case_index: usize,
103    arg_helpers: Vec<String>,
104    expected_helper: String,
105}
106
107fn synthesize_helpers(stages: &[a::Stage]) -> Helpers {
108    let mut helper_stages = Vec::new();
109    let mut cases = Vec::new();
110
111    for stage in stages {
112        let a::Stage::FnDecl(fd) = stage else { continue };
113        if fd.examples.is_empty() {
114            continue;
115        }
116        // v1: skip generics. Helper synthesis would need to pick a
117        // concrete instantiation; defer until there's a real need.
118        if !fd.type_params.is_empty() {
119            continue;
120        }
121        // Pure-only is already enforced by ExamplesOnEffectfulFn in slice 1,
122        // but we double-check here so a future regression doesn't lead the
123        // VM to invoke a real effect handler during `lex check`.
124        if !fd.effects.is_empty() {
125            continue;
126        }
127        for (k, ex) in fd.examples.iter().enumerate() {
128            let mut arg_helpers = Vec::with_capacity(ex.args.len());
129            for (i, arg) in ex.args.iter().enumerate() {
130                let helper_name = format!("__ex_{}_{}_arg_{}", fd.name, k, i);
131                helper_stages.push(zero_arg_helper(&helper_name, fd.params[i].ty.clone(), arg.clone()));
132                arg_helpers.push(helper_name);
133            }
134            let expected_helper = format!("__ex_{}_{}_expected", fd.name, k);
135            helper_stages.push(zero_arg_helper(
136                &expected_helper,
137                fd.return_type.clone(),
138                ex.expected.clone(),
139            ));
140            cases.push(Case {
141                fn_name: fd.name.clone(),
142                case_index: k,
143                arg_helpers,
144                expected_helper,
145            });
146        }
147    }
148
149    Helpers { helper_stages, cases }
150}
151
152fn zero_arg_helper(name: &str, return_type: a::TypeExpr, body: a::CExpr) -> a::Stage {
153    a::Stage::FnDecl(a::FnDecl {
154        name: name.into(),
155        type_params: Vec::new(),
156        params: Vec::new(),
157        effects: Vec::new(),
158        effect_row_var: None,
159        return_type,
160        body,
161        examples: Vec::new(),
162    })
163}
164
165enum CaseOutcome {
166    Pass,
167    Mismatch { expected: String, got: String },
168    RuntimeError(String),
169}
170
171fn run_case(bc: &std::sync::Arc<lex_bytecode::Program>, case: &Case) -> CaseOutcome {
172    // Each VM invocation is a fresh instance — they share no state, and
173    // because we restrict to pure functions, there's nothing to share.
174    let mut arg_values: Vec<Value> = Vec::with_capacity(case.arg_helpers.len());
175    for helper in &case.arg_helpers {
176        match call_zero_arg(bc, helper) {
177            Ok(v) => arg_values.push(v),
178            Err(e) => return CaseOutcome::RuntimeError(format!("computing arg from `{helper}`: {e}")),
179        }
180    }
181    let expected = match call_zero_arg(bc, &case.expected_helper) {
182        Ok(v) => v,
183        Err(e) => return CaseOutcome::RuntimeError(format!("computing expected from `{}`: {e}", case.expected_helper)),
184    };
185    let got = match call_with_args(bc, &case.fn_name, arg_values) {
186        Ok(v) => v,
187        Err(e) => return CaseOutcome::RuntimeError(format!("calling `{}`: {e}", case.fn_name)),
188    };
189    if expected == got {
190        CaseOutcome::Pass
191    } else {
192        CaseOutcome::Mismatch {
193            expected: pretty_value(&expected),
194            got: pretty_value(&got),
195        }
196    }
197}
198
199fn call_zero_arg(bc: &std::sync::Arc<lex_bytecode::Program>, name: &str) -> Result<Value, String> {
200    call_with_args(bc, name, Vec::new())
201}
202
203fn call_with_args(
204    bc: &std::sync::Arc<lex_bytecode::Program>,
205    name: &str,
206    args: Vec<Value>,
207) -> Result<Value, String> {
208    let handler = DefaultHandler::new(Policy::pure()).with_program(std::sync::Arc::clone(bc));
209    let mut vm = Vm::with_handler(bc, Box::new(handler));
210    // Defensive: cap steps so a runaway example can't hang `lex check`.
211    vm.set_step_limit(1_000_000);
212    vm.call(name, args).map_err(|e| format!("{e:?}"))
213}
214
215/// Pretty-print a `Value` for inclusion in `ExampleMismatch` errors.
216/// We want the JSON envelope to carry something a human and an LLM can
217/// both read; `Debug` is verbose but unambiguous and matches the rest of
218/// Lex's diagnostic style.
219fn pretty_value(v: &Value) -> String {
220    match v {
221        Value::Int(n) => n.to_string(),
222        Value::Float(f) => f.to_string(),
223        Value::Bool(b) => b.to_string(),
224        Value::Str(s) => format!("{s:?}"),
225        Value::Unit => "()".into(),
226        Value::List(xs) => format!(
227            "[{}]",
228            xs.iter().map(pretty_value).collect::<Vec<_>>().join(", ")
229        ),
230        Value::Tuple(xs) => format!(
231            "({})",
232            xs.iter().map(pretty_value).collect::<Vec<_>>().join(", ")
233        ),
234        Value::Variant { name, args } if args.is_empty() => name.clone(),
235        Value::Variant { name, args } => format!(
236            "{}({})",
237            name,
238            args.iter().map(pretty_value).collect::<Vec<_>>().join(", ")
239        ),
240        Value::Record { fields: fs, .. } => format!(
241            "{{ {} }}",
242            fs.iter()
243                .map(|(k, v)| format!("{k}: {}", pretty_value(v)))
244                .collect::<Vec<_>>()
245                .join(", ")
246        ),
247        other => format!("{other:?}"),
248    }
249}
lex_runtime/examples.rs

lex_runtime/
examples.rs
