sim_lib_forge/
eval.rs

1use sim_codec_bridge::{
2    ask_profile_symbol, brief_profile_symbol, collab_profile_symbol, loom_profile_symbol,
3};
4use sim_kernel::{Cx, Error, Expr, NumberLiteral, Result, Symbol};
5use sim_value::build::entry;
6
7use crate::{CompiledIntent, Verifier, VerifyCatalog};
8
9/// Recorded model answer and token cost for one deterministic eval playback.
10#[derive(Clone, Debug, PartialEq)]
11pub struct EvalPlayback {
12    /// Answer returned by the recorded model playback.
13    pub answer: Expr,
14    /// Token cost charged to this playback.
15    pub tokens: u64,
16}
17
18impl EvalPlayback {
19    /// Builds one recorded playback row.
20    pub fn new(answer: Expr, tokens: u64) -> Self {
21        Self { answer, tokens }
22    }
23}
24
25/// Deterministic cassette rows for one eval case.
26#[derive(Clone, Debug, PartialEq)]
27pub struct EvalCassette {
28    /// Compiler or lift token cost charged when the artifact is not cached.
29    pub compiler_tokens: u64,
30    /// Raw prose baseline playback.
31    pub raw: EvalPlayback,
32    /// Compiled artifact playback.
33    pub compiled: EvalPlayback,
34    /// Downshifted compiled playback.
35    pub downshifted: EvalPlayback,
36}
37
38impl EvalCassette {
39    /// Builds the deterministic playback set for one case.
40    pub fn new(
41        compiler_tokens: u64,
42        raw: EvalPlayback,
43        compiled: EvalPlayback,
44        downshifted: EvalPlayback,
45    ) -> Self {
46        Self {
47            compiler_tokens,
48            raw,
49            compiled,
50            downshifted,
51        }
52    }
53}
54
55/// One committed FORGE eval case.
56#[derive(Clone, Debug, PartialEq)]
57pub struct EvalCase {
58    /// Stable case id.
59    pub name: Symbol,
60    /// BRIDGE profile covered by this case.
61    pub profile: Symbol,
62    /// Source prose under test.
63    pub prose: String,
64    /// Concrete arguments supplied to the compiled intent.
65    pub args: Expr,
66    /// Ground-truth answer expected by the verifier set.
67    pub gold_answer: Expr,
68    /// Semantic verifier ids that must pass for this case.
69    pub verifiers: Vec<Symbol>,
70    /// Recorded, network-free model playbacks for this case.
71    pub cassette: EvalCassette,
72}
73
74impl EvalCase {
75    /// Builds one eval case.
76    pub fn new(
77        name: Symbol,
78        profile: Symbol,
79        prose: impl Into<String>,
80        args: Expr,
81        gold_answer: Expr,
82        verifiers: Vec<Symbol>,
83        cassette: EvalCassette,
84    ) -> Self {
85        Self {
86            name,
87            profile,
88            prose: prose.into(),
89            args,
90            gold_answer,
91            verifiers,
92            cassette,
93        }
94    }
95}
96
97/// One eval arm configuration.
98#[derive(Clone, Debug, PartialEq, Eq)]
99pub struct EvalArm {
100    /// Stable report name.
101    pub name: String,
102    /// Whether this arm runs a compiled artifact instead of raw prose.
103    pub compiled: bool,
104    /// Whether this arm uses a golden artifact cache hit.
105    pub cached: bool,
106    /// Whether this arm uses the downshifted execution playback.
107    pub downshift: bool,
108    /// Whether this arm is served entirely by identical-request replay.
109    pub replay: bool,
110}
111
112impl EvalArm {
113    /// Raw prose baseline arm.
114    pub fn raw_prose(name: impl Into<String>) -> Self {
115        Self {
116            name: name.into(),
117            compiled: false,
118            cached: false,
119            downshift: false,
120            replay: false,
121        }
122    }
123
124    /// Compiled arm that pays the lift/compiler cost for every case.
125    pub fn compiled_uncached(name: impl Into<String>) -> Self {
126        Self {
127            name: name.into(),
128            compiled: true,
129            cached: false,
130            downshift: false,
131            replay: false,
132        }
133    }
134
135    /// Compiled arm that hits a golden artifact and pays no compiler cost.
136    pub fn compiled_cached(name: impl Into<String>) -> Self {
137        Self {
138            name: name.into(),
139            compiled: true,
140            cached: true,
141            downshift: false,
142            replay: false,
143        }
144    }
145
146    /// Compiled cached arm that executes on the downshifted playback.
147    pub fn compiled_cached_downshifted(name: impl Into<String>) -> Self {
148        Self {
149            name: name.into(),
150            compiled: true,
151            cached: true,
152            downshift: true,
153            replay: false,
154        }
155    }
156
157    /// Identical-request replay arm.
158    pub fn identical_request_replay(name: impl Into<String>) -> Self {
159        Self {
160            name: name.into(),
161            compiled: true,
162            cached: true,
163            downshift: false,
164            replay: true,
165        }
166    }
167}
168
169/// Metrics aggregated for one eval arm.
170#[derive(Clone, Debug, Default, PartialEq)]
171pub struct ArmMetrics {
172    /// Fraction of cases whose semantic verifiers passed.
173    pub accuracy: f64,
174    /// Total token cost recorded by the arm.
175    pub tokens: u64,
176    /// Lift/compiler model calls made by the arm.
177    pub compiler_calls: u64,
178    /// Execution model calls made by the arm.
179    pub execution_calls: u64,
180    /// Identical-request replay hits recorded by the arm.
181    pub replay_hits: u64,
182}
183
184/// Eval report across all requested arms.
185#[derive(Clone, Debug, Default, PartialEq)]
186pub struct EvalReport {
187    /// Ordered arm metrics.
188    pub arms: Vec<(String, ArmMetrics)>,
189}
190
191impl EvalReport {
192    /// Returns metrics for a named arm.
193    pub fn metrics(&self, name: &str) -> Option<&ArmMetrics> {
194        self.arms
195            .iter()
196            .find_map(|(arm, metrics)| (arm == name).then_some(metrics))
197    }
198}
199
200/// Returns the standard FORGE eval arms.
201pub fn standard_eval_arms() -> Vec<EvalArm> {
202    vec![
203        EvalArm::raw_prose("raw-prose-baseline"),
204        EvalArm::compiled_uncached("compiled-uncached"),
205        EvalArm::compiled_cached("compiled-cached"),
206        EvalArm::compiled_cached_downshifted("compiled-cached-downshifted"),
207        EvalArm::identical_request_replay("identical-request-replay"),
208    ]
209}
210
211/// Returns the committed network-free FORGE eval corpus.
212pub fn standard_eval_corpus() -> Vec<EvalCase> {
213    vec![
214        corpus_case(CorpusSpec {
215            id: "brief-status",
216            profile: brief_profile_symbol(),
217            prose: "Summarize the release status in one word.",
218            args: Expr::String("release status is green".to_owned()),
219            gold_answer: Expr::String("green".to_owned()),
220            tokens: TokenSpec::new(42, 118, 64, 31),
221        }),
222        corpus_case(CorpusSpec {
223            id: "ask-count",
224            profile: ask_profile_symbol(),
225            prose: "Count the accepted review checks.",
226            args: Expr::Vector(vec![int(1), int(1), int(1), int(1)]),
227            gold_answer: int(4),
228            tokens: TokenSpec::new(36, 104, 58, 28),
229        }),
230        corpus_case(CorpusSpec {
231            id: "loom-plan",
232            profile: loom_profile_symbol(),
233            prose: "Order the two rollout rows.",
234            args: Expr::Vector(vec![
235                Expr::String("audit".to_owned()),
236                Expr::String("ship".to_owned()),
237            ]),
238            gold_answer: Expr::Vector(vec![
239                Expr::String("audit".to_owned()),
240                Expr::String("ship".to_owned()),
241            ]),
242            tokens: TokenSpec::new(48, 132, 72, 36),
243        }),
244        corpus_case(CorpusSpec {
245            id: "collab-vote",
246            profile: collab_profile_symbol(),
247            prose: "Approve only if the verifier passes.",
248            args: Expr::Bool(true),
249            gold_answer: Expr::Bool(true),
250            tokens: TokenSpec::new(44, 126, 66, 33),
251        }),
252    ]
253}
254
255/// Runs the corpus against the requested eval arms.
256pub fn run_eval(cx: &mut Cx, corpus: &[EvalCase], arms: &[EvalArm]) -> Result<EvalReport> {
257    if corpus.is_empty() {
258        return Err(Error::Eval("eval corpus is empty".to_owned()));
259    }
260    if arms.is_empty() {
261        return Err(Error::Eval("eval arms are empty".to_owned()));
262    }
263
264    let mut report = EvalReport { arms: Vec::new() };
265    for arm in arms {
266        let mut metrics = ArmMetrics::default();
267        let mut passed = 0_u64;
268        for case in corpus {
269            let outcome = run_case(cx, case, arm)?;
270            metrics.tokens = metrics.tokens.saturating_add(outcome.tokens);
271            metrics.compiler_calls = metrics
272                .compiler_calls
273                .saturating_add(outcome.compiler_calls);
274            metrics.execution_calls = metrics
275                .execution_calls
276                .saturating_add(outcome.execution_calls);
277            metrics.replay_hits = metrics.replay_hits.saturating_add(outcome.replay_hits);
278            if outcome.passed {
279                passed = passed.saturating_add(1);
280            }
281        }
282        metrics.accuracy = passed as f64 / corpus.len() as f64;
283        report.arms.push((arm.name.clone(), metrics));
284    }
285    Ok(report)
286}
287
288struct CaseOutcome {
289    passed: bool,
290    tokens: u64,
291    compiler_calls: u64,
292    execution_calls: u64,
293    replay_hits: u64,
294}
295
296fn run_case(cx: &mut Cx, case: &EvalCase, arm: &EvalArm) -> Result<CaseOutcome> {
297    if case.verifiers.is_empty() {
298        return Err(Error::Eval(format!(
299            "eval case {} declares no verifiers",
300            case.name
301        )));
302    }
303
304    let mut tokens = 0_u64;
305    let mut compiler_calls = 0_u64;
306    let mut execution_calls = 0_u64;
307    let mut replay_hits = 0_u64;
308    if arm.compiled && !arm.cached && !arm.replay {
309        compiler_calls = 1;
310        tokens = tokens.saturating_add(case.cassette.compiler_tokens);
311    }
312
313    let answer = if arm.replay {
314        replay_hits = 1;
315        &case.cassette.compiled.answer
316    } else {
317        execution_calls = 1;
318        let playback = playback_for(case, arm);
319        tokens = tokens.saturating_add(playback.tokens);
320        &playback.answer
321    };
322
323    Ok(CaseOutcome {
324        passed: verify_case(cx, case, answer)?,
325        tokens,
326        compiler_calls,
327        execution_calls,
328        replay_hits,
329    })
330}
331
332fn playback_for<'a>(case: &'a EvalCase, arm: &EvalArm) -> &'a EvalPlayback {
333    if !arm.compiled {
334        &case.cassette.raw
335    } else if arm.downshift {
336        &case.cassette.downshifted
337    } else {
338        &case.cassette.compiled
339    }
340}
341
342fn verify_case(cx: &mut Cx, case: &EvalCase, answer: &Expr) -> Result<bool> {
343    let mut catalog = VerifyCatalog::new();
344    for verifier in &case.verifiers {
345        catalog.register_verifier(
346            verifier.clone(),
347            Verifier::Assertion {
348                predicate: equals_predicate(case.gold_answer.clone()),
349            },
350        );
351    }
352    let intent = CompiledIntent {
353        name: case.name.clone(),
354        verifiers: case.verifiers.clone(),
355        ..CompiledIntent::default()
356    };
357    Ok(catalog.verify_answer(cx, &intent, answer)?.accepted())
358}
359
360fn equals_predicate(expected: Expr) -> Expr {
361    Expr::Map(vec![
362        entry(
363            "predicate",
364            Expr::Symbol(Symbol::qualified("forge", "equals")),
365        ),
366        entry("expected", expected),
367    ])
368}
369
370struct CorpusSpec {
371    id: &'static str,
372    profile: Symbol,
373    prose: &'static str,
374    args: Expr,
375    gold_answer: Expr,
376    tokens: TokenSpec,
377}
378
379struct TokenSpec {
380    compiler: u64,
381    raw: u64,
382    compiled: u64,
383    downshifted: u64,
384}
385
386impl TokenSpec {
387    fn new(compiler: u64, raw: u64, compiled: u64, downshifted: u64) -> Self {
388        Self {
389            compiler,
390            raw,
391            compiled,
392            downshifted,
393        }
394    }
395}
396
397fn corpus_case(spec: CorpusSpec) -> EvalCase {
398    EvalCase::new(
399        Symbol::qualified("forge/eval", spec.id),
400        spec.profile,
401        spec.prose,
402        spec.args,
403        spec.gold_answer.clone(),
404        vec![Symbol::qualified(
405            "forge/eval",
406            format!("{}-gold", spec.id).as_str(),
407        )],
408        EvalCassette::new(
409            spec.tokens.compiler,
410            EvalPlayback::new(spec.gold_answer.clone(), spec.tokens.raw),
411            EvalPlayback::new(spec.gold_answer.clone(), spec.tokens.compiled),
412            EvalPlayback::new(spec.gold_answer, spec.tokens.downshifted),
413        ),
414    )
415}
416
417fn int(value: i64) -> Expr {
418    Expr::Number(NumberLiteral {
419        domain: Symbol::qualified("number", "i64"),
420        canonical: value.to_string(),
421    })
422}
sim_lib_forge/eval.rs

sim_lib_forge/
eval.rs
