sim_lib_openai_server/plan/
eval.rs

1use std::mem;
2
3use sim_kernel::{Cx, Error, Expr, Result, Symbol};
4use sim_lib_agent_runner_core::{ModelRequest, ModelResponse};
5
6use crate::{
7    capabilities::{openai_gateway_plan_capability, openai_gateway_plan_remote_capability},
8    plan::{
9        address::resolve_atom_address,
10        eval_context::{EvalContext, PlanPrivacy},
11        eval_helpers::{
12            branch_id, child_args, field, is_slow_fixture, keyword_atom, keyword_value,
13            verifier_accepts,
14        },
15        fixtures::{
16            fixture_echo_response, fixture_static_response, fixture_tool_call_response,
17            request_text, response_summary, response_text,
18        },
19        shape::{check_plan, plan_parts},
20    },
21    runtime::{
22        OpenAiFederation, OpenAiPlanCache, OpenAiRunnerRegistry, PlanCacheKey, PlanCacheMode,
23        PlanCacheWriteTarget,
24    },
25};
26
27/// Result of evaluating a plan: the final response expression and the trace of
28/// branch events recorded during evaluation.
29#[derive(Clone, Debug, PartialEq)]
30pub struct PlanEvalReport {
31    /// Final model response, encoded as an [`Expr`].
32    pub response: Expr,
33    /// Ordered branch lifecycle events emitted while evaluating the plan.
34    pub events: Vec<PlanEvalEvent>,
35}
36
37/// A single event recorded during plan evaluation, such as a branch start or end.
38#[derive(Clone, Debug, PartialEq, Eq)]
39pub struct PlanEvalEvent {
40    /// Event kind symbol (for example `branch-start` or `branch-end`).
41    pub kind: Symbol,
42    /// Structured event payload describing the branch and its outcome.
43    pub payload: Expr,
44}
45
46/// Evaluates a plan against a request and returns only the final response expression.
47pub fn eval_plan(cx: &mut Cx, plan: &Expr, request: &Expr) -> Result<Expr> {
48    eval_plan_report(cx, plan, request).map(|report| report.response)
49}
50
51/// Evaluates a plan and returns the response together with its event trace.
52pub fn eval_plan_report(cx: &mut Cx, plan: &Expr, request: &Expr) -> Result<PlanEvalReport> {
53    eval_plan_report_inner(cx, plan, request, None, None, None)
54}
55
56/// Evaluates a plan with a plan cache available for `plan/cache` combinators.
57pub fn eval_plan_report_with_cache(
58    cx: &mut Cx,
59    plan: &Expr,
60    request: &Expr,
61    cache: &mut OpenAiPlanCache,
62) -> Result<PlanEvalReport> {
63    eval_plan_report_inner(cx, plan, request, Some(cache), None, None)
64}
65
66/// Evaluates a plan with both a plan cache and a runner registry for local inference.
67pub fn eval_plan_report_with_cache_and_runners(
68    cx: &mut Cx,
69    plan: &Expr,
70    request: &Expr,
71    cache: &mut OpenAiPlanCache,
72    runners: &OpenAiRunnerRegistry,
73) -> Result<PlanEvalReport> {
74    eval_plan_report_inner(cx, plan, request, Some(cache), Some(runners), None)
75}
76
77/// Evaluates a plan with a federation handle for `gateway/*` remote atoms.
78pub fn eval_plan_report_with_federation(
79    cx: &mut Cx,
80    plan: &Expr,
81    request: &Expr,
82    federation: &OpenAiFederation,
83) -> Result<PlanEvalReport> {
84    eval_plan_report_inner(cx, plan, request, None, None, Some(federation))
85}
86
87/// Evaluates a plan with a plan cache, runner registry, and federation handle
88/// all available.
89pub fn eval_plan_report_with_cache_runners_and_federation(
90    cx: &mut Cx,
91    plan: &Expr,
92    request: &Expr,
93    cache: &mut OpenAiPlanCache,
94    runners: &OpenAiRunnerRegistry,
95    federation: &OpenAiFederation,
96) -> Result<PlanEvalReport> {
97    eval_plan_report_inner(
98        cx,
99        plan,
100        request,
101        Some(cache),
102        Some(runners),
103        Some(federation),
104    )
105}
106
107fn eval_plan_report_inner(
108    cx: &mut Cx,
109    plan: &Expr,
110    request: &Expr,
111    cache: Option<&mut OpenAiPlanCache>,
112    runners: Option<&OpenAiRunnerRegistry>,
113    federation: Option<&OpenAiFederation>,
114) -> Result<PlanEvalReport> {
115    check_plan(plan)?;
116    let (name, _) = plan_parts(plan)?;
117    if name != "atom" {
118        cx.require(&openai_gateway_plan_capability())?;
119    }
120    let request_model = ModelRequest::try_from(request.clone())?;
121    let context = EvalContext::from_request(request);
122    let mut evaluator = PlanEvaluator {
123        cx,
124        request: request_model,
125        cache,
126        runners,
127        federation,
128        events: Vec::new(),
129        next_branch: 0,
130    };
131    let response = evaluator.eval(plan, context)?;
132    Ok(PlanEvalReport {
133        response: Expr::from(response),
134        events: evaluator.events,
135    })
136}
137
138struct PlanEvaluator<'a> {
139    cx: &'a mut Cx,
140    request: ModelRequest,
141    cache: Option<&'a mut OpenAiPlanCache>,
142    runners: Option<&'a OpenAiRunnerRegistry>,
143    federation: Option<&'a OpenAiFederation>,
144    events: Vec<PlanEvalEvent>,
145    next_branch: u64,
146}
147
148impl PlanEvaluator<'_> {
149    fn eval(&mut self, plan: &Expr, context: EvalContext) -> Result<ModelResponse> {
150        let (name, args) = plan_parts(plan)?;
151        if name == "atom" {
152            let [Expr::String(address)] = args else {
153                return Err(Error::Eval("plan/atom expects one address".to_owned()));
154            };
155            return self.eval_atom(address, context);
156        }
157        let children = child_args(args);
158        match name {
159            "race" => self.eval_race(&children, context),
160            "fallback" => self.eval_fallback(&children, context),
161            "chain" => self.eval_chain(&children, context),
162            "budget" => self.eval_wrapped(name, children[0], context.budgeted(args)),
163            "market" => self.eval_wrapped(name, children[0], context),
164            "local" => self.eval_wrapped(name, children[0], context.local()),
165            "remote" => self.eval_remote(children[0], context),
166            "trace" => self.eval_wrapped(name, children[0], context.trace()),
167            "verify" => self.eval_verify(args, &children, context),
168            "debate" => self.eval_debate(args, &children, context),
169            "cache" => self.eval_cache(args, children[0], context),
170            _ => Err(Error::Eval(format!("unknown plan combinator {name}"))),
171        }
172    }
173
174    fn eval_atom(&mut self, address: &str, context: EvalContext) -> Result<ModelResponse> {
175        let descriptor = resolve_atom_address(address)?;
176        if !descriptor.fixture {
177            if context.privacy == PlanPrivacy::LocalOnly && descriptor.head != "skill" {
178                return Err(Error::Eval(format!(
179                    "local-only privacy rejects remote model {address}"
180                )));
181            }
182            return match descriptor.head.as_str() {
183                "openai" | "ollama" | "skill" => self
184                    .runners
185                    .ok_or_else(|| Error::Eval(format!("model_not_found: {address}")))?
186                    .infer(self.cx, address, self.request.clone()),
187                "gateway" => self
188                    .federation
189                    .ok_or_else(|| Error::Eval(format!("model_not_found: {address}")))?
190                    .infer(
191                        self.cx,
192                        address,
193                        &self.request,
194                        &context.federation_policy(),
195                    ),
196                _ => Err(Error::Eval(format!("model_not_found: {address}"))),
197            };
198        }
199        match descriptor.address.as_str() {
200            "fixture/echo" | "fixture/slow-echo" => {
201                Ok(fixture_echo_response(address, &self.request))
202            }
203            "fixture/a" => Ok(fixture_static_response(address, "fixture a transcript")),
204            "fixture/b" => Ok(fixture_static_response(address, "fixture b transcript")),
205            "fixture/judge" => Ok(fixture_static_response(
206                address,
207                &format!("judged: {}", request_text(&self.request)),
208            )),
209            "fixture/tool-call" => Ok(fixture_tool_call_response(address, &self.request, false)),
210            "fixture/repeat-tool-call" => {
211                Ok(fixture_tool_call_response(address, &self.request, true))
212            }
213            "fixture/always-ok" => Ok(fixture_static_response(address, "ok")),
214            "fixture/always-fail" => Ok(fixture_static_response(address, "fail")),
215            "fixture/fail" => Err(Error::Eval("fixture/fail failed".to_owned())),
216            _ => Ok(fixture_static_response(
217                address,
218                &request_text(&self.request),
219            )),
220        }
221    }
222
223    fn eval_race(&mut self, children: &[&Expr], context: EvalContext) -> Result<ModelResponse> {
224        let mut winner = None;
225        let mut first_error = None;
226        for child in children {
227            let branch = self.branch_start("race", child);
228            if children.len() > 1 && is_slow_fixture(child) {
229                self.branch_end(
230                    branch,
231                    "cancelled",
232                    Expr::String("cancelled by faster branch".to_owned()),
233                );
234                continue;
235            }
236            match self.eval(child, context.clone()) {
237                Ok(response) if winner.is_none() => {
238                    self.branch_end(branch, "winner", response_summary(&response));
239                    winner = Some(response);
240                }
241                Ok(response) => {
242                    self.branch_end(branch, "cancelled", response_summary(&response));
243                }
244                Err(err) => {
245                    self.branch_end(branch, "error", Expr::String(err.to_string()));
246                    first_error.get_or_insert(err);
247                }
248            }
249        }
250        winner.ok_or_else(|| {
251            first_error.unwrap_or_else(|| Error::Eval("plan/race had no winning branch".to_owned()))
252        })
253    }
254
255    fn eval_fallback(&mut self, children: &[&Expr], context: EvalContext) -> Result<ModelResponse> {
256        let mut first_error = None;
257        for child in children {
258            let branch = self.branch_start("fallback", child);
259            match self.eval(child, context.clone()) {
260                Ok(response) => {
261                    self.branch_end(branch, "accepted", response_summary(&response));
262                    return Ok(response);
263                }
264                Err(err) => {
265                    self.branch_end(branch, "error", Expr::String(err.to_string()));
266                    first_error.get_or_insert(err);
267                }
268            }
269        }
270        Err(first_error.unwrap_or_else(|| Error::Eval("plan/fallback had no children".to_owned())))
271    }
272
273    fn eval_chain(&mut self, children: &[&Expr], context: EvalContext) -> Result<ModelResponse> {
274        let saved = self.request.clone();
275        let mut last = None;
276        for child in children {
277            let response = self.eval_child("chain", child, context.clone())?;
278            self.request = ModelRequest::new(Expr::String(response_text(&response)), Vec::new());
279            last = Some(response);
280        }
281        self.request = saved;
282        last.ok_or_else(|| Error::Eval("plan/chain had no children".to_owned()))
283    }
284
285    fn eval_wrapped(
286        &mut self,
287        combinator: &str,
288        child: &Expr,
289        context: EvalContext,
290    ) -> Result<ModelResponse> {
291        self.eval_child(combinator, child, context)
292    }
293
294    fn eval_remote(&mut self, child: &Expr, context: EvalContext) -> Result<ModelResponse> {
295        if context.privacy == PlanPrivacy::LocalOnly {
296            return Err(Error::Eval(
297                "local-only privacy rejects plan/remote".to_owned(),
298            ));
299        }
300        self.cx.require(&openai_gateway_plan_remote_capability())?;
301        self.eval_child("remote", child, context)
302    }
303
304    fn eval_cache(
305        &mut self,
306        args: &[Expr],
307        child: &Expr,
308        context: EvalContext,
309    ) -> Result<ModelResponse> {
310        let mode = keyword_value(args, "mode")
311            .map(PlanCacheMode::from_expr)
312            .transpose()?
313            .unwrap_or_default();
314        if mode == PlanCacheMode::Disabled {
315            return self.eval_child("cache", child, context);
316        }
317
318        let request_expr = Expr::from(self.request.clone());
319        let key = PlanCacheKey::for_request_plan(&request_expr, child)?;
320        if mode.reads()
321            && let Some(cached) = self
322                .cache
323                .as_ref()
324                .and_then(|cache| cache.get(&key))
325                .cloned()
326        {
327            self.events.push(PlanEvalEvent {
328                kind: Symbol::new("cache-hit"),
329                payload: key.to_expr(),
330            });
331            return ModelResponse::try_from(cached);
332        }
333
334        let response = self.eval_child("cache", child, context)?;
335        if mode.writes()
336            && let Some(cache) = &mut self.cache
337        {
338            cache.put(
339                &mut *self.cx,
340                PlanCacheWriteTarget::Memory,
341                key,
342                Expr::from(response.clone()),
343            )?;
344        }
345        Ok(response)
346    }
347
348    fn eval_verify(
349        &mut self,
350        args: &[Expr],
351        children: &[&Expr],
352        context: EvalContext,
353    ) -> Result<ModelResponse> {
354        let generator = self.eval_child("verify/generator", children[0], context.clone())?;
355        let verifier = self.eval_child("verify/checker", children[1], context.clone())?;
356        if verifier_accepts(&verifier) {
357            return Ok(generator);
358        }
359        match keyword_atom(args, "on-fail").unwrap_or("error") {
360            "accept" => Ok(generator),
361            "error" | "escalate" => Err(Error::Eval(
362                "plan/verify checker rejected output".to_owned(),
363            )),
364            other => Err(Error::Eval(format!(
365                "unsupported plan/verify on-fail behavior {other}"
366            ))),
367        }
368    }
369
370    fn eval_debate(
371        &mut self,
372        args: &[Expr],
373        children: &[&Expr],
374        context: EvalContext,
375    ) -> Result<ModelResponse> {
376        let mut transcripts = Vec::new();
377        for child in children {
378            let response = self.eval_child("debate/side", child, context.clone())?;
379            transcripts.push(response_text(&response));
380        }
381        let judge_plan = keyword_value(args, "judge").cloned().unwrap_or_else(|| {
382            Expr::List(vec![
383                Expr::Symbol(Symbol::new("plan/atom")),
384                Expr::String("fixture/judge".to_owned()),
385            ])
386        });
387        let judge_input = transcripts.join(" | ");
388        self.with_request(
389            ModelRequest::new(Expr::String(judge_input), Vec::new()),
390            |evaluator| evaluator.eval_child("debate/judge", &judge_plan, context),
391        )
392    }
393
394    fn eval_child(
395        &mut self,
396        combinator: &str,
397        child: &Expr,
398        context: EvalContext,
399    ) -> Result<ModelResponse> {
400        let branch = self.branch_start(combinator, child);
401        match self.eval(child, context) {
402            Ok(response) => {
403                self.branch_end(branch, "completed", response_summary(&response));
404                Ok(response)
405            }
406            Err(err) => {
407                self.branch_end(branch, "error", Expr::String(err.to_string()));
408                Err(err)
409            }
410        }
411    }
412
413    fn with_request<T>(
414        &mut self,
415        request: ModelRequest,
416        f: impl FnOnce(&mut Self) -> Result<T>,
417    ) -> Result<T> {
418        let saved = mem::replace(&mut self.request, request);
419        let result = f(self);
420        self.request = saved;
421        result
422    }
423
424    fn branch_start(&mut self, combinator: &str, child: &Expr) -> u64 {
425        let branch = self.next_branch;
426        self.next_branch += 1;
427        self.events.push(PlanEvalEvent {
428            kind: Symbol::new("branch-start"),
429            payload: Expr::Map(vec![
430                field("branch-id", branch_id(branch)),
431                field(
432                    "combinator",
433                    Expr::Symbol(Symbol::new(combinator.to_owned())),
434                ),
435                field("plan", child.clone()),
436            ]),
437        });
438        branch
439    }
440
441    fn branch_end(&mut self, branch: u64, status: &str, result: Expr) {
442        self.events.push(PlanEvalEvent {
443            kind: Symbol::new("branch-end"),
444            payload: Expr::Map(vec![
445                field("branch-id", branch_id(branch)),
446                field("status", Expr::Symbol(Symbol::new(status.to_owned()))),
447                field("result", result),
448            ]),
449        });
450    }
451}
sim_lib_openai_server/plan/eval.rs

sim_lib_openai_server/plan/
eval.rs
