sim_lib_openai_server/plan/
eval.rs

1use std::mem;
2
3use sim_kernel::{Cx, Error, Expr, Result, Symbol};
4use sim_lib_agent_runner_core::{ModelRequest, ModelResponse};
5
6use crate::{
7    capabilities::{openai_gateway_plan_capability, openai_gateway_plan_remote_capability},
8    plan::{
9        address::resolve_atom_address,
10        eval_context::{EvalContext, PlanPrivacy},
11        eval_helpers::{
12            branch_id, child_args, field, is_slow_fixture, keyword_atom, keyword_value,
13            verifier_accepts,
14        },
15        fixtures::{
16            fixture_echo_response, fixture_static_response, fixture_tool_call_response,
17            request_text, response_summary, response_text,
18        },
19        shape::{check_plan, plan_parts},
20    },
21    runtime::{
22        OpenAiFederation, OpenAiPlanCache, OpenAiRunnerRegistry, PlanCacheKey, PlanCacheMode,
23        PlanCacheWriteTarget,
24    },
25};
26
27/// Result of evaluating a plan: the final response expression and the trace of
28/// branch events recorded during evaluation.
29#[derive(Clone, Debug, PartialEq)]
30pub struct PlanEvalReport {
31    /// Final model response, encoded as an [`Expr`].
32    pub response: Expr,
33    /// Ordered branch lifecycle events emitted while evaluating the plan.
34    pub events: Vec<PlanEvalEvent>,
35}
36
37/// A single event recorded during plan evaluation, such as a branch start or end.
38#[derive(Clone, Debug, PartialEq, Eq)]
39pub struct PlanEvalEvent {
40    /// Event kind symbol (for example `branch-start` or `branch-end`).
41    pub kind: Symbol,
42    /// Structured event payload describing the branch and its outcome.
43    pub payload: Expr,
44}
45
46/// Evaluates a plan against a request and returns only the final response expression.
47pub fn eval_plan(cx: &mut Cx, plan: &Expr, request: &Expr) -> Result<Expr> {
48    eval_plan_report(cx, plan, request).map(|report| report.response)
49}
50
51/// Evaluates a plan and returns the response together with its event trace.
52pub fn eval_plan_report(cx: &mut Cx, plan: &Expr, request: &Expr) -> Result<PlanEvalReport> {
53    eval_plan_report_inner(cx, plan, request, None, None, None)
54}
55
56/// Evaluates a plan with a plan cache available for `plan/cache` combinators.
57pub fn eval_plan_report_with_cache(
58    cx: &mut Cx,
59    plan: &Expr,
60    request: &Expr,
61    cache: &mut OpenAiPlanCache,
62) -> Result<PlanEvalReport> {
63    eval_plan_report_inner(cx, plan, request, Some(cache), None, None)
64}
65
66/// Evaluates a plan with both a plan cache and a runner registry for local inference.
67pub fn eval_plan_report_with_cache_and_runners(
68    cx: &mut Cx,
69    plan: &Expr,
70    request: &Expr,
71    cache: &mut OpenAiPlanCache,
72    runners: &OpenAiRunnerRegistry,
73) -> Result<PlanEvalReport> {
74    eval_plan_report_inner(cx, plan, request, Some(cache), Some(runners), None)
75}
76
77/// Evaluates a plan with a federation handle for `gateway/*` remote atoms.
78pub fn eval_plan_report_with_federation(
79    cx: &mut Cx,
80    plan: &Expr,
81    request: &Expr,
82    federation: &OpenAiFederation,
83) -> Result<PlanEvalReport> {
84    eval_plan_report_inner(cx, plan, request, None, None, Some(federation))
85}
86
87/// Evaluates a plan with a plan cache, runner registry, and federation handle
88/// all available.
89pub fn eval_plan_report_with_cache_runners_and_federation(
90    cx: &mut Cx,
91    plan: &Expr,
92    request: &Expr,
93    cache: &mut OpenAiPlanCache,
94    runners: &OpenAiRunnerRegistry,
95    federation: &OpenAiFederation,
96) -> Result<PlanEvalReport> {
97    eval_plan_report_inner(
98        cx,
99        plan,
100        request,
101        Some(cache),
102        Some(runners),
103        Some(federation),
104    )
105}
106
107fn eval_plan_report_inner(
108    cx: &mut Cx,
109    plan: &Expr,
110    request: &Expr,
111    cache: Option<&mut OpenAiPlanCache>,
112    runners: Option<&OpenAiRunnerRegistry>,
113    federation: Option<&OpenAiFederation>,
114) -> Result<PlanEvalReport> {
115    check_plan(plan)?;
116    let (name, _) = plan_parts(plan)?;
117    if name != "atom" {
118        cx.require(&openai_gateway_plan_capability())?;
119    }
120    let request_model = ModelRequest::try_from(request.clone())?;
121    let context = EvalContext::from_request(request);
122    let mut evaluator = PlanEvaluator {
123        cx,
124        request: request_model,
125        cache,
126        runners,
127        federation,
128        events: Vec::new(),
129        next_branch: 0,
130    };
131    let response = evaluator.eval(plan, context)?;
132    Ok(PlanEvalReport {
133        response: Expr::from(response),
134        events: evaluator.events,
135    })
136}
137
138struct PlanEvaluator<'a> {
139    cx: &'a mut Cx,
140    request: ModelRequest,
141    cache: Option<&'a mut OpenAiPlanCache>,
142    runners: Option<&'a OpenAiRunnerRegistry>,
143    federation: Option<&'a OpenAiFederation>,
144    events: Vec<PlanEvalEvent>,
145    next_branch: u64,
146}
147
148impl PlanEvaluator<'_> {
149    fn eval(&mut self, plan: &Expr, context: EvalContext) -> Result<ModelResponse> {
150        let (name, args) = plan_parts(plan)?;
151        if name == "atom" {
152            let [Expr::String(address)] = args else {
153                return Err(Error::Eval("plan/atom expects one address".to_owned()));
154            };
155            return self.eval_atom(address, context);
156        }
157        let children = child_args(args);
158        match name {
159            "race" => self.eval_race(&children, context),
160            "fallback" => self.eval_fallback(&children, context),
161            "chain" => self.eval_chain(&children, context),
162            "budget" => self.eval_wrapped(name, children[0], context.budgeted(args)),
163            "market" => self.eval_wrapped(name, children[0], context),
164            "local" => self.eval_wrapped(name, children[0], context.local()),
165            "remote" => self.eval_remote(children[0], context),
166            "trace" => self.eval_wrapped(name, children[0], context.trace()),
167            "verify" => self.eval_verify(args, &children, context),
168            "debate" => self.eval_debate(args, &children, context),
169            "cache" => self.eval_cache(args, children[0], context),
170            _ => Err(Error::Eval(format!("unknown plan combinator {name}"))),
171        }
172    }
173
174    fn eval_atom(&mut self, address: &str, context: EvalContext) -> Result<ModelResponse> {
175        let descriptor = resolve_atom_address(address)?;
176        if descriptor.is_gateway() {
177            if context.privacy == PlanPrivacy::LocalOnly {
178                return Err(Error::Eval(format!(
179                    "local-only privacy rejects remote model {address}"
180                )));
181            }
182            return self
183                .federation
184                .ok_or_else(|| Error::Eval(format!("model_not_found: {address}")))?
185                .infer(
186                    self.cx,
187                    address,
188                    &self.request,
189                    &context.federation_policy(),
190                );
191        }
192        if descriptor.is_runner_backed() {
193            let runners = self
194                .runners
195                .ok_or_else(|| Error::Eval(format!("model_not_found: {address}")))?;
196            let card = runners
197                .card_for(address)
198                .ok_or_else(|| Error::Eval(format!("model_not_found: {address}")))?;
199            if context.privacy == PlanPrivacy::LocalOnly
200                && !locality_is_allowed_for_local_only(&card.locality)
201            {
202                return Err(Error::Eval(format!(
203                    "local-only privacy rejects remote model {address}"
204                )));
205            }
206            return runners.infer(self.cx, address, self.request.clone());
207        }
208        match descriptor.address.as_str() {
209            "fixture/echo" | "fixture/slow-echo" => {
210                Ok(fixture_echo_response(address, &self.request))
211            }
212            "fixture/a" => Ok(fixture_static_response(address, "fixture a transcript")),
213            "fixture/b" => Ok(fixture_static_response(address, "fixture b transcript")),
214            "fixture/judge" => Ok(fixture_static_response(
215                address,
216                &format!("judged: {}", request_text(&self.request)),
217            )),
218            "fixture/tool-call" => Ok(fixture_tool_call_response(address, &self.request, false)),
219            "fixture/repeat-tool-call" => {
220                Ok(fixture_tool_call_response(address, &self.request, true))
221            }
222            "fixture/always-ok" => Ok(fixture_static_response(address, "ok")),
223            "fixture/always-fail" => Ok(fixture_static_response(address, "fail")),
224            "fixture/fail" => Err(Error::Eval("fixture/fail failed".to_owned())),
225            _ => Ok(fixture_static_response(
226                address,
227                &request_text(&self.request),
228            )),
229        }
230    }
231
232    fn eval_race(&mut self, children: &[&Expr], context: EvalContext) -> Result<ModelResponse> {
233        let mut winner = None;
234        let mut first_error = None;
235        for child in children {
236            let branch = self.branch_start("race", child);
237            if children.len() > 1 && is_slow_fixture(child) {
238                self.branch_end(
239                    branch,
240                    "cancelled",
241                    Expr::String("cancelled by faster branch".to_owned()),
242                );
243                continue;
244            }
245            match self.eval(child, context.clone()) {
246                Ok(response) if winner.is_none() => {
247                    self.branch_end(branch, "winner", response_summary(&response));
248                    winner = Some(response);
249                }
250                Ok(response) => {
251                    self.branch_end(branch, "cancelled", response_summary(&response));
252                }
253                Err(err) => {
254                    self.branch_end(branch, "error", Expr::String(err.to_string()));
255                    first_error.get_or_insert(err);
256                }
257            }
258        }
259        winner.ok_or_else(|| {
260            first_error.unwrap_or_else(|| Error::Eval("plan/race had no winning branch".to_owned()))
261        })
262    }
263
264    fn eval_fallback(&mut self, children: &[&Expr], context: EvalContext) -> Result<ModelResponse> {
265        let mut first_error = None;
266        for child in children {
267            let branch = self.branch_start("fallback", child);
268            match self.eval(child, context.clone()) {
269                Ok(response) => {
270                    self.branch_end(branch, "accepted", response_summary(&response));
271                    return Ok(response);
272                }
273                Err(err) => {
274                    self.branch_end(branch, "error", Expr::String(err.to_string()));
275                    first_error.get_or_insert(err);
276                }
277            }
278        }
279        Err(first_error.unwrap_or_else(|| Error::Eval("plan/fallback had no children".to_owned())))
280    }
281
282    fn eval_chain(&mut self, children: &[&Expr], context: EvalContext) -> Result<ModelResponse> {
283        let saved = self.request.clone();
284        let mut last = None;
285        for child in children {
286            let response = self.eval_child("chain", child, context.clone())?;
287            self.request = ModelRequest::new(Expr::String(response_text(&response)), Vec::new());
288            last = Some(response);
289        }
290        self.request = saved;
291        last.ok_or_else(|| Error::Eval("plan/chain had no children".to_owned()))
292    }
293
294    fn eval_wrapped(
295        &mut self,
296        combinator: &str,
297        child: &Expr,
298        context: EvalContext,
299    ) -> Result<ModelResponse> {
300        self.eval_child(combinator, child, context)
301    }
302
303    fn eval_remote(&mut self, child: &Expr, context: EvalContext) -> Result<ModelResponse> {
304        if context.privacy == PlanPrivacy::LocalOnly {
305            return Err(Error::Eval(
306                "local-only privacy rejects plan/remote".to_owned(),
307            ));
308        }
309        self.cx.require(&openai_gateway_plan_remote_capability())?;
310        self.eval_child("remote", child, context)
311    }
312
313    fn eval_cache(
314        &mut self,
315        args: &[Expr],
316        child: &Expr,
317        context: EvalContext,
318    ) -> Result<ModelResponse> {
319        let mode = keyword_value(args, "mode")
320            .map(PlanCacheMode::from_expr)
321            .transpose()?
322            .unwrap_or_default();
323        if mode == PlanCacheMode::Disabled {
324            return self.eval_child("cache", child, context);
325        }
326
327        let request_expr = Expr::from(self.request.clone());
328        let key = PlanCacheKey::for_request_plan(&request_expr, child)?;
329        if mode.reads()
330            && let Some(cached) = self
331                .cache
332                .as_ref()
333                .and_then(|cache| cache.get(&key))
334                .cloned()
335        {
336            self.events.push(PlanEvalEvent {
337                kind: Symbol::new("cache-hit"),
338                payload: key.to_expr(),
339            });
340            return ModelResponse::try_from(cached);
341        }
342
343        let response = self.eval_child("cache", child, context)?;
344        if mode.writes()
345            && let Some(cache) = &mut self.cache
346        {
347            cache.put(
348                &mut *self.cx,
349                PlanCacheWriteTarget::Memory,
350                key,
351                Expr::from(response.clone()),
352            )?;
353        }
354        Ok(response)
355    }
356
357    fn eval_verify(
358        &mut self,
359        args: &[Expr],
360        children: &[&Expr],
361        context: EvalContext,
362    ) -> Result<ModelResponse> {
363        let generator = self.eval_child("verify/generator", children[0], context.clone())?;
364        let verifier = self.eval_child("verify/checker", children[1], context.clone())?;
365        if verifier_accepts(&verifier) {
366            return Ok(generator);
367        }
368        match keyword_atom(args, "on-fail").unwrap_or("error") {
369            "accept" => Ok(generator),
370            "error" | "escalate" => Err(Error::Eval(
371                "plan/verify checker rejected output".to_owned(),
372            )),
373            other => Err(Error::Eval(format!(
374                "unsupported plan/verify on-fail behavior {other}"
375            ))),
376        }
377    }
378
379    fn eval_debate(
380        &mut self,
381        args: &[Expr],
382        children: &[&Expr],
383        context: EvalContext,
384    ) -> Result<ModelResponse> {
385        let mut transcripts = Vec::new();
386        for child in children {
387            let response = self.eval_child("debate/side", child, context.clone())?;
388            transcripts.push(response_text(&response));
389        }
390        let judge_plan = keyword_value(args, "judge").cloned().unwrap_or_else(|| {
391            Expr::List(vec![
392                Expr::Symbol(Symbol::new("plan/atom")),
393                Expr::String("fixture/judge".to_owned()),
394            ])
395        });
396        let judge_input = transcripts.join(" | ");
397        self.with_request(
398            ModelRequest::new(Expr::String(judge_input), Vec::new()),
399            |evaluator| evaluator.eval_child("debate/judge", &judge_plan, context),
400        )
401    }
402
403    fn eval_child(
404        &mut self,
405        combinator: &str,
406        child: &Expr,
407        context: EvalContext,
408    ) -> Result<ModelResponse> {
409        let branch = self.branch_start(combinator, child);
410        match self.eval(child, context) {
411            Ok(response) => {
412                self.branch_end(branch, "completed", response_summary(&response));
413                Ok(response)
414            }
415            Err(err) => {
416                self.branch_end(branch, "error", Expr::String(err.to_string()));
417                Err(err)
418            }
419        }
420    }
421
422    fn with_request<T>(
423        &mut self,
424        request: ModelRequest,
425        f: impl FnOnce(&mut Self) -> Result<T>,
426    ) -> Result<T> {
427        let saved = mem::replace(&mut self.request, request);
428        let result = f(self);
429        self.request = saved;
430        result
431    }
432
433    fn branch_start(&mut self, combinator: &str, child: &Expr) -> u64 {
434        let branch = self.next_branch;
435        self.next_branch += 1;
436        self.events.push(PlanEvalEvent {
437            kind: Symbol::new("branch-start"),
438            payload: Expr::Map(vec![
439                field("branch-id", branch_id(branch)),
440                field(
441                    "combinator",
442                    Expr::Symbol(Symbol::new(combinator.to_owned())),
443                ),
444                field("plan", child.clone()),
445            ]),
446        });
447        branch
448    }
449
450    fn branch_end(&mut self, branch: u64, status: &str, result: Expr) {
451        self.events.push(PlanEvalEvent {
452            kind: Symbol::new("branch-end"),
453            payload: Expr::Map(vec![
454                field("branch-id", branch_id(branch)),
455                field("status", Expr::Symbol(Symbol::new(status.to_owned()))),
456                field("result", result),
457            ]),
458        });
459    }
460}
461
462fn locality_is_allowed_for_local_only(locality: &Symbol) -> bool {
463    matches!(
464        locality.name.as_ref(),
465        "local" | "agent" | "agent-backed" | "fabric" | "in-process" | "process"
466    )
467}
sim_lib_openai_server/plan/eval.rs

sim_lib_openai_server/plan/
eval.rs
