deepstrike_sdk/runtime/
eval.rs

1//! `judge()` — one-shot quality scoring against a goal + criteria.
2//!
3//! Rust port of node/src/runtime/eval.ts. Wraps the kernel's `gen_eval` free functions
4//! (re-exported from `deepstrike_core::harness::eval`) into a small surface for callers
5//! that just want "does this result meet the criteria?" without standing up
6//! `AttemptLoop`.
7
8use deepstrike_core::context::renderer::RenderedContext;
9use deepstrike_core::harness::eval::{
10    EvalResult, build_eval_messages as core_build_eval_messages,
11    parse_verdict as core_parse_verdict, verdict_output_schema as core_verdict_output_schema,
12};
13use deepstrike_core::types::message::{Content, Message, Role};
14use futures::StreamExt;
15
16use crate::Result;
17use crate::providers::{LLMProvider, StreamEvent};
18
19/// Re-export the kernel's `Criterion` so callers don't need to import from `deepstrike_core`.
20pub use deepstrike_core::harness::eval::Criterion;
21
22/// The verdict shape parsed from the judge LLM. Same struct as the kernel's `EvalResult`.
23pub type Verdict = EvalResult;
24
25/// Build the kernel's eval prompt for (goal, criteria, result). Pure — does not call an LLM.
26pub fn build_eval_messages(goal: &str, criteria: &[Criterion], result: &str) -> Vec<Message> {
27    core_build_eval_messages(goal, criteria, result, 1, false)
28}
29
30/// Parse a Verdict from raw judge-LLM text.
31pub fn parse_verdict(text: &str) -> Verdict {
32    core_parse_verdict(text)
33}
34
35/// The JSON Schema the kernel expects judge output to conform to.
36pub fn verdict_output_schema() -> serde_json::Value {
37    core_verdict_output_schema(false)
38}
39
40/// One-shot judge: render the eval prompt, stream the provider, parse the verdict.
41/// Errors when the provider returns no text.
42pub async fn judge(
43    provider: &dyn LLMProvider,
44    goal: &str,
45    criteria: &[Criterion],
46    result: &str,
47) -> Result<Verdict> {
48    let msgs = build_eval_messages(goal, criteria, result);
49    let system_text: String = msgs
50        .iter()
51        .filter(|m| m.role == Role::System)
52        .filter_map(|m| match &m.content {
53            Content::Text(s) => Some(s.clone()),
54            _ => None,
55        })
56        .collect::<Vec<_>>()
57        .join("\n\n");
58    let turns: Vec<Message> = msgs
59        .into_iter()
60        .filter(|m| m.role != Role::System)
61        .collect();
62    let ctx = RenderedContext {
63        system_text,
64        system_stable: String::new(),
65        system_knowledge: String::new(),
66        state_turn: None,
67        turns,
68        frozen_prefix_len: None,
69        budget_overflow: None,
70    };
71
72    let mut text = String::new();
73    let mut stream = provider.stream(&ctx, &[], None, None).await?;
74    while let Some(evt) = stream.next().await {
75        if let Ok(StreamEvent::TextDelta { delta }) = evt {
76            text.push_str(&delta);
77        }
78    }
79    if text.is_empty() {
80        return Err(crate::Error::Other(
81            "judge: provider produced no text".to_string(),
82        ));
83    }
84    Ok(parse_verdict(&text))
85}
deepstrike_sdk/runtime/eval.rs

deepstrike_sdk/runtime/
eval.rs
