deepstrike_sdk/runtime/
eval.rs

1//! `judge()` — one-shot quality scoring against a goal + criteria.
2//!
3//! Rust port of node/src/runtime/eval.ts. Wraps the kernel's `gen_eval` free functions
4//! (re-exported from `deepstrike_core::harness::eval`) into a small surface for callers
5//! that just want "does this result meet the criteria?" without standing up
6//! `HarnessLoop`.
7
8use deepstrike_core::context::renderer::RenderedContext;
9use deepstrike_core::harness::eval::{
10    build_eval_messages as core_build_eval_messages, parse_verdict as core_parse_verdict,
11    verdict_output_schema as core_verdict_output_schema, EvalResult,
12};
13use deepstrike_core::types::message::{Content, Message, Role};
14use futures::StreamExt;
15
16use crate::providers::{LLMProvider, StreamEvent};
17use crate::Result;
18
19/// Re-export the kernel's `Criterion` so callers don't need to import from `deepstrike_core`.
20pub use deepstrike_core::harness::eval::Criterion;
21
22/// The verdict shape parsed from the judge LLM. Same struct as the kernel's `EvalResult`.
23pub type Verdict = EvalResult;
24
25/// Build the kernel's eval prompt for (goal, criteria, result). Pure — does not call an LLM.
26pub fn build_eval_messages(goal: &str, criteria: &[Criterion], result: &str) -> Vec<Message> {
27    core_build_eval_messages(goal, criteria, result, 1, false)
28}
29
30/// Parse a Verdict from raw judge-LLM text.
31pub fn parse_verdict(text: &str) -> Verdict {
32    core_parse_verdict(text)
33}
34
35/// The JSON Schema the kernel expects judge output to conform to.
36pub fn verdict_output_schema() -> serde_json::Value {
37    core_verdict_output_schema(false)
38}
39
40/// One-shot judge: render the eval prompt, stream the provider, parse the verdict.
41/// Errors when the provider returns no text.
42pub async fn judge(
43    provider: &dyn LLMProvider,
44    goal: &str,
45    criteria: &[Criterion],
46    result: &str,
47) -> Result<Verdict> {
48    let msgs = build_eval_messages(goal, criteria, result);
49    let system_text: String = msgs
50        .iter()
51        .filter(|m| m.role == Role::System)
52        .filter_map(|m| match &m.content {
53            Content::Text(s) => Some(s.clone()),
54            _ => None,
55        })
56        .collect::<Vec<_>>()
57        .join("\n\n");
58    let turns: Vec<Message> = msgs.into_iter().filter(|m| m.role != Role::System).collect();
59    let ctx = RenderedContext {
60        system_text,
61        system_stable: String::new(),
62        system_knowledge: String::new(),
63        state_turn: None,
64        turns,
65        frozen_prefix_len: None,
66    };
67
68    let mut text = String::new();
69    let mut stream = provider.stream(&ctx, &[], None, None).await?;
70    while let Some(evt) = stream.next().await {
71        if let Ok(StreamEvent::TextDelta { delta }) = evt {
72            text.push_str(&delta);
73        }
74    }
75    if text.is_empty() {
76        return Err(crate::Error::Other("judge: provider produced no text".to_string()));
77    }
78    Ok(parse_verdict(&text))
79}
deepstrike_sdk/runtime/eval.rs

deepstrike_sdk/runtime/
eval.rs
