deepstrike_sdk/
harness_loop.rs

1use deepstrike_core::context::renderer::RenderedContext;
2use deepstrike_core::types::message::{Message, Role};
3use futures::StreamExt;
4
5use crate::Result;
6use crate::harness::{HarnessEvent, HarnessOutcome, HarnessRequest, QualityGate, Verdict};
7use crate::providers::{LLMProvider, StreamEvent};
8use crate::run_event::RunEvent;
9use crate::runtime::RuntimeRunner;
10
11fn rendered_context_from_messages(messages: Vec<Message>) -> RenderedContext {
12    let mut system_parts = Vec::new();
13    let mut turns = Vec::new();
14
15    for message in messages {
16        if message.role == Role::System {
17            if let Some(text) = message.content.as_text() {
18                system_parts.push(text.to_owned());
19            }
20        } else {
21            turns.push(message);
22        }
23    }
24
25    let system_text = system_parts.join("\n\n");
26    RenderedContext {
27        system_text: system_text.clone(),
28        system_stable: system_text,
29        system_knowledge: String::new(),
30        turns,
31        state_turn: None,
32        frozen_prefix_len: None,
33    }
34}
35
36async fn collect_run_runtime(
37    runner: &RuntimeRunner,
38    req: &HarnessRequest,
39) -> Result<(String, u32, u64, String)> {
40    let criteria_texts: Vec<String> = req.criteria.iter().map(|c| c.text.clone()).collect();
41    let mut text = String::new();
42    let mut iterations = 0u32;
43    let mut total_tokens = 0u64;
44    let mut status = "error".to_string();
45    let mut stream = runner
46        .run_streaming(&req.goal, &criteria_texts, req.extensions.as_ref(), None)
47        .await?;
48    while let Some(evt) = stream.next().await {
49        match evt? {
50            RunEvent::TextDelta(d) => text.push_str(&d),
51            RunEvent::Done {
52                iterations: i,
53                total_tokens: t,
54                status: s,
55            } => {
56                iterations = i;
57                total_tokens = t;
58                status = s;
59            }
60            _ => {}
61        }
62    }
63    Ok((text, iterations, total_tokens, status))
64}
65
66/// SinglePassHarness — run once, always passes.
67pub struct SinglePassHarness<'a> {
68    runner: &'a RuntimeRunner,
69}
70
71impl<'a> SinglePassHarness<'a> {
72    pub fn new(runner: &'a RuntimeRunner) -> Self {
73        Self { runner }
74    }
75
76    pub async fn run(&self, request: HarnessRequest) -> Result<HarnessOutcome> {
77        let (text, iterations, total_tokens, status) =
78            collect_run_runtime(self.runner, &request).await?;
79        Ok(HarnessOutcome {
80            result: text,
81            passed: true,
82            iterations,
83            total_tokens,
84            status,
85            overall_score: 1.0,
86            feedback: None,
87            details: vec![],
88        })
89    }
90}
91
92/// EvalLoopHarness — retry until QualityGate passes.
93///
94/// I3.4 (A1): prefer [`HarnessLoop`] with a `verdict_fn` for host-defined judgment.
95/// `EvalLoopHarness::stream` does NOT honor the `gate` (only `run` does); `HarnessLoop` runs the
96/// eval loop uniformly across both surfaces.
97#[deprecated(since = "0.2.23", note = "Use HarnessLoop with `with_verdict_fn` for host-defined judgment. EvalLoopHarness.stream() does not honor gate.")]
98pub struct EvalLoopHarness<'a, G: QualityGate> {
99    runner: &'a RuntimeRunner,
100    gate: G,
101    max_attempts: usize,
102}
103
104impl<'a, G: QualityGate> EvalLoopHarness<'a, G> {
105    pub fn new(runner: &'a RuntimeRunner, gate: G, max_attempts: usize) -> Self {
106        Self {
107            runner,
108            gate,
109            max_attempts,
110        }
111    }
112
113    pub async fn run(&self, request: HarnessRequest) -> Result<HarnessOutcome> {
114        let mut outcome = HarnessOutcome {
115            result: String::new(),
116            passed: false,
117            iterations: 0,
118            total_tokens: 0,
119            status: "error".into(),
120            overall_score: 0.0,
121            feedback: None,
122            details: vec![],
123        };
124        for _ in 0..self.max_attempts {
125            let (text, iterations, total_tokens, status) =
126                collect_run_runtime(self.runner, &request).await?;
127            outcome = HarnessOutcome {
128                result: text,
129                passed: false,
130                iterations,
131                total_tokens,
132                status,
133                overall_score: 0.0,
134                feedback: None,
135                details: vec![],
136            };
137            if self.gate.evaluate(&request, &outcome).await? {
138                outcome.passed = true;
139                return Ok(outcome);
140            }
141        }
142        Ok(outcome)
143    }
144}
145
146/// I3.2 (A2/A3): host-supplied judgment for each attempt's result. Mirrors the Node SDK
147/// `VerdictFn`. Returning `Some(Verdict)` short-circuits the built-in LLM eval; returning `None`
148/// defers to it. Sync-only in Rust today — when async hosts need it, lift to a boxed future.
149pub struct VerdictCtx<'a> {
150    pub goal: &'a str,
151    pub criteria: &'a [crate::harness::Criterion],
152    pub attempt: u32,
153    pub result: &'a str,
154}
155pub type VerdictFn = std::sync::Arc<dyn Fn(VerdictCtx<'_>) -> Option<Verdict> + Send + Sync>;
156
157/// HarnessLoop — LLM-as-judge with feedback injection and skill extraction.
158pub struct HarnessLoop<'a> {
159    runner: &'a RuntimeRunner,
160    eval_provider: Box<dyn LLMProvider>,
161    max_attempts: usize,
162    skill_dir: Option<std::path::PathBuf>,
163    verdict_fn: Option<VerdictFn>,
164}
165
166impl<'a> HarnessLoop<'a> {
167    pub fn new(
168        runner: &'a RuntimeRunner,
169        eval_provider: impl LLMProvider + 'static,
170        max_attempts: usize,
171        skill_dir: Option<std::path::PathBuf>,
172    ) -> Self {
173        Self {
174            runner,
175            eval_provider: Box::new(eval_provider),
176            max_attempts,
177            skill_dir,
178            verdict_fn: None,
179        }
180    }
181
182    /// I3.2 (A2/A3): plug in a host-supplied verdict closure. Returning `Some(Verdict)` skips the
183    /// LLM eval; returning `None` defers. Pure addition — not setting it is byte-equivalent to
184    /// the prior LLM-eval-only path.
185    pub fn with_verdict_fn(mut self, f: VerdictFn) -> Self {
186        self.verdict_fn = Some(f);
187        self
188    }
189
190    pub fn run_streaming<'b>(
191        &'b self,
192        request: HarnessRequest,
193    ) -> impl futures::Stream<Item = Result<HarnessEvent>> + 'b {
194        use deepstrike_core::harness::eval::{build_eval_messages, parse_verdict, Criterion};
195
196        async_stream::stream! {
197            let mut current_goal = request.goal.clone();
198            let criteria_texts: Vec<String> = request.criteria.iter().map(|c| c.text.clone()).collect();
199
200            for attempt in 1..=self.max_attempts as u32 {
201                let mut last_result = String::new();
202                let mut last_iterations = 0u32;
203                let mut last_total_tokens = 0u64;
204                let mut last_status = "error".to_string();
205
206                let goal_for_run = current_goal.clone();
207                let stream_result = self.runner
208                    .run_streaming(&goal_for_run, &criteria_texts, request.extensions.as_ref(), None)
209                    .await;
210                let mut stream = match stream_result {
211                    Ok(s) => s,
212                    Err(e) => { yield Err(e); return; }
213                };
214
215                while let Some(evt) = stream.next().await {
216                    match evt {
217                        Ok(RunEvent::TextDelta(d)) => {
218                            last_result.push_str(&d);
219                            yield Ok(HarnessEvent::Token(d));
220                        }
221                        Ok(RunEvent::ToolCall { id, name }) => yield Ok(HarnessEvent::ToolCall { id, name }),
222                        Ok(RunEvent::ToolResult { call_id, content, is_error, .. }) => {
223                            yield Ok(HarnessEvent::ToolResult { call_id, content, is_error });
224                        }
225                        Ok(RunEvent::Done { iterations, total_tokens, status }) => {
226                            last_iterations = iterations;
227                            last_total_tokens = total_tokens;
228                            last_status = status;
229                        }
230                        Ok(_) => {}
231                        Err(e) => { yield Err(e); return; }
232                    }
233                }
234
235                yield Ok(HarnessEvent::Supervising);
236
237                // I3.2 (A2/A3): host-supplied verdict_fn short-circuits the LLM eval. None ⇒ defer.
238                let mut verdict: Option<Verdict> = None;
239                let mut skill_candidate_from_eval = None;
240                if let Some(f) = self.verdict_fn.clone() {
241                    let ctx = VerdictCtx {
242                        goal: &request.goal,
243                        criteria: &request.criteria,
244                        attempt,
245                        result: &last_result,
246                    };
247                    verdict = f(ctx);
248                }
249                let verdict = if let Some(v) = verdict {
250                    v
251                } else {
252                    // #6 (0.5.0): eval/verdict compute is the kernel's stateless free functions (was the
253                    // EvalPipeline state machine). Build the eval prompt, call the eval LLM, parse the verdict.
254                    let eval_criteria: Vec<Criterion> = request.criteria.iter().map(|c| Criterion {
255                        text: c.text.clone(), required: c.required, weight: c.weight,
256                    }).collect();
257                    let messages = build_eval_messages(&request.goal, &eval_criteria, &last_result, attempt, true);
258
259                    let mut eval_text = String::new();
260                    let context = rendered_context_from_messages(messages);
261                    let eval_state = self.eval_provider.create_run_state();
262                    let mut eval_stream = match self.eval_provider.stream(&context, &[], None, eval_state.as_ref()).await {
263                        Ok(s) => s,
264                        Err(e) => { yield Err(e); return; }
265                    };
266                    while let Some(evt) = eval_stream.next().await {
267                        if let Ok(StreamEvent::TextDelta { delta }) = evt {
268                            eval_text.push_str(&delta);
269                        }
270                    }
271
272                    let eval_result = parse_verdict(&eval_text);
273                    skill_candidate_from_eval = eval_result.skill_candidate.clone();
274                    Verdict {
275                        passed: eval_result.passed,
276                        overall_score: eval_result.overall_score,
277                        feedback: eval_result.feedback.clone(),
278                        details: eval_result.details.iter().map(|d| crate::harness::CriterionResult {
279                            criterion: d.criterion.clone(), passed: d.passed, score: d.score, feedback: d.feedback.clone(),
280                        }).collect(),
281                    }
282                };
283
284                if verdict.passed {
285                    if let Some(sc) = skill_candidate_from_eval {
286                        if let Some(dir) = &self.skill_dir {
287                            let mut fm = format!("---\nname: {}\ndescription: {}\n", sc.name, sc.description);
288                            if let Some(wtu) = &sc.when_to_use { fm.push_str(&format!("when_to_use: {}\n", wtu)); }
289                            fm.push_str("---\n\n");
290                            fm.push_str(&sc.content);
291                            if let Err(e) = tokio::fs::write(dir.join(format!("{}.md", sc.name)), fm).await {
292                                yield Err(e.into()); return;
293                            }
294                        }
295                    }
296                    yield Ok(HarnessEvent::Done { verdict, iterations: last_iterations, total_tokens: last_total_tokens, status: last_status });
297                    return;
298                }
299
300                yield Ok(HarnessEvent::Revising { verdict: verdict.clone() });
301                current_goal = format!("{}\n\n[Attempt {} feedback: {}]", request.goal, attempt, verdict.feedback);
302            }
303
304            yield Ok(HarnessEvent::MaxAttemptsReached);
305        }
306    }
307}
deepstrike_sdk/harness_loop.rs

deepstrike_sdk/
harness_loop.rs
