oneharness_core/domain/
normalize.rs

1//! Best-effort extraction of a harness's final assistant text from its raw
2//! stdout. Pure: no I/O. The execution envelope (exit code, stdout, stderr,
3//! duration) is always guaranteed; `text` is a convenience, and its method is
4//! recorded so a consumer can tell extraction apart from raw passthrough.
5
6use crate::domain::report::OutputFormat;
7use serde_json::Value;
8
9/// A successfully extracted final message and the method used to find it.
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct Extracted {
12    pub text: String,
13    pub source: String,
14}
15
16/// Object keys, in priority order, that commonly hold a harness's final text.
17const TEXT_KEYS: &[&str] = &["result", "text", "message", "output", "content", "response"];
18
19/// Extract the final text from `stdout` according to the harness's format.
20/// Returns `None` when nothing usable can be found (the caller then leaves
21/// `text` null and consumers fall back to the raw `stdout`).
22pub fn extract(stdout: &str, fmt: OutputFormat) -> Option<Extracted> {
23    match fmt {
24        OutputFormat::Text => extract_text(stdout),
25        OutputFormat::Json => extract_json(stdout),
26        OutputFormat::StreamJson => extract_stream_json(stdout),
27    }
28}
29
30fn extract_text(stdout: &str) -> Option<Extracted> {
31    let t = stdout.trim();
32    (!t.is_empty()).then(|| Extracted {
33        text: t.to_string(),
34        source: "raw".to_string(),
35    })
36}
37
38fn extract_json(stdout: &str) -> Option<Extracted> {
39    // The common case is a single JSON document carrying the final answer in a
40    // known key (Claude Code's terminal `result`). Try that first.
41    if let Ok(value) = serde_json::from_str::<Value>(stdout.trim()) {
42        if let Some((text, key)) = json_text(&value) {
43            return Some(Extracted {
44                text,
45                source: format!("json:{key}"),
46            });
47        }
48    }
49    // OpenCode also requests `--format json` but emits *line-delimited* events,
50    // not one document, so the single-parse above fails (or finds no top-level
51    // text key). Its visible answer lives in `text` parts; recover it from those.
52    extract_opencode_parts(stdout)
53        // Codex `exec --json` emits JSONL whose final answer is an `agent_message`
54        // item (used when `--events` upgrades codex to its JSON event stream).
55        .or_else(|| extract_codex_agent_message(stdout))
56        // Qwen `--output-format json` emits one JSON *array* of Anthropic-style
57        // messages; the answer is the last assistant message's `text` blocks.
58        .or_else(|| extract_content_block_text(stdout))
59}
60
61/// Codex `exec --json`: the final assistant text is the last `item.completed`
62/// whose `item.type == "agent_message"`, under `item.text`. `None` when no such
63/// item is present. Sourced from a real codex transcript.
64fn extract_codex_agent_message(stdout: &str) -> Option<Extracted> {
65    let mut last: Option<String> = None;
66    for value in json_lines(stdout) {
67        if value.get("type").and_then(Value::as_str) != Some("item.completed") {
68            continue;
69        }
70        let Some(item) = value.get("item") else {
71            continue;
72        };
73        if item.get("type").and_then(Value::as_str) != Some("agent_message") {
74            continue;
75        }
76        if let Some(t) = item.get("text").and_then(Value::as_str) {
77            if !t.trim().is_empty() {
78                last = Some(t.to_string());
79            }
80        }
81    }
82    last.map(|text| Extracted {
83        text,
84        source: "json:codex-agent-message".to_string(),
85    })
86}
87
88/// Anthropic content-block text (Qwen / Claude stream-json): the last assistant
89/// message's `text` blocks, joined — the visible final answer when there is no
90/// terminal `result` string. `message.content[]` (or a top-level `content[]`)
91/// entries of `type:"text"` with a non-empty `text`. `None` when none is found.
92fn extract_content_block_text(stdout: &str) -> Option<Extracted> {
93    let mut last: Option<String> = None;
94    for value in json_lines(stdout) {
95        let content = value
96            .get("message")
97            .and_then(|m| m.get("content"))
98            .or_else(|| value.get("content"))
99            .and_then(Value::as_array);
100        let Some(blocks) = content else { continue };
101        let joined = blocks
102            .iter()
103            .filter(|b| b.get("type").and_then(Value::as_str) == Some("text"))
104            .filter_map(|b| b.get("text").and_then(Value::as_str))
105            .filter(|t| !t.trim().is_empty())
106            .collect::<Vec<_>>()
107            .join("\n");
108        if !joined.is_empty() {
109            last = Some(joined);
110        }
111    }
112    last.map(|text| Extracted {
113        text,
114        source: "content-blocks:text".to_string(),
115    })
116}
117
118/// JSON values in `stdout`: a top-level array flattened to its elements (Qwen's
119/// `json` mode), else each parseable line (JSONL / stream-json). Mirrors
120/// `events::json_candidates` so the text and event paths see the same shapes.
121fn json_lines(stdout: &str) -> Vec<Value> {
122    if let Ok(Value::Array(items)) = serde_json::from_str::<Value>(stdout.trim()) {
123        return items;
124    }
125    stdout
126        .lines()
127        .filter_map(|line| serde_json::from_str::<Value>(line.trim()).ok())
128        .collect()
129}
130
131/// OpenCode's `run --format json` streams one JSON event per line. The assistant's
132/// visible answer is carried by its `text` parts: events whose `part` object has
133/// `type: "text"` and a `text` string. Other parts — `step-start`/`step-finish`,
134/// `reasoning`, and `tool` — are not the final answer and are skipped. A single
135/// turn can emit several text parts across steps (e.g. a line of prose, a tool
136/// call, then more prose), so they are joined in document order. The source is
137/// recorded as `opencode-parts` so a consumer can tell this reconstruction apart
138/// from a single-document `result`. `None` when no text part is present.
139fn extract_opencode_parts(stdout: &str) -> Option<Extracted> {
140    let mut texts: Vec<String> = Vec::new();
141    for line in stdout.lines() {
142        let line = line.trim();
143        if line.is_empty() {
144            continue;
145        }
146        let Ok(value) = serde_json::from_str::<Value>(line) else {
147            continue;
148        };
149        let Some(part) = value.get("part").and_then(Value::as_object) else {
150            continue;
151        };
152        if part.get("type").and_then(Value::as_str) != Some("text") {
153            continue;
154        }
155        if let Some(text) = part.get("text").and_then(Value::as_str) {
156            if !text.trim().is_empty() {
157                texts.push(text.to_string());
158            }
159        }
160    }
161    (!texts.is_empty()).then(|| Extracted {
162        text: texts.join("\n"),
163        source: "json:opencode-parts".to_string(),
164    })
165}
166
167/// Scan line-delimited JSON events and return the last usable text, preferring
168/// a terminal `result` event (the shape harnesses use for their final answer).
169fn extract_stream_json(stdout: &str) -> Option<Extracted> {
170    let mut last: Option<(String, &'static str)> = None;
171    let mut last_result: Option<String> = None;
172    for line in stdout.lines() {
173        let line = line.trim();
174        if line.is_empty() {
175            continue;
176        }
177        let Ok(value) = serde_json::from_str::<Value>(line) else {
178            continue;
179        };
180        if let Some((text, key)) = json_text(&value) {
181            if key == "result" {
182                last_result = Some(text.clone());
183            }
184            last = Some((text, key));
185        }
186    }
187    if let Some(text) = last_result {
188        return Some(Extracted {
189            text,
190            source: "stream-json:result".to_string(),
191        });
192    }
193    // Qwen's stream-json carries no terminal `result` string; its answer is the
194    // last assistant message's Anthropic content-block `text`. Prefer that over a
195    // stray top-level text key from an intermediate event.
196    if let Some(extracted) = extract_content_block_text(stdout) {
197        return Some(extracted);
198    }
199    last.map(|(text, key)| Extracted {
200        text,
201        source: format!("stream-json:{key}"),
202    })
203}
204
205/// Pull a non-empty string out of a JSON value: a bare string, or the first
206/// matching key of an object. Returns the text and the key that matched.
207fn json_text(value: &Value) -> Option<(String, &'static str)> {
208    match value {
209        Value::String(s) if !s.trim().is_empty() => Some((s.clone(), "string")),
210        Value::Object(map) => {
211            for key in TEXT_KEYS {
212                if let Some(Value::String(s)) = map.get(*key) {
213                    if !s.trim().is_empty() {
214                        return Some((s.clone(), key));
215                    }
216                }
217            }
218            None
219        }
220        _ => None,
221    }
222}
223
224#[cfg(test)]
225mod tests {
226    use super::*;
227
228    #[test]
229    fn text_format_trims_raw_stdout() {
230        let got = extract("  hi there\n\n", OutputFormat::Text).unwrap();
231        assert_eq!(got.text, "hi there");
232        assert_eq!(got.source, "raw");
233    }
234
235    #[test]
236    fn empty_text_yields_none() {
237        assert!(extract("   \n", OutputFormat::Text).is_none());
238    }
239
240    #[test]
241    fn json_prefers_result_field() {
242        let raw = r#"{"type":"result","result":"the answer","is_error":false}"#;
243        let got = extract(raw, OutputFormat::Json).unwrap();
244        assert_eq!(got.text, "the answer");
245        assert_eq!(got.source, "json:result");
246    }
247
248    #[test]
249    fn json_falls_back_to_other_known_keys() {
250        let got = extract(r#"{"message":"hello"}"#, OutputFormat::Json).unwrap();
251        assert_eq!(got.text, "hello");
252        assert_eq!(got.source, "json:message");
253    }
254
255    #[test]
256    fn bare_json_string_is_extracted() {
257        let got = extract(r#""just a string""#, OutputFormat::Json).unwrap();
258        assert_eq!(got.text, "just a string");
259        assert_eq!(got.source, "json:string");
260    }
261
262    #[test]
263    fn unparseable_json_yields_none() {
264        assert!(extract("not json at all", OutputFormat::Json).is_none());
265    }
266
267    #[test]
268    fn stream_json_takes_terminal_result_event() {
269        let raw = concat!(
270            "{\"type\":\"system\"}\n",
271            "{\"type\":\"assistant\",\"text\":\"thinking\"}\n",
272            "{\"type\":\"result\",\"result\":\"final\"}\n",
273        );
274        let got = extract(raw, OutputFormat::StreamJson).unwrap();
275        assert_eq!(got.text, "final");
276        assert_eq!(got.source, "stream-json:result");
277    }
278
279    #[test]
280    fn stream_json_falls_back_to_last_text_without_result() {
281        let raw = "{\"type\":\"assistant\",\"text\":\"first\"}\n{\"type\":\"assistant\",\"text\":\"second\"}\n";
282        let got = extract(raw, OutputFormat::StreamJson).unwrap();
283        assert_eq!(got.text, "second");
284        assert_eq!(got.source, "stream-json:text");
285    }
286
287    // A real `opencode run --format json` transcript (captured from OpenCode
288    // 1.17.3 against claude-haiku-4-5): JSONL with a `step_start`, a `text`
289    // event carrying the answer under `part.text`, and a `step_finish`. This is
290    // the shape that made `extract_json`'s single-document parse fail and left
291    // `text` null before the `opencode-parts` fallback.
292    const OPENCODE_RUN_JSONL: &str = concat!(
293        r#"{"type":"step_start","timestamp":1781179518088,"sessionID":"ses_1496d7d5effenFlBINeoCqBVk8","part":{"id":"prt_eb6928c85001I7CgUE9rTv6VJ0","messageID":"msg_eb69284ee001f9gKyD7PZudPCV","sessionID":"ses_1496d7d5effenFlBINeoCqBVk8","type":"step-start"}}"#,
294        "\n",
295        r#"{"type":"text","timestamp":1781179518140,"sessionID":"ses_1496d7d5effenFlBINeoCqBVk8","part":{"id":"prt_eb6928c87001tfso5uqcn63dxP","messageID":"msg_eb69284ee001f9gKyD7PZudPCV","sessionID":"ses_1496d7d5effenFlBINeoCqBVk8","type":"text","text":"PING-123","time":{"start":1781179518087,"end":1781179518139}}}"#,
296        "\n",
297        r#"{"type":"step_finish","timestamp":1781179518188,"sessionID":"ses_1496d7d5effenFlBINeoCqBVk8","part":{"id":"prt_eb6928ce6001n1g7PPB9g6pC3D","reason":"stop","messageID":"msg_eb69284ee001f9gKyD7PZudPCV","sessionID":"ses_1496d7d5effenFlBINeoCqBVk8","type":"step-finish","tokens":{"total":8186,"input":3,"output":7,"reasoning":0,"cache":{"write":8176,"read":0}},"cost":0.010258}}"#,
298        "\n",
299    );
300
301    #[test]
302    fn opencode_jsonl_extracts_text_part_under_json_format() {
303        // OpenCode requests `--format json` (OutputFormat::Json) but streams JSONL,
304        // so this exercises the fallback inside `extract_json`, not stream-json.
305        let got = extract(OPENCODE_RUN_JSONL, OutputFormat::Json).unwrap();
306        assert_eq!(got.text, "PING-123");
307        assert_eq!(got.source, "json:opencode-parts");
308    }
309
310    #[test]
311    fn opencode_jsonl_joins_text_parts_and_skips_tool_and_step_parts() {
312        // A real tool-using turn (captured the same way): two `text` parts around
313        // a `tool` part, plus step-start/step-finish. The final text is the two
314        // text parts joined in order; the tool/step parts are not the answer.
315        let raw = concat!(
316            r#"{"type":"step_start","sessionID":"ses_x","part":{"id":"p0","type":"step-start"}}"#,
317            "\n",
318            r#"{"type":"text","sessionID":"ses_x","part":{"id":"p1","type":"text","text":"I'll run that shell command for you."}}"#,
319            "\n",
320            r#"{"type":"tool_use","sessionID":"ses_x","part":{"id":"p2","type":"tool","tool":"bash","state":{"status":"completed","output":"HELLO-FROM-TOOL"}}}"#,
321            "\n",
322            r#"{"type":"text","sessionID":"ses_x","part":{"id":"p3","type":"text","text":"The command printed: `HELLO-FROM-TOOL`\n\nFINI-42"}}"#,
323            "\n",
324            r#"{"type":"step_finish","sessionID":"ses_x","part":{"id":"p4","type":"step-finish","cost":0.01,"tokens":{"input":3,"output":7}}}"#,
325            "\n",
326        );
327        let got = extract(raw, OutputFormat::Json).unwrap();
328        assert_eq!(
329            got.text,
330            "I'll run that shell command for you.\nThe command printed: `HELLO-FROM-TOOL`\n\nFINI-42"
331        );
332        assert_eq!(got.source, "json:opencode-parts");
333        // The tool's output is excluded, not surfaced as the answer.
334        assert!(!got.text.contains("step-finish"));
335    }
336
337    #[test]
338    fn opencode_jsonl_with_no_text_parts_yields_none() {
339        // A turn that produced only a tool call and step events has no visible
340        // answer to extract; `text` stays null and the consumer falls back to
341        // stdout rather than oneharness fabricating something.
342        let raw = concat!(
343            r#"{"type":"step_start","sessionID":"ses_x","part":{"id":"p0","type":"step-start"}}"#,
344            "\n",
345            r#"{"type":"step_finish","sessionID":"ses_x","part":{"id":"p1","type":"step-finish","cost":0.01}}"#,
346            "\n",
347        );
348        assert!(extract(raw, OutputFormat::Json).is_none());
349    }
350
351    #[test]
352    fn claude_single_document_still_wins_over_jsonl_fallback() {
353        // Guard the no-regression promise: a single-document `result` (Claude
354        // Code) must keep extracting via `json:result`, never the opencode path.
355        let raw = r#"{"type":"result","result":"the answer","is_error":false}"#;
356        let got = extract(raw, OutputFormat::Json).unwrap();
357        assert_eq!(got.text, "the answer");
358        assert_eq!(got.source, "json:result");
359    }
360
361    #[test]
362    fn opencode_parts_skip_blank_unparseable_and_partless_lines() {
363        // The JSONL scan must skip noise without crashing or mis-extracting: a
364        // blank line, a non-JSON line, a JSON object with no `part`, a `part`
365        // that is not a text part, and a text part whose text is whitespace —
366        // then still recover the one real answer that follows them.
367        let raw = concat!(
368            "\n",
369            "   \n",
370            "not json at all\n",
371            r#"{"type":"noise"}"#,
372            "\n",
373            r#"{"type":"reasoning","part":{"type":"reasoning","text":"thinking out loud"}}"#,
374            "\n",
375            r#"{"type":"text","part":{"type":"text","text":"   "}}"#,
376            "\n",
377            r#"{"type":"text","part":{"type":"text","text":"REAL-ANSWER"}}"#,
378            "\n",
379        );
380        let got = extract(raw, OutputFormat::Json).unwrap();
381        assert_eq!(got.text, "REAL-ANSWER");
382        assert_eq!(got.source, "json:opencode-parts");
383    }
384
385    #[test]
386    fn stream_json_skips_blank_and_unparseable_lines() {
387        // stream-json events interleaved with a blank line and a non-JSON line;
388        // the scan ignores both and still returns the last usable text.
389        let raw = concat!(
390            "\n",
391            "<<< not json >>>\n",
392            "{\"type\":\"assistant\",\"text\":\"only-line\"}\n",
393        );
394        let got = extract(raw, OutputFormat::StreamJson).unwrap();
395        assert_eq!(got.text, "only-line");
396        assert_eq!(got.source, "stream-json:text");
397    }
398
399    #[test]
400    fn stream_json_with_no_text_yields_none() {
401        // Events that carry no extractable text leave `text` null rather than
402        // fabricating an answer.
403        let raw = "{\"type\":\"system\"}\n{\"type\":\"ping\"}\n";
404        assert!(extract(raw, OutputFormat::StreamJson).is_none());
405    }
406
407    #[test]
408    fn json_text_ignores_non_string_and_empty_key_values() {
409        // A known key whose value is the wrong type (number) or blank is not a
410        // valid answer; a non-string/non-object document yields nothing at all.
411        assert!(extract(r#"{"result":42}"#, OutputFormat::Json).is_none());
412        assert!(extract(r#"{"result":"   "}"#, OutputFormat::Json).is_none());
413        assert!(extract("12345", OutputFormat::Json).is_none());
414    }
415}
oneharness_core/domain/normalize.rs

oneharness_core/domain/
normalize.rs
