verify_enrichment/
verify_enrichment.rs

1//! Live check of the RunEvent enrichment against the REAL agent CLIs.
2//!
3//! Runs a harness with a tool-triggering prompt and prints every `RunEvent`,
4//! so you can watch Session / Usage / ToolStart.input / ToolEnd.output arrive
5//! from the actual CLI wire format — not the synthetic JSON the unit tests
6//! use. (Claude's tool *input* is expected to be `None`: it streams args
7//! incrementally; codex delivers `command` inline, so it shows input.)
8//!
9//! `cargo run --example verify_enrichment -- [claude|codex|bob] ["prompt"]`
10//! (requires the CLI installed + signed in.)
11
12use std::sync::{mpsc::sync_channel, Arc};
13
14use harness::{default_registry, ReasoningEffort, RunCallback, RunEvent, RunMode, RunRequest, RunTuning};
15
16fn main() -> Result<(), String> {
17    let id = std::env::args().nth(1).unwrap_or_else(|| "claude".to_owned());
18    let prompt = std::env::args().nth(2).unwrap_or_else(|| default_prompt(&id));
19
20    let reg = default_registry();
21    let h = reg
22        .by_id(&id)
23        .ok_or_else(|| format!("unknown/disabled harness: {id}"))?;
24
25    // Cheapest viable settings (per the "don't burn tokens" rule).
26    let tuning = match id.as_str() {
27        "claude" => RunTuning { model: Some("haiku".to_owned()), ..RunTuning::default() },
28        "codex" => RunTuning { effort: Some(ReasoningEffort::Low), ..RunTuning::default() },
29        _ => RunTuning::default(),
30    };
31
32    let (tx, rx) = sync_channel::<RunEvent>(256);
33    let on_event: RunCallback = Arc::new(move |ev| {
34        let _ = tx.send(ev);
35    });
36
37    eprintln!("── running {id} ──");
38    let _handle = h.run(
39        RunRequest {
40            run_id: "verify".into(),
41            prompt,
42            cwd: Some(std::env::current_dir().map_err(|e| e.to_string())?),
43            mode: RunMode::Ask,
44            tuning,
45            resume: None,
46        },
47        on_event,
48    )
49    .map_err(|e| e.to_string())?;
50
51    let (mut session, mut tool_input, mut tool_output, mut usage) = (false, false, false, false);
52    for ev in rx {
53        print_event(&ev);
54        match &ev {
55            RunEvent::Session { session_id, model, .. } => {
56                session = session_id.is_some() || model.is_some();
57            }
58            RunEvent::ToolStart { input, .. } => tool_input |= input.is_some(),
59            RunEvent::ToolEnd { output, .. } => tool_output |= output.is_some(),
60            RunEvent::Usage { input_tokens, output_tokens, total_tokens, .. } => {
61                usage = input_tokens.is_some() || output_tokens.is_some() || total_tokens.is_some();
62            }
63            RunEvent::Exited { .. } => break,
64            _ => {}
65        }
66    }
67    eprintln!(
68        "\n── enrichment seen from real {id}: session={session} tool_input={tool_input} tool_output={tool_output} usage={usage}"
69    );
70    Ok(())
71}
72
73fn default_prompt(id: &str) -> String {
74    match id {
75        "codex" => "Run the command `ls` to list the current directory, then tell me how many entries there are in one short sentence.".to_owned(),
76        _ => "Use the Read tool to read the file Cargo.toml, then tell me the workspace resolver version in one short sentence.".to_owned(),
77    }
78}
79
80fn print_event(ev: &RunEvent) {
81    let line = match ev {
82        RunEvent::Text { delta, .. } => format!("Text({:?})", trunc(delta, 40)),
83        RunEvent::Thinking { delta, .. } => format!("Thinking({:?})", trunc(delta, 40)),
84        RunEvent::Session { session_id, model, .. } => {
85            format!("Session(session_id={session_id:?}, model={model:?})")
86        }
87        RunEvent::ToolStart { name, input, .. } => {
88            format!("ToolStart(name={name:?}, input={:?})", input.as_deref().map(|s| trunc(s, 60)))
89        }
90        RunEvent::ToolEnd { ok, output, .. } => {
91            format!("ToolEnd(ok={ok}, output={:?})", output.as_deref().map(|s| trunc(s, 60)))
92        }
93        RunEvent::Usage { input_tokens, output_tokens, total_tokens, .. } => {
94            format!("Usage(in={input_tokens:?}, out={output_tokens:?}, total={total_tokens:?})")
95        }
96        other => format!("{other:?}"),
97    };
98    eprintln!("  {line}");
99}
100
101fn trunc(s: &str, n: usize) -> String {
102    let s = s.replace('\n', "\\n");
103    if s.chars().count() > n {
104        format!("{}…", s.chars().take(n).collect::<String>())
105    } else {
106        s
107    }
108}
verify_enrichment/verify_enrichment.rs

verify_enrichment/
verify_enrichment.rs
