verify_enrichment/
verify_enrichment.rs

1//! Live check of the RunEvent enrichment against the REAL agent CLIs.
2//!
3//! Runs a harness with a tool-triggering prompt and prints every `RunEvent`,
4//! so you can watch Session / Usage / ToolStart.input / ToolEnd.output arrive
5//! from the actual CLI wire format — not the synthetic JSON the unit tests
6//! use. (Claude's tool *input* is expected to be `None`: it streams args
7//! incrementally; codex delivers `command` inline, so it shows input.)
8//!
9//! `cargo run --example verify_enrichment -- [claude|codex|bob] ["prompt"]`
10//! (requires the CLI installed + signed in.)
11
12use std::sync::{mpsc::sync_channel, Arc};
13
14use harness::{default_registry, ReasoningEffort, RunCallback, RunEvent, RunMode, RunRequest, RunTuning};
15
16fn main() -> Result<(), String> {
17    let id = std::env::args().nth(1).unwrap_or_else(|| "claude".to_owned());
18    let prompt = std::env::args().nth(2).unwrap_or_else(|| default_prompt(&id));
19
20    let reg = default_registry();
21    let h = reg
22        .by_id(&id)
23        .ok_or_else(|| format!("unknown/disabled harness: {id}"))?;
24
25    // Cheapest viable settings (per the "don't burn tokens" rule).
26    let tuning = match id.as_str() {
27        "claude" => RunTuning { model: Some("haiku".to_owned()), ..RunTuning::default() },
28        "codex" => RunTuning { effort: Some(ReasoningEffort::Low), ..RunTuning::default() },
29        _ => RunTuning::default(),
30    };
31
32    let (tx, rx) = sync_channel::<RunEvent>(256);
33    let on_event: RunCallback = Arc::new(move |ev| {
34        let _ = tx.send(ev);
35    });
36
37    eprintln!("── running {id} ──");
38    let _handle = h.run(
39        RunRequest {
40            run_id: "verify".into(),
41            prompt,
42            cwd: Some(std::env::current_dir().map_err(|e| e.to_string())?),
43            mode: RunMode::Ask,
44            tuning,
45        },
46        on_event,
47    )
48    .map_err(|e| e.to_string())?;
49
50    let (mut session, mut tool_input, mut tool_output, mut usage) = (false, false, false, false);
51    for ev in rx {
52        print_event(&ev);
53        match &ev {
54            RunEvent::Session { session_id, model, .. } => {
55                session = session_id.is_some() || model.is_some();
56            }
57            RunEvent::ToolStart { input, .. } => tool_input |= input.is_some(),
58            RunEvent::ToolEnd { output, .. } => tool_output |= output.is_some(),
59            RunEvent::Usage { input_tokens, output_tokens, total_tokens, .. } => {
60                usage = input_tokens.is_some() || output_tokens.is_some() || total_tokens.is_some();
61            }
62            RunEvent::Exited { .. } => break,
63            _ => {}
64        }
65    }
66    eprintln!(
67        "\n── enrichment seen from real {id}: session={session} tool_input={tool_input} tool_output={tool_output} usage={usage}"
68    );
69    Ok(())
70}
71
72fn default_prompt(id: &str) -> String {
73    match id {
74        "codex" => "Run the command `ls` to list the current directory, then tell me how many entries there are in one short sentence.".to_owned(),
75        _ => "Use the Read tool to read the file Cargo.toml, then tell me the workspace resolver version in one short sentence.".to_owned(),
76    }
77}
78
79fn print_event(ev: &RunEvent) {
80    let line = match ev {
81        RunEvent::Text { delta, .. } => format!("Text({:?})", trunc(delta, 40)),
82        RunEvent::Thinking { delta, .. } => format!("Thinking({:?})", trunc(delta, 40)),
83        RunEvent::Session { session_id, model, .. } => {
84            format!("Session(session_id={session_id:?}, model={model:?})")
85        }
86        RunEvent::ToolStart { name, input, .. } => {
87            format!("ToolStart(name={name:?}, input={:?})", input.as_deref().map(|s| trunc(s, 60)))
88        }
89        RunEvent::ToolEnd { ok, output, .. } => {
90            format!("ToolEnd(ok={ok}, output={:?})", output.as_deref().map(|s| trunc(s, 60)))
91        }
92        RunEvent::Usage { input_tokens, output_tokens, total_tokens, .. } => {
93            format!("Usage(in={input_tokens:?}, out={output_tokens:?}, total={total_tokens:?})")
94        }
95        other => format!("{other:?}"),
96    };
97    eprintln!("  {line}");
98}
99
100fn trunc(s: &str, n: usize) -> String {
101    let s = s.replace('\n', "\\n");
102    if s.chars().count() > n {
103        format!("{}…", s.chars().take(n).collect::<String>())
104    } else {
105        s
106    }
107}
verify_enrichment/verify_enrichment.rs

verify_enrichment/
verify_enrichment.rs
