warden/reports/
tools.rs

1//! `warden report tools` — tool call frequency, and the failure rate warden
2//! deliberately refuses to invent.
3//!
4//! The ingested event carries the `tool_use` blocks a model emitted; it does not
5//! carry the tool *result*, so whether a call succeeded is not in the store.
6//! A `0%` failure rate would be a fabrication, so the column renders `–` and a
7//! note says why: unsupported columns are greyed out rather than printing a
8//! misleading `0`.
9
10use std::collections::BTreeMap;
11
12use crate::output::{Cell, Report, Table};
13use crate::store::Scanner;
14
15use super::{count, scan, ReportCtx, ReportError};
16
17#[derive(Default)]
18struct ToolUse {
19    calls: u64,
20    events: u64,
21    targets: std::collections::BTreeSet<String>,
22}
23
24pub fn build(scanner: &Scanner, ctx: &ReportCtx) -> Result<Report, ReportError> {
25    let scanned = scan(scanner, ctx)?;
26
27    let mut by_tool: BTreeMap<String, ToolUse> = BTreeMap::new();
28    let mut total_calls = 0u64;
29    for event in &scanned.events {
30        let mut seen: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new();
31        for call in &event.tool_calls {
32            total_calls += 1;
33            let entry = by_tool.entry(call.tool_name.clone()).or_default();
34            entry.calls += 1;
35            if let Some(target) = &call.tool_target {
36                entry.targets.insert(target.clone());
37            }
38            if seen.insert(call.tool_name.as_str()) {
39                entry.events += 1;
40            }
41        }
42    }
43
44    let mut ranked: Vec<(String, ToolUse)> = by_tool.into_iter().collect();
45    ranked.sort_by(|a, b| b.1.calls.cmp(&a.1.calls).then_with(|| a.0.cmp(&b.0)));
46
47    let mut table = Table::new(["tool", "calls", "share", "turns", "targets", "fail rate"]);
48    let mut rows = Vec::new();
49
50    for (tool, use_) in &ranked {
51        let share = if total_calls == 0 {
52            0.0
53        } else {
54            use_.calls as f64 * 100.0 / total_calls as f64
55        };
56        table.push(vec![
57            Cell::text(tool),
58            count(use_.calls),
59            Cell::Float(share, 1),
60            count(use_.events),
61            count(use_.targets.len() as u64),
62            // Not derivable from what is ingested. Never `0%`.
63            Cell::Unsupported,
64        ]);
65        rows.push(serde_json::json!({
66            "tool": tool,
67            "calls": use_.calls,
68            "share_pct": (share * 10.0).round() / 10.0,
69            "turns": use_.events,
70            "distinct_targets": use_.targets.len(),
71            "failures": serde_json::Value::Null,
72            "fail_rate": serde_json::Value::Null,
73        }));
74    }
75
76    let mut notes = scanned.notes;
77    notes.push(
78        "fail rate is blank, not 0%: warden ingests the tool calls a model made, not the results \
79         they returned, so success and failure are not in the store and would have to be invented",
80    );
81    notes.push(
82        "share is the percentage of all tool calls in the period; turns counts the events that \
83         used the tool at least once",
84    );
85
86    Ok(Report::new("tools", ctx.window, table)
87        .with_json_rows(rows)
88        .with_notes(notes.finish()))
89}
90
91#[cfg(test)]
92mod tests {
93    use super::super::testkit::*;
94    use super::*;
95    use crate::cli::TimeWindow;
96    use crate::output::{Style, UNSUPPORTED};
97
98    fn report() -> Report {
99        let (_dir, paths) = store(&[
100            with_tools(
101                used("a", ms(2026, 8, 4, 8), "acme", "m", 10, 1),
102                &[("Read", "src/lib.rs"), ("Read", "src/main.rs")],
103            ),
104            with_tools(
105                used("b", ms(2026, 8, 4, 9), "acme", "m", 10, 1),
106                &[("Edit", "src/lib.rs")],
107            ),
108        ]);
109        build(
110            &Scanner::new(paths),
111            &ReportCtx::new(TimeWindow::all(), None, true),
112        )
113        .unwrap()
114    }
115
116    #[test]
117    fn counts_calls_and_ranks_the_busiest_tool_first() {
118        let report = report();
119        assert_eq!(report.json_rows[0]["tool"], "Read");
120        assert_eq!(report.json_rows[0]["calls"], 2);
121        assert_eq!(report.json_rows[0]["turns"], 1, "two calls in one event");
122        assert_eq!(report.json_rows[0]["distinct_targets"], 2);
123        assert_eq!(report.json_rows[0]["share_pct"], 66.7);
124        assert_eq!(report.json_rows[1]["tool"], "Edit");
125    }
126
127    #[test]
128    fn failure_is_null_and_never_a_fabricated_zero() {
129        let report = report();
130        for row in &report.json_rows {
131            assert!(row["fail_rate"].is_null());
132            assert!(row["failures"].is_null());
133        }
134        let rendered = report.table.render(Style::plain());
135        assert!(rendered.contains(UNSUPPORTED), "{rendered}");
136        assert!(!rendered.contains("0.0%"), "{rendered}");
137        assert!(!rendered.contains(" 0%"), "{rendered}");
138        assert!(
139            report
140                .notes
141                .iter()
142                .any(|n| n.contains("fail rate is blank, not 0%")),
143            "{:?}",
144            report.notes
145        );
146    }
147
148    #[test]
149    fn a_period_with_no_tool_calls_renders_an_empty_table_not_a_panic() {
150        let (_dir, paths) = store(&[used("a", ms(2026, 8, 4, 8), "acme", "m", 10, 1)]);
151        let report = build(
152            &Scanner::new(paths),
153            &ReportCtx::new(TimeWindow::all(), None, true),
154        )
155        .unwrap();
156        assert!(report.table.is_empty());
157        assert_eq!(report.table.render(Style::plain()).lines().count(), 1);
158    }
159}
warden/reports/tools.rs

warden/reports/
tools.rs
