agent_sdk_eval/
usage.rs

1//! Deterministic usage reports derived from trace metrics.
2
3use serde::{Deserialize, Serialize};
4
5use agent_sdk_core::{AgentError, RunTrace};
6
7use crate::{EvaluationScope, TraceMetrics};
8
9#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
10/// Provider/tool usage report derived from durable trace evidence.
11pub struct UsageReport {
12    /// Scope this usage report describes.
13    pub scope: EvaluationScope,
14    /// Number of journal records inspected.
15    pub record_count: usize,
16    /// Number of runs represented in the source trace.
17    pub run_count: usize,
18    /// Number of turns represented in the source trace.
19    pub turn_count: usize,
20    /// Number of completed provider calls.
21    pub provider_call_count: u64,
22    /// Sum of provider input tokens.
23    pub provider_input_tokens: u64,
24    /// Sum of provider output tokens.
25    pub provider_output_tokens: u64,
26    /// Sum of provider total tokens.
27    pub provider_total_tokens: u64,
28    /// Number of distinct tool calls.
29    pub tool_call_count: u64,
30    /// Number of completed tool calls.
31    pub tool_completed_count: u64,
32    /// Number of failed, timed out, cancelled, denied, unknown, or recovery-required tool calls.
33    pub tool_non_success_count: u64,
34    /// Elapsed time across the scope when durable timestamps support it.
35    pub elapsed_ms: Option<u64>,
36    /// Sum of per-tool elapsed milliseconds when available.
37    pub tool_total_elapsed_ms: Option<u64>,
38    /// Limitations found while deriving usage.
39    pub limitations: Vec<String>,
40}
41
42impl UsageReport {
43    /// Builds a usage report from a run trace.
44    pub fn from_run_trace(trace: &RunTrace) -> Result<Self, AgentError> {
45        Self::from_trace_metrics(TraceMetrics::from_run_trace(trace)?)
46    }
47
48    /// Builds a usage report from precomputed trace metrics.
49    pub fn from_trace_metrics(metrics: TraceMetrics) -> Result<Self, AgentError> {
50        let mut limitations = Vec::new();
51        if metrics.record_count == 0 {
52            limitations.push("usage report has no journal records".to_string());
53        }
54        if metrics.provider_call_count > 0 && metrics.provider_total_tokens == 0 {
55            limitations.push("provider usage did not include token counts".to_string());
56        }
57        if metrics.elapsed_ms.is_none() {
58            limitations
59                .push("scope elapsed time is unavailable from durable timestamps".to_string());
60        }
61        Ok(Self {
62            scope: metrics.scope,
63            record_count: metrics.record_count,
64            run_count: metrics.run_count,
65            turn_count: metrics.turn_count,
66            provider_call_count: metrics.provider_call_count,
67            provider_input_tokens: metrics.provider_input_tokens,
68            provider_output_tokens: metrics.provider_output_tokens,
69            provider_total_tokens: metrics.provider_total_tokens,
70            tool_call_count: metrics.tool_call_count,
71            tool_completed_count: metrics.tool_completed_count,
72            tool_non_success_count: metrics
73                .tool_failed_count
74                .saturating_add(metrics.tool_timed_out_count)
75                .saturating_add(metrics.tool_cancelled_count)
76                .saturating_add(metrics.tool_denied_count)
77                .saturating_add(metrics.tool_unknown_count)
78                .saturating_add(metrics.tool_recovery_required_count),
79            elapsed_ms: metrics.elapsed_ms,
80            tool_total_elapsed_ms: metrics.tool_total_elapsed_ms,
81            limitations,
82        })
83    }
84}
agent_sdk_eval/usage.rs

agent_sdk_eval/
usage.rs
