use std::collections::{BTreeMap, BTreeSet};
use serde::{Deserialize, Serialize};
use crate::REPORT_VERSION;
use crate::findings::{Confidence, Finding, FindingKind, add_ms};
use crate::measured::{Measured, Source};
use crate::normalize::{Api, Normalized, SkipReason};
use crate::record::Record;
use crate::table::{Client, Row, RowKind, Table, ToolUsage};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Report {
pub v: u32,
pub window: Option<Window>,
pub coverage: Coverage,
pub servers: Vec<ServerFields>,
pub time: Time,
pub composition: CompositionTotals,
pub clients: Vec<Client>,
pub tools: Vec<ToolUsage>,
pub findings: Vec<Finding>,
pub rows: Vec<Row>,
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct Window {
pub start_ms: i64,
pub end_ms: i64,
pub days: f64,
}
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
pub struct Coverage {
pub records: u64,
pub analysed: u64,
pub skipped: SkippedCounts,
pub paths: BTreeMap<String, u64>,
pub bad_lines: u64,
pub turns: u64,
pub conversations: u64,
pub turns_with_reuse: u64,
pub turns_with_timing: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
pub struct SkippedCounts {
pub not_chat: u64,
pub error: u64,
pub non_2xx: u64,
pub unparsed: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ServerFields {
pub api: Api,
pub fields: Vec<CountField>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CountField {
PromptTokens,
CompletionTokens,
ReusedTokens,
EvaluatedTokens,
PrefillMs,
DecodeMs,
LoadMs,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Time {
pub prefill_ms: Option<Measured<f64>>,
pub lost_ms: Option<Measured<f64>>,
pub lost_tokens: u64,
pub by_cause: BTreeMap<String, CauseTotal>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CauseTotal {
pub tokens: u64,
pub ms: Option<Measured<f64>>,
pub rows: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
pub struct CompositionTotals {
pub system: u64,
pub tools: u64,
pub history: u64,
pub new_content: u64,
}
const DAY_MS: f64 = 86_400_000.0;
const SUMMARY_CAUSES: usize = 3;
const SUMMARY_FINDINGS: usize = 3;
const SUMMARY_CLIENTS: usize = 3;
pub(crate) fn assemble(
records: &[Record],
normalized: &Normalized,
table: Table,
findings: Vec<Finding>,
) -> Report {
let window = match (
records.iter().map(|r| r.t_start_ms).min(),
records.iter().map(|r| r.t_end_ms.max(r.t_start_ms)).max(),
) {
(Some(start_ms), Some(end_ms)) => Some(Window {
start_ms,
end_ms,
days: (end_ms - start_ms) as f64 / DAY_MS,
}),
_ => None,
};
Report {
v: REPORT_VERSION,
window,
coverage: coverage(records, normalized, &table.rows),
servers: servers(&table.rows),
time: time(&table.rows),
composition: composition(&table.rows),
clients: table.clients,
tools: table.tools,
findings,
rows: table.rows,
}
}
fn coverage(records: &[Record], normalized: &Normalized, rows: &[Row]) -> Coverage {
let mut coverage = Coverage {
records: records.len() as u64,
analysed: normalized.exchanges.len() as u64,
..Coverage::default()
};
for skipped in &normalized.skipped {
match skipped.reason {
SkipReason::NotChat => {
coverage.skipped.not_chat += 1;
let path = skipped.path.split('?').next().unwrap_or_default();
*coverage.paths.entry(path.to_string()).or_default() += 1;
}
SkipReason::Error => coverage.skipped.error += 1,
SkipReason::Non2xx => coverage.skipped.non_2xx += 1,
SkipReason::Unparsed => coverage.skipped.unparsed += 1,
}
}
coverage.conversations = rows
.iter()
.filter_map(|r| r.conversation)
.collect::<BTreeSet<_>>()
.len() as u64;
for row in rows.iter().filter(|r| r.kind == RowKind::Turn) {
coverage.turns += 1;
coverage.turns_with_reuse += u64::from(row.counts.reused_tokens.is_some());
coverage.turns_with_timing += u64::from(row.counts.prefill_ms.is_some());
}
coverage
}
fn servers(rows: &[Row]) -> Vec<ServerFields> {
let mut apis: BTreeMap<Api, BTreeSet<CountField>> = BTreeMap::new();
for row in rows {
let fields = apis.entry(row.api).or_default();
let c = &row.counts;
let reported = [
(CountField::PromptTokens, c.prompt_tokens.map(|m| m.source)),
(
CountField::CompletionTokens,
c.completion_tokens.map(|m| m.source),
),
(CountField::ReusedTokens, c.reused_tokens.map(|m| m.source)),
(
CountField::EvaluatedTokens,
c.evaluated_tokens.map(|m| m.source),
),
(CountField::PrefillMs, c.prefill_ms.map(|m| m.source)),
(CountField::DecodeMs, c.decode_ms.map(|m| m.source)),
(CountField::LoadMs, c.load_ms.map(|m| m.source)),
];
for (field, source) in reported {
if source == Some(Source::Server) {
fields.insert(field);
}
}
}
apis.into_iter()
.map(|(api, fields)| ServerFields {
api,
fields: fields.into_iter().collect(),
})
.collect()
}
fn time(rows: &[Row]) -> Time {
let mut prefill_ms = None;
let mut lost_ms: Option<Measured<f64>> = None;
let mut lost_tokens = 0u64;
let mut by_cause: BTreeMap<String, CauseTotal> = BTreeMap::new();
for row in rows {
if let Some(prefill) = row.counts.prefill_ms {
prefill_ms = Some(add_ms(prefill_ms, prefill));
}
if !row.loss_measured {
continue;
}
if let Some(prefill) = row.counts.prefill_ms {
lost_ms = Some(add_ms(
lost_ms,
Measured {
value: 0.0,
source: prefill.source,
},
));
}
for loss in [&row.lost_client, &row.lost_external].into_iter().flatten() {
lost_tokens += u64::from(loss.tokens);
let total = by_cause.entry(loss.cause.clone()).or_insert(CauseTotal {
tokens: 0,
ms: None,
rows: 0,
});
total.tokens += u64::from(loss.tokens);
total.rows += 1;
if let Some(ms) = loss.ms {
total.ms = Some(add_ms(total.ms, ms));
lost_ms = Some(add_ms(lost_ms, ms));
}
}
}
Time {
prefill_ms,
lost_ms,
lost_tokens,
by_cause,
}
}
fn composition(rows: &[Row]) -> CompositionTotals {
let mut totals = CompositionTotals::default();
for row in rows {
let c = &row.composition;
totals.system += u64::from(c.system.value);
totals.tools += u64::from(c.tools.value);
totals.history += u64::from(c.history.value);
totals.new_content += u64::from(c.new_content.value);
}
totals
}
impl Report {
pub fn summary(&self) -> String {
let mut lines = Vec::new();
let turns = self.coverage.turns;
let span = self
.window
.map(|w| {
format!(
" over {}",
duration(w.end_ms.saturating_sub(w.start_ms) as f64)
)
})
.unwrap_or_default();
lines.push(format!(
"llmp: {} requests{span}, {} analysed, {} in {}",
self.coverage.records,
self.coverage.analysed,
plural(turns, "turn"),
plural(self.coverage.conversations, "conversation")
));
let several = self.clients.len() > 1;
if several {
let shown: Vec<String> = self
.clients
.iter()
.take(SUMMARY_CLIENTS)
.map(|c| {
format!(
"{} ({}, {})",
c.client,
plural(u64::from(c.requests), "request"),
plural(u64::from(c.conversations), "conversation")
)
})
.collect();
let more = self.clients.len().saturating_sub(SUMMARY_CLIENTS);
let tail = if more > 0 {
format!(" · {more} more")
} else {
String::new()
};
lines.push(format!("clients: {}{tail}", shown.join(" · ")));
}
lines.push(self.time_line());
let mut causes: Vec<(&String, &CauseTotal)> = self.time.by_cause.iter().collect();
causes.sort_by(|a, b| {
let ms = |c: &CauseTotal| c.ms.map_or(-1.0, |m| m.value);
ms(b.1)
.total_cmp(&ms(a.1))
.then(b.1.tokens.cmp(&a.1.tokens))
.then(a.0.cmp(b.0))
});
if !causes.is_empty() {
let top: Vec<String> = causes
.iter()
.take(SUMMARY_CAUSES)
.map(|(cause, total)| {
format!("{} {}", cause_label(cause), size(total.tokens, total.ms))
})
.collect();
lines.push(format!(" by cause: {}", top.join(" · ")));
}
lines.push(self.coverage_line());
if !self.findings.is_empty() {
lines.push("findings:".into());
for finding in self.findings.iter().take(SUMMARY_FINDINGS) {
let client = match (&finding.client, several) {
(Some(client), true) => format!(" [{client}]"),
_ => String::new(),
};
lines.push(format!(
" [{}] {}{client}",
confidence_label(finding.confidence),
label(finding)
));
}
if self.findings.len() > SUMMARY_FINDINGS {
lines.push(format!(
" … {} more with --json",
self.findings.len() - SUMMARY_FINDINGS
));
}
}
let mut text = lines.join("\n");
text.push('\n');
text
}
fn time_line(&self) -> String {
match (self.time.lost_ms, self.time.prefill_ms) {
(Some(lost), Some(prefill)) if prefill.value > 0.0 => format!(
"time lost re-processing prompts: {}{} of {} prefill ({:.0}%), {} tokens",
duration(lost.value),
source_note(lost.source),
duration(prefill.value),
100.0 * lost.value / prefill.value,
thousands(self.time.lost_tokens)
),
(Some(lost), _) => format!(
"time lost re-processing prompts: {}{}, {} tokens",
duration(lost.value),
source_note(lost.source),
thousands(self.time.lost_tokens)
),
(None, _) => "time lost re-processing prompts: not measured (no turn had the server's reuse and timing counters)".into(),
}
}
fn coverage_line(&self) -> String {
let c = &self.coverage;
let mut skipped = Vec::new();
for (count, name) in [
(c.skipped.not_chat, "not chat"),
(c.skipped.error, "failed"),
(c.skipped.non_2xx, "non-2xx"),
(c.skipped.unparsed, "unreadable"),
] {
if count > 0 {
skipped.push(format!("{count} {name}"));
}
}
let mut line = format!("coverage: {} of {} records analysed", c.analysed, c.records);
if !skipped.is_empty() {
line.push_str(&format!(" ({})", skipped.join(", ")));
}
if c.bad_lines > 0 {
line.push_str(&format!(", {}", plural(c.bad_lines, "bad line")));
}
if c.turns > 0 {
line.push_str(&format!(
"; reuse reported on {} of {} turns, timing on {}",
c.turns_with_reuse, c.turns, c.turns_with_timing
));
}
line
}
}
fn label(finding: &Finding) -> String {
let model = finding.model.as_deref().unwrap_or("?");
let evidence = &finding.evidence;
let text = |key: &str| {
evidence
.get(key)
.and_then(|v| v.as_str())
.unwrap_or("?")
.to_string()
};
let number = |key: &str| evidence.get(key).and_then(|v| v.as_u64()).unwrap_or(0);
let lost = match (finding.lost_tokens, finding.lost_ms) {
(Some(tokens), ms) => format!(": {} lost", size(u64::from(tokens), ms)),
(None, _) => ", loss not measured".into(),
};
let context = finding
.context_tokens
.map(|t| thousands(u64::from(t)))
.unwrap_or_default();
match finding.kind {
FindingKind::SideCallOnConversationModel => format!(
"{} on the conversation model {model}{lost}",
plural(number("side_calls"), "side-call")
),
FindingKind::PrefixBreak => format!(
"prompt prefix broken by {} on {} of {} links ({model}){lost}",
cause_label(&text("break_cause")),
number("broken"),
number("links")
),
FindingKind::ServerCacheLoss => format!(
"{} on {} ({model}){lost}",
cause_label(&text("kind")),
plural(finding.rows.len() as u64, "turn")
),
FindingKind::OtherConversationEviction => format!(
"cache lost to other conversations on {} ({model}){lost}",
plural(finding.rows.len() as u64, "turn")
),
FindingKind::OtherModelEviction => format!(
"cache lost to other models on {} ({model}){lost}",
plural(finding.rows.len() as u64, "turn")
),
FindingKind::UnusedTool => format!(
"tool {} offered on {}, never called ({context} tokens each)",
text("tool"),
plural(number("offered"), "request")
),
FindingKind::LargeToolSchema => format!(
"tool {} schema is {} tokens ({context} over {})",
text("tool"),
thousands(number("schema_tokens")),
number("threshold")
),
FindingKind::LargeSystemPrompt => format!(
"system prompt of {model} is {} tokens ({context} over {})",
thousands(number("system_tokens")),
number("threshold")
),
}
}
fn cause_label(cause: &str) -> &str {
match cause {
"tools_changed" => "tool list changes",
"system_changed" => "system prompt changes",
"history_rewritten" => "history rewrites",
"side_call" => "side-calls",
"other_conversation" => "other conversations",
"other_model" => "other models",
"server_empty" => "server cache emptied",
"server_head_only" => "server kept only the head",
"server_rollback" => "server cache rolled back",
other => other,
}
}
fn confidence_label(confidence: Confidence) -> &'static str {
match confidence {
Confidence::High => "high",
Confidence::Medium => "medium",
Confidence::Low => "low",
}
}
fn source_note(source: Source) -> &'static str {
match source {
Source::Server | Source::Tokenizer => "",
Source::Proxy => " (proxy-timed, upper bound)",
Source::Estimate => " (estimated)",
}
}
fn size(tokens: u64, ms: Option<Measured<f64>>) -> String {
match ms {
Some(ms) => format!("{} ({} tokens)", duration(ms.value), thousands(tokens)),
None => format!("{} tokens", thousands(tokens)),
}
}
fn duration(ms: f64) -> String {
let ms = ms.max(0.0);
if ms < 1000.0 {
return format!("{ms:.0} ms");
}
let seconds = ms / 1000.0;
if seconds < 60.0 {
return format!("{seconds:.1} s");
}
let total = seconds.round() as u64;
let (hours, minutes, seconds) = (total / 3600, total / 60 % 60, total % 60);
if hours > 0 {
format!("{hours}h {minutes}m")
} else {
format!("{minutes}m {seconds}s")
}
}
fn plural(n: u64, noun: &str) -> String {
if n == 1 {
format!("1 {noun}")
} else {
format!("{} {noun}s", thousands(n))
}
}
fn thousands(n: u64) -> String {
let digits = n.to_string();
let mut out = String::new();
for (i, c) in digits.chars().enumerate() {
if i > 0 && (digits.len() - i).is_multiple_of(3) {
out.push(',');
}
out.push(c);
}
out
}