use std::cmp::Ordering;
use std::collections::{BTreeMap, BTreeSet};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value, json};
use crate::Config;
use crate::measured::{Measured, Source};
use crate::table::{BreakCause, Cause, Loss, Row, RowKind, Shape, Table};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Finding {
pub kind: FindingKind,
pub model: Option<String>,
pub client: Option<String>,
pub rows: Vec<u64>,
pub lost_tokens: Option<u32>,
pub lost_ms: Option<Measured<f64>>,
pub context_tokens: Option<u32>,
pub confidence: Confidence,
pub evidence: serde_json::Value,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FindingKind {
SideCallOnConversationModel,
PrefixBreak,
ServerCacheLoss,
OtherConversationEviction,
OtherModelEviction,
UnusedTool,
LargeToolSchema,
LargeSystemPrompt,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Confidence {
High,
Medium,
Low,
}
const PREFIX_BREAK_SHARE: f64 = 0.25;
const PREFIX_BREAK_MIN_SESSION: usize = 3;
const MIN_ROWS: usize = 2;
const HIGH_ROWS: usize = 5;
const HIGH_DAYS: f64 = 14.0;
const MEDIUM_DAYS: f64 = 7.0;
const HIGH_SESSIONS: usize = 5;
const DAY_MS: f64 = 86_400_000.0;
#[derive(Default)]
struct LossSum {
tokens: u64,
ms: Option<Measured<f64>>,
untimed: u32,
}
impl LossSum {
fn add(&mut self, loss: &Loss) {
self.tokens += u64::from(loss.tokens);
match loss.ms {
Some(ms) => self.ms = Some(add_ms(self.ms, ms)),
None => self.untimed += 1,
}
}
}
pub(crate) fn add_ms(total: Option<Measured<f64>>, ms: Measured<f64>) -> Measured<f64> {
match total {
Some(total) => Measured {
value: total.value + ms.value,
source: total.source.max(ms.source),
},
None => ms,
}
}
pub(crate) fn find(table: &Table, config: &Config) -> Vec<Finding> {
let models: BTreeSet<&str> = table.rows.iter().map(|r| r.model.as_str()).collect();
let by_id: BTreeMap<u64, &Row> = table.rows.iter().map(|r| (r.id, r)).collect();
let mut findings = Vec::new();
for model in models {
let rows: Vec<&Row> = table.rows.iter().filter(|r| r.model == model).collect();
findings.extend(side_call_on_conversation_model(model, &rows));
findings.extend(server_cache_losses(model, &rows));
findings.extend(other_conversation_eviction(model, &rows, &by_id));
findings.extend(other_model_eviction(model, &rows, &by_id));
let clients: BTreeSet<&str> = rows.iter().map(|r| r.client.as_str()).collect();
for client in clients {
let mine: Vec<&Row> = rows
.iter()
.copied()
.filter(|r| r.client == client)
.collect();
findings.extend(prefix_breaks(model, client, &mine));
findings.extend(large_system_prompt(
model,
client,
&mine,
config.thresholds.system_tokens,
));
}
}
findings.extend(tool_findings(table, config.thresholds.tool_schema_tokens));
findings.sort_by(order);
findings
}
fn loss_finding(
kind: FindingKind,
(model, client): (&str, Option<&str>),
rows: Vec<u64>,
measured: usize,
sum: LossSum,
mut evidence: Map<String, Value>,
) -> Finding {
if sum.untimed > 0 {
evidence.insert("untimed_rows".into(), Value::from(sum.untimed));
}
Finding {
kind,
model: Some(model.to_string()),
client: client.map(str::to_string),
rows,
lost_tokens: (measured > 0).then(|| u32::try_from(sum.tokens).unwrap_or(u32::MAX)),
lost_ms: sum.ms,
context_tokens: None,
confidence: counter_confidence(measured),
evidence: Value::Object(evidence),
}
}
fn counter_confidence(measured: usize) -> Confidence {
match measured {
0 => Confidence::Low,
n if n < HIGH_ROWS => Confidence::Medium,
_ => Confidence::High,
}
}
fn side_call_on_conversation_model(model: &str, rows: &[&Row]) -> Option<Finding> {
let side_calls: Vec<&Row> = rows
.iter()
.copied()
.filter(|r| r.kind == RowKind::SideCall)
.collect();
if side_calls.is_empty() || !rows.iter().any(|r| r.kind == RowKind::Turn) {
return None;
}
let after: Vec<&Row> = rows
.iter()
.copied()
.filter(|r| r.kind == RowKind::Turn && !r.side_calls_between.is_empty())
.collect();
let measured = after.iter().filter(|r| r.loss_measured).count();
let mut sum = LossSum::default();
for row in &after {
if row.cause == Some(Cause::SideCall)
&& let Some(loss) = &row.lost_external
{
sum.add(loss);
}
}
let mut shapes: BTreeMap<&str, u32> = BTreeMap::new();
for row in &side_calls {
if let Some(shape) = row.side_call_shape {
*shapes.entry(shape_name(shape)).or_default() += 1;
}
}
let clients: BTreeSet<&str> = side_calls.iter().map(|r| r.client.as_str()).collect();
let evidence = json!({
"shapes": shapes,
"side_calls": side_calls.len(),
"turns_after_side_calls": after.len(),
"clients": clients,
});
Some(loss_finding(
FindingKind::SideCallOnConversationModel,
(model, None),
side_calls.iter().map(|r| r.id).collect(),
measured,
sum,
object(evidence),
))
}
fn prefix_breaks(model: &str, client: &str, rows: &[&Row]) -> Vec<Finding> {
let mut session_sizes: BTreeMap<u32, usize> = BTreeMap::new();
for row in rows {
*session_sizes.entry(row.session).or_default() += 1;
}
let links: Vec<&Row> = rows
.iter()
.copied()
.filter(|r| {
r.kind == RowKind::Turn
&& r.linked_to.is_some()
&& !r.new_conversation
&& session_sizes
.get(&r.session)
.is_some_and(|&n| n >= PREFIX_BREAK_MIN_SESSION)
})
.collect();
let mut findings = Vec::new();
for cause in [
BreakCause::ToolsChanged,
BreakCause::SystemChanged,
BreakCause::HistoryRewritten,
] {
let broken: Vec<&Row> = links
.iter()
.copied()
.filter(|r| r.break_cause == Some(cause) && !r.compaction)
.collect();
let share = broken.len() as f64 / links.len().max(1) as f64;
if broken.is_empty() || share < PREFIX_BREAK_SHARE {
continue;
}
let mut sum = LossSum::default();
for loss in broken.iter().filter_map(|r| r.lost_client.as_ref()) {
sum.add(loss);
}
let evidence = json!({
"break_cause": cause,
"links": links.len(),
"broken": broken.len(),
"share": share,
});
findings.push(loss_finding(
FindingKind::PrefixBreak,
(model, Some(client)),
broken.iter().map(|r| r.id).collect(),
broken.iter().filter(|r| r.loss_measured).count(),
sum,
object(evidence),
));
}
findings
}
fn server_cache_losses(model: &str, rows: &[&Row]) -> Vec<Finding> {
["server_empty", "server_head_only", "server_rollback"]
.into_iter()
.filter_map(|kind| {
let hit: Vec<&Row> = rows
.iter()
.copied()
.filter(|r| r.lost_external.as_ref().is_some_and(|l| l.cause == kind))
.collect();
if hit.len() < MIN_ROWS {
return None;
}
let mut sum = LossSum::default();
for loss in hit.iter().filter_map(|r| r.lost_external.as_ref()) {
sum.add(loss);
}
Some(loss_finding(
FindingKind::ServerCacheLoss,
(model, None),
hit.iter().map(|r| r.id).collect(),
hit.len(),
sum,
object(json!({ "kind": kind })),
))
})
.collect()
}
fn other_conversation_eviction(
model: &str,
rows: &[&Row],
by_id: &BTreeMap<u64, &Row>,
) -> Option<Finding> {
let hit: Vec<&Row> = rows
.iter()
.copied()
.filter(|r| r.cause == Some(Cause::OtherConversation))
.collect();
if hit.len() < MIN_ROWS {
return None;
}
let mut sum = LossSum::default();
for loss in hit.iter().filter_map(|r| r.lost_external.as_ref()) {
sum.add(loss);
}
let others: Vec<&Row> = hit
.iter()
.flat_map(|r| &r.other_conversations_between)
.filter_map(|id| by_id.get(id).copied())
.collect();
let conversations: BTreeSet<u32> = others.iter().filter_map(|r| r.conversation).collect();
let clients: BTreeSet<&str> = hit
.iter()
.copied()
.chain(others.iter().copied())
.map(|r| r.client.as_str())
.collect();
Some(loss_finding(
FindingKind::OtherConversationEviction,
(model, None),
hit.iter().map(|r| r.id).collect(),
hit.iter().filter(|r| r.loss_measured).count(),
sum,
object(json!({ "other_conversations": conversations.len(), "clients": clients })),
))
}
fn other_model_eviction(
model: &str,
rows: &[&Row],
by_id: &BTreeMap<u64, &Row>,
) -> Option<Finding> {
let hit: Vec<&Row> = rows
.iter()
.copied()
.filter(|r| r.cause == Some(Cause::OtherModel))
.collect();
if hit.len() < MIN_ROWS {
return None;
}
let mut sum = LossSum::default();
for loss in hit.iter().filter_map(|r| r.lost_external.as_ref()) {
sum.add(loss);
}
let others: BTreeSet<&str> = hit
.iter()
.flat_map(|r| &r.other_domains_between)
.filter_map(|id| by_id.get(id).map(|r| r.model.as_str()))
.collect();
Some(loss_finding(
FindingKind::OtherModelEviction,
(model, None),
hit.iter().map(|r| r.id).collect(),
hit.iter().filter(|r| r.loss_measured).count(),
sum,
object(json!({ "other_models": others })),
))
}
fn large_system_prompt(
model: &str,
client: &str,
rows: &[&Row],
threshold: u32,
) -> Option<Finding> {
let over: Vec<&Row> = rows
.iter()
.copied()
.filter(|r| r.composition.system.value > threshold)
.collect();
let largest = over.iter().map(|r| r.composition.system.value).max()?;
let exact = over.iter().all(|r| server_sized(r));
Some(Finding {
kind: FindingKind::LargeSystemPrompt,
model: Some(model.to_string()),
client: Some(client.to_string()),
rows: over.iter().map(|r| r.id).collect(),
lost_tokens: None,
lost_ms: None,
context_tokens: Some(largest - threshold),
confidence: if exact {
Confidence::High
} else {
Confidence::Medium
},
evidence: json!({ "system_tokens": largest, "threshold": threshold }),
})
}
fn server_sized(row: &Row) -> bool {
row.counts.prompt_tokens.is_some() || row.composition.system.source == Source::Tokenizer
}
fn tool_findings(table: &Table, threshold: u32) -> Vec<Finding> {
let mut findings = Vec::new();
for tool in &table.tools {
let rows: Vec<&Row> = table
.rows
.iter()
.filter(|r| r.client == tool.client)
.collect();
let exact = rows
.iter()
.filter(|r| r.composition.tools.value > 0)
.all(|r| server_sized(r) || r.composition.tools.source == Source::Tokenizer);
let start = rows.iter().map(|r| r.t_start_ms).min();
let end = rows.iter().map(|r| r.t_start_ms).max();
let days = match (start, end) {
(Some(start), Some(end)) => (end - start) as f64 / DAY_MS,
_ => 0.0,
};
let sessions = rows
.iter()
.map(|r| r.session)
.collect::<BTreeSet<_>>()
.len();
let absence = if days >= HIGH_DAYS && sessions >= HIGH_SESSIONS {
Confidence::High
} else if days >= MEDIUM_DAYS {
Confidence::Medium
} else {
Confidence::Low
};
if tool.called == 0 && tool.offered > 0 {
findings.push(Finding {
kind: FindingKind::UnusedTool,
model: None,
client: Some(tool.client.clone()),
rows: Vec::new(),
lost_tokens: None,
lost_ms: None,
context_tokens: Some(tool.schema_tokens),
confidence: absence,
evidence: json!({ "tool": tool.name, "offered": tool.offered, "days": days, "sessions": sessions }),
});
}
if tool.schema_tokens > threshold {
findings.push(Finding {
kind: FindingKind::LargeToolSchema,
model: None,
client: Some(tool.client.clone()),
rows: Vec::new(),
lost_tokens: None,
lost_ms: None,
context_tokens: Some(tool.schema_tokens - threshold),
confidence: if exact { Confidence::High } else { Confidence::Medium },
evidence: json!({ "tool": tool.name, "schema_tokens": tool.schema_tokens, "threshold": threshold }),
});
}
}
findings
}
fn order(a: &Finding, b: &Finding) -> Ordering {
let ms = |f: &Finding| f.lost_ms.map_or(-1.0, |m| m.value);
a.confidence
.cmp(&b.confidence)
.then_with(|| ms(b).total_cmp(&ms(a)))
.then_with(|| {
b.context_tokens
.unwrap_or(0)
.cmp(&a.context_tokens.unwrap_or(0))
})
.then_with(|| (a.kind as u8).cmp(&(b.kind as u8)))
.then_with(|| a.model.cmp(&b.model))
.then_with(|| a.client.cmp(&b.client))
.then_with(|| a.evidence.to_string().cmp(&b.evidence.to_string()))
}
fn shape_name(shape: Shape) -> &'static str {
match shape {
Shape::Summarise => "summarise",
Shape::Structured => "structured",
Shape::Oneshot => "oneshot",
}
}
fn object(value: Value) -> Map<String, Value> {
match value {
Value::Object(map) => map,
_ => Map::new(),
}
}