use std::collections::{BTreeMap, BTreeSet};
use serde::Serialize;
use super::tool_conformance::{
ToolConformanceCase, ToolConformanceReport, ToolProbeClassification, ToolProbeFallbackMode,
};
pub const TOOL_SCORECARD_SCHEMA_VERSION: u32 = 1;
pub const TOOL_SCORECARD_PLAN_SCHEMA_VERSION: u32 = 1;
#[derive(Debug, Clone, Serialize)]
pub struct ToolScorecardReport {
pub schema_version: u32,
pub route_count: usize,
pub summary: ToolScorecardSummary,
pub routes: Vec<ToolScorecardRoute>,
}
#[derive(Debug, Clone, Serialize)]
pub struct ToolScorecardSummary {
pub pass: usize,
pub warn: usize,
pub fail: usize,
pub best_route: Option<ToolScorecardRouteKey>,
}
#[derive(Debug, Clone, Serialize)]
pub struct ToolScorecardRouteKey {
pub provider: String,
pub model: String,
pub quality_score: u16,
}
#[derive(Debug, Clone, Serialize)]
pub struct ToolScorecardRoute {
pub provider: String,
pub model: String,
pub report_count: usize,
pub case_count: usize,
pub successful_cases: usize,
pub parseable_tool_call_cases: usize,
pub native_tool_call_cases: usize,
pub text_tool_call_cases: usize,
pub actionless_cases: usize,
pub empty_completion_cases: usize,
pub malformed_argument_cases: usize,
pub http_error_cases: usize,
pub transport_error_cases: usize,
pub pass_rate: f64,
pub parseable_tool_call_rate: f64,
pub empty_completion_rate: f64,
pub actionless_rate: f64,
pub quality_score: u16,
pub status: &'static str,
pub recommended_tool_mode: &'static str,
pub observed_wire_dialects: Vec<&'static str>,
pub classification_counts: BTreeMap<&'static str, usize>,
pub issues: Vec<&'static str>,
}
#[derive(Debug, Clone, Serialize)]
pub struct ToolScorecardPlan {
pub schema_version: u32,
pub kind: &'static str,
pub catalog: ToolScorecardCatalogProvenance,
pub route_count: usize,
pub case_count: usize,
pub required_case_count: usize,
pub batch_manifest_request_count: usize,
pub routes: Vec<ToolScorecardPlanRoute>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub batch_manifest_requests: Vec<ToolScorecardBatchManifestRequest>,
}
#[derive(Debug, Clone, Serialize)]
pub struct ToolScorecardCatalogProvenance {
pub schema_version: u32,
pub generated_by: String,
pub hash_blake3: String,
}
#[derive(Debug, Clone, Serialize)]
pub struct ToolScorecardPlanRoute {
pub provider: String,
pub model: String,
pub catalog_claim: ToolScorecardCatalogClaim,
pub cases: Vec<ToolScorecardPlanCase>,
}
#[derive(Debug, Clone, Serialize)]
pub struct ToolScorecardCatalogClaim {
pub preferred_tool_format: Option<String>,
pub tool_mode_parity: Option<String>,
pub native_tools: bool,
pub text_tools: bool,
pub text_tool_wire_format_supported: bool,
pub max_tools: Option<u32>,
pub supports_parallel_tool_calls: bool,
pub server_parser: String,
pub tool_search: Vec<String>,
pub batch_api: bool,
pub batch_wire_format: Option<String>,
pub batch_input_mode: Option<String>,
pub batch_discount_percent: Option<u32>,
pub provider_rate_limits: bool,
pub model_rate_limits: bool,
pub provider_rpm: Option<u32>,
pub pricing: bool,
pub provider_latency_p50_ms: Option<u64>,
}
#[derive(Debug, Clone, Serialize)]
pub struct ToolScorecardPlanCase {
pub id: &'static str,
pub description: &'static str,
pub requirement: &'static str,
pub requirement_reason: &'static str,
pub turn_count: u8,
pub batch_eligible: bool,
pub probe_focus: Vec<&'static str>,
}
#[derive(Debug, Clone, Serialize)]
pub struct ToolScorecardBatchManifestRequest {
pub request_id: String,
pub provider: String,
pub model: String,
pub case_id: &'static str,
pub batch_wire_format: Option<String>,
pub batch_input_mode: Option<String>,
}
#[derive(Debug, Default)]
struct RouteAccumulator {
provider: String,
model: String,
report_count: usize,
cases: Vec<ToolConformanceCase>,
}
pub fn scorecard_from_tool_reports(reports: Vec<ToolConformanceReport>) -> ToolScorecardReport {
let mut grouped: BTreeMap<(String, String), RouteAccumulator> = BTreeMap::new();
for report in reports {
let key = (report.provider.clone(), report.model.clone());
let entry = grouped.entry(key).or_insert_with(|| RouteAccumulator {
provider: report.provider,
model: report.model,
report_count: 0,
cases: Vec::new(),
});
entry.report_count += 1;
entry.cases.extend(report.cases);
}
let mut routes = grouped
.into_values()
.map(score_route)
.collect::<Vec<ToolScorecardRoute>>();
routes.sort_by(|left, right| {
right
.quality_score
.cmp(&left.quality_score)
.then_with(|| left.provider.cmp(&right.provider))
.then_with(|| left.model.cmp(&right.model))
});
let mut summary = ToolScorecardSummary {
pass: 0,
warn: 0,
fail: 0,
best_route: routes.first().map(|route| ToolScorecardRouteKey {
provider: route.provider.clone(),
model: route.model.clone(),
quality_score: route.quality_score,
}),
};
for route in &routes {
match route.status {
"pass" => summary.pass += 1,
"warn" => summary.warn += 1,
_ => summary.fail += 1,
}
}
ToolScorecardReport {
schema_version: TOOL_SCORECARD_SCHEMA_VERSION,
route_count: routes.len(),
summary,
routes,
}
}
pub fn tool_scorecard_plan_from_catalog(
route_filters: &[String],
include_batch_manifest: bool,
) -> Result<ToolScorecardPlan, String> {
let artifact = crate::provider_catalog::artifact();
let catalog_json = serde_json::to_vec(&artifact)
.map_err(|error| format!("error: failed to serialize provider catalog: {error}"))?;
let catalog_hash = format!("blake3:{}", blake3::hash(&catalog_json));
let requested_routes = parse_route_filters(route_filters)?;
let provider_by_id = artifact
.providers
.iter()
.map(|provider| (provider.id.as_str(), provider))
.collect::<BTreeMap<_, _>>();
let mut seen_routes = BTreeSet::new();
let mut plan_routes = Vec::new();
let mut batch_manifest_requests = Vec::new();
for model in &artifact.models {
let route_key = (model.provider.clone(), model.id.clone());
if !requested_routes.is_empty() && !requested_routes.contains(&route_key) {
continue;
}
seen_routes.insert(route_key);
let caps = crate::llm::capabilities::lookup(&model.provider, &model.id);
let provider = provider_by_id.get(model.provider.as_str());
let claim = ToolScorecardCatalogClaim {
preferred_tool_format: caps
.preferred_tool_format
.clone()
.or_else(|| model.tool_support.preferred_format.clone()),
tool_mode_parity: caps
.tool_mode_parity
.clone()
.or_else(|| model.tool_support.parity.clone()),
native_tools: caps.native_tools || model.tool_support.native,
text_tools: model.tool_support.text,
text_tool_wire_format_supported: caps.text_tool_wire_format_supported,
max_tools: caps.max_tools.or(model.tool_support.max_tools),
supports_parallel_tool_calls: caps.supports_parallel_tool_calls,
server_parser: caps.server_parser,
tool_search: caps.tool_search,
batch_api: caps.batch_api || model.batch.is_some(),
batch_wire_format: caps
.batch_wire_format
.clone()
.or_else(|| model.batch.as_ref().map(|batch| batch.wire_format.clone())),
batch_input_mode: caps
.batch_input_mode
.clone()
.or_else(|| model.batch.as_ref().map(|batch| batch.input_mode.clone())),
batch_discount_percent: caps.batch_discount_percent.or_else(|| {
model
.batch
.as_ref()
.and_then(|batch| batch.discount_percent)
}),
provider_rate_limits: provider
.and_then(|provider| provider.rate_limits.as_ref())
.is_some(),
model_rate_limits: model.rate_limits.is_some(),
provider_rpm: provider.and_then(|provider| provider.rpm),
pricing: model.pricing.is_some(),
provider_latency_p50_ms: provider.and_then(|provider| provider.latency_p50_ms),
};
let cases = fixed_micro_cases_for_claim(&claim);
if include_batch_manifest && claim.batch_api {
for case in &cases {
if !case.batch_eligible {
continue;
}
batch_manifest_requests.push(ToolScorecardBatchManifestRequest {
request_id: format!(
"tool-scorecard:{}:{}:{}",
model.provider, model.id, case.id
),
provider: model.provider.clone(),
model: model.id.clone(),
case_id: case.id,
batch_wire_format: claim.batch_wire_format.clone(),
batch_input_mode: claim.batch_input_mode.clone(),
});
}
}
plan_routes.push(ToolScorecardPlanRoute {
provider: model.provider.clone(),
model: model.id.clone(),
catalog_claim: claim,
cases,
});
}
if !requested_routes.is_empty() {
let missing = requested_routes
.difference(&seen_routes)
.map(|(provider, model)| format!("{provider}:{model}"))
.collect::<Vec<_>>();
if !missing.is_empty() {
return Err(format!(
"error: route(s) not found in provider catalog: {}",
missing.join(", ")
));
}
}
let case_count = plan_routes.iter().map(|route| route.cases.len()).sum();
let required_case_count = plan_routes
.iter()
.flat_map(|route| &route.cases)
.filter(|case| case.requirement == "required")
.count();
Ok(ToolScorecardPlan {
schema_version: TOOL_SCORECARD_PLAN_SCHEMA_VERSION,
kind: "plan",
catalog: ToolScorecardCatalogProvenance {
schema_version: artifact.schema_version,
generated_by: artifact.generated_by,
hash_blake3: catalog_hash,
},
route_count: plan_routes.len(),
case_count,
required_case_count,
batch_manifest_request_count: batch_manifest_requests.len(),
routes: plan_routes,
batch_manifest_requests,
})
}
fn parse_route_filters(filters: &[String]) -> Result<BTreeSet<(String, String)>, String> {
let mut routes = BTreeSet::new();
for filter in filters {
let Some((provider, model)) = filter.split_once(':') else {
return Err(format!(
"error: route filter '{filter}' must use provider:model"
));
};
let provider = provider.trim();
let model = model.trim();
if provider.is_empty() || model.is_empty() {
return Err(format!(
"error: route filter '{filter}' must include both provider and model"
));
}
routes.insert((provider.to_string(), model.to_string()));
}
Ok(routes)
}
fn fixed_micro_cases_for_claim(claim: &ToolScorecardCatalogClaim) -> Vec<ToolScorecardPlanCase> {
let parallel_requirement = if claim.supports_parallel_tool_calls {
("required", "route_claims_parallel_tool_calls")
} else {
("not_applicable", "route_does_not_claim_parallel_tool_calls")
};
vec![
ToolScorecardPlanCase {
id: "single_tool_call",
description: "single ordinary tool call with exact JSON arguments",
requirement: "required",
requirement_reason: "baseline_tool_call_quality",
turn_count: 1,
batch_eligible: true,
probe_focus: vec!["tool_choice", "json_arguments", "wire_dialect"],
},
ToolScorecardPlanCase {
id: "parallel_tool_calls",
description: "multiple tool calls in one assistant response",
requirement: parallel_requirement.0,
requirement_reason: parallel_requirement.1,
turn_count: 1,
batch_eligible: true,
probe_focus: vec!["parallel_dispatch", "tool_call_count", "argument_binding"],
},
ToolScorecardPlanCase {
id: "large_string_argument",
description: "large string/code argument with quotes, unicode, and heredoc-shaped text",
requirement: "required",
requirement_reason: "byte_fidelity_for_edit_payloads",
turn_count: 1,
batch_eligible: true,
probe_focus: vec!["byte_fidelity", "escaping", "unicode"],
},
ToolScorecardPlanCase {
id: "tool_result_followup",
description: "assistant continuation after receiving a tool result",
requirement: "required",
requirement_reason: "multi_turn_agent_loop_quality",
turn_count: 2,
batch_eligible: false,
probe_focus: vec!["tool_result_adjacency", "continuation", "action_vs_prose"],
},
ToolScorecardPlanCase {
id: "no_tool_answer_or_refusal",
description: "plain answer or refusal when no tool should be called",
requirement: "required",
requirement_reason: "spurious_tool_call_guard",
turn_count: 1,
batch_eligible: true,
probe_focus: vec!["no_tool", "refusal", "answer_quality"],
},
ToolScorecardPlanCase {
id: "unavailable_tool_repair",
description: "repair or reject a request for an unavailable tool",
requirement: "required",
requirement_reason: "unsafe_or_unavailable_tool_recovery",
turn_count: 1,
batch_eligible: true,
probe_focus: vec!["tool_name_repair", "no_unsafe_args", "recovery"],
},
ToolScorecardPlanCase {
id: "done_sentinel",
description: "completion-contract done sentinel emission",
requirement: "required",
requirement_reason: "agent_completion_contract",
turn_count: 1,
batch_eligible: true,
probe_focus: vec!["done_sentinel", "completion_contract"],
},
ToolScorecardPlanCase {
id: "parameter_edges",
description: "provider-advertised parameter edge behavior",
requirement: "required",
requirement_reason: "catalog_parameter_claims",
turn_count: 1,
batch_eligible: true,
probe_focus: vec!["temperature", "max_tokens", "tool_choice"],
},
]
}
fn score_route(acc: RouteAccumulator) -> ToolScorecardRoute {
let case_count = acc.cases.len();
let mut successful_cases = 0;
let mut parseable_tool_call_cases = 0;
let mut native_tool_call_cases = 0;
let mut text_tool_call_cases = 0;
let mut actionless_cases = 0;
let mut empty_completion_cases = 0;
let mut malformed_argument_cases = 0;
let mut http_error_cases = 0;
let mut transport_error_cases = 0;
let mut observed_wire_dialects = BTreeSet::new();
let mut classification_counts = BTreeMap::new();
for case in &acc.cases {
*classification_counts
.entry(classification_key(&case.classification))
.or_insert(0) += 1;
observed_wire_dialects.insert(wire_dialect_key(&case.classification));
if case.ok {
successful_cases += 1;
}
match case.classification {
ToolProbeClassification::StructuredNativeToolCall => {
parseable_tool_call_cases += 1;
native_tool_call_cases += 1;
}
ToolProbeClassification::ParseableHarnTextToolCall => {
parseable_tool_call_cases += 1;
text_tool_call_cases += 1;
}
ToolProbeClassification::ProseOnlyNonTool => {
actionless_cases += 1;
}
ToolProbeClassification::EmptySilent => {
actionless_cases += 1;
empty_completion_cases += 1;
}
ToolProbeClassification::MalformedJsonArguments => {
malformed_argument_cases += 1;
}
ToolProbeClassification::HttpError => {
http_error_cases += 1;
}
ToolProbeClassification::TransportError => {
transport_error_cases += 1;
}
ToolProbeClassification::RawModelToolTag => {}
}
}
let pass_rate = rate(successful_cases, case_count);
let parseable_tool_call_rate = rate(parseable_tool_call_cases, case_count);
let empty_completion_rate = rate(empty_completion_cases, case_count);
let actionless_rate = rate(actionless_cases, case_count);
let quality_score = ((pass_rate * 100.0).round() as u16).min(100);
let recommended_tool_mode = recommended_tool_mode(native_tool_call_cases, text_tool_call_cases);
let issues = route_issues(
case_count,
successful_cases,
recommended_tool_mode,
actionless_cases,
malformed_argument_cases,
http_error_cases,
transport_error_cases,
);
let status = route_status(recommended_tool_mode, successful_cases, case_count, &issues);
ToolScorecardRoute {
provider: acc.provider,
model: acc.model,
report_count: acc.report_count,
case_count,
successful_cases,
parseable_tool_call_cases,
native_tool_call_cases,
text_tool_call_cases,
actionless_cases,
empty_completion_cases,
malformed_argument_cases,
http_error_cases,
transport_error_cases,
pass_rate,
parseable_tool_call_rate,
empty_completion_rate,
actionless_rate,
quality_score,
status,
recommended_tool_mode,
observed_wire_dialects: observed_wire_dialects.into_iter().collect(),
classification_counts,
issues,
}
}
fn rate(numerator: usize, denominator: usize) -> f64 {
if denominator == 0 {
0.0
} else {
numerator as f64 / denominator as f64
}
}
fn recommended_tool_mode(native_cases: usize, text_cases: usize) -> &'static str {
if native_cases > 0 {
ToolProbeFallbackMode::Native.as_str()
} else if text_cases > 0 {
ToolProbeFallbackMode::Text.as_str()
} else {
ToolProbeFallbackMode::Disabled.as_str()
}
}
fn route_status(
recommended_tool_mode: &str,
successful_cases: usize,
case_count: usize,
issues: &[&'static str],
) -> &'static str {
if recommended_tool_mode == "disabled" || case_count == 0 || successful_cases == 0 {
"fail"
} else if successful_cases < case_count || !issues.is_empty() {
"warn"
} else {
"pass"
}
}
fn route_issues(
case_count: usize,
successful_cases: usize,
recommended_tool_mode: &str,
actionless_cases: usize,
malformed_argument_cases: usize,
http_error_cases: usize,
transport_error_cases: usize,
) -> Vec<&'static str> {
let mut issues = Vec::new();
if case_count == 0 {
issues.push("no_cases");
}
if recommended_tool_mode == "disabled" {
issues.push("tool_calling_disabled");
}
if successful_cases > 0 && successful_cases < case_count {
issues.push("partial_tool_call_pass_rate");
}
if actionless_cases > 0 {
issues.push("empty_or_actionless_completion");
}
if malformed_argument_cases > 0 {
issues.push("malformed_tool_arguments");
}
if http_error_cases > 0 {
issues.push("provider_http_errors");
}
if transport_error_cases > 0 {
issues.push("transport_errors");
}
issues
}
fn classification_key(classification: &ToolProbeClassification) -> &'static str {
match classification {
ToolProbeClassification::StructuredNativeToolCall => "structured_native_tool_call",
ToolProbeClassification::ParseableHarnTextToolCall => "parseable_harn_text_tool_call",
ToolProbeClassification::RawModelToolTag => "raw_model_tool_tag",
ToolProbeClassification::ProseOnlyNonTool => "prose_only_non_tool",
ToolProbeClassification::MalformedJsonArguments => "malformed_json_arguments",
ToolProbeClassification::EmptySilent => "empty_silent",
ToolProbeClassification::HttpError => "http_error",
ToolProbeClassification::TransportError => "transport_error",
}
}
fn wire_dialect_key(classification: &ToolProbeClassification) -> &'static str {
match classification {
ToolProbeClassification::StructuredNativeToolCall => "native_tool_calls",
ToolProbeClassification::ParseableHarnTextToolCall => "harn_text_tool_calls",
ToolProbeClassification::RawModelToolTag => "raw_model_tool_tag",
ToolProbeClassification::ProseOnlyNonTool => "prose",
ToolProbeClassification::MalformedJsonArguments => "malformed_tool_args",
ToolProbeClassification::EmptySilent => "empty",
ToolProbeClassification::HttpError => "http_error",
ToolProbeClassification::TransportError => "transport_error",
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::llm::tool_conformance::{
ToolCallingConformanceSummary, ToolProbeMode, ToolProbeStatus,
};
#[test]
fn scorecard_ranks_successful_native_route_first() {
let pass = report(
"anthropic",
"claude",
vec![case(
ToolProbeClassification::StructuredNativeToolCall,
true,
)],
);
let fail = report(
"fireworks",
"gpt-oss",
vec![case(ToolProbeClassification::EmptySilent, false)],
);
let scorecard = scorecard_from_tool_reports(vec![fail, pass]);
assert_eq!(scorecard.route_count, 2);
assert_eq!(scorecard.summary.pass, 1);
assert_eq!(scorecard.summary.fail, 1);
assert_eq!(scorecard.routes[0].provider, "anthropic");
assert_eq!(scorecard.routes[0].status, "pass");
assert_eq!(scorecard.routes[0].recommended_tool_mode, "native");
assert_eq!(
scorecard.routes[1].issues,
vec!["tool_calling_disabled", "empty_or_actionless_completion"]
);
}
#[test]
fn scorecard_plan_filters_catalog_routes_and_names_required_cases() {
let plan =
tool_scorecard_plan_from_catalog(&[String::from("anthropic:claude-sonnet-4-6")], true)
.expect("plan from catalog");
assert_eq!(plan.schema_version, TOOL_SCORECARD_PLAN_SCHEMA_VERSION);
assert_eq!(plan.kind, "plan");
assert_eq!(plan.route_count, 1);
assert_eq!(plan.routes[0].provider, "anthropic");
assert_eq!(plan.routes[0].model, "claude-sonnet-4-6");
assert!(plan.catalog.hash_blake3.starts_with("blake3:"));
let case_ids = plan.routes[0]
.cases
.iter()
.map(|case| case.id)
.collect::<Vec<_>>();
assert!(case_ids.contains(&"single_tool_call"));
assert!(case_ids.contains(&"large_string_argument"));
assert!(case_ids.contains(&"tool_result_followup"));
assert!(case_ids.contains(&"done_sentinel"));
assert_eq!(plan.case_count, plan.routes[0].cases.len());
assert!(plan.required_case_count >= 7);
}
#[test]
fn scorecard_plan_rejects_unknown_route_filters() {
let err = tool_scorecard_plan_from_catalog(&[String::from("missing:nope")], false)
.expect_err("unknown route should fail");
assert!(err.contains("missing:nope"), "{err}");
}
fn report(
provider: &str,
model: &str,
cases: Vec<ToolConformanceCase>,
) -> ToolConformanceReport {
ToolConformanceReport {
schema_version: 1,
provider: provider.to_string(),
model: model.to_string(),
base_url: None,
tool_name: "echo_marker".to_string(),
marker: "marker".to_string(),
cases,
tool_calling: ToolCallingConformanceSummary {
native: ToolProbeStatus::Unknown,
text: ToolProbeStatus::Unknown,
streaming_native: ToolProbeStatus::Unknown,
fallback_mode: ToolProbeFallbackMode::Disabled,
failure_reason: None,
},
}
}
fn case(classification: ToolProbeClassification, ok: bool) -> ToolConformanceCase {
ToolConformanceCase {
mode: ToolProbeMode::NonStreaming,
ok,
classification,
fallback_mode: ToolProbeFallbackMode::Native,
failure_reason: None,
http_status: None,
elapsed_ms: Some(1),
native_tool_call_count: usize::from(ok),
text_tool_call_count: 0,
parser_errors: Vec::new(),
protocol_violations: Vec::new(),
content_sample: None,
}
}
}