use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use crate::llm_config::{self, ProviderDef};
#[path = "tool_conformance_helpers.rs"]
mod helpers;
#[path = "tool_conformance_request.rs"]
mod request;
use super::usage_normalization::extract_probe_usage;
pub use super::usage_normalization::ToolProbeUsage;
pub(super) use helpers::{aggregate_stream_text, probe_tool_registry};
use request::{probe_request_body, validate_probe_request_body};
pub const TOOL_CONFORMANCE_SCHEMA_VERSION: u32 = 1;
pub const TOOL_CONFORMANCE_REQUEST_SCHEMA_VERSION: u32 = 3;
pub const TOOL_CONFORMANCE_REQUEST_AUDIT_SCHEMA_VERSION: u32 = 4;
pub const TOOL_PROBE_TOOL_NAME: &str = "echo_marker";
pub const DEFAULT_TOOL_PROBE_MARKER: &str = "harn_tool_probe_marker";
#[derive(Debug, Clone)]
pub struct ToolConformanceProbeOptions {
pub provider: String,
pub model: String,
pub base_url: Option<String>,
pub modes: Vec<ToolProbeMode>,
pub probe_case: ToolProbeCase,
pub marker: String,
pub repeat: usize,
pub timeout_secs: u64,
}
impl ToolConformanceProbeOptions {
pub fn new(provider: impl Into<String>, model: impl Into<String>) -> Self {
Self {
provider: provider.into(),
model: model.into(),
base_url: None,
modes: vec![ToolProbeMode::NonStreaming, ToolProbeMode::Streaming],
probe_case: ToolProbeCase::SingleToolCall,
marker: DEFAULT_TOOL_PROBE_MARKER.to_string(),
repeat: 1,
timeout_secs: 120,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ToolProbeMode {
NonStreaming,
Streaming,
}
impl ToolProbeMode {
pub fn as_str(self) -> &'static str {
match self {
Self::NonStreaming => "non_streaming",
Self::Streaming => "streaming",
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ToolProbeCase {
#[default]
SingleToolCall,
ParallelToolCalls,
LargeStringArgument,
ToolResultFollowup,
SignedThinkingToolResultFollowup,
NoToolAnswerOrRefusal,
UnavailableToolRepair,
DoneSentinel,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ToolProbeRequestProfile {
#[default]
CatalogDefault,
ParameterEdges,
}
impl ToolProbeRequestProfile {
pub fn as_str(self) -> &'static str {
match self {
Self::CatalogDefault => "catalog_default",
Self::ParameterEdges => "parameter_edges",
}
}
pub fn catalog_request_audit_profiles() -> Vec<Self> {
vec![Self::CatalogDefault, Self::ParameterEdges]
}
}
impl ToolProbeCase {
pub fn as_str(self) -> &'static str {
match self {
Self::SingleToolCall => "single_tool_call",
Self::ParallelToolCalls => "parallel_tool_calls",
Self::LargeStringArgument => "large_string_argument",
Self::ToolResultFollowup => "tool_result_followup",
Self::SignedThinkingToolResultFollowup => "signed_thinking_tool_result_followup",
Self::NoToolAnswerOrRefusal => "no_tool_answer_or_refusal",
Self::UnavailableToolRepair => "unavailable_tool_repair",
Self::DoneSentinel => "done_sentinel",
}
}
pub fn catalog_request_audit_cases() -> Vec<Self> {
vec![
Self::SingleToolCall,
Self::ParallelToolCalls,
Self::LargeStringArgument,
Self::ToolResultFollowup,
Self::SignedThinkingToolResultFollowup,
Self::NoToolAnswerOrRefusal,
Self::UnavailableToolRepair,
Self::DoneSentinel,
]
}
pub fn is_live_applicable(self, provider: &str, model: &str) -> bool {
match self {
Self::SignedThinkingToolResultFollowup => {
crate::llm::tool_scorecard::signed_thinking_tool_history_supported(provider, model)
}
_ => true,
}
}
fn expected_value(self, marker: &str) -> String {
match self {
Self::SingleToolCall => marker.to_string(),
Self::ParallelToolCalls => marker.to_string(),
Self::LargeStringArgument => format!(
"marker={marker}\nquoted=\"value\"\njson={{\"marker\":{marker:?},\"nested\":[1,true]}}\nheredoc=<<EOF\n{marker}\nEOF\nunicode=\\u{{2603}}\\u{{1F680}}\nend"
),
Self::ToolResultFollowup => format!("tool_result_followup:{marker}"),
Self::SignedThinkingToolResultFollowup => {
format!("signed_thinking_tool_result_followup:{marker}")
}
Self::NoToolAnswerOrRefusal => format!("direct_answer:{marker}"),
Self::UnavailableToolRepair => format!("unavailable_tool:{marker}"),
Self::DoneSentinel => format!("<done>{marker}</done>"),
}
}
fn requires_probe_tool(self) -> bool {
matches!(
self,
Self::SingleToolCall | Self::ParallelToolCalls | Self::LargeStringArgument
)
}
fn request_uses_probe_tool(self) -> bool {
self.requires_probe_tool()
|| matches!(
self,
Self::ToolResultFollowup | Self::SignedThinkingToolResultFollowup
)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolConformanceRequestReport {
pub schema_version: u32,
pub provider: String,
pub model: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub base_url: Option<String>,
#[serde(default)]
pub probe_case: ToolProbeCase,
#[serde(default)]
pub request_profile: ToolProbeRequestProfile,
pub tool_name: String,
pub marker: String,
pub expected_value: String,
pub requests: Vec<ToolConformanceRequestCase>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolConformanceRequestCase {
pub mode: ToolProbeMode,
pub request_body: Value,
pub validation: ToolConformanceRequestValidation,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolConformanceRequestValidation {
pub dialect: String,
pub status: ToolConformanceRequestValidationStatus,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub issues: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolConformanceRequestAuditReport {
pub schema_version: u32,
pub catalog_model_count: usize,
pub route_count: usize,
pub probe_cases: Vec<String>,
pub request_profiles: Vec<String>,
pub modes: Vec<String>,
pub request_count: usize,
pub validation_pass_count: usize,
pub validation_fail_count: usize,
pub not_applicable_count: usize,
pub dialect_counts: BTreeMap<String, usize>,
pub provider_counts: BTreeMap<String, usize>,
pub routes: Vec<ToolConformanceRequestAuditRoute>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub failures: Vec<ToolConformanceRequestAuditFailure>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub not_applicable: Vec<ToolConformanceRequestAuditNotApplicable>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolConformanceRequestAuditRoute {
pub provider: String,
pub model: String,
pub request_count: usize,
pub validation_pass_count: usize,
pub validation_fail_count: usize,
pub not_applicable_count: usize,
pub dialect_counts: BTreeMap<String, usize>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolConformanceRequestAuditFailure {
pub provider: String,
pub model: String,
pub probe_case: String,
pub request_profile: String,
pub mode: String,
pub dialect: String,
pub issues: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolConformanceRequestAuditNotApplicable {
pub provider: String,
pub model: String,
pub probe_case: String,
pub request_profile: String,
pub mode: String,
pub dialect: String,
pub reason: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ToolConformanceRequestValidationStatus {
Pass,
Fail,
NotApplicable,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ToolProbeClassification {
StructuredNativeToolCall,
ParseableHarnTextToolCall,
DirectAnswerNoTool,
UnavailableToolRepair,
DoneSentinel,
RawModelToolTag,
ProseOnlyNonTool,
MalformedJsonArguments,
EmptySilent,
HttpError,
TransportError,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ToolProbeStatus {
Pass,
Fail,
Unknown,
}
impl ToolProbeStatus {
pub fn as_str(&self) -> &'static str {
match self {
Self::Pass => "pass",
Self::Fail => "fail",
Self::Unknown => "unknown",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ToolProbeFallbackMode {
Native,
Text,
Disabled,
}
impl ToolProbeFallbackMode {
pub fn as_str(&self) -> &'static str {
match self {
Self::Native => "native",
Self::Text => "text",
Self::Disabled => "disabled",
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolConformanceReport {
pub schema_version: u32,
pub provider: String,
pub model: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub base_url: Option<String>,
#[serde(default)]
pub probe_case: ToolProbeCase,
pub tool_name: String,
pub marker: String,
#[serde(default)]
pub expected_value: String,
pub cases: Vec<ToolConformanceCase>,
pub tool_calling: ToolCallingConformanceSummary,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolCallingConformanceSummary {
pub native: ToolProbeStatus,
pub text: ToolProbeStatus,
pub streaming_native: ToolProbeStatus,
pub fallback_mode: ToolProbeFallbackMode,
#[serde(skip_serializing_if = "Option::is_none")]
pub failure_reason: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ToolConformanceCase {
pub mode: ToolProbeMode,
pub ok: bool,
pub classification: ToolProbeClassification,
pub fallback_mode: ToolProbeFallbackMode,
#[serde(skip_serializing_if = "Option::is_none")]
pub failure_reason: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub http_status: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub elapsed_ms: Option<u64>,
pub native_tool_call_count: usize,
pub text_tool_call_count: usize,
#[serde(skip_serializing_if = "Option::is_none")]
pub usage: Option<ToolProbeUsage>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub parser_errors: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub protocol_violations: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub content_sample: Option<String>,
}
impl ToolConformanceCase {
fn transport_error(mode: ToolProbeMode, message: String, elapsed_ms: Option<u64>) -> Self {
Self {
mode,
ok: false,
classification: ToolProbeClassification::TransportError,
fallback_mode: ToolProbeFallbackMode::Disabled,
failure_reason: Some(message),
http_status: None,
elapsed_ms,
native_tool_call_count: 0,
text_tool_call_count: 0,
usage: None,
parser_errors: Vec::new(),
protocol_violations: Vec::new(),
content_sample: None,
}
}
fn http_error(
mode: ToolProbeMode,
status: u16,
message: String,
elapsed_ms: Option<u64>,
) -> Self {
Self {
mode,
ok: false,
classification: ToolProbeClassification::HttpError,
fallback_mode: ToolProbeFallbackMode::Disabled,
failure_reason: Some(message),
http_status: Some(status),
elapsed_ms,
native_tool_call_count: 0,
text_tool_call_count: 0,
usage: None,
parser_errors: Vec::new(),
protocol_violations: Vec::new(),
content_sample: None,
}
}
}
pub async fn run_tool_conformance_probe(
options: ToolConformanceProbeOptions,
) -> ToolConformanceReport {
let model = llm_config::resolve_model_info(&options.model);
let provider = if options.provider.trim().is_empty() {
model.provider.clone()
} else {
options.provider.clone()
};
let model_id = resolved_probe_model_id(&model.id);
let base_url = options.base_url.clone().or_else(|| {
llm_config::provider_config(&provider).map(|def| llm_config::resolve_base_url(&def))
});
let mut cases = Vec::new();
let modes = normalized_modes(&options.modes);
let expected_value = options.probe_case.expected_value(&options.marker);
for _ in 0..options.repeat.max(1) {
for mode in &modes {
cases.push(
execute_live_probe_case(
&provider,
&model_id,
base_url.as_deref(),
*mode,
options.probe_case,
&expected_value,
options.timeout_secs,
)
.await,
);
}
}
report_from_cases(
provider,
model_id,
base_url,
options.probe_case,
options.marker,
expected_value,
cases,
)
}
fn resolved_probe_model_id(selector: &str) -> String {
llm_config::wire_model_id(selector)
}
pub fn classify_tool_conformance_fixture(
provider: impl Into<String>,
model: impl Into<String>,
mode: ToolProbeMode,
marker: impl Into<String>,
raw: &str,
) -> ToolConformanceReport {
classify_tool_conformance_fixture_for_case(
provider,
model,
mode,
ToolProbeCase::SingleToolCall,
marker,
raw,
)
}
pub fn classify_tool_conformance_fixture_for_case(
provider: impl Into<String>,
model: impl Into<String>,
mode: ToolProbeMode,
probe_case: ToolProbeCase,
marker: impl Into<String>,
raw: &str,
) -> ToolConformanceReport {
let marker = marker.into();
let expected_value = probe_case.expected_value(&marker);
let provider = provider.into();
let model = model.into();
let response = serde_json::from_str::<Value>(raw).unwrap_or_else(|_| json!({ "content": raw }));
let usage = extract_probe_usage(&provider, &model, &response);
let case = classify_tool_probe_response(
mode,
&response,
probe_case,
&expected_value,
None,
None,
usage,
);
report_from_cases(
provider,
model,
None,
probe_case,
marker,
expected_value,
vec![case],
)
}
pub fn tool_conformance_request_report(
provider: impl Into<String>,
model: impl Into<String>,
base_url: Option<String>,
modes: Vec<ToolProbeMode>,
probe_case: ToolProbeCase,
request_profile: ToolProbeRequestProfile,
marker: impl Into<String>,
) -> Result<ToolConformanceRequestReport, String> {
let provider = provider.into();
let model = model.into();
let marker = marker.into();
let expected_value = probe_case.expected_value(&marker);
let mut requests = Vec::new();
for mode in normalized_modes(&modes) {
let request_body = probe_request_body(
&provider,
&model,
mode,
probe_case,
request_profile,
&expected_value,
)?;
requests.push(ToolConformanceRequestCase {
mode,
validation: validate_probe_request_body(
&provider,
&model,
probe_case,
request_profile,
&request_body,
),
request_body,
});
}
Ok(ToolConformanceRequestReport {
schema_version: TOOL_CONFORMANCE_REQUEST_SCHEMA_VERSION,
provider,
model,
base_url,
probe_case,
request_profile,
tool_name: TOOL_PROBE_TOOL_NAME.to_string(),
marker,
expected_value,
requests,
})
}
pub fn tool_conformance_request_report_json(
provider: impl Into<String>,
model: impl Into<String>,
base_url: Option<String>,
modes: Vec<ToolProbeMode>,
probe_case: ToolProbeCase,
request_profile: ToolProbeRequestProfile,
marker: impl Into<String>,
) -> Result<String, String> {
let report = tool_conformance_request_report(
provider,
model,
base_url,
modes,
probe_case,
request_profile,
marker,
)?;
serde_json::to_string_pretty(&report).map_err(|error| {
format!("internal error: failed to render tool-probe request report: {error}")
})
}
pub fn tool_conformance_request_catalog_audit(
probe_cases: Vec<ToolProbeCase>,
request_profiles: Vec<ToolProbeRequestProfile>,
modes: Vec<ToolProbeMode>,
) -> ToolConformanceRequestAuditReport {
let probe_cases = if probe_cases.is_empty() {
ToolProbeCase::catalog_request_audit_cases()
} else {
probe_cases
};
let modes = normalized_modes(&modes);
let request_profiles = if request_profiles.is_empty() {
ToolProbeRequestProfile::catalog_request_audit_profiles()
} else {
normalized_request_profiles(&request_profiles)
};
let catalog_model_count = llm_config::model_catalog_entries().len();
let routing_routes = crate::provider_catalog::artifact().routing_routes;
let mut request_count = 0usize;
let mut validation_pass_count = 0usize;
let mut validation_fail_count = 0usize;
let mut not_applicable_count = 0usize;
let mut dialect_counts = BTreeMap::new();
let mut provider_counts = BTreeMap::new();
let mut routes = Vec::new();
let mut failures = Vec::new();
let mut not_applicable = Vec::new();
for catalog_route in &routing_routes {
let mut route = ToolConformanceRequestAuditRoute {
provider: catalog_route.provider.clone(),
model: catalog_route.model.clone(),
request_count: 0,
validation_pass_count: 0,
validation_fail_count: 0,
not_applicable_count: 0,
dialect_counts: BTreeMap::new(),
};
for probe_case in &probe_cases {
for request_profile in &request_profiles {
for mode in &modes {
route.request_count += 1;
request_count += 1;
*provider_counts
.entry(catalog_route.provider.clone())
.or_insert(0) += 1;
match tool_conformance_request_report(
catalog_route.provider.clone(),
catalog_route.model.clone(),
None,
vec![*mode],
*probe_case,
*request_profile,
DEFAULT_TOOL_PROBE_MARKER,
) {
Ok(report) => {
for request in report.requests {
let dialect = request.validation.dialect.clone();
*dialect_counts.entry(dialect.clone()).or_insert(0) += 1;
*route.dialect_counts.entry(dialect.clone()).or_insert(0) += 1;
match request.validation.status {
ToolConformanceRequestValidationStatus::Pass => {
route.validation_pass_count += 1;
validation_pass_count += 1;
}
ToolConformanceRequestValidationStatus::Fail => {
route.validation_fail_count += 1;
validation_fail_count += 1;
failures.push(ToolConformanceRequestAuditFailure {
provider: catalog_route.provider.clone(),
model: catalog_route.model.clone(),
probe_case: probe_case.as_str().to_string(),
request_profile: request_profile.as_str().to_string(),
mode: mode.as_str().to_string(),
dialect,
issues: request.validation.issues,
});
}
ToolConformanceRequestValidationStatus::NotApplicable => {
route.not_applicable_count += 1;
not_applicable_count += 1;
not_applicable.push(
ToolConformanceRequestAuditNotApplicable {
provider: catalog_route.provider.clone(),
model: catalog_route.model.clone(),
probe_case: probe_case.as_str().to_string(),
request_profile: request_profile
.as_str()
.to_string(),
mode: mode.as_str().to_string(),
dialect,
reason: request
.validation
.issues
.first()
.cloned()
.unwrap_or_else(|| {
"probe case is not applicable to this route"
.to_string()
}),
},
);
}
}
}
}
Err(error) => {
route.validation_fail_count += 1;
validation_fail_count += 1;
failures.push(ToolConformanceRequestAuditFailure {
provider: catalog_route.provider.clone(),
model: catalog_route.model.clone(),
probe_case: probe_case.as_str().to_string(),
request_profile: request_profile.as_str().to_string(),
mode: mode.as_str().to_string(),
dialect: "request_build".to_string(),
issues: vec![error],
});
}
}
}
}
}
routes.push(route);
}
ToolConformanceRequestAuditReport {
schema_version: TOOL_CONFORMANCE_REQUEST_AUDIT_SCHEMA_VERSION,
catalog_model_count,
route_count: routes.len(),
probe_cases: probe_cases
.into_iter()
.map(|probe_case| probe_case.as_str().to_string())
.collect(),
request_profiles: request_profiles
.into_iter()
.map(|request_profile| request_profile.as_str().to_string())
.collect(),
modes: modes
.into_iter()
.map(|mode| mode.as_str().to_string())
.collect(),
request_count,
validation_pass_count,
validation_fail_count,
not_applicable_count,
dialect_counts,
provider_counts,
routes,
failures,
not_applicable,
}
}
pub fn report_satisfies_required_probe(report: &ToolConformanceReport, requirement: &str) -> bool {
match requirement {
"tool_probe" | "tool_call_probe" => {
report.tool_calling.fallback_mode != ToolProbeFallbackMode::Disabled
&& report.cases.iter().any(|case| case.ok)
}
"native_tool_probe" => report.tool_calling.native == ToolProbeStatus::Pass,
"streaming_tool_probe" => report.tool_calling.streaming_native == ToolProbeStatus::Pass,
_ => false,
}
}
fn normalized_modes(modes: &[ToolProbeMode]) -> Vec<ToolProbeMode> {
if modes.is_empty() {
return vec![ToolProbeMode::NonStreaming, ToolProbeMode::Streaming];
}
let mut out = Vec::new();
for mode in modes {
if !out.contains(mode) {
out.push(*mode);
}
}
out
}
fn normalized_request_profiles(
profiles: &[ToolProbeRequestProfile],
) -> Vec<ToolProbeRequestProfile> {
if profiles.is_empty() {
return ToolProbeRequestProfile::catalog_request_audit_profiles();
}
let mut out = Vec::new();
for profile in profiles {
if !out.contains(profile) {
out.push(*profile);
}
}
out
}
fn report_from_cases(
provider: String,
model: String,
base_url: Option<String>,
probe_case: ToolProbeCase,
marker: String,
expected_value: String,
cases: Vec<ToolConformanceCase>,
) -> ToolConformanceReport {
let summary = summarize_cases(&cases);
ToolConformanceReport {
schema_version: TOOL_CONFORMANCE_SCHEMA_VERSION,
provider,
model,
base_url,
probe_case,
tool_name: TOOL_PROBE_TOOL_NAME.to_string(),
marker,
expected_value,
cases,
tool_calling: summary,
}
}
fn summarize_cases(cases: &[ToolConformanceCase]) -> ToolCallingConformanceSummary {
let native = summarize_native_mode(cases, ToolProbeMode::NonStreaming);
let streaming_native = summarize_native_mode(cases, ToolProbeMode::Streaming);
let text = summarize_text_mode(cases);
let fallback_mode =
if native == ToolProbeStatus::Pass || streaming_native == ToolProbeStatus::Pass {
ToolProbeFallbackMode::Native
} else if text == ToolProbeStatus::Pass {
ToolProbeFallbackMode::Text
} else {
ToolProbeFallbackMode::Disabled
};
let failure_reason = if fallback_mode == ToolProbeFallbackMode::Disabled {
cases.iter().find_map(|case| case.failure_reason.clone())
} else {
None
};
ToolCallingConformanceSummary {
native,
text,
streaming_native,
fallback_mode,
failure_reason,
}
}
fn summarize_native_mode(cases: &[ToolConformanceCase], mode: ToolProbeMode) -> ToolProbeStatus {
let mut saw_mode = false;
let mut all_passed = true;
for case in cases.iter().filter(|case| case.mode == mode) {
saw_mode = true;
if !(case.ok && case.classification == ToolProbeClassification::StructuredNativeToolCall) {
all_passed = false;
}
}
match (saw_mode, all_passed) {
(false, _) => ToolProbeStatus::Unknown,
(true, true) => ToolProbeStatus::Pass,
(true, false) => ToolProbeStatus::Fail,
}
}
fn summarize_text_mode(cases: &[ToolConformanceCase]) -> ToolProbeStatus {
let mut saw_text = false;
let mut saw_passing_mode = false;
for mode in [ToolProbeMode::NonStreaming, ToolProbeMode::Streaming] {
let mut saw_mode = false;
let mut saw_text_in_mode = false;
let mut all_mode_cases_passed = true;
for case in cases.iter().filter(|case| case.mode == mode) {
saw_mode = true;
saw_text_in_mode |= case.classification
== ToolProbeClassification::ParseableHarnTextToolCall
|| case.text_tool_call_count > 0;
if !(case.ok
&& case.classification == ToolProbeClassification::ParseableHarnTextToolCall)
{
all_mode_cases_passed = false;
}
}
saw_text |= saw_text_in_mode;
if saw_mode && saw_text_in_mode && all_mode_cases_passed {
saw_passing_mode = true;
}
}
if !saw_text {
return ToolProbeStatus::Unknown;
}
if saw_passing_mode {
ToolProbeStatus::Pass
} else {
ToolProbeStatus::Fail
}
}
async fn execute_live_probe_case(
provider: &str,
model: &str,
base_url: Option<&str>,
mode: ToolProbeMode,
probe_case: ToolProbeCase,
marker: &str,
timeout_secs: u64,
) -> ToolConformanceCase {
let clock = harn_clock::RealClock::arc();
let started_ms = clock.monotonic_ms();
let Some(def) = llm_config::provider_config(provider) else {
return ToolConformanceCase::transport_error(
mode,
format!("unknown provider: {provider}"),
Some(elapsed_ms(&*clock, started_ms)),
);
};
let base_url = base_url
.filter(|value| !value.trim().is_empty())
.map(str::to_string)
.unwrap_or_else(|| llm_config::resolve_base_url(&def));
let url = match chat_url(&def, &base_url) {
Ok(url) => url,
Err(message) => {
return ToolConformanceCase::transport_error(
mode,
message,
Some(elapsed_ms(&*clock, started_ms)),
);
}
};
let body = match probe_request_body(
provider,
model,
mode,
probe_case,
ToolProbeRequestProfile::CatalogDefault,
marker,
) {
Ok(body) => body,
Err(message) => {
return ToolConformanceCase::transport_error(
mode,
message,
Some(elapsed_ms(&*clock, started_ms)),
);
}
};
let client = if mode == ToolProbeMode::Streaming {
crate::llm::streaming_client_for_base_url(&base_url)
} else {
crate::llm::blocking_client_for_base_url(&base_url)
};
let api_key = crate::llm::resolve_api_key(provider).unwrap_or_default();
let request = client
.post(&url)
.header("Content-Type", "application/json")
.timeout(std::time::Duration::from_secs(timeout_secs))
.json(&body);
let mut request = crate::llm::api::apply_auth_headers(request, &api_key, Some(&def));
for (name, value) in &def.extra_headers {
request = request.header(name.as_str(), value.as_str());
}
let response = match request.send().await {
Ok(response) => response,
Err(error) => {
return ToolConformanceCase::transport_error(
mode,
format!("provider request failed: {error}"),
Some(elapsed_ms(&*clock, started_ms)),
);
}
};
let status = response.status();
let text = match response.text().await {
Ok(text) => text,
Err(error) => {
return ToolConformanceCase::transport_error(
mode,
format!("provider response was unreadable: {error}"),
Some(elapsed_ms(&*clock, started_ms)),
);
}
};
let elapsed = Some(elapsed_ms(&*clock, started_ms));
if !status.is_success() {
return ToolConformanceCase::http_error(
mode,
status.as_u16(),
sample_failure(&text, "provider returned non-success HTTP status"),
elapsed,
);
}
let response_value = if mode == ToolProbeMode::Streaming {
aggregate_stream_text(&text, provider)
} else {
serde_json::from_str::<Value>(&text).unwrap_or_else(|_| json!({ "content": text }))
};
let usage = extract_probe_usage(provider, model, &response_value);
classify_tool_probe_response(
mode,
&response_value,
probe_case,
marker,
Some(status.as_u16()),
elapsed,
usage,
)
}
fn probe_marker_present(calls: &[Value], marker: &str) -> bool {
probe_values_present(calls, &[marker.to_string()])
}
fn probe_values_present(calls: &[Value], expected_values: &[String]) -> bool {
expected_values
.iter()
.all(|expected| probe_value_count(calls, expected) > 0)
}
fn probe_value_count(calls: &[Value], marker: &str) -> usize {
calls.iter().any(|call| {
call.get("name").and_then(Value::as_str) == Some(TOOL_PROBE_TOOL_NAME)
&& call
.get("arguments")
.and_then(|args| args.get("value"))
.and_then(Value::as_str)
== Some(marker)
}) as usize
}
fn expected_tool_values(probe_case: ToolProbeCase, expected_value: &str) -> Vec<String> {
match probe_case {
ToolProbeCase::ParallelToolCalls => vec![
format!("{expected_value}:first"),
format!("{expected_value}:second"),
],
_ => vec![expected_value.to_string()],
}
}
fn classify_tool_probe_response(
mode: ToolProbeMode,
response: &Value,
probe_case: ToolProbeCase,
expected_value: &str,
http_status: Option<u16>,
elapsed_ms: Option<u64>,
usage: Option<ToolProbeUsage>,
) -> ToolConformanceCase {
let native = extract_native_tool_calls(response);
let native_count = native.len();
let mut malformed_native = false;
let expected_tool_values = expected_tool_values(probe_case, expected_value);
let mut native_probe_calls = Vec::new();
for call in &native {
if call.name == TOOL_PROBE_TOOL_NAME {
match &call.arguments {
Some(Value::Object(map)) => {
if let Some(value) = map.get("value").and_then(Value::as_str) {
native_probe_calls.push(value.to_string());
}
}
_ => malformed_native = true,
}
}
}
let native_pass = probe_case.requires_probe_tool()
&& expected_tool_values
.iter()
.all(|expected| native_probe_calls.contains(expected))
&& (probe_case != ToolProbeCase::ParallelToolCalls
|| (native_probe_calls.len() == expected_tool_values.len() && !malformed_native));
if native_pass {
return ToolConformanceCase {
mode,
ok: true,
classification: ToolProbeClassification::StructuredNativeToolCall,
fallback_mode: ToolProbeFallbackMode::Native,
failure_reason: None,
http_status,
elapsed_ms,
native_tool_call_count: native_count,
text_tool_call_count: 0,
usage,
parser_errors: Vec::new(),
protocol_violations: Vec::new(),
content_sample: content_sample(response),
};
}
let content = extract_content(response);
let tools = probe_tool_registry();
let tagged = crate::llm::tools::parse_text_tool_calls_with_tools(&content, Some(&tools));
let parsed = if probe_marker_present(&tagged.calls, expected_value) {
tagged
} else {
let fenced = crate::llm::tools::parse_fenced_json_tool_calls(&content);
if probe_marker_present(&fenced.calls, expected_value) {
fenced
} else {
tagged
}
};
let text_count = parsed.calls.len();
if !probe_case.requires_probe_tool() {
return classify_no_tool_probe_response(
mode,
probe_case,
expected_value,
&content,
(native_count, text_count),
parsed,
(http_status, elapsed_ms),
usage,
);
}
let text_pass = if probe_case == ToolProbeCase::ParallelToolCalls {
probe_values_present(&parsed.calls, &expected_tool_values)
&& parsed.calls.len() == expected_tool_values.len()
} else {
probe_marker_present(&parsed.calls, expected_value)
};
if text_pass {
return ToolConformanceCase {
mode,
ok: true,
classification: ToolProbeClassification::ParseableHarnTextToolCall,
fallback_mode: ToolProbeFallbackMode::Text,
failure_reason: None,
http_status,
elapsed_ms,
native_tool_call_count: native_count,
text_tool_call_count: text_count,
usage,
parser_errors: parsed.errors,
protocol_violations: parsed.violations,
content_sample: sample_content(&content),
};
}
let (classification, failure_reason) = if malformed_native || !parsed.errors.is_empty() {
(
ToolProbeClassification::MalformedJsonArguments,
Some(first_non_empty(
parsed.errors.first().cloned(),
"malformed_tool_arguments",
)),
)
} else if content.trim().is_empty() && native_count == 0 {
(
ToolProbeClassification::EmptySilent,
Some("empty_silent_response".to_string()),
)
} else if has_raw_model_tool_tag(&content) {
(
ToolProbeClassification::RawModelToolTag,
Some("raw_tool_tag_no_structured_calls".to_string()),
)
} else {
(
ToolProbeClassification::ProseOnlyNonTool,
Some("no_executable_tool_call".to_string()),
)
};
ToolConformanceCase {
mode,
ok: false,
classification,
fallback_mode: ToolProbeFallbackMode::Disabled,
failure_reason,
http_status,
elapsed_ms,
native_tool_call_count: native_count,
text_tool_call_count: text_count,
usage,
parser_errors: parsed.errors,
protocol_violations: parsed.violations,
content_sample: sample_content(&content),
}
}
fn classify_no_tool_probe_response(
mode: ToolProbeMode,
probe_case: ToolProbeCase,
expected_value: &str,
content: &str,
counts: (usize, usize),
parsed: crate::llm::tools::TextToolParseResult,
timing: (Option<u16>, Option<u64>),
usage: Option<ToolProbeUsage>,
) -> ToolConformanceCase {
let (native_count, text_count) = counts;
let (http_status, elapsed_ms) = timing;
let unexpected_tool = native_count > 0 || text_count > 0;
let empty = content.trim().is_empty() && !unexpected_tool;
let ok = !unexpected_tool && !empty && content.contains(expected_value);
let classification = if unexpected_tool {
ToolProbeClassification::RawModelToolTag
} else if empty {
ToolProbeClassification::EmptySilent
} else if ok {
match probe_case {
ToolProbeCase::NoToolAnswerOrRefusal => ToolProbeClassification::DirectAnswerNoTool,
ToolProbeCase::UnavailableToolRepair => ToolProbeClassification::UnavailableToolRepair,
ToolProbeCase::DoneSentinel => ToolProbeClassification::DoneSentinel,
_ => ToolProbeClassification::ProseOnlyNonTool,
}
} else if has_raw_model_tool_tag(content) {
ToolProbeClassification::RawModelToolTag
} else {
ToolProbeClassification::ProseOnlyNonTool
};
let failure_reason = (!ok).then(|| {
if unexpected_tool {
"unexpected_tool_call".to_string()
} else if empty {
"empty_silent_response".to_string()
} else {
format!("missing_expected_text:{expected_value}")
}
});
ToolConformanceCase {
mode,
ok,
classification,
fallback_mode: ToolProbeFallbackMode::Disabled,
failure_reason,
http_status,
elapsed_ms,
native_tool_call_count: native_count,
text_tool_call_count: text_count,
usage,
parser_errors: parsed.errors,
protocol_violations: parsed.violations,
content_sample: sample_content(content),
}
}
fn chat_url(def: &ProviderDef, base_url: &str) -> Result<String, String> {
let endpoint = if def.chat_endpoint.trim().is_empty() {
"/v1/chat/completions"
} else {
def.chat_endpoint.as_str()
};
let url = if endpoint.starts_with("http://") || endpoint.starts_with("https://") {
endpoint.to_string()
} else if endpoint.starts_with('/') {
format!("{}{}", base_url.trim_end_matches('/'), endpoint)
} else {
format!("{}/{}", base_url.trim_end_matches('/'), endpoint)
};
reqwest::Url::parse(&url)
.map(|_| url.clone())
.map_err(|error| format!("invalid provider chat URL '{url}': {error}"))
}
#[derive(Debug)]
struct NativeToolCall {
name: String,
arguments: Option<Value>,
}
fn extract_native_tool_calls(response: &Value) -> Vec<NativeToolCall> {
let mut calls = Vec::new();
visit_native_tool_call_arrays(response, &mut calls);
calls
}
fn visit_native_tool_call_arrays(value: &Value, calls: &mut Vec<NativeToolCall>) {
match value {
Value::Object(map) => {
if let Some(call) = parse_anthropic_tool_use_object(map) {
calls.push(call);
}
if let Some(tool_calls) = map.get("tool_calls").and_then(Value::as_array) {
for item in tool_calls {
if let Some(call) = parse_native_tool_call(item) {
calls.push(call);
}
}
}
for child in map.values() {
visit_native_tool_call_arrays(child, calls);
}
}
Value::Array(items) => {
for item in items {
visit_native_tool_call_arrays(item, calls);
}
}
_ => {}
}
}
fn parse_anthropic_tool_use_object(
object: &serde_json::Map<String, Value>,
) -> Option<NativeToolCall> {
if object.get("type").and_then(Value::as_str) != Some("tool_use") {
return None;
}
let name = object
.get("name")
.and_then(Value::as_str)
.filter(|name| !name.is_empty())?
.to_string();
let arguments = object
.get("input")
.or_else(|| object.get("arguments"))
.cloned()
.unwrap_or_else(|| json!({}));
Some(NativeToolCall {
name,
arguments: Some(arguments),
})
}
fn parse_native_tool_call(item: &Value) -> Option<NativeToolCall> {
let obj = item.as_object()?;
let function = obj.get("function").and_then(Value::as_object);
let name = function
.and_then(|function| function.get("name"))
.or_else(|| obj.get("name"))
.and_then(Value::as_str)?
.to_string();
match crate::llm::tools::parse_text_tool_call_from_native_name(&name) {
crate::llm::tools::NativeToolNameTextCall::Parsed { name, arguments } => {
return Some(NativeToolCall {
name,
arguments: Some(arguments),
});
}
crate::llm::tools::NativeToolNameTextCall::Malformed { name, .. } => {
return Some(NativeToolCall {
name,
arguments: None,
});
}
crate::llm::tools::NativeToolNameTextCall::NotCall => {}
}
let raw_args = function
.and_then(|function| function.get("arguments"))
.or_else(|| obj.get("arguments"));
let arguments = match raw_args {
Some(Value::String(raw)) => serde_json::from_str::<Value>(raw).ok(),
Some(value @ Value::Object(_)) => Some(value.clone()),
Some(_) => None,
None => Some(json!({})),
};
Some(NativeToolCall { name, arguments })
}
fn extract_content(response: &Value) -> String {
let mut parts = Vec::new();
visit_content(response, &mut parts);
parts
.into_iter()
.filter(|part| !part.trim().is_empty())
.collect::<Vec<_>>()
.join("\n")
}
fn visit_content(value: &Value, parts: &mut Vec<String>) {
match value {
Value::Object(map) => {
if object_is_private_reasoning_content(map) {
return;
}
for key in ["content", "response", "text"] {
if let Some(text) = map.get(key).and_then(Value::as_str) {
parts.push(text.to_string());
}
}
for (key, child) in map {
if field_is_private_reasoning(key) {
continue;
}
visit_content(child, parts);
}
}
Value::Array(items) => {
for item in items {
visit_content(item, parts);
}
}
_ => {}
}
}
fn object_is_private_reasoning_content(map: &serde_json::Map<String, Value>) -> bool {
let block_type = map.get("type").and_then(Value::as_str).unwrap_or("");
if matches!(block_type, "reasoning" | "thinking" | "reasoning_summary") {
return true;
}
matches!(
map.get("visibility").and_then(Value::as_str),
Some("private" | "internal")
)
}
fn field_is_private_reasoning(field: &str) -> bool {
matches!(
field,
"analysis"
| "reasoning"
| "reasoning_content"
| "reasoning_details"
| "reasoning_summary"
| "thinking"
| "thinking_summary"
)
}
fn has_raw_model_tool_tag(content: &str) -> bool {
let lowered = content.to_ascii_lowercase();
lowered.contains("<tool_call")
|| lowered.contains("<toolcall")
|| lowered.contains("tool_code:")
|| lowered.contains("tool_call:")
|| lowered.contains("call:")
|| lowered.contains("<function")
}
fn content_sample(response: &Value) -> Option<String> {
sample_content(&extract_content(response))
}
fn sample_content(content: &str) -> Option<String> {
let trimmed = content.trim();
if trimmed.is_empty() {
None
} else {
Some(trimmed.chars().take(240).collect())
}
}
fn sample_failure(text: &str, fallback: &str) -> String {
let trimmed = text.trim();
if trimmed.is_empty() {
fallback.to_string()
} else {
format!(
"{fallback}: {}",
trimmed.chars().take(240).collect::<String>()
)
}
}
fn first_non_empty(value: Option<String>, fallback: &str) -> String {
value
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| fallback.to_string())
}
fn elapsed_ms(clock: &dyn harn_clock::Clock, started_ms: i64) -> u64 {
clock.monotonic_ms().saturating_sub(started_ms).max(0) as u64
}
#[cfg(test)]
#[path = "tool_conformance_tests.rs"]
mod tests;