use std::collections::BTreeMap;
use std::path::Path;
use std::process;
use harn_vm::llm::capabilities::ProviderCapabilityMatrixRow;
use harn_vm::provider_catalog::{CatalogModel, ModelToolEmpiricalParity, ProviderCatalogArtifact};
use crate::cli::CheckOutputFormat;
use crate::commands::tool_mode_parity::{self, ToolModeParityOverlay};
use crate::json_envelope::{to_string_pretty, JsonEnvelope};
pub(crate) const PROVIDER_MATRIX_SCHEMA_VERSION: u32 = 1;
const DEFAULT_TOOL_MODE_PARITY_OVERLAY_PATH: &str =
".harn-runs/coding-agent-bench/latest/tool_mode_parity_overlay.toml";
const FEATURES: &[&str] = &[
"thinking",
"vision",
"audio",
"pdf",
"streaming",
"files_api",
"json_schema",
"xml_scaffolding",
"markdown_scaffolding",
"assistant_prefill",
"role_developer",
"xml_tools",
"native_tools",
"text_tools",
"native_tool_default",
"text_tool_default",
"native_json",
"delimited_output",
"xml_tagged_output",
"tools",
"cache",
];
pub(crate) fn run(format: CheckOutputFormat, filter: Option<&str>, deprecated_json_format: bool) {
let rows = filtered_rows(filter);
let catalog = load_catalog_for_docs().unwrap_or_else(|error| {
eprintln!("error: {error}");
process::exit(2);
});
match format {
CheckOutputFormat::Text => print_text(&rows),
CheckOutputFormat::Json => print_json(&rows, deprecated_json_format),
CheckOutputFormat::Markdown => print!("{}", generate_markdown(&rows, &catalog)),
}
}
pub(crate) fn load_catalog_for_docs() -> Result<ProviderCatalogArtifact, String> {
let mut catalog = harn_vm::provider_catalog::artifact();
if let Some(overlay) = load_default_tool_mode_parity_overlay()? {
apply_empirical_parity_overlay(&mut catalog, &overlay);
}
Ok(catalog)
}
pub(crate) fn generate_markdown(
rows: &[ProviderCapabilityMatrixRow],
catalog: &ProviderCatalogArtifact,
) -> String {
let mut out = String::new();
out.push_str("# Provider capability matrix\n\n");
out.push_str("<!-- GENERATED by `harn providers matrix` -- do not edit by hand. -->\n");
out.push_str("<!-- Source of truth: crates/harn-vm/src/llm/capabilities.toml. -->\n\n");
out.push_str("<!-- markdownlint-disable MD013 -->\n\n");
out.push_str(
"This table is generated from Harn's live provider capability rules. `Model pattern` is the `model_match` rule used by the runtime; first match wins within each provider. `Version min` is the optional inclusive lower bound for provider-specific model versions.\n\n",
);
out.push_str("Regenerate with `make gen-provider-matrix` and verify with `make check-provider-matrix`.\n\n");
out.push_str(
"| Provider | Model pattern | Version min | Thinking | Vision | Audio | PDF | Streaming | Files API | JSON schema | Prompt | Output mode | Prefill | Role | Tool prompt | Thinking blocks | Default tools | Native tools | Text tools | Parity | Tools | Cache |\n",
);
out.push_str(
"|---|---|---|---|---:|---:|---:|---:|---:|---|---|---|---:|---|---|---|---|---:|---:|---|---:|---:|\n",
);
for row in rows {
out.push_str(&format!(
"| `{}` | `{}` | {} | {} | {} | {} | {} | {} | {} | {} | {} | `{}` | {} | `{}` | `{}` | `{}` | `{}` | {} | {} | `{}` | {} | {} |\n",
row.provider,
row.model,
markdown_cell(&version_min_cell(row)),
markdown_cell(&thinking_cell(row)),
yes_no(row.vision),
yes_no(row.audio),
yes_no(row.pdf),
yes_no(row.streaming),
yes_no(row.files_api_supported),
markdown_cell(&json_schema_cell(row)),
markdown_cell(&scaffolding_cell(row)),
row.structured_output_mode,
yes_no(row.supports_assistant_prefill),
instruction_role_cell(row),
tool_prompt_cell(row),
row.thinking_block_style,
row.preferred_tool_format,
yes_no(row.native_tools),
yes_no(row.text_tools),
row.tool_mode_parity,
yes_no(row.tools),
yes_no(row.cache),
));
}
out.push_str("\n## Tool-format recommendations by catalog model\n\n");
out.push_str(
"This section starts from the checked-in provider catalog. Recommended format follows the live capability matrix, and the empirical columns are layered from `.harn-runs/coding-agent-bench/latest/tool_mode_parity_overlay.toml` when that overlay exists locally. Rows without sampled evidence show `data not yet collected`.\n\n",
);
out.push_str(
"| Provider | Model | Recommended format | Parity | Native pass | Text pass | Samples | Last evaluated | Confidence | Evidence |\n",
);
out.push_str("|---|---|---|---|---|---|---:|---|---|---|\n");
for model in catalogued_models(catalog) {
let empirical = model.tool_support.empirical_parity.as_ref();
out.push_str(&format!(
"| `{}` | `{}` | `{}` | `{}` | {} | {} | {} | {} | {} | {} |\n",
model.provider,
escape_markdown_pipes(&model.id),
recommended_tool_format_cell(model),
parity_cell(model),
empirical
.map(pass_rate_cell_native)
.unwrap_or_else(|| "-".to_string()),
empirical
.map(pass_rate_cell_text)
.unwrap_or_else(|| "-".to_string()),
empirical
.map(|row| row.sample_size.to_string())
.unwrap_or_else(|| "-".to_string()),
empirical
.map(|row| format!("`{}`", row.last_evaluated))
.unwrap_or_else(|| "-".to_string()),
empirical
.map(|row| format!("`{}`", row.confidence))
.unwrap_or_else(|| "-".to_string()),
empirical_status_cell(empirical),
));
}
out
}
pub(crate) fn filtered_rows(filter: Option<&str>) -> Vec<ProviderCapabilityMatrixRow> {
let rows = harn_vm::llm::capabilities::matrix_rows();
let Some(feature) = filter else {
return rows;
};
let normalized = normalize_feature(feature);
if !FEATURES.contains(&normalized.as_str()) {
eprintln!(
"error: unknown provider matrix feature `{feature}`. Expected one of: {}",
FEATURES.join(", ")
);
process::exit(2);
}
rows.into_iter()
.filter(|row| row_supports_feature(row, &normalized))
.collect()
}
fn load_default_tool_mode_parity_overlay() -> Result<Option<ToolModeParityOverlay>, String> {
let path = Path::new(DEFAULT_TOOL_MODE_PARITY_OVERLAY_PATH);
if !path.exists() {
return Ok(None);
}
tool_mode_parity::read_overlay(path).map(Some)
}
fn apply_empirical_parity_overlay(
catalog: &mut ProviderCatalogArtifact,
overlay: &ToolModeParityOverlay,
) {
let empirical_by_model: BTreeMap<(String, String), ModelToolEmpiricalParity> = overlay
.rows
.iter()
.map(|row| {
(
(row.provider.clone(), row.model.clone()),
ModelToolEmpiricalParity {
verdict: row.tool_mode_parity.clone(),
preferred_format: row.preferred_tool_format.clone(),
confidence: row.confidence.clone(),
sample_size: row.sample_size.min(u32::MAX as usize) as u32,
last_evaluated: row.last_updated.clone(),
native_pass_rate: row.native.pass_rate,
text_pass_rate: row.text.pass_rate,
verifier_divergence_rate: row.verifier_divergence_rate,
},
)
})
.collect();
for model in &mut catalog.models {
model.tool_support.empirical_parity = empirical_by_model
.get(&(model.provider.clone(), model.id.clone()))
.cloned();
}
}
fn catalogued_models(catalog: &ProviderCatalogArtifact) -> Vec<&CatalogModel> {
let mut models = catalog.models.iter().collect::<Vec<_>>();
models.sort_by(|left, right| {
left.provider
.cmp(&right.provider)
.then_with(|| left.id.cmp(&right.id))
});
models
}
fn normalize_feature(feature: &str) -> String {
feature.trim().to_lowercase().replace('-', "_")
}
fn row_supports_feature(row: &ProviderCapabilityMatrixRow, feature: &str) -> bool {
match feature {
"thinking" => !row.thinking.is_empty(),
"vision" => row.vision,
"audio" => row.audio,
"pdf" => row.pdf,
"streaming" => row.streaming,
"files_api" => row.files_api_supported,
"json_schema" => row.json_schema.is_some(),
"xml_scaffolding" => row.prefers_xml_scaffolding,
"markdown_scaffolding" => row.prefers_markdown_scaffolding,
"assistant_prefill" => row.supports_assistant_prefill,
"role_developer" => row.prefers_role_developer,
"xml_tools" => row.prefers_xml_tools,
"native_tools" => row.native_tools,
"text_tools" => row.text_tools,
"native_tool_default" => row.preferred_tool_format == "native",
"text_tool_default" => row.preferred_tool_format == "text",
"native_json" => row.structured_output_mode == "native_json",
"delimited_output" => row.structured_output_mode == "delimited",
"xml_tagged_output" => row.structured_output_mode == "xml_tagged",
"tools" => row.tools,
"cache" => row.cache,
_ => false,
}
}
fn print_text(rows: &[ProviderCapabilityMatrixRow]) {
let table_rows: Vec<[String; 21]> = rows
.iter()
.map(|row| {
[
provider_model_cell(row),
version_min_cell(row),
thinking_cell(row),
yes_no(row.vision).to_string(),
yes_no(row.audio).to_string(),
yes_no(row.pdf).to_string(),
yes_no(row.streaming).to_string(),
yes_no(row.files_api_supported).to_string(),
json_schema_cell(row),
scaffolding_cell(row),
row.structured_output_mode.clone(),
yes_no(row.supports_assistant_prefill).to_string(),
instruction_role_cell(row),
tool_prompt_cell(row),
row.thinking_block_style.clone(),
row.preferred_tool_format.clone(),
yes_no(row.native_tools).to_string(),
yes_no(row.text_tools).to_string(),
row.tool_mode_parity.clone(),
yes_no(row.tools).to_string(),
yes_no(row.cache).to_string(),
]
})
.collect();
let headers = [
"provider/model".to_string(),
"version_min".to_string(),
"thinking".to_string(),
"vision".to_string(),
"audio".to_string(),
"pdf".to_string(),
"streaming".to_string(),
"files_api".to_string(),
"json_schema".to_string(),
"prompt".to_string(),
"output_mode".to_string(),
"prefill".to_string(),
"role".to_string(),
"tool_prompt".to_string(),
"thinking_blocks".to_string(),
"default_tools".to_string(),
"native_tools".to_string(),
"text_tools".to_string(),
"parity".to_string(),
"tools".to_string(),
"cache".to_string(),
];
let mut widths: Vec<usize> = headers.iter().map(String::len).collect();
for row in &table_rows {
for (index, value) in row.iter().enumerate() {
widths[index] = widths[index].max(value.len());
}
}
print_row(&headers, &widths);
for row in table_rows {
print_row(&row, &widths);
}
}
fn recommended_tool_format_cell(model: &CatalogModel) -> String {
if let Some(empirical) = model.tool_support.empirical_parity.as_ref() {
return empirical.preferred_format.clone();
}
if let Some(preferred) = model.tool_support.preferred_format.as_ref() {
return preferred.clone();
}
if model.tool_support.native {
return "native".to_string();
}
if model.tool_support.text {
return "text".to_string();
}
"not supported".to_string()
}
fn parity_cell(model: &CatalogModel) -> String {
model
.tool_support
.empirical_parity
.as_ref()
.map(|empirical| empirical.verdict.clone())
.or_else(|| model.tool_support.parity.clone())
.unwrap_or_else(|| "unknown".to_string())
}
fn empirical_status_cell(empirical: Option<&ModelToolEmpiricalParity>) -> String {
if empirical.is_some() {
"`sampled`".to_string()
} else {
"`data not yet collected`".to_string()
}
}
fn pass_rate_cell_native(empirical: &ModelToolEmpiricalParity) -> String {
pass_rate_cell(empirical.native_pass_rate)
}
fn pass_rate_cell_text(empirical: &ModelToolEmpiricalParity) -> String {
pass_rate_cell(empirical.text_pass_rate)
}
fn pass_rate_cell(value: f64) -> String {
format!("`{:.1}%`", value * 100.0)
}
fn print_row<const N: usize>(cells: &[String; N], widths: &[usize]) {
for (index, cell) in cells.iter().enumerate() {
if index > 0 {
print!(" ");
}
print!("{cell:<width$}", width = widths[index]);
}
println!();
}
fn print_json(rows: &[ProviderCapabilityMatrixRow], deprecated_json_format: bool) {
let mut envelope = JsonEnvelope::ok(PROVIDER_MATRIX_SCHEMA_VERSION, rows);
if deprecated_json_format {
envelope = envelope.with_warning(
"deprecated.flag",
"`harn check --provider-matrix --format=json` is deprecated; use `--json`",
);
}
println!("{}", to_string_pretty(&envelope));
}
fn thinking_cell(row: &ProviderCapabilityMatrixRow) -> String {
if row.thinking.is_empty() {
"no".to_string()
} else {
row.thinking.join(",")
}
}
fn provider_model_cell(row: &ProviderCapabilityMatrixRow) -> String {
if row
.model
.strip_prefix(row.provider.as_str())
.is_some_and(|rest| rest.starts_with('/'))
{
row.model.clone()
} else {
format!("{}/{}", row.provider, row.model)
}
}
fn version_min_cell(row: &ProviderCapabilityMatrixRow) -> String {
let Some(parts) = row.version_min.as_ref() else {
return "any".to_string();
};
match parts.as_slice() {
[major, minor] => format!(">={major}.{minor}"),
_ => format!(
">={}",
parts
.iter()
.map(u32::to_string)
.collect::<Vec<_>>()
.join(".")
),
}
}
fn json_schema_cell(row: &ProviderCapabilityMatrixRow) -> String {
row.json_schema.clone().unwrap_or_else(|| "no".to_string())
}
fn scaffolding_cell(row: &ProviderCapabilityMatrixRow) -> String {
match (
row.prefers_xml_scaffolding,
row.prefers_markdown_scaffolding,
) {
(true, true) => "xml,markdown".to_string(),
(true, false) => "xml".to_string(),
(false, true) => "markdown".to_string(),
(false, false) => "plain".to_string(),
}
}
fn instruction_role_cell(row: &ProviderCapabilityMatrixRow) -> String {
if row.prefers_role_developer {
"developer".to_string()
} else {
"system".to_string()
}
}
fn tool_prompt_cell(row: &ProviderCapabilityMatrixRow) -> String {
if row.prefers_xml_tools {
"xml".to_string()
} else {
"json".to_string()
}
}
fn yes_no(value: bool) -> &'static str {
if value {
"yes"
} else {
"no"
}
}
fn markdown_cell(value: &str) -> String {
if value == "no" {
value.to_string()
} else {
format!("`{value}`")
}
}
fn escape_markdown_pipes(value: &str) -> String {
value.replace('|', "\\|")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn json_schema_filter_only_keeps_supported_rows() {
let rows = filtered_rows(Some("json-schema"));
assert!(!rows.is_empty());
assert!(rows.iter().all(|row| row.json_schema.is_some()));
}
#[test]
fn markdown_is_generated_from_matrix_rows() {
let rows = harn_vm::llm::capabilities::matrix_rows();
let catalog = harn_vm::provider_catalog::artifact();
let markdown = generate_markdown(&rows, &catalog);
assert!(markdown.contains("Provider capability matrix"));
assert!(markdown.contains("Source of truth"));
assert!(markdown.contains("harn providers matrix"));
assert!(markdown.contains(
"| Provider | Model pattern | Version min | Thinking | Vision | Audio | PDF | Streaming | Files API | JSON schema | Prompt | Output mode | Prefill | Role | Tool prompt | Thinking blocks | Default tools | Native tools | Text tools | Parity | Tools | Cache |"
));
assert!(markdown.contains("Tool-format recommendations by catalog model"));
}
#[test]
fn markdown_includes_empirical_and_unknown_model_rows() {
let rows = harn_vm::llm::capabilities::matrix_rows();
let mut catalog = harn_vm::provider_catalog::artifact();
catalog.models[0].tool_support.preferred_format = Some("native".to_string());
catalog.models[0].tool_support.empirical_parity =
Some(harn_vm::provider_catalog::ModelToolEmpiricalParity {
verdict: "text_better".to_string(),
preferred_format: "text".to_string(),
confidence: "high".to_string(),
sample_size: 6,
last_evaluated: "2026-05-24".to_string(),
native_pass_rate: 0.25,
text_pass_rate: 0.75,
verifier_divergence_rate: 0.0,
});
let sampled_model_id = catalog.models[0].id.clone();
let unsampled_model_id = catalog.models[1].id.clone();
let markdown = generate_markdown(&rows, &catalog);
assert!(markdown.contains(&format!("`{sampled_model_id}` | `text` | `text_better` | `25.0%` | `75.0%` | 6 | `2026-05-24` | `high` | `sampled`")));
assert!(markdown.contains(&unsampled_model_id));
assert!(markdown.contains("`data not yet collected`"));
}
}