use std::process;
use harn_vm::llm::capabilities::ProviderCapabilityMatrixRow;
use crate::cli::CheckOutputFormat;
const FEATURES: &[&str] = &[
"thinking",
"vision",
"audio",
"pdf",
"streaming",
"files_api",
"json_schema",
"xml_scaffolding",
"markdown_scaffolding",
"assistant_prefill",
"role_developer",
"xml_tools",
"native_json",
"delimited_output",
"xml_tagged_output",
"tools",
"cache",
];
pub(crate) fn run(format: CheckOutputFormat, filter: Option<&str>) {
let rows = filtered_rows(filter);
match format {
CheckOutputFormat::Text => print_text(&rows),
CheckOutputFormat::Json => print_json(&rows),
CheckOutputFormat::Markdown => print!("{}", generate_markdown(&rows)),
}
}
pub(crate) fn generate_markdown(rows: &[ProviderCapabilityMatrixRow]) -> String {
let mut out = String::new();
out.push_str("# Provider Capability Matrix\n\n");
out.push_str("<!-- GENERATED by `harn check --provider-matrix --format markdown` -- do not edit by hand. -->\n");
out.push_str("<!-- Source of truth: crates/harn-vm/src/llm/capabilities.toml. -->\n\n");
out.push_str("<!-- markdownlint-disable MD013 -->\n\n");
out.push_str(
"This table is generated from Harn's live provider capability rules. `Model pattern` is the `model_match` rule used by the runtime; first match wins within each provider. `Version min` is the optional inclusive lower bound for provider-specific model versions.\n\n",
);
out.push_str("Regenerate with `make gen-provider-matrix` and verify with `make check-provider-matrix`.\n\n");
out.push_str(
"| Provider | Model pattern | Version min | Thinking | Vision | Audio | PDF | Streaming | Files API | JSON schema | Prompt | Output mode | Prefill | Role | Tool prompt | Thinking blocks | Tools | Cache |\n",
);
out.push_str(
"|---|---|---|---|---:|---:|---:|---:|---:|---|---|---|---:|---|---|---|---:|---:|\n",
);
for row in rows {
out.push_str(&format!(
"| `{}` | `{}` | {} | {} | {} | {} | {} | {} | {} | {} | {} | `{}` | {} | `{}` | `{}` | `{}` | {} | {} |\n",
row.provider,
row.model,
markdown_cell(&version_min_cell(row)),
markdown_cell(&thinking_cell(row)),
yes_no(row.vision),
yes_no(row.audio),
yes_no(row.pdf),
yes_no(row.streaming),
yes_no(row.files_api_supported),
markdown_cell(&json_schema_cell(row)),
markdown_cell(&scaffolding_cell(row)),
row.structured_output_mode,
yes_no(row.supports_assistant_prefill),
instruction_role_cell(row),
tool_prompt_cell(row),
row.thinking_block_style,
yes_no(row.tools),
yes_no(row.cache),
));
}
out
}
fn filtered_rows(filter: Option<&str>) -> Vec<ProviderCapabilityMatrixRow> {
let rows = harn_vm::llm::capabilities::matrix_rows();
let Some(feature) = filter else {
return rows;
};
let normalized = normalize_feature(feature);
if !FEATURES.contains(&normalized.as_str()) {
eprintln!(
"error: unknown provider matrix feature `{feature}`. Expected one of: {}",
FEATURES.join(", ")
);
process::exit(2);
}
rows.into_iter()
.filter(|row| row_supports_feature(row, &normalized))
.collect()
}
fn normalize_feature(feature: &str) -> String {
feature.trim().to_lowercase().replace('-', "_")
}
fn row_supports_feature(row: &ProviderCapabilityMatrixRow, feature: &str) -> bool {
match feature {
"thinking" => !row.thinking.is_empty(),
"vision" => row.vision,
"audio" => row.audio,
"pdf" => row.pdf,
"streaming" => row.streaming,
"files_api" => row.files_api_supported,
"json_schema" => row.json_schema.is_some(),
"xml_scaffolding" => row.prefers_xml_scaffolding,
"markdown_scaffolding" => row.prefers_markdown_scaffolding,
"assistant_prefill" => row.supports_assistant_prefill,
"role_developer" => row.prefers_role_developer,
"xml_tools" => row.prefers_xml_tools,
"native_json" => row.structured_output_mode == "native_json",
"delimited_output" => row.structured_output_mode == "delimited",
"xml_tagged_output" => row.structured_output_mode == "xml_tagged",
"tools" => row.tools,
"cache" => row.cache,
_ => false,
}
}
fn print_text(rows: &[ProviderCapabilityMatrixRow]) {
let table_rows: Vec<[String; 17]> = rows
.iter()
.map(|row| {
[
provider_model_cell(row),
version_min_cell(row),
thinking_cell(row),
yes_no(row.vision).to_string(),
yes_no(row.audio).to_string(),
yes_no(row.pdf).to_string(),
yes_no(row.streaming).to_string(),
yes_no(row.files_api_supported).to_string(),
json_schema_cell(row),
scaffolding_cell(row),
row.structured_output_mode.clone(),
yes_no(row.supports_assistant_prefill).to_string(),
instruction_role_cell(row),
tool_prompt_cell(row),
row.thinking_block_style.clone(),
yes_no(row.tools).to_string(),
yes_no(row.cache).to_string(),
]
})
.collect();
let headers = [
"provider/model".to_string(),
"version_min".to_string(),
"thinking".to_string(),
"vision".to_string(),
"audio".to_string(),
"pdf".to_string(),
"streaming".to_string(),
"files_api".to_string(),
"json_schema".to_string(),
"prompt".to_string(),
"output_mode".to_string(),
"prefill".to_string(),
"role".to_string(),
"tool_prompt".to_string(),
"thinking_blocks".to_string(),
"tools".to_string(),
"cache".to_string(),
];
let mut widths: Vec<usize> = headers.iter().map(String::len).collect();
for row in &table_rows {
for (index, value) in row.iter().enumerate() {
widths[index] = widths[index].max(value.len());
}
}
print_row(&headers, &widths);
for row in table_rows {
print_row(&row, &widths);
}
}
fn print_row<const N: usize>(cells: &[String; N], widths: &[usize]) {
for (index, cell) in cells.iter().enumerate() {
if index > 0 {
print!(" ");
}
print!("{cell:<width$}", width = widths[index]);
}
println!();
}
fn print_json(rows: &[ProviderCapabilityMatrixRow]) {
println!(
"{}",
serde_json::to_string_pretty(rows).unwrap_or_else(|error| {
eprintln!("error: failed to serialize provider matrix: {error}");
process::exit(1);
})
);
}
fn thinking_cell(row: &ProviderCapabilityMatrixRow) -> String {
if row.thinking.is_empty() {
"no".to_string()
} else {
row.thinking.join(",")
}
}
fn provider_model_cell(row: &ProviderCapabilityMatrixRow) -> String {
if row
.model
.strip_prefix(row.provider.as_str())
.is_some_and(|rest| rest.starts_with('/'))
{
row.model.clone()
} else {
format!("{}/{}", row.provider, row.model)
}
}
fn version_min_cell(row: &ProviderCapabilityMatrixRow) -> String {
let Some(parts) = row.version_min.as_ref() else {
return "any".to_string();
};
match parts.as_slice() {
[major, minor] => format!(">={major}.{minor}"),
_ => format!(
">={}",
parts
.iter()
.map(u32::to_string)
.collect::<Vec<_>>()
.join(".")
),
}
}
fn json_schema_cell(row: &ProviderCapabilityMatrixRow) -> String {
row.json_schema.clone().unwrap_or_else(|| "no".to_string())
}
fn scaffolding_cell(row: &ProviderCapabilityMatrixRow) -> String {
match (
row.prefers_xml_scaffolding,
row.prefers_markdown_scaffolding,
) {
(true, true) => "xml,markdown".to_string(),
(true, false) => "xml".to_string(),
(false, true) => "markdown".to_string(),
(false, false) => "plain".to_string(),
}
}
fn instruction_role_cell(row: &ProviderCapabilityMatrixRow) -> String {
if row.prefers_role_developer {
"developer".to_string()
} else {
"system".to_string()
}
}
fn tool_prompt_cell(row: &ProviderCapabilityMatrixRow) -> String {
if row.prefers_xml_tools {
"xml".to_string()
} else {
"json".to_string()
}
}
fn yes_no(value: bool) -> &'static str {
if value {
"yes"
} else {
"no"
}
}
fn markdown_cell(value: &str) -> String {
if value == "no" {
value.to_string()
} else {
format!("`{value}`")
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn json_schema_filter_only_keeps_supported_rows() {
let rows = filtered_rows(Some("json-schema"));
assert!(!rows.is_empty());
assert!(rows.iter().all(|row| row.json_schema.is_some()));
}
#[test]
fn markdown_is_generated_from_matrix_rows() {
let rows = harn_vm::llm::capabilities::matrix_rows();
let markdown = generate_markdown(&rows);
assert!(markdown.contains("Provider Capability Matrix"));
assert!(markdown.contains("Source of truth"));
assert!(markdown.contains("harn check --provider-matrix --format markdown"));
assert!(markdown.contains(
"| Provider | Model pattern | Version min | Thinking | Vision | Audio | PDF | Streaming | Files API | JSON schema | Prompt | Output mode | Prefill | Role | Tool prompt | Thinking blocks | Tools | Cache |"
));
}
}