import { cli_locale } from "std/cli/localization"
import { cli_message } from "std/cli/localization/providers/tool_scorecard"
import { safe_bool, safe_dict, safe_int, safe_list, safe_string } from "std/cli/render"
/**
* `harn provider tool-scorecard` render layer.
*
* The scorecard aggregator stays in Rust because it consumes typed
* `ToolConformanceReport` fixtures and will later compare those observations
* with provider catalog rows. This script only renders the JSON envelope.
*
* Inputs (from the dispatch shim):
* HARN_PROVIDER_TOOL_SCORECARD_PAYLOAD_JSON — JSON envelope matching the
* `ToolScorecardReport` shape.
* HARN_PROVIDER_TOOL_SCORECARD_PAYLOAD_PRETTY — pretty-printed companion,
* forwarded verbatim in JSON mode to preserve serde float fidelity.
* HARN_OUTPUT_JSON — "1" for the JSON envelope, else rendered text.
* HARN_PROVIDER_TOOL_SCORECARD_MARKDOWN — "1" for Markdown instead of the
* compact human summary when JSON mode is off.
*/
fn __join_strings(values: list, fallback: string) -> string {
let out = ""
let first = true
for value in values {
if type_of(value) != "string" {
continue
}
if !first {
out = out + ","
}
out = out + value
first = false
}
if out == "" {
return fallback
}
return out
}
fn __join_case_ids(values: list, fallback: string) -> string {
let out = ""
let first = true
for value in values {
if type_of(value) != "dict" {
continue
}
const id = safe_string(value["id"], "")
if id == "" {
continue
}
if !first {
out = out + ","
}
out = out + id
first = false
}
if out == "" {
return fallback
}
return out
}
fn __case_execution_count(values: list, status: string) -> int {
let count = 0
for value in values {
if type_of(value) != "dict" {
continue
}
const execution = safe_dict(value["execution"])
if safe_string(execution["status"], "") == status {
count = count + 1
}
}
return count
}
fn __join_dict_field(values: list, key: string, fallback: string) -> string {
let out = ""
let first = true
for value in values {
if type_of(value) != "dict" {
continue
}
const item = safe_string(value[key], "")
if item == "" {
continue
}
if !first {
out = out + ","
}
out = out + item
first = false
}
if out == "" {
return fallback
}
return out
}
fn __render_human(__cli_locale: string, report: dict) -> string {
const summary = safe_dict(report["summary"])
const fitness = safe_dict(report["fitness"])
let out = cli_message(
__cli_locale,
"provider tool-call scorecard: routes={0} pass={1} warn={2} fail={3} trusted={4} needs_review={5} quarantined={6} format_recommendations={7}\n",
[
to_string(to_string(safe_int(report["route_count"], 0))),
to_string(to_string(safe_int(summary["pass"], 0))),
to_string(to_string(safe_int(summary["warn"], 0))),
to_string(to_string(safe_int(summary["fail"], 0))),
to_string(to_string(safe_int(summary["trusted"], 0))),
to_string(to_string(safe_int(summary["needs_review"], 0))),
to_string(to_string(safe_int(summary["quarantined"], 0))),
to_string(to_string(len(safe_list(fitness["recommendations"])))),
],
)
const best = safe_dict(summary["best_route"])
const best_provider = safe_string(best["provider"], "")
if best_provider != "" {
out = cli_message(
__cli_locale,
"{0}best={1} {2} score={3}\n",
[
to_string(out),
to_string(best_provider),
to_string(safe_string(best["model"], "")),
to_string(to_string(safe_int(best["quality_score"], 0))),
],
)
}
const routes = safe_list(report["routes"])
for route in routes {
if type_of(route) != "dict" {
continue
}
const cases = safe_int(route["case_count"], 0)
const passed = safe_int(route["successful_cases"], 0)
const catalog_mismatches = safe_list(route["catalog_mismatches"])
out = cli_message(
__cli_locale,
"{0} {1} {2} score={3} status={4} trust={5} mode={6} pass={7}/{8} parseable={9}/{10} actionless={11} issues={12} trust_reasons={13} catalog_mismatches={14}\n",
[
to_string(out),
to_string(safe_string(route["provider"], "")),
to_string(safe_string(route["model"], "")),
to_string(to_string(safe_int(route["quality_score"], 0))),
to_string(safe_string(route["status"], "")),
to_string(safe_string(route["trust_status"], "")),
to_string(safe_string(route["recommended_tool_mode"], "")),
to_string(to_string(passed)),
to_string(to_string(cases)),
to_string(to_string(safe_int(route["parseable_tool_call_cases"], 0))),
to_string(to_string(cases)),
to_string(to_string(safe_int(route["actionless_cases"], 0))),
to_string(__join_strings(safe_list(route["issues"]), "-")),
to_string(__join_strings(safe_list(route["trust_reasons"]), "-")),
to_string(__join_dict_field(catalog_mismatches, "code", "-")),
],
)
}
return out
}
fn __render_report_markdown(report: dict) -> string {
const summary = safe_dict(report["summary"])
let out = "# Provider Tool-Call Scorecard\n\n"
out = out + "- Routes: " + to_string(safe_int(report["route_count"], 0)) + "\n"
out = out + "- Pass: " + to_string(safe_int(summary["pass"], 0)) + "\n"
out = out + "- Warn: " + to_string(safe_int(summary["warn"], 0)) + "\n"
out = out + "- Fail: " + to_string(safe_int(summary["fail"], 0)) + "\n"
out = out + "- Trusted: " + to_string(safe_int(summary["trusted"], 0)) + "\n"
out = out + "- Needs review: "
+ to_string(safe_int(summary["needs_review"], 0))
+ "\n"
out = out + "- Quarantined: "
+ to_string(safe_int(summary["quarantined"], 0))
+ "\n"
const best = safe_dict(summary["best_route"])
const best_provider = safe_string(best["provider"], "")
if best_provider != "" {
out = out + "- Best route: `" + best_provider + " "
+ safe_string(best["model"], "")
+ "` (score "
+ to_string(safe_int(best["quality_score"], 0))
+ ")\n"
}
out = out + "\n"
out = out
+ "| Provider | Model | Score | Status | Trust | Mode | Pass | Parseable | Issues | Trust reasons | Catalog mismatches |\n"
out = out + "| --- | --- | ---: | --- | --- | --- | ---: | ---: | --- | --- | --- |\n"
const routes = safe_list(report["routes"])
for route in routes {
if type_of(route) != "dict" {
continue
}
const cases = safe_int(route["case_count"], 0)
const catalog_mismatches = safe_list(route["catalog_mismatches"])
out = out + "| "
+ safe_string(route["provider"], "")
+ " | "
+ safe_string(route["model"], "")
+ " | "
+ to_string(safe_int(route["quality_score"], 0))
+ " | "
+ safe_string(route["status"], "")
+ " | "
+ safe_string(route["trust_status"], "")
+ " | "
+ safe_string(route["recommended_tool_mode"], "")
+ " | "
+ to_string(safe_int(route["successful_cases"], 0))
+ "/"
+ to_string(cases)
+ " | "
+ to_string(safe_int(route["parseable_tool_call_cases"], 0))
+ "/"
+ to_string(cases)
+ " | "
+ __join_strings(safe_list(route["issues"]), "-")
+ " | "
+ __join_strings(safe_list(route["trust_reasons"]), "-")
+ " | "
+ __join_dict_field(catalog_mismatches, "code", "-")
+ " |\n"
}
return out
}
fn __render_plan_human(__cli_locale: string, report: dict) -> string {
const catalog = safe_dict(report["catalog"])
let out = cli_message(
__cli_locale,
"provider tool-call scorecard plan: routes={0} unscorecardable_providers={1} cases={2} required={3} executable={4} live_tool_probes={5} offline_request_checks={6} not_applicable={7} batch_manifest_requests={8} readiness_commands={9}\n",
[
to_string(to_string(safe_int(report["route_count"], 0))),
to_string(to_string(safe_int(report["unscorecardable_provider_count"], 0))),
to_string(to_string(safe_int(report["case_count"], 0))),
to_string(to_string(safe_int(report["required_case_count"], 0))),
to_string(to_string(safe_int(report["executable_case_count"], 0))),
to_string(to_string(safe_int(report["live_tool_probe_case_count"], 0))),
to_string(to_string(safe_int(report["offline_request_case_count"], 0))),
to_string(to_string(safe_int(report["not_applicable_case_count"], 0))),
to_string(to_string(safe_int(report["batch_manifest_request_count"], 0))),
to_string(to_string(safe_int(report["readiness_command_count"], 0))),
],
)
out = cli_message(
__cli_locale,
"{0}catalog={1} schema={2}\n",
[
to_string(out),
to_string(safe_string(catalog["hash_blake3"], "-")),
to_string(to_string(safe_int(catalog["schema_version"], 0))),
],
)
const routes = safe_list(report["routes"])
for route in routes {
if type_of(route) != "dict" {
continue
}
const claim = safe_dict(route["catalog_claim"])
const cases = safe_list(route["cases"])
const readiness = safe_dict(route["readiness"])
out = cli_message(
__cli_locale,
"{0} {1} {2} trust={3} trust_reasons={4} readiness={5} preferred={6} parity={7} native={8} text={9} parallel={10} batch={11} executable={12} missing_runner={13} cases={14}\n",
[
to_string(out),
to_string(safe_string(route["provider"], "")),
to_string(safe_string(route["model"], "")),
to_string(safe_string(route["trust_status"], "")),
to_string(__join_strings(safe_list(route["trust_reasons"]), "-")),
to_string(safe_string(readiness["status"], "")),
to_string(safe_string(claim["preferred_tool_format"], "-")),
to_string(safe_string(claim["tool_mode_parity"], "-")),
to_string(to_string(safe_bool(claim["native_tools"], false))),
to_string(to_string(safe_bool(claim["text_tools"], false))),
to_string(to_string(safe_bool(claim["supports_parallel_tool_calls"], false))),
to_string(to_string(safe_bool(claim["batch_api"], false))),
to_string(to_string(__case_execution_count(cases, "executable"))),
to_string(to_string(__case_execution_count(cases, "missing_runner"))),
to_string(__join_case_ids(cases, "-")),
],
)
}
const provider_summaries = safe_list(report["provider_summaries"])
for provider in provider_summaries {
if type_of(provider) != "dict" {
continue
}
out = cli_message(
__cli_locale,
"{0} provider-summary {1} routes={2} cases={3} required={4} executable={5} live_tool_probes={6} offline_request_checks={7} readiness_commands={8} batch_manifest_requests={9} not_applicable={10}\n",
[
to_string(out),
to_string(safe_string(provider["provider"], "")),
to_string(to_string(safe_int(provider["route_count"], 0))),
to_string(to_string(safe_int(provider["case_count"], 0))),
to_string(to_string(safe_int(provider["required_case_count"], 0))),
to_string(to_string(safe_int(provider["executable_case_count"], 0))),
to_string(to_string(safe_int(provider["live_tool_probe_case_count"], 0))),
to_string(to_string(safe_int(provider["offline_request_case_count"], 0))),
to_string(to_string(safe_int(provider["readiness_command_count"], 0))),
to_string(to_string(safe_int(provider["batch_manifest_request_count"], 0))),
to_string(to_string(safe_int(provider["not_applicable_case_count"], 0))),
],
)
}
const unscorecardable = safe_list(report["unscorecardable_providers"])
for provider in unscorecardable {
if type_of(provider) != "dict" {
continue
}
out = cli_message(
__cli_locale,
"{0} unscorecardable {1} reason={2} models={3} active_models={4} local_runtime={5} auth_required={6} credential_env_names={7}\n",
[
to_string(out),
to_string(safe_string(provider["provider"], "")),
to_string(safe_string(provider["reason"], "")),
to_string(to_string(safe_int(provider["model_count"], 0))),
to_string(to_string(safe_int(provider["active_model_count"], 0))),
to_string(to_string(safe_bool(provider["local_runtime"], false))),
to_string(to_string(safe_bool(provider["auth_required"], false))),
to_string(__join_strings(safe_list(provider["credential_env_names"]), "-")),
],
)
}
return out
}
fn __render_plan_markdown(report: dict) -> string {
const catalog = safe_dict(report["catalog"])
let out = "# Provider Tool-Call Scorecard Plan\n\n"
out = out + "- Routes: " + to_string(safe_int(report["route_count"], 0)) + "\n"
out = out + "- Unscorecardable providers: "
+ to_string(safe_int(report["unscorecardable_provider_count"], 0))
+ "\n"
out = out + "- Cases: " + to_string(safe_int(report["case_count"], 0)) + "\n"
out = out + "- Required cases: "
+ to_string(safe_int(report["required_case_count"], 0))
+ "\n"
out = out + "- Executable cases: "
+ to_string(safe_int(report["executable_case_count"], 0))
+ "\n"
out = out + "- Live tool probes: "
+ to_string(safe_int(report["live_tool_probe_case_count"], 0))
+ "\n"
out = out + "- Offline request checks: "
+ to_string(safe_int(report["offline_request_case_count"], 0))
+ "\n"
out = out + "- Not applicable cases: "
+ to_string(safe_int(report["not_applicable_case_count"], 0))
+ "\n"
out = out + "- Batch manifest requests: "
+ to_string(safe_int(report["batch_manifest_request_count"], 0))
+ "\n"
out = out + "- Readiness commands: "
+ to_string(safe_int(report["readiness_command_count"], 0))
+ "\n"
out = out + "- Catalog: `"
+ safe_string(catalog["hash_blake3"], "-")
+ "` (schema "
+ to_string(safe_int(catalog["schema_version"], 0))
+ ")\n\n"
out = out
+ "| Provider | Model | Trust | Trust reasons | Readiness | Preferred | Parity | Native | Text | Parallel | Batch | Executable | Missing runner | Cases |\n"
out = out
+ "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---: | ---: | --- |\n"
const routes = safe_list(report["routes"])
for route in routes {
if type_of(route) != "dict" {
continue
}
const claim = safe_dict(route["catalog_claim"])
const cases = safe_list(route["cases"])
const readiness = safe_dict(route["readiness"])
out = out + "| "
+ safe_string(route["provider"], "")
+ " | "
+ safe_string(route["model"], "")
+ " | "
+ safe_string(route["trust_status"], "")
+ " | "
+ __join_strings(safe_list(route["trust_reasons"]), "-")
+ " | "
+ safe_string(readiness["status"], "")
+ " | "
+ safe_string(claim["preferred_tool_format"], "-")
+ " | "
+ safe_string(claim["tool_mode_parity"], "-")
+ " | "
+ to_string(safe_bool(claim["native_tools"], false))
+ " | "
+ to_string(safe_bool(claim["text_tools"], false))
+ " | "
+ to_string(safe_bool(claim["supports_parallel_tool_calls"], false))
+ " | "
+ to_string(safe_bool(claim["batch_api"], false))
+ " | "
+ to_string(__case_execution_count(cases, "executable"))
+ " | "
+ to_string(__case_execution_count(cases, "missing_runner"))
+ " | "
+ __join_case_ids(cases, "-")
+ " |\n"
}
const provider_summaries = safe_list(report["provider_summaries"])
if len(provider_summaries) > 0 {
out = out
+ "\n## Provider Scheduling Summary\n\n"
+ "| Provider | Routes | Cases | Required | Executable | Live tool probes | Offline request checks | Readiness commands | Batch manifest requests | Not applicable |\n"
out = out + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n"
for provider in provider_summaries {
if type_of(provider) != "dict" {
continue
}
out = out + "| "
+ safe_string(provider["provider"], "")
+ " | "
+ to_string(safe_int(provider["route_count"], 0))
+ " | "
+ to_string(safe_int(provider["case_count"], 0))
+ " | "
+ to_string(safe_int(provider["required_case_count"], 0))
+ " | "
+ to_string(safe_int(provider["executable_case_count"], 0))
+ " | "
+ to_string(safe_int(provider["live_tool_probe_case_count"], 0))
+ " | "
+ to_string(safe_int(provider["offline_request_case_count"], 0))
+ " | "
+ to_string(safe_int(provider["readiness_command_count"], 0))
+ " | "
+ to_string(safe_int(provider["batch_manifest_request_count"], 0))
+ " | "
+ to_string(safe_int(provider["not_applicable_case_count"], 0))
+ " |\n"
}
}
const unscorecardable = safe_list(report["unscorecardable_providers"])
if len(unscorecardable) > 0 {
out = out
+ "\n## Unscorecardable Providers\n\n"
+ "| Provider | Reason | Models | Active models | Local runtime | Auth required | Credential env names |\n"
out = out + "| --- | --- | ---: | ---: | --- | --- | --- |\n"
for provider in unscorecardable {
if type_of(provider) != "dict" {
continue
}
out = out + "| "
+ safe_string(provider["provider"], "")
+ " | "
+ safe_string(provider["reason"], "")
+ " | "
+ to_string(safe_int(provider["model_count"], 0))
+ " | "
+ to_string(safe_int(provider["active_model_count"], 0))
+ " | "
+ to_string(safe_bool(provider["local_runtime"], false))
+ " | "
+ to_string(safe_bool(provider["auth_required"], false))
+ " | "
+ __join_strings(safe_list(provider["credential_env_names"]), "-")
+ " |\n"
}
}
return out
}
fn main(harness: Harness) -> int {
const __cli_locale = cli_locale(harness.env)
const raw = harness.env.get_or("HARN_PROVIDER_TOOL_SCORECARD_PAYLOAD_JSON", "")
if raw == "" {
harness.stdio.eprintln(
"internal error: HARN_PROVIDER_TOOL_SCORECARD_PAYLOAD_JSON not set by dispatch shim",
)
return 70
}
const json_mode = harness.env.get_or("HARN_OUTPUT_JSON", "0") == "1"
const markdown_mode = harness.env.get_or("HARN_PROVIDER_TOOL_SCORECARD_MARKDOWN", "0") == "1"
if json_mode {
const pretty = harness.env.get_or("HARN_PROVIDER_TOOL_SCORECARD_PAYLOAD_PRETTY", raw)
harness.stdio.println(pretty)
} else {
const report = try {
json_parse(raw)
} catch (e) {
harness.stdio.eprintln(
"internal error: failed to parse provider-tool-scorecard payload: " + e.message,
)
return 70
}
const kind = safe_string(report["kind"], "report")
const text = if markdown_mode && kind == "plan" {
__render_plan_markdown(report)
} else if markdown_mode {
__render_report_markdown(report)
} else if kind == "plan" {
__render_plan_human(__cli_locale, report)
} else {
__render_human(__cli_locale, report)
}
const trimmed = if len(text) > 0 && text[len(text) - 1] == "\n" {
text[0:len(text) - 1]
} else {
text
}
harness.stdio.println(trimmed)
}
return 0
}