/**
* `harn provider option-probe` empirically binds one portable generation
* option claim to one concrete provider endpoint.
*
* The normal runtime admits and shapes options from the capability catalog.
* That is correct for product calls and circular evidence for a probe. The
* Rust dispatch shim therefore selects a typed probe contract for exactly the
* chosen option. Option extraction carries that contract through spawned
* transport work. Every unrelated catalog guard remains active.
*/
import { parse, parser } from "std/cli/argparse"
pub import {
ProviderOptionProbeAttempt,
ProviderOptionProbeReport,
ProviderOptionProbeVerdict,
ProviderPortableProbeOption,
provider_option_probe_catalog_claim,
provider_option_probe_route,
} from "std/cli/providers/contracts"
import { provider_portable_option_catalog_field } from "std/cli/providers/contracts"
const __OPTIONS = [
"temperature",
"top_p",
"top_k",
"seed",
"frequency_penalty",
"presence_penalty",
"stop",
]
fn __probe_verdict(value: string) -> ProviderOptionProbeVerdict {
if value == "accepted" {
return "accepted"
}
if value == "rejected" {
return "rejected"
}
if value == "inconclusive" {
return "inconclusive"
}
if value == "gated_locally" {
return "gated_locally"
}
if value == "excluded_by_account_policy" {
return "excluded_by_account_policy"
}
throw "invalid provider option probe verdict: ${value}"
}
fn __attempt(
llm: HarnessLlm,
provider: string,
model: string,
option: ProviderPortableProbeOption,
max_tokens: int,
) -> unknown {
const prompt = "Reply with the single word: ok"
if option == "temperature" {
return try {
llm.call(
prompt,
nil,
{provider: provider, model: model, max_tokens: max_tokens, stream: false, temperature: 0.2},
)
}
}
if option == "top_p" {
return try {
llm.call(
prompt,
nil,
{provider: provider, model: model, max_tokens: max_tokens, stream: false, top_p: 0.9},
)
}
}
if option == "top_k" {
return try {
llm.call(
prompt,
nil,
{provider: provider, model: model, max_tokens: max_tokens, stream: false, top_k: 1},
)
}
}
if option == "seed" {
return try {
llm.call(
prompt,
nil,
{provider: provider, model: model, max_tokens: max_tokens, stream: false, seed: 1},
)
}
}
if option == "frequency_penalty" {
return try {
llm.call(
prompt,
nil,
{
provider: provider,
model: model,
max_tokens: max_tokens,
stream: false,
frequency_penalty: 0.1,
},
)
}
}
if option == "presence_penalty" {
return try {
llm.call(
prompt,
nil,
{
provider: provider,
model: model,
max_tokens: max_tokens,
stream: false,
presence_penalty: 0.1,
},
)
}
}
return try {
llm.call(
prompt,
nil,
{
provider: provider,
model: model,
max_tokens: max_tokens,
stream: false,
stop: ["harn-option-probe-stop"],
},
)
}
}
fn __failure_class(error: unknown) -> string {
if type_of(error) != "dict" || to_string(error?.origin ?? "") == "local" {
return "local"
}
const kind = to_string(error?.kind ?? "")
const reason = to_string(error?.reason ?? "")
const message = to_string(error?.message ?? "")
if reason == "auth_failure" {
return "unauthorized"
}
// OpenRouter refuses every endpoint the account's privacy settings exclude.
// That is account state, not evidence about the option or the catalog.
if reason == "model_unavailable"
&& starts_with(message, "openrouter HTTP 404 ")
&& contains(message, "guardrail restrictions and data policy") {
return "account_policy"
}
if kind == "transient" {
return "transient"
}
if reason == "invalid_request" {
return "rejected"
}
if contains(["model_unavailable", "context_overflow", "invalid_response"], reason) {
return "unrelated"
}
return "unrelated"
}
/**
* Classifies whether a request reached the provider and whether it accepted the selected option.
*
* @effects: []
* @errors: []
*/
pub fn provider_option_probe_classify(outcome: unknown) -> ProviderOptionProbeAttempt {
if !is_err(outcome) {
const usage = unwrap(outcome)?.usage
return {
verdict: __probe_verdict("accepted"),
measured: true,
request_count: 1,
input_tokens: to_int(usage?.input_tokens),
output_tokens: to_int(usage?.output_tokens),
cost_usd: to_float(usage?.cost_usd),
}
}
const raw = unwrap_err(outcome)
if type_of(raw) == "dict" && to_string(raw?.reason ?? "") == "empty_generation" {
return {
verdict: __probe_verdict("accepted"),
measured: true,
request_count: 1,
reason: "served_empty",
}
}
const message = if type_of(raw) == "dict" {
to_string(raw?.message ?? raw)
} else {
to_string(raw)
}
const class = __failure_class(raw)
if class == "account_policy" {
return {
verdict: __probe_verdict("excluded_by_account_policy"),
measured: false,
request_count: 1,
failure_class: class,
reason: to_string(raw?.reason ?? ""),
error: message,
cost_usd: 0.0,
}
}
if class == "local" {
return {
verdict: __probe_verdict("gated_locally"),
measured: false,
request_count: 0,
failure_class: class,
error: message,
}
}
if class == "rejected" {
return {
verdict: __probe_verdict("rejected"),
measured: true,
request_count: 1,
reason: to_string(raw?.reason ?? "invalid_request"),
error: message,
cost_usd: 0.0,
}
}
return {
verdict: __probe_verdict("inconclusive"),
measured: false,
request_count: 1,
failure_class: class,
reason: if type_of(raw) == "dict" {
to_string(raw?.reason ?? "")
} else {
""
},
error: message,
}
}
fn __control(llm: HarnessLlm, provider: string, model: string, max_tokens: int) -> unknown {
return try {
llm.call(
"Reply with the single word: ok",
nil,
{provider: provider, model: model, max_tokens: max_tokens, stream: false},
)
}
}
fn __append_accounting(
attempt: ProviderOptionProbeAttempt,
added: ProviderOptionProbeAttempt,
) -> ProviderOptionProbeAttempt {
const priced = (attempt.observed_cost_count
?? if attempt.cost_usd != nil {
1
} else {
0
})
+ (added.observed_cost_count ?? if added.cost_usd != nil {
1
} else {
0
})
const usage = (attempt.observed_usage_count
?? if attempt.input_tokens != nil || attempt.output_tokens != nil {
1
} else {
0
})
+ (added.observed_usage_count
?? if added.input_tokens != nil || added.output_tokens != nil {
1
} else {
0
})
return attempt
+ {
request_count: attempt.request_count + added.request_count,
retry_count: (attempt.retry_count ?? 0) + (added.retry_count ?? 0),
observed_cost_count: priced,
observed_usage_count: usage,
cost_usd: if priced > 0 {
(attempt.cost_usd ?? 0.0) + (added.cost_usd ?? 0.0)
} else {
nil
},
input_tokens: if usage > 0 {
(attempt.input_tokens ?? 0) + (added.input_tokens ?? 0)
} else {
nil
},
output_tokens: if usage > 0 {
(attempt.output_tokens ?? 0) + (added.output_tokens ?? 0)
} else {
nil
},
}
}
/**
* Combines a retry result with the first request's accounting.
*
* @effects: []
* @errors: []
*/
pub fn provider_option_probe_retry_result(
first: ProviderOptionProbeAttempt,
second: ProviderOptionProbeAttempt,
) -> ProviderOptionProbeAttempt {
const combined = __append_accounting(second, first)
return combined + {retry_count: (combined.retry_count ?? 0) + 1}
}
/**
* Attributes a rejection to the probed option only when the same request without the option succeeds.
*
* A route that refuses every request (mandatory reasoning, a retired model,
* a bad key) would otherwise turn each option into a confident "unsupported".
*
* @effects: []
* @errors: []
*/
pub fn provider_option_probe_confirm_rejection(
attempt: ProviderOptionProbeAttempt,
control: ProviderOptionProbeAttempt,
) -> ProviderOptionProbeAttempt {
const combined = __append_accounting(attempt, control)
if control.verdict == "accepted" && control.measured && control.request_count > 0 {
return combined
}
return combined
+ {
verdict: __probe_verdict("inconclusive"),
measured: false,
failure_class: "control_failed",
reason: attempt.reason ?? "",
error: attempt.error ?? "",
control_error: control.error ?? to_string(control.verdict),
}
}
/**
* Builds the endpoint-keyed claim, observation, and drift receipt for one measured request.
*
* @effects: []
* @errors: []
*/
pub fn provider_option_probe_report(
provider: string,
model: string,
route: string,
option: ProviderPortableProbeOption,
claimed: bool,
ungated: bool,
attempt: ProviderOptionProbeAttempt,
) -> ProviderOptionProbeReport {
const observed = if !attempt.measured || attempt.request_count == 0 {
nil
} else if attempt.verdict == "accepted" {
true
} else if attempt.verdict == "rejected" {
false
} else {
nil
}
const status: "match" | "drift" | "unmeasured" = if observed == nil {
"unmeasured"
} else if observed == claimed {
"match"
} else {
"drift"
}
return {
schema_version: "harn.provider_option_probe.v1",
endpoint: {provider: provider, model: model, route: route},
option: option,
catalog: {field: provider_portable_option_catalog_field(option), claimed_supported: claimed},
probe: {
ungated: ungated,
request_count: attempt.request_count,
measured_count: if observed != nil {
1
} else {
0
},
verdict: attempt.verdict,
attempt: attempt,
},
diff: {status: status, claimed_supported: claimed, observed_supported: observed},
}
}
fn __render(report: ProviderOptionProbeReport) -> string {
return report.endpoint.provider + ":" + report.endpoint.model + " " + report.option + " claim="
+ to_string(report.diff.claimed_supported)
+ " observed="
+ to_string(report.diff.observed_supported)
+ " "
+ report.diff.status
}
fn main(harness: Harness) {
const parsed = parse(
parser(
{
name: "provider_option_probe",
args: [
{kind: "flag", name: "provider", long: "--provider", required: true},
{kind: "flag", name: "model", long: "--model", required: true},
{kind: "flag", name: "option", long: "--option", required: true},
{kind: "flag", name: "max_tokens", long: "--max-tokens", parse: "int", default: 8},
{kind: "switch", name: "plan", long: "--plan"},
{kind: "switch", name: "ungated", long: "--ungated"},
{kind: "switch", name: "fail_on_drift", long: "--fail-on-drift"},
],
},
),
argv,
)
if is_err(parsed) {
throw unwrap_err(parsed).message
}
const args = unwrap(parsed).options
const option_text = to_string(args.option)
if !contains(__OPTIONS, option_text) {
throw "option-probe: --option must be one of: " + join(__OPTIONS, ", ")
}
const option: ProviderPortableProbeOption = option_text
const provider = to_string(args.provider)
const model = to_string(args.model)
const caps = harness.llm.provider_capabilities(provider, model)
const route = provider_option_probe_route(caps)
const claimed = provider_option_probe_catalog_claim(caps, option)
const ungated = args.ungated ?? false
if args.plan {
const plan = {
schema_version: "harn.provider_option_probe_plan.v1",
endpoint: {provider: provider, model: model, route: route},
option: option,
catalog: {field: provider_portable_option_catalog_field(option), claimed_supported: claimed},
request_count: 1,
ungated: ungated,
}
harness.stdio.println(json_stringify_pretty(plan))
return
}
let attempt = provider_option_probe_classify(
__attempt(harness.llm, provider, model, option, args.max_tokens),
)
// One retry after a transport timeout: a scale-to-zero endpoint can spend
// the whole first request cold-starting (Fireworks Nemotron Lightning timed
// out at ~120 s, then answered every option within seconds).
if attempt.verdict == "inconclusive" && (attempt.reason ?? "") == "timeout" {
const second = provider_option_probe_classify(
__attempt(harness.llm, provider, model, option, args.max_tokens),
)
attempt = provider_option_probe_retry_result(attempt, second)
}
if attempt.verdict == "rejected" {
attempt = provider_option_probe_confirm_rejection(
attempt,
provider_option_probe_classify(__control(harness.llm, provider, model, args.max_tokens)),
)
}
const report = provider_option_probe_report(
provider,
model,
route,
option,
claimed,
ungated,
attempt,
)
if harness.env.get_or("HARN_OUTPUT_JSON", "0") == "1" {
harness.stdio.println(json_stringify_pretty(report))
} else {
harness.stdio.println(__render(report))
}
if args.fail_on_drift {
if report.diff.status == "drift" {
harness.runtime.exit(1)
}
if report.diff.status == "unmeasured" {
harness.runtime.exit(2)
}
}
}