// std/agent/judge_internals — private helpers shared by `std/agent/judge`
// (verify_completion + turn_end_condition) and `std/agent/step_judge`. NOT
// intended as a public surface: every export is prefixed `__judge_` and
// the file is excluded from the docs sweep. Keep shared judge plumbing here:
// LLM-option overrides, structured-output schema field names, the shared
// checkpoint-invocation preamble, and the "raw_verdict -> {vetoed, feedback}"
// classifier.
import { agent_typed_output_checkpoint } from "std/agent/primitives"
import { agent_emit_event } from "std/agent/state"
/**
* Keys on `judge_cfg` that override `opts.llm_options` when present.
* Listed once here so adding (or renaming) an LLM tuning knob is a
* single-line change instead of a 2x sweep.
*/
const __JUDGE_LLM_OVERRIDE_KEYS = ["temperature", "max_tokens", "top_p", "tool_format", "effort"]
const __COMPLETION_JUDGE_DEFAULT_TIMEOUT_MS = 60000
const __COMPLETION_JUDGE_DEFAULT_DEADLINE_RESERVE_MS = 5000
pub type CompletionJudgeAdmission = {
schema?: "harn.completion_judge_admission.v1",
checkpoint_name?: string,
admitted: bool,
remaining_ms: int?,
operation_budget_ms: int?,
deadline_reserve_ms: int?,
provider_timeout_ms?: int?,
prompt_chars?: int,
prompt_token_estimate?: int | float,
outcome: string,
duration_ms?: int | float,
}
pub type JudgeCheckpointRun = {
checkpoint: dict,
duration_ms: int | float,
admission: CompletionJudgeAdmission,
}
pub type JudgeCheckpointRequest = {
judge_cfg: dict,
opts: dict,
payload: dict,
checkpoint_name: string,
system: string,
user: string,
schema: dict,
}
/**
* Open the completion-judge window. Paired with `__judge_emit_decision`, which
* closes it.
*
* A judge call is a model round trip with no tool row and no streamed text, so
* without this a host has nothing to show between the agent's last edit and the
* verdict. Fires after the invocation cap but before deadline admission: a
* refused admission therefore closes the window with an unavailable decision.
*
* @effects: [agent]
* @errors: []
* @api_stability: internal
*/
pub fn __judge_emit_started(
agent: HarnessAgent,
session_id: string,
iteration: int,
trigger: string?,
) -> nil {
agent_emit_event(agent, session_id, "judge_started", {iteration: iteration, trigger: trigger})
return nil
}
/**
* Close the completion-judge window with its normalized host projection.
*
* @effects: [agent]
* @errors: []
* @api_stability: internal
*/
pub fn __judge_emit_decision(
agent: HarnessAgent,
session_id: string,
iteration: int,
verdict: dict,
) -> nil {
agent_emit_event(
agent,
session_id,
"judge_decision",
{
iteration: iteration,
verdict: verdict?.verdict
?? if verdict.vetoed {
"continue"
} else {
"done"
},
reasoning: verdict?.reasoning ?? "",
next_step: verdict?.next_step ?? "",
judge_duration_ms: verdict?.judge_duration_ms ?? 0,
source: verdict?.source ?? "unknown",
trigger: verdict?.trigger,
reason: verdict?.reason,
gap_class: verdict?.gap_class,
confirm: verdict?.confirm ?? !verdict.vetoed,
converted_from: verdict?.converted_from,
escalation_recommended: verdict?.escalation_recommended,
escalation_target: verdict?.escalation_target,
judge_outcome: verdict?.judge_outcome,
terminal_evidence_preserved: verdict?.terminal_evidence_preserved ?? false,
},
)
return nil
}
/**
* Pure completion-judge deadline admission core.
*
* @effects: []
* @errors: []
*/
pub fn __judge_completion_admission(
now_ms: int,
deadline_at_ms: int?,
operation_budget_ms: int,
reserve_ms: int,
) -> CompletionJudgeAdmission {
const remaining_ms = if deadline_at_ms == nil {
nil
} else {
to_int(deadline_at_ms) - to_int(now_ms)
}
const admitted = remaining_ms == nil
|| remaining_ms
>= to_int(operation_budget_ms) + to_int(reserve_ms)
return {
admitted: admitted,
remaining_ms: remaining_ms,
operation_budget_ms: operation_budget_ms,
deadline_reserve_ms: reserve_ms,
outcome: if admitted {
"admitted"
} else {
"deadline_budget_insufficient"
},
}
}
/**
* Apply per-judge LLM overrides on top of an already-built `llm_opts`
* dict. Mirrors the `for key in [...]` block that lived inline at both
* call sites before v0.8.43.
*
* @effects: []
* @errors: []
* @api_stability: internal
*/
pub fn __judge_apply_llm_overrides(llm_opts: dict, judge_cfg: dict) -> dict {
let out = llm_opts
for key in __JUDGE_LLM_OVERRIDE_KEYS {
if judge_cfg[key] != nil {
out = out + {[key]: judge_cfg[key]}
}
}
return out
}
/**
* Shared checkpoint-invocation preamble for the completion (`std/agent/judge`)
* and step (`std/agent/step_judge`) judges: build the per-judge `llm_opts`
* (base `opts.llm_options` + model/provider/schema/session/system, then the
* `__judge_apply_llm_overrides` tuning knobs) and time the named
* `agent_typed_output_checkpoint` call. Returns `{checkpoint, duration_ms}`;
* the callers keep their own divergent fail-open defaults and verdict shaping.
*
* @effects: [llm.call]
* @errors: []
* @api_stability: internal
*/
/**
* The setting that names the model a judge grades with.
*
* One string, defined once, because it appears in the refusal a caller reads
* and in the receipt a host inspects, and those two must not drift.
*/
const __JUDGE_MODEL_SETTING = "model"
/**
* The catalog ladder that names one default judge model per provider family.
*
* Owned by the runtime model catalog, not by this module: volatile model
* identity belongs in catalog source data, and policy code carries only the
* stable role name.
*/
const __JUDGE_ROLE_LADDER = "judge"
/**
* The provider family a judge grades in.
*
* The judge's own configured provider wins; otherwise the run's provider. This
* is the ACTOR's provider, deliberately, and it is not the same thing as the
* actor's model: a cheap judge from the same family is a real second opinion,
* whereas the actor's own model asked its own question twice is not. When
* neither is set, the provider inferred from the actor's model id is the last
* resort; `model_info` returns nil rather than throwing for an unknown id.
*
* @effects: []
* @errors: []
* @api_stability: internal
* @example: __judge_provider_family(harness.llm, judge_cfg, opts)
*/
pub fn __judge_provider_family(llm: HarnessLlm, judge_cfg: dict, opts: dict) -> string? {
const declared = trim(to_string(judge_cfg?.provider ?? opts?.provider ?? ""))
if declared != "" {
return declared
}
const actor_model = trim(to_string(opts?.model ?? ""))
if actor_model == "" {
return nil
}
const inferred = trim(to_string(llm.model_info(actor_model)?.provider ?? ""))
if inferred == "" {
return nil
}
return inferred
}
/**
* The catalog's default judge model for one provider family, or `nil`.
*
* The `judge` ladder is a per-family table, not a fallback chain: resolution
* matches the family and stops. Falling through to another provider's row
* would dispatch a judge against a provider the caller never chose, and a
* caller who set no key for it would read a credential error as a verdict.
*
* @effects: []
* @errors: []
* @api_stability: internal
* @example: __judge_catalog_role_model(harness.llm, "anthropic")
*/
pub fn __judge_catalog_role_model(llm: HarnessLlm, provider: string?) -> string? {
if provider == nil || provider == "" {
return nil
}
const ladder = llm.model_ladder(__JUDGE_ROLE_LADDER)
for step in ladder?.steps ?? [] {
if to_string(step?.provider ?? "") == provider {
const model = trim(to_string(step?.model ?? ""))
if model != "" {
return model
}
}
}
return nil
}
/**
* The model this judge will grade with, and where that decision came from.
*
* Two sources, in order, and `nil` when neither answers:
*
* 1. `judge_cfg.model` — what the caller asked for, source `judge_config`.
* 2. the catalog `judge` role for this provider family, source
* `catalog_role`.
*
* It deliberately never falls back to `opts.model`, which is the ACTOR's model
* for this run: a run graded by the model that produced it is not a check, it
* is the same model asked the same question twice, and it is the mechanical
* cause of a cheap judge accepting a blocked run as done (harn#7937,
* harn#7910).
*
* Rung 2 exists so that the simplest possible harness — `turn_end_condition:
* true` and nothing else — still grades, which is the whole point of the
* shorthand. It is a real rung, not a placeholder: the `judge` ladder ships
* with a row for every provider family Harn routes to.
*
* @effects: []
* @errors: []
* @api_stability: internal
* @example: __judge_resolve_model(harness.llm, judge_cfg, opts)
*/
pub fn __judge_resolve_model(
llm: HarnessLlm,
judge_cfg: dict,
opts: dict,
) -> {model: string, source: string}? {
const configured = trim(to_string(judge_cfg?.[__JUDGE_MODEL_SETTING] ?? ""))
if configured != "" {
return {model: configured, source: "judge_config"}
}
const family = __judge_provider_family(llm, judge_cfg, opts)
const role_model = __judge_catalog_role_model(llm, family)
if role_model != nil {
return {model: role_model, source: "catalog_role"}
}
return nil
}
/**
* What a caller is told when no judge model is configured.
*
* Names the setting and the reason, so the reader does not have to guess which
* of three judge configurations is missing one, and does not read the refusal
* as a transient failure worth retrying.
*
* @effects: []
* @errors: []
* @api_stability: internal
* @example: __judge_model_unconfigured_message("agent.completion_judge")
*/
pub fn __judge_model_unconfigured_message(checkpoint_name: string, provider: string?) -> string {
const family = if provider == nil || provider == "" {
"this run's provider could not be determined"
} else {
"provider `" + to_string(provider) + "` has no `" + __JUDGE_ROLE_LADDER
+ "` role in the model catalog"
}
return checkpoint_name
+ " did not run: no judge model is configured and "
+ family
+ ". Set `"
+ __JUDGE_MODEL_SETTING
+ "` on this judge's configuration (verify_completion_judge, turn_end_condition, or step_judge), "
+ "or add a `[model_ladders."
+ __JUDGE_ROLE_LADDER
+ "]` step for that provider. "
+ "A judge deliberately does not inherit the run's own model: a run graded by the model that produced it is not a check."
}
pub fn __judge_run_checkpoint(
harness: Harness,
request: JudgeCheckpointRequest,
) -> JudgeCheckpointRun {
const judge_cfg = request.judge_cfg
const opts = request.opts
const payload = request.payload
const checkpoint_name = request.checkpoint_name
const system = request.system
const user = request.user
const schema = request.schema
const base = opts?.llm_options ?? {}
// Resolve the judge's model BEFORE anything else this function does. A
// refusal has to happen before the deadline receipt, the window event, and
// the provider call, or the run pays for a judge that was never eligible to
// grade it.
const resolved_judge = __judge_resolve_model(harness.llm, judge_cfg, opts)
if resolved_judge == nil {
const refusal = {
schema: "harn.completion_judge_admission.v1",
checkpoint_name: checkpoint_name,
admitted: false,
judge_model: nil,
judge_model_source: "unconfigured",
outcome: "model_unconfigured",
}
if checkpoint_name == "agent.completion_judge" {
agent_emit_event(harness.agent, payload.session_id, "typed_checkpoint", refusal)
}
return {
checkpoint: {
ok: false,
status: "model_unconfigured",
error: __judge_model_unconfigured_message(
checkpoint_name,
__judge_provider_family(harness.llm, judge_cfg, opts),
),
receipt: refusal,
},
duration_ms: 0,
admission: refusal,
}
}
const mock_scope = if checkpoint_name == "agent.completion_judge" {
"completion.judge"
} else if checkpoint_name == "agent.step_judge" {
"step.judge"
} else {
"default"
}
const operation_budget_ms = if checkpoint_name == "agent.completion_judge" {
max(1, to_int(judge_cfg?.operation_timeout_ms) ?? __COMPLETION_JUDGE_DEFAULT_TIMEOUT_MS)
} else {
nil
}
const provider_timeout_ms = if checkpoint_name == "agent.completion_judge" {
min(
operation_budget_ms,
max(1, to_int(judge_cfg?.timeout_ms ?? base?.timeout_ms) ?? operation_budget_ms),
)
} else {
to_int(judge_cfg?.timeout_ms ?? base?.timeout_ms)
}
const deadline_reserve_ms = if checkpoint_name == "agent.completion_judge" {
max(
0,
to_int(judge_cfg?.deadline_reserve_ms)
?? __COMPLETION_JUDGE_DEFAULT_DEADLINE_RESERVE_MS,
)
} else {
nil
}
const deadline_at_ms = to_int(opts?._deadline_at_ms)
const admission_now_ms = harness.clock.monotonic_ms()
const deadline_admission = if checkpoint_name == "agent.completion_judge" {
__judge_completion_admission(
admission_now_ms,
deadline_at_ms,
operation_budget_ms,
deadline_reserve_ms,
)
} else {
{
admitted: true,
remaining_ms: nil,
operation_budget_ms: nil,
deadline_reserve_ms: nil,
outcome: "not_applicable",
}
}
const admitted = deadline_admission.admitted
const admission = deadline_admission
+ {
schema: "harn.completion_judge_admission.v1",
checkpoint_name: checkpoint_name,
admitted: admitted,
provider_timeout_ms: provider_timeout_ms,
prompt_chars: len(system) + len(user),
prompt_token_estimate: ceil((len(system) + len(user)) / 4.0),
// Which model actually graded this run, on the receipt a host already
// reads. Without it, "the judge agreed" cannot be told apart from "the
// actor agreed with itself" after the fact.
judge_model: resolved_judge.model,
judge_model_source: resolved_judge.source,
}
if checkpoint_name == "agent.completion_judge" {
agent_emit_event(harness.agent, payload.session_id, "typed_checkpoint", admission)
}
if !admitted {
return {
checkpoint: {
ok: false,
status: "admission_refused",
error:
"completion judge was not started because its request budget cannot fit before the terminal deadline",
receipt: admission,
},
duration_ms: 0,
admission: admission,
}
}
const completion_timeout_opts = if checkpoint_name == "agent.completion_judge" {
{timeout_ms: provider_timeout_ms, operation_timeout_ms: operation_budget_ms}
} else {
{}
}
const llm_opts = __judge_apply_llm_overrides(
base
+ {
model: resolved_judge.model,
provider: judge_cfg?.provider ?? opts?.provider,
output: {schema: schema, strict: true, validation: "error"},
session_id: payload.session_id,
system: system,
mock_scope: base?.mock_scope ?? mock_scope,
_call_stage: "verify",
}
+ completion_timeout_opts,
judge_cfg,
)
const started = harness.clock.monotonic_ms()
const checkpoint = agent_typed_output_checkpoint(
harness.agent,
harness.llm,
checkpoint_name,
user,
schema,
llm_opts,
)
const duration_ms = harness.clock.monotonic_ms() - started
if checkpoint_name == "agent.completion_judge" {
const usage = checkpoint?.usage ?? checkpoint?.final_result?.usage ?? {}
const cache_read_tokens = usage?.cache_read_tokens ?? 0
const cache_write_tokens = usage?.cache_write_tokens ?? 0
agent_emit_event(
harness.agent,
payload.session_id,
"typed_checkpoint",
{
schema: "harn.completion_judge_cache.v1",
eligible: system != "" && llm_opts?.cache != false,
stable_prefix_hash: "sha256:" + sha256(system),
stable_prefix_chars: len(system),
cache_read_tokens: cache_read_tokens,
cache_write_tokens: cache_write_tokens,
cache_used: cache_read_tokens > 0,
},
)
agent_emit_event(
harness.agent,
payload.session_id,
"typed_checkpoint",
admission
+ {
outcome: if checkpoint?.ok ?? false {
"completed"
} else {
to_string(checkpoint?.status ?? "error")
},
duration_ms: duration_ms,
},
)
}
return {checkpoint: checkpoint, duration_ms: duration_ms, admission: admission}
}
/**
* JSON structural characters can never be part of a legitimate verdict
* token, so a captured verdict containing one was mangled upstream.
*/
const __JUDGE_VERDICT_JSON_JUNK = ["\"", ",", "{", "}", ":", "\\"]
/**
* Normalize a captured judge verdict to its leading token. Structured
* judges occasionally emit sloppy JSON (double commas, run-on key/value
* pairs) that the structured-call repair layer salvages by capturing
* trailing JSON junk into the verdict string — observed live in
* `judge_decision` events as `continue",,` and `continue", "reasoning":`.
* Cut at the first JSON structural character and trim; verdicts without
* JSON junk (including multi-word prose verdicts) pass through unchanged.
*
* @effects: []
* @errors: []
* @api_stability: internal
* @example: __judge_verdict_token("continue\",,")
*/
pub fn __judge_verdict_token(raw_verdict: string?) {
const normalized = lowercase(trim(to_string(raw_verdict ?? "")))
let cut = len(normalized)
for junk in __JUDGE_VERDICT_JSON_JUNK {
const idx = normalized.index_of(junk)
if idx >= 0 && idx < cut {
cut = idx
}
}
if cut == len(normalized) {
return normalized
}
return trim(normalized[0:cut])
}
/**
* Punctuation that cheap models hang off the leading verdict word
* (`"done."`, `"done!"`, `"pass:"`, `"(done)"`, quotes/asterisks from
* markdown emphasis). Stripped from both ends of the leading token before
* the allow-list compare so a decorated `done` still classifies as DONE.
*/
const __JUDGE_TOKEN_PUNCT = ".,;:!?\"'`*_-)("
/**
* Reduce a normalized verdict to its leading whitespace-delimited word with
* surrounding punctuation stripped. Cheap models decorate enum values
* (`"done."`, `"yes, complete"`, `"done — all tests pass"`); the verdict's
* *intent* is its first word, so classification keys on that word instead of
* requiring whole-string equality. `"not done yet"` reduces to `"not"`
* (still a veto), while `"done."` reduces to `"done"` (a pass).
*
* @effects: []
* @errors: []
* @api_stability: internal
* @example: __judge_leading_word("done.")
*/
pub fn __judge_leading_word(normalized: string) {
let first = ""
for piece in normalized.split(" ") {
if first == "" && trim(piece) != "" {
first = trim(piece)
}
}
// Strip leading punctuation.
let start = 0
while start < len(first) && __JUDGE_TOKEN_PUNCT.contains(first.char_at(start)) {
start = start + 1
}
// Strip trailing punctuation.
let stop = len(first)
while stop > start && __JUDGE_TOKEN_PUNCT.contains(first.char_at(stop - 1)) {
stop = stop - 1
}
return first[start:stop]
}
/**
* Classify a raw verdict string against an allow-list of pass tokens and
* compose the `{vetoed, feedback?}` outcome. `feedback_candidates` is
* tried in order: the first non-nil, non-empty entry wins. Falls back
* to `feedback_default` when nothing else is set.
*
* Used by `agent_step_judge` (pass tokens like "pass"/"yes"/"approve") and
* by `agent_evaluate_completion` (pass tokens like "done"/"complete").
* The raw verdict is normalized through `__judge_verdict_token` first, then
* reduced to its leading word so a decorated enum value (`"done."`,
* `"done — all tests pass"`) still classifies. The stored/emitted `verdict`
* field carries the clean leading word.
*
* @effects: []
* @errors: []
* @api_stability: internal
* @example: __judge_classify_verdict("pass", ["pass", "yes"], [critique], default)
*/
pub fn __judge_classify_verdict(
raw_verdict: string?,
pass_tokens: any,
feedback_candidates: list,
feedback_default: any,
) {
const normalized = __judge_leading_word(__judge_verdict_token(raw_verdict))
if contains(pass_tokens, normalized) {
return {vetoed: false, verdict: normalized}
}
let feedback = ""
for candidate in feedback_candidates {
if feedback == "" && candidate != nil && to_string(candidate) != "" {
feedback = to_string(candidate)
}
}
if feedback == "" {
feedback = feedback_default
}
return {vetoed: true, feedback: feedback, verdict: normalized}
}
// -------------------------------------------------------------------------------------------------
// Completion-gate policy (pure). These implement the deterministic veto
// arithmetic that `std/agent/judge::completion_gate` rides on `verify_completion`.
// They are PURE (no store / no LLM / no host calls) so the parity fixtures can
// pin the source policy's decisions on synthetic verdict sets. All host facts
// (source-vs-cosmetic write classification, verifier verdict) enter as data —
// the split from the ownership principle "Harn owns orchestration
// policy; hosts supply facts". NEVER key any decision on a done-sentinel string
// (ledger ⛔#3): the gate reads only write/verify facts.
// -------------------------------------------------------------------------------------------------
/**
* Reduce a list of host write facts to `{source_write_count, cosmetic_write_count,
* other_write_count}`. Each write is classified by `classify_write(path, diff?)`
* when the callback is supplied, else by the write's own `kind` field, else
* conservatively as `"source"` (an unclassified write counts as source progress,
* so the evidence gate never manufactures a false veto — ledger ⛔#5's spirit).
* Only the literal kind `"cosmetic"` is excluded from source progress; any other
* non-`"source"` kind (e.g. `"test"`, `"doc"`) is counted as `other`.
*
* @effects: []
* @errors: []
* @api_stability: internal
*/
pub fn __completion_gate_classify_writes(writes: list?, classify_write: any) {
let source = 0
let cosmetic = 0
let other = 0
for write in writes ?? [] {
const path = to_string(write?.path ?? "")
const kind = if classify_write != nil && path != "" {
to_string(classify_write(path, write?.diff) ?? "source")
} else {
to_string(write?.kind ?? "source")
}
if kind == "cosmetic" {
cosmetic = cosmetic + 1
} else if kind == "source" {
source = source + 1
} else {
other = other + 1
}
}
return {source_write_count: source, cosmetic_write_count: cosmetic, other_write_count: other}
}
/**
* Combine one or more host verifier verdicts (`{ok, findings?}`) into a single
* verdict. A caller-supplied `veto_combine(verdicts)` overrides the default; the
* default is explicit AND-of-oracles arithmetic — the combined verdict is green
* only when EVERY verdict is green, and red findings are concatenated. Returns
* nil when there is no verdict to combine (verifier state unknown).
*
* @effects: []
* @errors: []
* @api_stability: internal
*/
pub fn __completion_gate_combine_verify(verify: any, veto_combine: any) {
if verify == nil {
return nil
}
const verdicts = if type_of(verify) == "list" {
verify
} else {
[verify]
}
if len(verdicts) == 0 {
return nil
}
if veto_combine != nil {
return veto_combine(verdicts)
}
let all_ok = true
let findings = []
let command = ""
for verdict in verdicts {
if command == "" {
command = trim(to_string(verdict?.command ?? ""))
}
if !(verdict?.ok ?? false) {
all_ok = false
const finding = to_string(verdict?.findings ?? "")
if finding != "" {
findings = findings.appending(finding)
}
}
}
return {ok: all_ok, findings: join(findings, "; "), command: command}
}
/** What a deterministic verifier oracle was observed to do, or that it did not run. */
pub type CompletionVerificationObservation = "passed" | "failed" | "not_run"
/**
* The deterministic completion gate's own reading of the verifier oracle for one
* boundary. It is threaded onto the evidence snapshot so a model judge is SHOWN
* the answer the runtime already computed rather than asked to infer it from a
* bounded packet that may not carry the verifying call at all.
*
* `observed` is a positive reading only when it says `passed`. `not_run` covers
* both "no oracle was configured" and "the gate could not read facts", which is
* why no consumer may treat the gate's `confirm` as a substitute for this field.
*/
pub type CompletionVerificationState = {
oracle_expected: bool,
command: string,
observed: CompletionVerificationObservation,
observed_at_evidence_index?: int?,
}
/**
* The kind of gap a `continue` names. This is what CONSTRAINS a refusal: the
* judge keeps sole authority over artifacts, manner and negative clauses, and
* authorization. It loses authority only over `failed_verification`, the single
* question a deterministic oracle may already have answered. An absent or
* unrecognized value reads as `other`, which never yields authority.
*/
pub type CompletionJudgeGapClass = "missing_artifact" \
| "unmet_manner_clause" \
| "failed_verification" \
| "unresolved_authorization" \
| "other"
/** The gap classes a model `continue` may name. See `CompletionJudgeGapClass`. */
pub const COMPLETION_JUDGE_GAP_CLASSES = [
"missing_artifact",
"unmet_manner_clause",
"failed_verification",
"unresolved_authorization",
"other",
]
/**
* Normalize a judge-reported gap class. Absent, blank, and unrecognized values
* all read as `other` — the class that grants no authority — so the tolerant
* direction and the safe direction are the same direction.
*
* @effects: []
* @errors: []
* @api_stability: internal
* @example: __completion_judge_gap_class("Failed_Verification")
*/
pub fn __completion_judge_gap_class(value: string?) -> CompletionJudgeGapClass {
const named = lowercase(trim(to_string(value ?? "")))
if named == "missing_artifact"
|| named == "unmet_manner_clause"
|| named == "failed_verification"
|| named == "unresolved_authorization" {
return named
}
return "other"
}
/**
* Normalize a gate-reported verification reading. The gate is a host-supplied
* closure, so its result is untrusted: an unrecognized `observed` is refused
* outright rather than defaulted, because the only value that grants authority
* downstream is a positive one and a coerced default could manufacture it.
* Returns nil when there is no usable reading.
*
* @effects: []
* @errors: []
* @api_stability: internal
* @example: __completion_verification_state_read({observed: "passed"})
*/
pub fn __completion_verification_state_read(value: unknown) -> CompletionVerificationState? {
if type_of(value) != "dict" {
return nil
}
const named = lowercase(trim(to_string(value?.observed ?? "")))
const observed = if named == "passed" {
"passed"
} else if named == "failed" {
"failed"
} else if named == "not_run" {
"not_run"
} else {
nil
}
if observed == nil {
return nil
}
return {
oracle_expected: value?.oracle_expected == true,
command: to_string(value?.command ?? ""),
observed: observed,
observed_at_evidence_index: nil,
}
}
/**
* Where in a bounded packet the verification observation sits, or nil when the
* packet selected no verification action. A nil here alongside a `passed`
* reading is the exact diagnostic shape of a host that declared no
* `completion_evidence_role`: the gate saw the verifier, the packet did not.
*
* @effects: []
* @errors: []
* @api_stability: internal
* @example: __completion_verification_evidence_index([])
*/
pub fn __completion_verification_evidence_index(selected_actions: list) -> int? {
let index = nil
for selection in selected_actions {
if selection?.evidence_role == "verification" {
index = selection?.evidence_index
}
}
return index
}
/**
* Does a model `continue` naming a failed verification lose to the deterministic
* gate's reading?
*
* Every clause is required, and each blocks one specific way of getting this
* wrong:
*
* 1. `deterministic_invoked` — the gate stage actually ran. A declared plan is
* not proof that anyone answered, and an evaluation nobody adjudicated is
* not vetoed either, so silence would otherwise read as approval.
* 2. `observed == "passed"` — a POSITIVE reading. The gate also confirms when
* it could not read facts at all; keying on that confirm would let a gate
* that never looked overrule a legitimate refusal, which is the blanket
* weakening this arbitration exists to avoid, arriving from the other side.
* 3. `deterministic_reason != "facts_unavailable"` — the same hazard named
* directly, so a future gate that reports both a reading and that reason
* still cannot convert.
* 4. `gap_class == "failed_verification"` — the judge is only overruled on the
* one question a deterministic oracle has already answered.
*
* Anything short of all four leaves the judge's `continue` standing.
*
* @effects: []
* @errors: []
* @api_stability: internal
* @example: __completion_arbitration_converts("failed_verification", true, "verified", "passed")
*/
pub fn __completion_arbitration_converts(
gap_class: string,
deterministic_invoked: bool,
deterministic_reason: string,
observed: string,
) -> bool {
return gap_class == "failed_verification"
&& deterministic_invoked
&& deterministic_reason != "facts_unavailable"
&& observed == "passed"
}
/**
* Default rendered feedback strings for each vetoing ladder rule. The MECHANISM
* (evidence gate + post-write-verify gate) is generic; these strings are host
* DOMAIN content (host-authored coding prose), so `completion_gate`'s
* `feedback_templates` option lets a host override any of them by key. Keys:
* - `no_source_write_cosmetic` — evidence gate tripped, only cosmetic writes seen.
* - `no_source_write_absent` — evidence gate tripped, no writes seen.
* - `verification_after_write_red` — a red verifier after a write. The `{findings}`
* token is substituted with the rendered findings detail (` Findings: <x>.` or
* empty), so an override keeps the dynamic findings unless it drops the token.
* - `failed_verification` — the opt-in streak-aware form before escalation.
* - `repeated_verification_failures` — the opt-in escalated form. Supports both
* `{attempts}` and `{findings}` tokens.
* - `missing_verification` — a source write with a configured-but-unrun verifier.
*/
const __COMPLETION_GATE_FEEDBACK_DEFAULTS = {
failed_verification:
"Verification is still failing after your change.{findings} Fix the specific failure and re-run the verifier before finishing.",
missing_verification:
"You changed source but have not run the verifier yet. Run the narrowest verification for your change before finishing.",
no_source_write_absent:
"This task expects a source-code change, but no source file has been written yet. Make the required change before finishing.",
no_source_write_cosmetic:
"This task expects a source-code change, but only cosmetic / non-source writes have been made. Make the required change to a source file before finishing.",
repeated_verification_failures:
"Verification has failed {attempts} consecutive times after your changes.{findings} Stop re-running the same check unchanged: inspect the failure, make a targeted repair, then verify again.",
verification_after_write_red:
"Verification is still failing after your change.{findings} Fix the specific failure and re-run the verifier before finishing.",
}
/**
* The recognized `completion_gate` `feedback_templates` keys (the vetoing ladder
* rules a host may override). Exposed so `std/agent/judge` can validate an override
* dict loudly at config time.
*
* @effects: []
* @errors: []
* @api_stability: internal
*/
pub fn __completion_gate_feedback_keys() {
return keys(__COMPLETION_GATE_FEEDBACK_DEFAULTS)
}
/**
* Resolve one ladder feedback string: the host override (if any) else the default,
* with the `{findings}` and `{attempts}` tokens substituted. Only red-verifier
* rules pass non-empty values; for the other rules the defaults carry no tokens, so
* the substitutions are no-ops.
*/
fn __completion_gate_render_feedback(
templates: any,
key: any,
findings: string,
attempts: string = "",
) {
return replace(
replace(to_string(templates[key] ?? ""), "{findings}", findings),
"{attempts}",
attempts,
)
}
/** Build the strict red-verifier verdict, including opt-in streak escalation. */
fn __completion_gate_red_verdict(derived: dict, templates: any, detail: string) {
if derived?.consecutive_failed_after_write == nil {
return {
ok: false,
reason: "verification_after_write_red",
strict: true,
feedback: __completion_gate_render_feedback(
templates,
"verification_after_write_red",
detail,
),
}
}
const streak = max(to_int(derived?.consecutive_failed_after_write ?? 0) ?? 0, 0)
const threshold = max(to_int(derived?.escalation_threshold ?? 3) ?? 3, 1)
const converging = derived?.verification_failures_converging ?? false
const escalation_recommended = streak >= threshold && !converging
const reason = if escalation_recommended {
"repeated_verification_failures"
} else {
"failed_verification"
}
let verdict = {
ok: false,
reason: reason,
strict: true,
escalation_recommended: escalation_recommended,
feedback: __completion_gate_render_feedback(templates, reason, detail, to_string(streak)),
}
if escalation_recommended && derived?.escalation_target != nil {
verdict = verdict + {escalation_target: derived.escalation_target}
}
return verdict
}
/**
* The default completion-gate veto arithmetic: an ordered precedence ladder over
* derived facts, ported from the source `completion_gate_uncapped_status`. First
* matching rule wins.
*
* `derived` fields: `source_known` (whether write facts were supplied),
* `source_write_count`, `cosmetic_write_count`, `requires_write` (task needs a
* source change), `verify` (`{ok, findings}` or nil = unknown), `oracle_expected`
* (a verifier is configured), and optional streak-aware escalation facts:
* `consecutive_failed_after_write`, `verification_failures_converging`,
* `escalation_threshold`, and `escalation_target`.
*
* Returns `{ok, reason, strict?, feedback?}`. `strict: true` marks a veto that the
* per-session veto budget may NEVER convert to an allow — every post-write-not-green
* class (a red verifier, or a source write whose expected verifier has not produced a
* green verdict yet). This matches the source carve-out: budget exhaustion refuses to
* release the whole `saw_write && !verification_after_last_write` set, because a source
* write always needs a fresh green verifier before completion.
*
* `feedback_templates` (optional) overrides the rendered feedback string per rule
* (see `__COMPLETION_GATE_FEEDBACK_DEFAULTS`); nil/absent keeps the byte-identical
* defaults.
*
* @effects: []
* @errors: []
* @api_stability: internal
*/
pub fn __completion_gate_ladder(derived: dict, feedback_templates: dict? = nil) {
const templates = __COMPLETION_GATE_FEEDBACK_DEFAULTS + (feedback_templates ?? {})
const source_known = derived?.source_known ?? false
const source_writes = to_int(derived?.source_write_count ?? 0) ?? 0
const cosmetic_writes = to_int(derived?.cosmetic_write_count ?? 0) ?? 0
const saw_source = source_known && source_writes > 0
const requires_write = derived?.requires_write ?? true
const verify = derived?.verify
const verify_known = verify != nil
const verify_ok = verify_known && (verify?.ok ?? false)
const findings = to_string(verify?.findings ?? "")
const oracle_expected = derived?.oracle_expected ?? false
// 1. Evidence requirement (ledger ⛔#5): a task that must change source may not
// finish with zero SOURCE writes. Cosmetic/test-scaffold writes are not progress.
if source_known && requires_write && !saw_source {
const key = if cosmetic_writes > 0 {
"no_source_write_cosmetic"
} else {
"no_source_write_absent"
}
const message = __completion_gate_render_feedback(templates, key, "")
return {ok: false, reason: "no_source_write", strict: false, feedback: message}
}
// 2. Post-write verification: a source write (or an unclassified write set) with
// a RED verifier blocks, and this class is strict — the budget can never let it
// through; only a fresh green verifier ends it.
if verify_known && !verify_ok && (saw_source || !source_known) {
const detail = if findings != "" {
" Findings: " + findings + "."
} else {
""
}
return __completion_gate_red_verdict(derived, templates, detail)
}
// 3. Green verifier -> completion is honorable.
if verify_ok {
return {
ok: true,
reason: if saw_source {
"verified_after_write"
} else {
"verified"
},
}
}
// 4. Source written, a verifier is configured, but it has not run yet. Strict:
// Never budget-release a post-write state without a fresh green verifier
// (`saw_write && !verification_after_last_write`), so this class — like the red
// one above — is never converted to an allow. Only a green verifier ends it.
if saw_source && oracle_expected && !verify_known {
return {
ok: false,
reason: "missing_verification",
strict: true,
feedback: __completion_gate_render_feedback(templates, "missing_verification", ""),
}
}
// 5. Nothing left to gate.
if source_known && !saw_source && !requires_write {
return {ok: true, reason: "no_workspace_write"}
}
if saw_source {
return {ok: true, reason: "wrote_source_no_oracle"}
}
// Facts too thin for a deterministic verdict (unknown writes, no verifier
// signal): the deterministic gate ABSTAINS and lets any configured LLM judge
// decide. Never a silent fabricated pass — the caller surfaces the degraded mode.
return {ok: true, reason: "gate_abstain"}
}
/**
* Apply the per-session veto budget to a ladder verdict. Pure: the caller reads
* `vetoes_used` from the store and persists the charge. After `max_vetoes`
* non-strict vetoes the gate converts a would-be veto into an attributable allow
* (`veto_budget_exhausted`), so a run that a weak model can never satisfy still
* ends with a named reason instead of riding vetoes to the wall-clock timeout.
* A `strict` veto (post-write-red) is NEVER converted. `max_vetoes <= 0` disables
* the budget (unbounded vetoes).
*
* Returns `{verdict, charge}` — `charge: true` means the caller should increment
* the session veto counter.
*
* @effects: []
* @errors: []
* @api_stability: internal
*/
pub fn __completion_gate_apply_budget(verdict: dict, vetoes_used: int, max_vetoes: int) {
if verdict?.ok ?? false {
return {verdict: verdict, charge: false}
}
const strict = verdict?.strict ?? false
if !strict && max_vetoes > 0 && vetoes_used >= max_vetoes {
return {
verdict: {ok: true, reason: "veto_budget_exhausted", converted_from: verdict?.reason ?? ""},
charge: false,
}
}
return {verdict: verdict, charge: true}
}
// A judge that cannot be STARTED is a scheduling fact the caller already
// terminates on. A judge that ran and could not be READ is different, and it is
// the one this bound exists for: its reply may be malformed for a turn and
// well-formed on the next attempt.
//
// One re-ask, not more. The sibling deadline path retries zero times, and a
// judge whose reply does not parse is usually broken the same way every time,
// so further attempts buy a recovery that rarely happens at the price of real
// provider calls on every run that is already failing.
const __COMPLETION_JUDGE_ERROR_ATTEMPTS: int = 2
/**
* True when the judge never produced a reading because it could not run inside
* its deadline, as opposed to running and returning something unreadable.
*
* @effects: []
* @errors: []
* @api_stability: internal
* @example: __judge_checkpoint_unavailable(checkpoint)
*/
pub fn __judge_checkpoint_unavailable(checkpoint: dict) -> bool {
return checkpoint?.status == "admission_refused"
|| checkpoint?.status == "model_unconfigured"
|| checkpoint?.status == "timeout"
|| checkpoint?.error_category == "timeout"
|| checkpoint?.error?.status == "timeout"
}
// A missing setting never becomes present on a retry, and a bounded
// retry loop that treats it as readable spends three refusals to reach
// the same answer.
/**
* Run a judge checkpoint, retrying a reply that could not be read up to a fixed
* bound.
*
* An exit authority that cannot answer must still be able to end a run. Before
* this bound existed, an unreadable reply was recorded and then read as
* `continue`, so a judge that never parsed could veto forever and the run could
* only stop by exhausting an unrelated budget. Retrying is bounded and the
* caller turns exhaustion into a typed terminal.
*
* A checkpoint that succeeded, or that could not be started at all, returns on
* the first attempt: neither is retryable here.
*
* @effects: [llm, event]
* @errors: []
* @api_stability: internal
* @example: __judge_run_checkpoint_bounded(harness, request)
*/
pub fn __judge_run_checkpoint_bounded(
harness: Harness,
request: JudgeCheckpointRequest,
) -> JudgeCheckpointRun {
let ran = __judge_run_checkpoint(harness, request)
let attempt = 1
while attempt < __COMPLETION_JUDGE_ERROR_ATTEMPTS {
if (ran.checkpoint?.ok ?? false) || __judge_checkpoint_unavailable(ran.checkpoint) {
return ran
}
ran = __judge_run_checkpoint(harness, request)
attempt = attempt + 1
}
return ran
}