{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://slow-powers.dev/schemas/evals.schema.json",
"title": "Skill Evaluation Definition",
"description": "Defines a set of test cases for evaluating a skill. Lives at <skill>/evals/evals.json.",
"type": "object",
"required": ["skill_name", "evals"],
"additionalProperties": false,
"properties": {
"skill_name": {
"type": "string",
"description": "Name of the skill being evaluated. Should match the skill directory name."
},
"evals": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#/definitions/eval" }
}
},
"definitions": {
"eval": {
"type": "object",
"required": ["id", "prompt", "expected_output"],
"additionalProperties": false,
"properties": {
"id": {
"type": "string",
"pattern": "^[a-z0-9][a-z0-9-]*$",
"description": "Stable kebab-case identifier. Used as directory name in the workspace tree."
},
"prompt": {
"type": "string",
"minLength": 1,
"description": "The user-facing message the subagent receives. Should read like a realistic user request."
},
"turns": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#/definitions/scriptedTurn" },
"description": "Ordered scripted user follow-ups. Each delivered turn resumes the same agent session; absence preserves one-shot dispatch."
},
"expected_output": {
"type": "string",
"minLength": 1,
"description": "Human-readable description of what a successful response looks like."
},
"files": {
"type": "array",
"items": { "type": "string" },
"description": "Fixture file paths relative to the skill's evals/ directory. Copied into the subagent's input directory before dispatch."
},
"runs": {
"type": "integer",
"minimum": 1,
"description": "Runs per condition for this eval, for variance reduction; overrides the --runs flag. Defaults to the flag's value (1 unless raised)."
},
"isolation": {
"type": "string",
"enum": ["shared", "isolated"],
"default": "shared",
"description": "Legacy isolation hint retained for config compatibility. Canonical runs already give every (eval, condition, run) dispatch a private environment for diff-scope capture, so 'shared' and 'isolated' currently have the same effective isolation."
},
"skill_should_trigger": {
"type": "boolean",
"default": true,
"description": "Whether the skill-under-test is expected to fire on this eval. Defaults to true. Set false for negative evals where correct behavior is NOT invoking the skill (e.g. an over-trigger guard); such evals are excluded from the skill-invocation rate and its validity warning."
},
"assertions": {
"type": "array",
"items": { "$ref": "#/definitions/assertion" },
"description": "Pass/fail criteria, added after iteration 1 when you know what outputs look like."
}
}
},
"scriptedTurn": {
"type": "object",
"required": ["prompt", "deliver_when"],
"additionalProperties": false,
"properties": {
"prompt": {
"type": "string",
"minLength": 1,
"description": "The user message to deliver as the next conversation turn."
},
"deliver_when": {
"type": "string",
"enum": ["always", "agent_asks"],
"description": "Deliver unconditionally, or only when the preceding assistant response contains a question mark."
},
"agent_response_matches": {
"type": "string",
"minLength": 1,
"description": "Optional Rust regex that must also match the preceding assistant response in agent_asks mode."
}
},
"allOf": [
{
"if": {
"properties": {
"deliver_when": { "const": "always" }
},
"required": ["deliver_when"]
},
"then": {
"not": {
"required": ["agent_response_matches"]
}
}
}
]
},
"assertion": {
"oneOf": [
{ "$ref": "#/definitions/transcriptCheck" },
{ "$ref": "#/definitions/llmJudge" },
{ "$ref": "#/definitions/commandCheck" },
{ "$ref": "#/definitions/diffScope" }
]
},
"transcriptCheck": {
"type": "object",
"required": ["id", "type", "check"],
"additionalProperties": false,
"properties": {
"id": { "type": "string" },
"type": { "const": "transcript_check" },
"check": {
"type": "string",
"description": "Runner check kind: tool_invocation_matches, or assistant_message_matches for a scripted conversation."
},
"pattern": {
"type": "string",
"description": "Rust regex matched against a rendered tool invocation or assistant message."
},
"must_precede": {
"type": "string",
"enum": ["completion_claim", "first_write", "any"],
"description": "Where in the run the match must occur. 'completion_claim' = before the final assistant message; 'first_write' = before the first write/patch tool call (passes when no write occurs); 'any' = anywhere in the run."
}
}
},
"llmJudge": {
"type": "object",
"required": ["id", "type", "rubric"],
"additionalProperties": false,
"properties": {
"id": { "type": "string" },
"type": { "const": "llm_judge" },
"rubric": {
"type": "string",
"minLength": 1,
"description": "The question the judge model answers. Should be answerable with PASS/FAIL + evidence."
},
"model": {
"type": "string",
"description": "Optional judge model override. When absent, defaults to the run-level judge model recorded in conditions.json, or the harness default when no run-level model was selected."
}
}
},
"commandCheck": {
"type": "object",
"required": ["id", "type", "command"],
"additionalProperties": false,
"properties": {
"id": { "type": "string" },
"type": { "const": "command_check" },
"setup_files": {
"type": "array",
"items": { "type": "string", "minLength": 1 },
"description": "Optional held-out paths relative to the skill's evals/ directory. The runner injects them into the task environment only during ingest."
},
"command": {
"type": "string",
"minLength": 1,
"description": "Trusted eval-author command executed by the runner in the task environment after agent dispatch."
},
"env": {
"type": "object",
"minProperties": 1,
"propertyNames": {
"type": "string",
"minLength": 1
},
"additionalProperties": {
"type": "string"
},
"description": "Optional environment-variable overrides applied to the command. Values are strings; an empty string is allowed."
},
"matrix": {
"type": "object",
"minProperties": 1,
"propertyNames": {
"type": "string",
"minLength": 1
},
"additionalProperties": {
"type": "array",
"minItems": 1,
"uniqueItems": true,
"items": {
"type": "string"
}
},
"description": "Optional environment matrix. The runner executes one command per Cartesian-product cell; matrix values override same-named env values."
},
"expect_exit_code": {
"type": "integer",
"minimum": -2147483648,
"maximum": 2147483647,
"default": 0,
"description": "Expected process exit code. Defaults to 0."
},
"expect_stdout": {
"type": "string",
"description": "Optional regex matched against the command's complete stdout."
}
}
},
"diffScope": {
"type": "object",
"required": ["id", "type"],
"anyOf": [
{ "required": ["max_files_touched"] },
{ "required": ["max_lines_changed"] }
],
"additionalProperties": false,
"properties": {
"id": { "type": "string" },
"type": { "const": "diff_scope" },
"max_files_touched": {
"type": "integer",
"minimum": 0,
"description": "Maximum number of changed, deleted, or newly-created files allowed in the final task environment. Framework files under .eval-magic-outputs are excluded."
},
"max_lines_changed": {
"type": "integer",
"minimum": 0,
"description": "Maximum total byte-lines added plus byte-lines removed allowed. Diffing uses Myers operations with zero-context hunks."
}
}
}
}
}