{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://slow-powers.dev/schemas/evals.schema.json",
"title": "Skill Evaluation Definition",
"description": "Defines a set of test cases for evaluating a skill. Lives at <skill>/evals/evals.json.",
"type": "object",
"required": ["skill_name", "evals"],
"additionalProperties": false,
"properties": {
"skill_name": {
"type": "string",
"description": "Name of the skill being evaluated. Should match the skill directory name."
},
"evals": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#/definitions/eval" }
}
},
"definitions": {
"eval": {
"type": "object",
"required": ["id", "prompt", "expected_output"],
"additionalProperties": false,
"properties": {
"id": {
"type": "string",
"pattern": "^[a-z0-9][a-z0-9-]*$",
"description": "Stable kebab-case identifier. Used as directory name in the workspace tree."
},
"prompt": {
"type": "string",
"minLength": 1,
"description": "The user-facing message the subagent receives. Should read like a realistic user request."
},
"expected_output": {
"type": "string",
"minLength": 1,
"description": "Human-readable description of what a successful response looks like."
},
"files": {
"type": "array",
"items": { "type": "string" },
"description": "Fixture file paths relative to the skill's evals/ directory. Copied into the subagent's input directory before dispatch."
},
"runs": {
"type": "integer",
"minimum": 1,
"description": "Runs per condition for this eval, for variance reduction; overrides the --runs flag. Defaults to the flag's value (1 unless raised)."
},
"skill_should_trigger": {
"type": "boolean",
"default": true,
"description": "Whether the skill-under-test is expected to fire on this eval. Defaults to true. Set false for negative evals where correct behavior is NOT invoking the skill (e.g. an over-trigger guard); such evals are excluded from the skill-invocation rate and its validity warning."
},
"assertions": {
"type": "array",
"items": { "$ref": "#/definitions/assertion" },
"description": "Pass/fail criteria, added after iteration 1 when you know what outputs look like."
}
}
},
"assertion": {
"oneOf": [
{ "$ref": "#/definitions/transcriptCheck" },
{ "$ref": "#/definitions/llmJudge" }
]
},
"transcriptCheck": {
"type": "object",
"required": ["id", "type", "check"],
"additionalProperties": false,
"properties": {
"id": { "type": "string" },
"type": { "const": "transcript_check" },
"check": {
"type": "string",
"description": "Name of a transcript-check kind handled by the runner's grader, e.g. tool_invocation_matches."
},
"pattern": {
"type": "string",
"description": "Regex (or substring) the check uses to match tool invocations."
},
"must_precede": {
"type": "string",
"enum": ["completion_claim", "any"],
"description": "Where in the run the matched invocation must occur. 'completion_claim' = before the final message. 'any' = anywhere in the run."
}
}
},
"llmJudge": {
"type": "object",
"required": ["id", "type", "rubric"],
"additionalProperties": false,
"properties": {
"id": { "type": "string" },
"type": { "const": "llm_judge" },
"rubric": {
"type": "string",
"minLength": 1,
"description": "The question the judge model answers. Should be answerable with PASS/FAIL + evidence."
},
"model": {
"type": "string",
"description": "Optional judge model override. When absent, defaults to the run-level judge model recorded in conditions.json, or the harness default when no run-level model was selected."
}
}
}
}
}