{
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "https://slow-powers.dev/schemas/evals.schema.json",
"title": "Skill Evaluation Definition",
"description": "Defines a set of test cases for evaluating a skill. Lives at <skill>/evals/evals.json.",
"type": "object",
"required": ["skill_name", "evals"],
"additionalProperties": false,
"properties": {
"skill_name": {
"type": "string",
"description": "Name of the skill being evaluated. Should match the skill directory name."
},
"evals": {
"type": "array",
"minItems": 1,
"items": { "$ref": "#/definitions/eval" }
}
},
"definitions": {
"eval": {
"type": "object",
"required": ["id", "prompt", "expected_output"],
"additionalProperties": false,
"properties": {
"id": {
"type": "string",
"pattern": "^[a-z0-9][a-z0-9-]*$",
"description": "Stable kebab-case identifier. Used as directory name in the workspace tree."
},
"prompt": {
"type": "string",
"minLength": 1,
"description": "The user-facing message the subagent receives. Should read like a realistic user request."
},
"expected_output": {
"type": "string",
"minLength": 1,
"description": "Human-readable description of what a successful response looks like."
},
"files": {
"type": "array",
"items": { "type": "string" },
"description": "Fixture file paths relative to the skill's evals/ directory. Copied into the subagent's input directory before dispatch."
},
"runs": {
"type": "integer",
"minimum": 1,
"description": "Runs per condition for this eval, for variance reduction; overrides the --runs flag. Defaults to the flag's value (1 unless raised)."
},
"isolation": {
"type": "string",
"enum": ["shared", "isolated"],
"default": "shared",
"description": "Isolation hint for run batching. 'isolated' forces this eval into its own group so it never shares an env with another eval (for confounds the framework can't detect from fixture conflicts, e.g. the agent mutates a shared fixture). Defaults to 'shared'. Evals whose fixtures conflict are auto-isolated regardless."
},
"skill_should_trigger": {
"type": "boolean",
"default": true,
"description": "Whether the skill-under-test is expected to fire on this eval. Defaults to true. Set false for negative evals where correct behavior is NOT invoking the skill (e.g. an over-trigger guard); such evals are excluded from the skill-invocation rate and its validity warning."
},
"assertions": {
"type": "array",
"items": { "$ref": "#/definitions/assertion" },
"description": "Pass/fail criteria, added after iteration 1 when you know what outputs look like."
}
}
},
"assertion": {
"oneOf": [
{ "$ref": "#/definitions/transcriptCheck" },
{ "$ref": "#/definitions/llmJudge" }
]
},
"transcriptCheck": {
"type": "object",
"required": ["id", "type", "check"],
"additionalProperties": false,
"properties": {
"id": { "type": "string" },
"type": { "const": "transcript_check" },
"check": {
"type": "string",
"description": "Name of a transcript-check kind handled by the runner's grader, e.g. tool_invocation_matches."
},
"pattern": {
"type": "string",
"description": "Regex (or substring) the check uses to match tool invocations."
},
"must_precede": {
"type": "string",
"enum": ["completion_claim", "any"],
"description": "Where in the run the matched invocation must occur. 'completion_claim' = before the final message. 'any' = anywhere in the run."
}
}
},
"llmJudge": {
"type": "object",
"required": ["id", "type", "rubric"],
"additionalProperties": false,
"properties": {
"id": { "type": "string" },
"type": { "const": "llm_judge" },
"rubric": {
"type": "string",
"minLength": 1,
"description": "The question the judge model answers. Should be answerable with PASS/FAIL + evidence."
},
"model": {
"type": "string",
"description": "Optional judge model override. When absent, defaults to the run-level judge model recorded in conditions.json, or the harness default when no run-level model was selected."
}
}
}
}
}