eval-magic 0.5.0

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://slow-powers.dev/schemas/grading.schema.json",
  "title": "Grading Result",
  "description": "Output of grading one (eval, condition) pair. Lives at <workspace>/iteration-N/eval-<id>/<condition>/grading.json.",
  "type": "object",
  "required": ["assertion_results", "summary"],
  "additionalProperties": false,
  "properties": {
    "assertion_results": {
      "type": "array",
      "items": {
        "type": "object",
        "required": ["id", "passed", "evidence"],
        "additionalProperties": false,
        "properties": {
          "id": {
            "type": "string",
            "description": "Matches the assertion id in evals.json."
          },
          "passed": { "type": "boolean" },
          "evidence": {
            "type": "string",
            "description": "Direct quote or specific reference from the run record. Vague summaries are not evidence."
          },
          "confidence": {
            "type": "number",
            "minimum": 0,
            "maximum": 1,
            "description": "Judge confidence. Low confidence (< 0.7) flags this result for human review. Always 1.0 for transcript_check results."
          },
          "grader": {
            "type": "string",
            "enum": ["transcript_check", "llm_judge"],
            "description": "Which grader produced this result."
          }
        }
      }
    },
    "summary": {
      "type": "object",
      "required": ["passed", "failed", "total", "pass_rate"],
      "additionalProperties": false,
      "properties": {
        "passed": { "type": "integer", "minimum": 0 },
        "failed": { "type": "integer", "minimum": 0 },
        "total": { "type": "integer", "minimum": 0 },
        "pass_rate": { "type": "number", "minimum": 0, "maximum": 1 }
      }
    },
    "meta_results": {
      "type": "array",
      "description": "Framework-injected meta-assertions (e.g. skill-invocation check). Reserved id prefix: __ (double underscore). Tracked separately from substantive assertion_results so they do not pollute the skill effectiveness pass_rate.",
      "items": {
        "type": "object",
        "required": ["id", "passed", "evidence"],
        "additionalProperties": false,
        "properties": {
          "id": { "type": "string" },
          "passed": { "type": "boolean" },
          "evidence": { "type": "string" },
          "confidence": { "type": "number", "minimum": 0, "maximum": 1 },
          "grader": {
            "type": "string",
            "enum": ["transcript_check", "llm_judge"]
          }
        }
      }
    },
    "meta_summary": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "passed": { "type": "integer", "minimum": 0 },
        "failed": { "type": "integer", "minimum": 0 },
        "total": { "type": "integer", "minimum": 0 },
        "skill_invoked": {
          "description": "True when the skill-invocation meta-check passed; false when the judge found no evidence the skill influenced behavior; null when no skill was loaded for this run.",
          "type": ["boolean", "null"]
        }
      }
    }
  }
}