eval-magic 0.6.1

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://slow-powers.dev/schemas/guard-denials.schema.json",
  "title": "Guard Denial Report",
  "description": "Output collected during detect-stray-writes from each guarded task environment. Records every blocked tool call because each denial may alter eval behavior. Lives at <workspace>/iteration-N/guard-denials.json.",
  "type": "object",
  "required": ["generated", "iteration", "total_denials", "tasks"],
  "additionalProperties": false,
  "properties": {
    "generated": { "type": "string", "description": "ISO timestamp" },
    "iteration": { "type": "integer", "minimum": 1 },
    "total_denials": { "type": "integer", "minimum": 0 },
    "tasks": {
      "type": "array",
      "description": "One entry per (eval, condition, optional run index) with at least one guard denial.",
      "items": {
        "type": "object",
        "required": [
          "eval_id",
          "condition",
          "denial_count",
          "denials"
        ],
        "additionalProperties": false,
        "properties": {
          "eval_id": { "type": "string" },
          "condition": { "type": "string" },
          "run_index": {
            "type": "integer",
            "minimum": 1,
            "description": "1-based run index within the (eval, condition) cell; absent for single-run cells."
          },
          "denial_count": { "type": "integer", "minimum": 1 },
          "denials": {
            "type": "array",
            "minItems": 1,
            "items": { "$ref": "#/definitions/denial" }
          }
        }
      }
    }
  },
  "definitions": {
    "denial": {
      "type": "object",
      "required": [
        "timestamp",
        "harness",
        "tool",
        "reason",
        "resolved_targets",
        "input_keys"
      ],
      "additionalProperties": false,
      "properties": {
        "timestamp": { "type": "string", "description": "ISO timestamp" },
        "harness": { "type": "string" },
        "tool": { "type": "string" },
        "reason": { "type": "string" },
        "resolved_targets": {
          "type": "array",
          "items": { "type": "string" }
        },
        "input_keys": {
          "type": "array",
          "uniqueItems": true,
          "items": { "type": "string" }
        }
      }
    }
  }
}