eval-magic 0.6.0

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://slow-powers.dev/schemas/command-check.schema.json",
  "title": "Command Check Result",
  "description": "Runner-owned result for one command_check assertion before finalize folds it into grading.json.",
  "type": "object",
  "required": [
    "id",
    "passed",
    "evidence",
    "expected_exit_code",
    "actual_exit_code",
    "stdout",
    "stderr"
  ],
  "additionalProperties": false,
  "properties": {
    "id": { "type": "string" },
    "passed": { "type": "boolean" },
    "evidence": { "type": "string" },
    "expected_exit_code": {
      "type": "integer",
      "minimum": -2147483648,
      "maximum": 2147483647
    },
    "actual_exit_code": {
      "type": ["integer", "null"],
      "minimum": -2147483648,
      "maximum": 2147483647,
      "description": "The single command's exit code, null for signal termination or a matrix result."
    },
    "stdout": {
      "type": "string",
      "description": "Lossy UTF-8 stdout diagnostic, truncated to at most 2 KiB; empty for a matrix result."
    },
    "stderr": {
      "type": "string",
      "description": "Lossy UTF-8 stderr diagnostic, truncated to at most 2 KiB; empty for a matrix result."
    },
    "cells": {
      "type": "array",
      "minItems": 1,
      "description": "Per-cell results for a matrix command check. Absent for a non-matrix check.",
      "items": {
        "type": "object",
        "required": [
          "env",
          "passed",
          "evidence",
          "actual_exit_code",
          "stdout",
          "stderr"
        ],
        "additionalProperties": false,
        "properties": {
          "env": {
            "type": "object",
            "minProperties": 1,
            "additionalProperties": {
              "type": "string"
            },
            "description": "Effective command-check environment overrides for this cell."
          },
          "passed": { "type": "boolean" },
          "evidence": { "type": "string" },
          "actual_exit_code": {
            "type": ["integer", "null"],
            "minimum": -2147483648,
            "maximum": 2147483647
          },
          "stdout": {
            "type": "string",
            "description": "Lossy UTF-8 stdout diagnostic, truncated to at most 2 KiB."
          },
          "stderr": {
            "type": "string",
            "description": "Lossy UTF-8 stderr diagnostic, truncated to at most 2 KiB."
          }
        }
      }
    }
  }
}