eval-magic 0.6.0

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://slow-powers.dev/schemas/conversation.schema.json",
  "title": "Scripted Conversation Completion",
  "description": "Runner-owned completion artifact for one scripted multi-turn eval task.",
  "type": "object",
  "required": ["status", "delivered_followups", "events"],
  "additionalProperties": false,
  "properties": {
    "status": {
      "type": "string",
      "enum": ["completed", "stopped"]
    },
    "delivered_followups": {
      "type": "integer",
      "minimum": 0
    },
    "stop_reason": {
      "type": "string",
      "enum": ["agent_did_not_ask", "agent_response_mismatch"]
    },
    "stopped_before_followup": {
      "type": "integer",
      "minimum": 1
    },
    "events": {
      "type": "array",
      "minItems": 2,
      "items": {
        "oneOf": [
          { "$ref": "#/definitions/userMessage" },
          { "$ref": "#/definitions/assistantMessage" },
          { "$ref": "#/definitions/conversationTool" }
        ]
      }
    }
  },
  "allOf": [
    {
      "if": {
        "properties": { "status": { "const": "stopped" } },
        "required": ["status"]
      },
      "then": {
        "required": ["stop_reason", "stopped_before_followup"]
      }
    },
    {
      "if": {
        "properties": { "status": { "const": "completed" } },
        "required": ["status"]
      },
      "then": {
        "not": {
          "anyOf": [
            { "required": ["stop_reason"] },
            { "required": ["stopped_before_followup"] }
          ]
        }
      }
    }
  ],
  "definitions": {
    "userMessage": {
      "type": "object",
      "required": ["type", "ordinal", "round", "text"],
      "additionalProperties": false,
      "properties": {
        "type": { "const": "user_message" },
        "ordinal": { "type": "integer", "minimum": 0 },
        "round": { "type": "integer", "minimum": 1 },
        "text": { "type": "string" }
      }
    },
    "assistantMessage": {
      "type": "object",
      "required": ["type", "ordinal", "round", "text"],
      "additionalProperties": false,
      "properties": {
        "type": { "const": "assistant_message" },
        "ordinal": { "type": "integer", "minimum": 0 },
        "round": { "type": "integer", "minimum": 1 },
        "text": { "type": "string" }
      }
    },
    "conversationTool": {
      "type": "object",
      "required": ["type", "ordinal", "round", "name"],
      "additionalProperties": false,
      "properties": {
        "type": { "const": "tool_invocation" },
        "ordinal": { "type": "integer", "minimum": 0 },
        "round": { "type": "integer", "minimum": 1 },
        "name": { "type": "string" },
        "args": { "type": ["object", "string", "array", "null"] },
        "result": { "type": ["string", "object", "null"] }
      }
    }
  }
}