eval-magic 0.3.4

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation
{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://slow-powers.dev/schemas/run-record.schema.json",
  "title": "Portable Run Record",
  "description": "Captures one subagent run. Harness-agnostic — each harness writes an adapter from its native transcript format to this shape. Downstream grading reads only this file.",
  "type": "object",
  "required": [
    "eval_id",
    "condition",
    "skill_path",
    "prompt",
    "files",
    "final_message",
    "tool_invocations"
  ],
  "additionalProperties": false,
  "properties": {
    "eval_id": {
      "type": "string",
      "description": "Matches the eval's id in evals.json."
    },
    "condition": {
      "type": "string",
      "description": "Reserved names: with_skill, without_skill, old_skill, new_skill."
    },
    "skill_path": {
      "type": ["string", "null"],
      "description": "Absolute path to the SKILL.md the subagent could load, or null if no skill was provided (without_skill condition)."
    },
    "prompt": {
      "type": "string",
      "description": "The user prompt as dispatched to the subagent."
    },
    "files": {
      "type": "array",
      "items": { "type": "string" },
      "description": "Fixture files the subagent had access to (absolute paths inside the run's workspace)."
    },
    "final_message": {
      "type": "string",
      "description": "The agent's final user-facing text output."
    },
    "tool_invocations": {
      "type": "array",
      "description": "Ordered list of tool calls during the run.",
      "items": {
        "type": "object",
        "required": ["name", "ordinal"],
        "additionalProperties": false,
        "properties": {
          "name": {
            "type": "string",
            "description": "Tool name as recorded by the harness (e.g. Bash, Read, run_command). Adapters should preserve original names."
          },
          "args": {
            "description": "Tool arguments. Object for structured tools, string for raw command-style tools.",
            "type": ["object", "string", "array", "null"]
          },
          "result": {
            "description": "Tool output, if captured. Truncate long outputs to ~2KB.",
            "type": ["string", "object", "null"]
          },
          "ordinal": {
            "type": "integer",
            "minimum": 0,
            "description": "0-indexed position in the run. Used by must_precede checks."
          }
        }
      }
    },
    "total_tokens": {
      "type": ["integer", "null"],
      "description": "From the harness's task completion event, or derived from the persisted transcript by record-runs (usage summed across unique message ids, including cache creation/read tokens — a different accounting than the completion event). Canonical timing lives in the sibling timing.json, whose `source` field records which origin produced it. May be null if neither source is available."
    },
    "duration_ms": {
      "type": ["integer", "null"],
      "description": "From the harness's task completion event, or derived from the persisted transcript by record-runs (wall clock between the first and last transcript timestamps). Canonical timing lives in the sibling timing.json. May be null if neither source is available."
    },
    "run_index": {
      "type": "integer",
      "minimum": 1,
      "description": "1-based run index within a multi-run (eval, condition) cell; absent for single-run cells."
    }
  }
}