eval-magic 0.5.0

One-stop CLI for running skill evals — measure whether an agent skill actually shifts behavior.
Documentation
{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "$id": "https://slow-powers.dev/schemas/stray-writes.schema.json",
  "title": "Stray-Write Report",
  "description": "Output of evals:detect-stray-writes. Flags subagent file writes / mutating commands that landed outside a run's outputs dir. Lives at <workspace>/iteration-N/stray-writes.json.",
  "type": "object",
  "required": ["generated", "iteration", "totals", "runs"],
  "additionalProperties": false,
  "properties": {
    "generated": { "type": "string", "description": "ISO timestamp" },
    "iteration": { "type": "integer" },
    "totals": {
      "type": "object",
      "required": ["violations", "warnings", "live_source_reads"],
      "additionalProperties": false,
      "properties": {
        "violations": { "type": "integer" },
        "warnings": { "type": "integer" },
        "live_source_reads": { "type": "integer" }
      }
    },
    "runs": {
      "type": "array",
      "description": "One entry per (eval, condition) run that had at least one finding.",
      "items": {
        "type": "object",
        "required": [
          "eval_id",
          "condition",
          "violations",
          "warnings",
          "live_source_reads"
        ],
        "additionalProperties": false,
        "properties": {
          "eval_id": { "type": "string" },
          "condition": { "type": "string" },
          "run_index": {
            "type": "integer",
            "minimum": 1,
            "description": "1-based run index within the (eval, condition) cell; absent for single-run cells."
          },
          "violations": {
            "type": "array",
            "description": "High-confidence: a write tool targeted a path outside the run's outputs dir.",
            "items": { "$ref": "#/definitions/finding" }
          },
          "warnings": {
            "type": "array",
            "description": "Heuristic: a Bash command matched a mutating pattern (install, git, sed -i, redirection) without referencing the outputs dir.",
            "items": { "$ref": "#/definitions/finding" }
          },
          "live_source_reads": {
            "type": "array",
            "description": "A read tool or Bash command accessed the live skill-under-test directory instead of the staged copy — the arm may be contaminated (staged-slug resolution race).",
            "items": { "$ref": "#/definitions/finding" }
          }
        }
      }
    }
  },
  "definitions": {
    "finding": {
      "type": "object",
      "required": ["tool", "ordinal", "reason"],
      "additionalProperties": false,
      "properties": {
        "tool": { "type": "string" },
        "path": {
          "type": "string",
          "description": "Target path for write-tool violations."
        },
        "command": {
          "type": "string",
          "description": "Command text for Bash warnings."
        },
        "ordinal": {
          "type": "integer",
          "description": "Position of the invocation in the run's tool_invocations."
        },
        "reason": { "type": "string" }
      }
    }
  }
}