1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
//! The `clap` derive command tree: the top-level parser, the shared/per-command
//! argument groups, and the subcommand enum.
//!
//! Flags are intentionally permissive (mostly optional); each handler tightens
//! them to what it actually requires (see the handlers in [`super::commands`]).
use clap::{Args, Parser, Subcommand};
/// Run skill evals — measure whether an agent skill actually shifts behavior.
///
/// An eval dispatches a fresh subagent twice per test case — once with the skill
/// loaded, once without (or old version vs. new) — and grades both outputs against
/// assertions. The pass-rate delta tells you whether the skill is worth shipping.
/// This CLI builds the workspace, stages skills for discovery, generates dispatch
/// prompts, assembles run records from transcripts, grades, and aggregates; your
/// agent harness supplies the one thing it never does itself: dispatching the
/// subagents.
///
/// The run loop is one canonical workflow in both modes:
///
/// run → dispatch agents → ingest → dispatch judges → finalize → teardown
///
/// The default target is the skill in the current directory. Pass
/// `--skill <path-or-name>` to select one skill from elsewhere. Pass
/// `--skill-dir <dir>` only when you want every other skill in that directory
/// staged as part of the eval environment. With no subcommand, the default
/// action is `run`.
#[derive(Debug, Parser)]
#[command(
name = "eval-magic",
version,
about = "Run skill evals — measure whether an agent skill actually shifts behavior.",
after_help = super::help::AFTER_HELP
)]
pub(crate) struct Cli {
/// Load a one-off harness descriptor file as the top registry layer.
///
/// The file merges field-by-field onto any registered harness with the same
/// `label` (or defines a new one), exactly like a project-local descriptor
/// in `.eval-magic/harnesses/`, and — when `--harness` is omitted — its
/// label becomes the invocation's default harness. Unlike discovered
/// descriptor files (skipped with a warning when broken), errors in this
/// explicitly named file are fatal.
#[arg(long, global = true, value_name = "PATH")]
pub harness_file: Option<String>,
#[command(subcommand)]
pub command: Option<Commands>,
}
/// Flags shared by most subcommands.
#[derive(Debug, Args)]
pub struct CommonArgs {
/// Optional directory of skills to stage as the eval environment.
///
/// Use this when the skill under test needs sibling skills available. The
/// skill-under-test is staged under a unique slug, and every *other* skill
/// folder inside this directory is staged under its natural name so
/// cross-references resolve. Omit it for the default single-skill isolated
/// run.
#[arg(long)]
pub skill_dir: Option<String>,
/// Skill under evaluation.
///
/// With `--skill-dir`, this is the child folder name, inferred when the
/// directory contains exactly one skill. Without `--skill-dir`, this is a
/// skill directory path, or a child directory name relative to the current
/// directory. Omit it when running from inside the skill directory.
#[arg(long)]
pub skill: Option<String>,
/// Iteration number for post-dispatch steps (defaults to latest existing).
#[arg(long)]
pub iteration: Option<u32>,
/// Comparison mode: `new-skill` (default, with vs. without) or `revision`
/// (old vs. new).
///
/// Mode A (`new-skill`) validates a brand-new skill against baseline behavior
/// with no skill loaded. Mode B (`revision`) tests a language change to an
/// existing skill: snapshot the old `SKILL.md` (see `snapshot`), then run both
/// variants against the same prompts. `revision` defaults `--baseline` to
/// `baseline`.
#[arg(long)]
pub mode: Option<String>,
/// Target harness: `claude-code` (default), `codex`, or `opencode`.
///
/// Claude Code and Codex both support staged skills, transcript ingest, and
/// the write guard (armed automatically). Each reads its own per-task events
/// file (`claude-events.jsonl`, `codex-events.jsonl`); Codex stages skills
/// under `.agents/skills`. OpenCode stages skills under `.opencode/skills`;
/// transcript ingest and the write guard are not yet wired for OpenCode.
/// The name is resolved against the harness descriptor registry after
/// parsing; an unknown name errors listing every registered harness.
#[arg(long)]
pub harness: Option<String>,
/// Workspace directory (defaults to `<cwd>/.eval-magic`).
///
/// The artifact root. Pass the same value to every command of a run, including
/// `teardown`.
#[arg(long)]
pub workspace_dir: Option<String>,
/// Restrict to these eval ids (comma-separated).
///
/// Mutually exclusive with `--skip`; every named id must exist or the run
/// aborts with the available ids listed. For cost-conscious reduced-set runs
/// without editing `evals.json`.
#[arg(long)]
pub only: Option<String>,
/// Skip these eval ids (comma-separated). Mutually exclusive with `--only`.
#[arg(long)]
pub skip: Option<String>,
/// Replace existing records rather than erroring.
#[arg(long)]
pub overwrite: bool,
}
/// `harness` groups the descriptor-inspection subcommands.
#[derive(Debug, Args)]
pub struct HarnessArgs {
#[command(subcommand)]
pub command: HarnessCommands,
}
#[derive(Debug, Subcommand)]
pub(crate) enum HarnessCommands {
/// Scaffold a commented descriptor template and notes skeleton for a new
/// harness.
///
/// Writes two files into the project-local descriptor layer:
/// `.eval-magic/harnesses/<name>.toml`, pre-filled with `label = "<name>"`
/// and every optional table present as a commented-out, inline-explained
/// example; and `.eval-magic/harnesses/<name>-notes.md`, the
/// verification-record skeleton where each filled-in value's source is
/// recorded. The scaffold is lint-clean exactly as written — `harness
/// lint` passes before a single field is filled in, and the file registers
/// `<name>` as a baseline harness immediately. Values must be verified
/// against the harness's own documentation or observed output, never
/// guessed. Scaffolding an already-registered name (e.g. the built-in
/// `claude-code`) is allowed with a note: descriptor layers merge
/// field-by-field, so the file overlays the registered harness instead of
/// defining a new one. The `[guard]` table is never scaffolded —
/// user-supplied descriptors may not declare it. The authoring guide is
/// docs/byoh.md; its "Upstreaming your descriptor" section covers
/// contributing the finished descriptor.
Init {
/// Label for the new harness (kebab-case, e.g. `cool-cli`); becomes
/// the descriptor's `label` and both file names.
name: String,
/// Print the rendered descriptor template to stdout instead of
/// writing files.
///
/// Prints only the template (no notes skeleton, no next steps), so
/// the output is redirectable — e.g. into a user-global layer
/// (`~/.config/eval-magic/harnesses/`) or a one-off `--harness-file`.
#[arg(long)]
stdout: bool,
/// Overwrite existing scaffold files.
#[arg(long)]
force: bool,
},
/// List every registered harness: name, source layer(s), enhancements.
///
/// One line per harness with the layers that contributed to it (built-in,
/// user, project, file — a merged descriptor shows all of them, e.g.
/// `built-in + project`) and the enhancements the resolved descriptor
/// declares (staging, skills-block, transcript, model-flag, guard,
/// shadow-preflight, dispatch-recipes; `baseline` when none). The session's
/// default harness — `claude-code`, or the `--harness-file` descriptor when
/// one is loaded — is marked `(default)`.
List,
/// Print one harness's resolved descriptor (after layer merging) as TOML.
///
/// The output is authorable: the provenance header lists every contributing
/// file as `#` comments, and the body is valid descriptor TOML you can copy
/// into a layer file (`.eval-magic/harnesses/<name>.toml`) and edit. Fields
/// at their baseline defaults are omitted. When reusing a guarded built-in's
/// output as a user layer, drop its `[guard]` table and
/// `run.supports_guard` first — user-supplied descriptors may not declare
/// them.
Show {
/// Registered harness name (see `harness list`).
name: String,
},
/// Validate a descriptor file, or every layer of a registered harness.
///
/// A file target runs the full load pipeline with one ✓/✗ line per check:
/// TOML syntax + schema (unknown fields, bad capability names), the
/// user-layer restrictions (`[guard]` and `run.supports_guard = true` stay
/// built-in-only; unguarded runs fall back to the detect-stray-writes
/// audit), and the cross-field invariants — merged onto the registered
/// harness with the same `label` when one exists, so a partial override is
/// checked against its real merge target. A name target re-lints every
/// discovered layer file strictly, surfacing descriptors that registry
/// initialization skipped with a warning. Exits non-zero when any check
/// fails. (A future `--probe` flag is reserved for a live dispatch probe
/// through the exec template.)
Lint {
/// Descriptor file path, or a registered harness name.
target: String,
},
}
/// `validate` only needs to know where to look.
#[derive(Debug, Args)]
pub struct ValidateArgs {
/// Directory whose child skills' `evals.json` files should be batch validated.
#[arg(long)]
pub skill_dir: Option<String>,
/// Skill directory to validate when `--skill-dir` is omitted.
#[arg(long)]
pub skill: Option<String>,
}
/// `init` writes the first eval scaffold for a skill.
#[derive(Debug, Args)]
pub struct InitArgs {
/// Optional directory containing the skill under evaluation.
///
/// Use this when the skill is an immediate child of a skills directory. If
/// omitted, `init` uses `--skill <path-or-name>` or the current directory.
/// `init` creates only the eval scaffold; it does not create the skill itself.
#[arg(long)]
pub skill_dir: Option<String>,
/// Skill under evaluation.
///
/// With `--skill-dir`, this is the child folder name, inferred when the
/// directory contains exactly one skill. Without `--skill-dir`, this is a
/// skill directory path, or a child directory name relative to the current
/// directory. This value becomes the generated `skill_name`.
#[arg(long)]
pub skill: Option<String>,
/// Stable kebab-case id for the first eval case.
///
/// If omitted, prompts interactively. The id is used as the workspace eval
/// directory name, so it must satisfy the eval schema's kebab-case pattern.
#[arg(long)]
pub id: Option<String>,
/// User-facing prompt the eval subagent receives.
///
/// If omitted, prompts interactively. Write this like a realistic user
/// request, not like an instruction to satisfy the eval.
#[arg(long)]
pub prompt: Option<String>,
/// Human-readable description of a successful response.
///
/// If omitted, prompts interactively. This seeds `expected_output`; add
/// concrete assertions after seeing iteration 1 outputs.
#[arg(long = "expected-output")]
pub expected_output: Option<String>,
/// Whether the skill is expected to trigger for this eval.
///
/// Defaults to true and is omitted from the generated JSON. Set false for
/// negative evals where correct behavior is not invoking the skill.
#[arg(long)]
pub skill_should_trigger: Option<bool>,
/// Overwrite an existing `<skill>/evals/evals.json`.
///
/// Refuses to overwrite existing evals by default and checks that before
/// prompting for seed fields.
#[arg(long)]
pub force: bool,
}
/// `grade` adds a finalize flag on top of the common set.
#[derive(Debug, Args)]
pub struct GradeArgs {
#[command(flatten)]
pub common: CommonArgs,
/// Merge judge responses instead of emitting judge tasks.
#[arg(long)]
pub finalize: bool,
}
/// `snapshot` adds a label and an optional git ref on top of the common set.
#[derive(Debug, Args)]
pub struct SnapshotArgs {
#[command(flatten)]
pub common: CommonArgs,
/// Label for the snapshot (its directory name under `snapshots/`).
#[arg(long)]
pub label: Option<String>,
/// Snapshot the skill as it existed at this git ref instead of the working
/// tree. (`ref` is a Rust keyword, so the field is `reference`.)
///
/// Reads the SKILL.md + sibling assets (excluding `evals/`) straight from git
/// without touching the working tree — the edit-first Mode B order: edit, then
/// `snapshot --ref HEAD`. Without `--ref`, snapshot reads the working tree.
#[arg(long = "ref")]
pub reference: Option<String>,
}
/// `promote-baseline` adds provenance flags (label + operator-declared models)
/// on top of the common set.
#[derive(Debug, Args)]
pub struct PromoteBaselineArgs {
#[command(flatten)]
pub common: CommonArgs,
/// Provenance label recorded in `BASELINE.md`.
///
/// Overrides a `label` recorded in the iteration's `conditions.json` (set via
/// `run --label`); when both are absent, `BASELINE.md` shows `(none)`.
#[arg(long)]
pub label: Option<String>,
/// Operator-declared agent model, recorded in `BASELINE.md`.
///
/// Overrides an `agent_model` recorded in the iteration's `conditions.json`
/// (set via `run --agent-model`); when both are absent, `BASELINE.md` shows
/// `unspecified`.
#[arg(long)]
pub agent_model: Option<String>,
/// Operator-declared judge model, recorded in `BASELINE.md`.
///
/// Overrides a `judge_model` recorded in the iteration's `conditions.json`
/// (set via `run --judge-model`); when both are absent, `BASELINE.md` shows
/// `unspecified`.
#[arg(long)]
pub judge_model: Option<String>,
}
/// `run` adds the build-time flags (mode/baseline selection, staging toggles,
/// guard, plan-mode, bootstrap) on top of the common set.
#[derive(Debug, Args)]
pub struct RunArgs {
#[command(flatten)]
pub common: CommonArgs,
/// Baseline snapshot label (defaults to `baseline` in `--mode revision`).
///
/// The snapshot label to use as the `old_skill` arm in revision mode (see
/// `snapshot`).
#[arg(long)]
pub baseline: Option<String>,
/// SessionStart-equivalent bootstrap file inlined into each dispatch.
///
/// A Markdown file prepended verbatim to every dispatch prompt inside a
/// `<session-start-context>` block — product-specific framing a SessionStart
/// hook would inject. It does NOT enumerate skills (the auto-built
/// available-skills block is the single source of the skill list). Omit it and
/// dispatches carry only that inventory.
#[arg(long)]
pub bootstrap: Option<String>,
/// Build the workspace but skip guard install and stop before next steps.
#[arg(long)]
pub dry_run: bool,
/// Inline each condition's SKILL.md into the dispatch prompt instead of
/// staging it under the harness skills dir.
///
/// For harnesses without project-local skill discovery. Forces the LLM-judge
/// meta-check tier and inlines only SKILL.md (not sibling skills or sibling
/// asset files); use the staged (default) path when the measured behavior
/// depends on sibling files. The isolated env (`env/`) is still built either
/// way — `--no-stage` only skips populating the harness skills dir. Also
/// disables the write guard (auto-armed or explicit) — it requires staging —
/// so no-stage runs are unguarded and rely on `detect-stray-writes` after
/// the fact.
#[arg(long)]
pub no_stage: bool,
/// Arm the write guard explicitly (it auto-arms when the harness supports it).
///
/// The write guard arms automatically on guard-capable built-in harnesses
/// whenever staging is active, so this flag only makes the request explicit:
/// where auto-arm quietly stays off (no declared guard, or `--no-stage`), an
/// explicit `--guard` warns, and a harness defined only by user-supplied
/// descriptors rejects it in preflight (rerun without it;
/// `detect-stray-writes` audits after the fact). Opt out with `--no-guard`.
///
/// The guard is a harness-native `PreToolUse` hook that *blocks* subagent
/// writes/installs outside the isolated run env (the agent-under-test's cwd)
/// while dispatches run. Its allowed roots are the env plus the OS temp dir, so
/// the guard boundary matches the same env that isolates the agent's reads.
/// Because the harness already cwd-bounds the agent's direct file tools to the
/// env, the guard's main remaining value is blocking Bash-subprocess escapes the
/// cwd boundary doesn't cover — `npm install`, `git worktree add`, `sed -i`,
/// redirects to absolute paths — and acting as a backstop when the isolated
/// session runs with relaxed permissions. The marker auto-expires after 6h
/// and is torn down at the next run; while armed the
/// hook fires on your own tool calls too. If it remains armed after `finalize`,
/// `finalize` reminds you to run `teardown` before editing source (which disarms
/// the cwd guard and every per-`(group, condition)` Cli env's guard). Requires
/// staging — with `--no-stage` the guard stays off and the run is unguarded.
/// Codex dispatches must include `--dangerously-bypass-hook-trust` so the
/// vetted project-local eval hook runs. Unguarded, stray writes are only
/// *detected* after the fact by `detect-stray-writes`, never blocked.
/// Under Claude Code the `PreToolUse` hook is staged in each env's
/// `.claude/settings.local.json`, and each `claude -p` dispatch loads it from
/// that cwd (`cd <eval-root>`), enforcing the eval boundary (the recipe never
/// passes `--bare`).
/// When invoking this from inside Codex, staging writes `.agents/skills` and
/// guarded runs also write `.codex/hooks.json`; Codex protects those paths in
/// its default workspace-write sandbox, so approval/escalation may be needed.
#[arg(long)]
pub guard: bool,
/// Opt out of the write guard for this run.
///
/// The guard arms automatically on harnesses that declare one (see
/// `--guard`). Pass this to run unguarded — e.g. when the skill under test
/// legitimately writes outside the isolated run env — and to silence the
/// unguarded-harness preflight warning. Unguarded, out-of-bounds writes are
/// only *detected* after the fact by `detect-stray-writes` (folded into
/// `ingest`), never blocked.
#[arg(long, conflicts_with = "guard")]
pub no_guard: bool,
/// Stage the skill-under-test under this verbatim name instead of the
/// conspicuous `slow-powers-eval-…` slug.
///
/// For name-confound experiments. Single-staging-condition modes only; refuses
/// to clobber an existing dir; registered for next-run cleanup.
#[arg(long)]
pub stage_name: Option<String>,
/// Inject the shared plan-mode profile as an operating-context layer.
///
/// Injects the shared, harness-agnostic plan-mode procedure
/// (`profiles/shared/plan-mode.md`) as a `<system-reminder>` in every
/// dispatch, identical across arms and harnesses. Opt-in, for
/// plan-mode-relevant skills. It is text the subagent reads, not a real
/// injected mode.
#[arg(long)]
pub plan_mode: bool,
/// Runs per condition cell, for variance reduction (default: 1).
///
/// Dispatches every eval N times per condition, so an iteration needs
/// `evals × 2 conditions × N` dispatches. Each run gets its own
/// `run-<k>/` directory under the condition (own `inputs/`, `outputs/`,
/// `run.json`, `timing.json`, `grading.json`) and a unique
/// `agent_description` carrying an `r<k>` segment. With N=1 the layout is
/// unchanged (artifacts sit directly in the condition directory). The
/// benchmark's per-condition `mean`/`stddev`/`n` then reflect all runs. A
/// per-eval `runs` field in evals.json overrides this flag for that eval.
#[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..))]
pub runs: u32,
/// Agent-under-test model for CLI dispatches; otherwise recorded as
/// provenance.
///
/// The run's dispatch recipes include the harness-native model flag when the
/// adapter supports one (e.g. Codex's `-m`, Claude Code's `--model`); otherwise
/// the value is persisted to `conditions.json` for `promote-baseline`.
#[arg(long)]
pub agent_model: Option<String>,
/// Default judge model for emitted judge tasks.
///
/// `grade` writes this into `judge-tasks.json` for judge tasks that do not
/// have an assertion-level `model` override, and Cli harness judge recipes
/// pass it through using the harness-native model flag. Also persists to
/// `conditions.json` for `promote-baseline`.
#[arg(long)]
pub judge_model: Option<String>,
/// Provenance label for this run, persisted into `conditions.json`.
///
/// Surfaced in `BASELINE.md` by `promote-baseline` (its own `--label` flag
/// still overrides).
#[arg(long)]
pub label: Option<String>,
}
/// Every subcommand on the CLI.
#[derive(Debug, Subcommand)]
pub(crate) enum Commands {
/// Build dispatches and run evals (the default action).
///
/// Builds the iteration workspace, snapshots the `SKILL.md`, stages skills, and
/// emits `dispatch.json` (machine-readable) alongside `dispatch-manifest.md`
/// (human-readable). Dispatch each task through the harness CLI (`claude -p`,
/// `codex exec`). Also writes `RUNBOOK.md`, a human-followable handoff for the
/// run ("Read and follow RUNBOOK.md").
Run(RunArgs),
/// Snapshot a workspace baseline.
///
/// Snapshots the skill as a Mode B baseline under
/// `<workspace>/<skill>/snapshots/<label>/`. Snapshots persist across
/// iterations; delete them by hand when no longer needed.
Snapshot(SnapshotArgs),
/// Tear down a workspace.
///
/// Disarms the guard, removes the staged skill set, and reclaims the workspace
/// artifacts that are safe to delete. Run it at the end of a run.
Teardown(CommonArgs),
/// Disarm the write guard.
///
/// Removes only the write guard (e.g. mid-run, before hand-editing files the
/// guard would block). The full `teardown` removes the guard AND the staged
/// skill set.
TeardownGuard(CommonArgs),
/// Ingest recorded transcripts into run records.
///
/// Fixed-order chain: record-runs → fill-transcripts → detect-stray-writes →
/// grade. Assembles each task's `run.json` + `timing.json`, scans for stray
/// writes, grades `transcript_check` assertions, then stops at the judge
/// hand-off, listing a judge task per `llm_judge` assertion. Requires
/// `--iteration`; reads each task's `outputs/<harness>-events.jsonl`.
/// Re-running after a fix is safe — every sub-step skips work already done.
Ingest(CommonArgs),
/// Finalize grading after judge responses are in.
///
/// Fixed-order chain: grade `--finalize` → aggregate. Merges the judge verdicts
/// and writes `benchmark.json`. If a live guard remains armed — the cwd guard, or
/// any per-`(group, condition)` Cli env guard — prints a `teardown` reminder before
/// source edits. Requires `--iteration`.
Finalize(CommonArgs),
/// Assemble run records from a dispatch and its transcripts.
///
/// Assembles a schema-valid `run.json` and backfills `timing.json` for every
/// task in a runner-built iteration, from `dispatch.json` +
/// `outputs/final-message.md` + each task's `outputs/<harness>-events.jsonl`.
/// Never clobbers existing records without `--overwrite`; transcript-derived
/// timing carries `"source": "transcript"`. Folded into `ingest`.
RecordRuns(CommonArgs),
/// Populate tool invocations from persisted transcripts.
///
/// Reads each task's `outputs/<harness>-events.jsonl` and populates
/// `tool_invocations` in `run.json`. Subsumed by `record-runs` for
/// runner-built iterations; still the tool for filling a pre-existing (hand- or
/// agent-written) `run.json`.
FillTranscripts(CommonArgs),
/// Detect writes outside the sandbox output boundary.
///
/// Scans each run's `tool_invocations` and writes `stray-writes.json`: write
/// tools targeting paths outside the run's outputs dir (violations), mutating
/// Bash heuristics (warnings), and live-source reads (an arm that read the live
/// skill instead of its staged copy). `aggregate` lifts all three into
/// `benchmark.json`'s `validity_warnings`.
DetectStrayWrites(CommonArgs),
/// Grade run records (transcript checks + LLM-judge task emission).
///
/// Evaluates `transcript_check` assertions directly (regex against
/// `tool_invocations`) and emits judge-task files for `llm_judge` assertions;
/// with `--finalize`, merges judge responses into per-run `grading.json`.
///
/// Injects the `__skill_invoked` meta-check — did the skill actually influence
/// behavior? It has two tiers, chosen automatically per run: code-based (where
/// the staged slug + transcript are available, as on Claude Code, it checks the
/// transcript for a `Skill` call matching the eval slug — deterministic and
/// free) and an LLM-judge fallback (where transcripts aren't available, a judge
/// compares the final message against the SKILL.md for behavioral fingerprints).
/// The meta-check does not count toward the substantive `pass_rate`.
Grade(GradeArgs),
/// Aggregate before/after benchmark deltas.
///
/// Reads grading + timing from an iteration and writes `benchmark.json` with
/// pass-rate / duration / token stats per condition, the delta, and
/// `validity_warnings`.
Aggregate(CommonArgs),
/// Scaffold a first `evals/evals.json` for a skill.
///
/// Creates `<skill>/evals/evals.json` with one schema-valid seed eval, then
/// prints the next run/ingest/finalize/promote commands. Prompts
/// interactively for any missing seed fields, and refuses to overwrite an
/// existing eval file unless `--force` is passed. This is scaffold-only: it
/// does not run agents, ingest transcripts, finalize, or promote results.
Init(InitArgs),
/// Promote a benchmark + gradings into a committed baseline.
PromoteBaseline(PromoteBaselineArgs),
/// Validate `evals.json` files against the bundled schemas.
Validate(ValidateArgs),
/// Inspect and validate harness descriptors (built-in and user-supplied).
///
/// Harnesses are described by layered TOML descriptor files: embedded
/// built-ins → user-global (`$EVAL_MAGIC_CONFIG_DIR`,
/// `$XDG_CONFIG_HOME/eval-magic`, or `~/.config/eval-magic`, each under
/// `harnesses/*.toml`) → project-local (`.eval-magic/harnesses/*.toml`) →
/// a one-off `--harness-file <path>`. A later file whose `label` matches an
/// earlier descriptor overrides individual fields (field-level merge, not
/// whole-file shadowing); a new `label` defines a new harness usable with
/// `--harness`. `list` surveys the registry, `show` prints one resolved
/// descriptor, and `lint` validates a descriptor file or registered name.
Harness(HarnessArgs),
/// Internal PreToolUse hook entry point. Invoked by the installed write-guard
/// hook as `eval-magic guard <marker>`, not by users; hidden from help.
#[command(hide = true)]
Guard {
/// Path to the guard marker file. Defaults to
/// `<cwd>/.claude/skills/.slow-powers-eval-guard.json`.
marker: Option<String>,
},
/// Internal Codex PreToolUse hook entry point. Invoked by the installed
/// write-guard hook as `eval-magic guard-codex <marker>`, not by users;
/// hidden from help.
#[command(hide = true)]
GuardCodex {
/// Path to the guard marker file. Defaults to
/// `<cwd>/.agents/skills/.slow-powers-eval-guard.json`.
marker: Option<String>,
},
/// Internal generic PreToolUse hook entry point. Invoked by the installed
/// write-guard hook as `eval-magic guard-hook --harness <name> <marker>`,
/// not by users; hidden from help. `guard` / `guard-codex` are frozen
/// aliases of this for the claude-code and codex harnesses.
#[command(hide = true, name = "guard-hook")]
GuardHook {
/// Harness whose embedded descriptor supplies the verdict shape; an
/// unknown name fails open (allows the call).
#[arg(long)]
harness: String,
/// Path to the guard marker file. Defaults to the harness's
/// `<skills_dir>/.slow-powers-eval-guard.json` under the cwd.
marker: Option<String>,
},
}