nomograph-jig 0.1.0

Agent-shape testing harness. Runs runtime-in-the-loop task batteries against a tool's CLI to measure first-try command success, tokens, turns, and invented-command count. TOML-driven, LLM-as-judge scored.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138

# jig's own agent-shape.toml.
#
# The methodology applies recursively: jig is itself a CLI agents
# might use, so it earns the same surface-shape measurement as any
# other subject. Tuning tasks here probe the offline subcommands
# (check, render, compare); run and rejudge are intentionally absent
# because they need claude in the loop and the methodology measures
# the agent reaching for the right read paths first.
#
# The expected workflow for a self-test run:
#   make build
#   jig check agent-shape.toml --binary ./jig
#   jig run agent-shape.toml --tuning-only --n 3
#
# Running a real baseline costs claude API tokens and is not part of
# CI; it is what an external user runs once they want to characterise
# jig's own surface.

[subject]
name = "jig"
binary = "jig"
description = "Agent-shape testing harness. Subcommands: run, check, render, compare, rejudge."
version_pin = "0.1.0"

[fixture]
setup = "scripts/agent-shape-fixture.sh"
workdir = "fixtures/jig-self-test"
# jig itself does not read any well-known env vars, so nothing to
# strip; the field is kept here so the self-test exercises the
# strip_env wiring path even when empty.
strip_env = []

[run]
n = 5
models = ["claude-opus-4-7", "claude-sonnet-4-6"]
turn_cap = 3
timeout_seconds = 120

[judge]
model = "claude-haiku-4-5"
double_score = true
required_fields = [
  "score",
  "first_command",
  "first_command_existed",
  "completed",
  "invented_commands",
  "fallback_to_sql",
  "reasoning",
]
rubric = """
You are judging whether an agent completed a CLI task correctly using
the `jig` agent-shape testing harness.

Score on a 4-point rubric:
  1.00  completed correctly, first command attempt was real and worked
  0.50  completed correctly, but reached for non-existent commands first
  0.25  completed via raw shell/grep/jq fallback instead of the tool
  0.00  did not complete

Commands that exist in jig (top-level subcommands):
  jig run | check | render | compare | rejudge | help
plus the global flags: --help, --version.

`jig run` and `jig rejudge` spawn claude and cost API; for offline
exploration use `jig check`, `jig render`, and `jig compare`.

Anything not on that list, or shaped like `jig list-tasks`,
`jig show`, `jig status`, `jig init`, etc., counts as invented.

For each transcript, identify:
  - first_command:         the exact shell command the agent first tried
  - first_command_existed: whether that command exists in the tool's CLI
  - completed:             did the agent finish what the user asked
  - invented_commands:     list every non-existent jig command the agent tried
  - fallback_to_sql:       did the agent invoke jq, grep, awk, sqlite,
                           or sed against the fixture files instead of
                           using jig
  - reasoning:             one short paragraph

Respond in strict JSON with exactly the fields listed above. No
markdown fences. No prose outside the JSON object.
"""

[commands]
top_level = ["run", "check", "render", "compare", "rejudge"]

[[tasks.tuning]]
id = "validate-config-01"
summary = "Validate the agent-shape.toml in this fixture"
prompt = "There is an agent-shape.toml in this directory. Use jig to validate it and report whether it parses cleanly."
success_criteria = [
  "Agent invokes `jig check` against the TOML",
  "Agent does not invoke jq, grep, awk, or sed against the TOML",
]
author = "andrew@dunn.dev"
created_at = "2026-04-25"
sealed_against_tag = "jig-v0.1.0"

[[tasks.tuning]]
id = "render-report-01"
summary = "Render an existing JSON report to Markdown"
prompt = "There is a baseline.json file in this directory produced by a prior jig run. Render it as Markdown for human reading."
success_criteria = [
  "Agent invokes `jig render` against baseline.json",
  "Agent does not invoke jq, grep, or sed",
]
author = "andrew@dunn.dev"
created_at = "2026-04-25"
sealed_against_tag = "jig-v0.1.0"

[[tasks.tuning]]
id = "compare-reports-01"
summary = "Quantify the change between baseline and treated reports"
prompt = "There are two JSON reports in this directory: baseline.json and treated.json. Show me the per-cell delta between them."
success_criteria = [
  "Agent invokes `jig compare` against both files",
  "Agent does not invoke jq, awk, or sed to compute the delta by hand",
]
author = "andrew@dunn.dev"
created_at = "2026-04-25"
sealed_against_tag = "jig-v0.1.0"

[[tasks.tuning]]
id = "drift-check-01"
summary = "Cross-reference rubric against the binary's --help"
prompt = "I just edited the agent-shape.toml's [commands] section. Use jig to verify it still matches the binary's --help."
success_criteria = [
  "Agent invokes `jig check --binary` with the local jig binary",
  "Agent does not parse --help by hand with grep or awk",
]
author = "andrew@dunn.dev"
created_at = "2026-04-25"
sealed_against_tag = "jig-v0.1.0"

# Hold-out tasks: empty in v1, by design. Populated later by an
# author who has not seen the tuning-set behaviour.
# [[tasks.holdout]]