nomograph-jig 0.1.0

Agent-shape testing harness. Runs runtime-in-the-loop task batteries against a tool's CLI to measure first-try command success, tokens, turns, and invented-command count. TOML-driven, LLM-as-judge scored.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

# Example agent-shape.toml
#
# This example targets a fictional CLI called `notes` so the schema
# can be read without context from any specific real tool. Replace
# the [subject] / [fixture] / [[tasks.tuning]] entries with your
# own when adopting jig.

[subject]
name = "notes"
binary = "notes"
description = "Plain-text note manager. Subcommands: add, list, search, tag, archive."
version_pin = "1.0.0"

[fixture]
# Idempotent setup: run before every trial so state is isolated.
setup = "scripts/fixture-setup.sh"
cleanup = "scripts/fixture-cleanup.sh"
workdir = "fixtures/notes-realistic"
# Strip caller-side env vars that would leak the developer's session
# into a trial. List the exact names the subject tool reads.
strip_env = ["NOTES_DIR", "NOTES_PROFILE"]

[run]
n = 10
models = ["claude-opus-4-7", "claude-sonnet-4-6"]
# Short turn cap: correction loops decay quickly after the first few
# turns, so a tight cap keeps cost bounded without truncating signal.
turn_cap = 3
timeout_seconds = 180

[judge]
model = "claude-haiku-4-5"
double_score = true
required_fields = [
  "score",
  "first_command",
  "first_command_existed",
  "completed",
  "invented_commands",
  "fallback_to_sql",
  "reasoning",
]
rubric = """
You are judging whether an agent completed a CLI task correctly.

Score on a 4-point rubric:
  1.00  completed correctly, first command attempt was real and worked
  0.50  completed correctly, but reached for non-existent commands first
  0.25  completed via raw shell fallback (grep, awk, sqlite, etc.)
        instead of using the tool
  0.00  did not complete

For each transcript, identify:
  - first_command:         the exact shell command the agent first tried
  - first_command_existed: whether that command exists in the tool's CLI
  - completed:             did the agent finish the stated task
  - invented_commands:     list every non-existent command the agent tried
  - fallback_to_sql:       did the agent ever invoke sqlite, psql, grep,
                           or awk on the tool's internal files
  - reasoning:             one short paragraph

Respond in strict JSON with exactly the fields listed above.
"""

# Tuning tasks: the designer iterates against these while shaping the
# tool. They will overfit. That is expected and documented.
[[tasks.tuning]]
id = "exploration-01"
summary = "What's in this notes store?"
prompt = "You are looking at an unfamiliar notes archive. Use the `notes` CLI to give a one-paragraph overview of what is in it."
success_criteria = [
  "Agent invokes `notes list` within the first 2 commands",
  "Agent does not grep the storage directory directly",
]
author = "example@nomograph"
created_at = "2026-01-01"
sealed_against_tag = "notes-v1.0.0"

[[tasks.tuning]]
id = "search-01"
summary = "Find a note by keyword"
prompt = "Find every note that mentions 'invoice' anywhere in its body."
success_criteria = [
  "Agent does not invoke raw grep on the storage directory",
  "Agent uses `notes search` if the tool has it",
]
author = "example@nomograph"
created_at = "2026-01-01"
sealed_against_tag = "notes-v1.0.0"

[[tasks.tuning]]
id = "archive-01"
summary = "Archive a specific note"
prompt = "There is a note titled 'Old onboarding doc' that should be archived. Archive it without deleting it."
success_criteria = [
  "Agent uses `notes archive` rather than moving the file directly",
]
author = "example@nomograph"
created_at = "2026-01-01"
sealed_against_tag = "notes-v1.0.0"

[[tasks.tuning]]
id = "tag-walk-01"
summary = "Enumerate notes carrying a specific tag"
prompt = "List every note tagged with `#meeting` so the user can review them in one pass."
success_criteria = [
  "Agent uses `notes list --tag` or `notes tag` if either exists",
  "Agent does not iterate the directory by hand",
]
author = "example@nomograph"
created_at = "2026-01-01"
sealed_against_tag = "notes-v1.0.0"

# Hold-out tasks: intentionally empty in the example. Populated by
# authors who have not seen tuning-set results, so the score on
# hold-out is an unbiased estimate of the tool's surface quality.
# [[tasks.holdout]]  (none yet)