# lean-ctx accuracy suite (#730)
#
# Hand-authored, hand-verified accuracy tasks for the deterministic A/B harness
# (`core::eval_ab`). Three shapes that mirror the competitive accuracy claim:
#
# * needle-in-haystack — one answer-bearing file among distractors
# * long-context QA — a single fact located in one of several longer docs
# * code-edit — a runnable POSIX-shell fix scored by its unit test
#
# Every gold answer is a REAL, verifiable fact about lean-ctx (no mock data); each
# corpus file was opened and the label confirmed. The QA answers are deliberately
# distinctive (numbers / proper nouns) so they survive aggressive compression — the
# committed test `lean_ctx_compression_preserves_every_qa_answer` enforces exactly
# that, model-free, as the deterministic floor of the "compressed >= raw" claim.
#
# Run the full quality gate against a live (or recorded) model:
# lean-ctx eval ab --suite rust/eval/accuracy-suite.ndjson --record rust/eval/accuracy-recording.json --gate --margin 0.02
# lean-ctx eval ab --suite rust/eval/accuracy-suite.ndjson --replay rust/eval/accuracy-recording.json --gate --margin 0.02
{"id":"needle-aggressiveness-range","domain":"qa","prompt":"What is the numeric value range of the lean-ctx aggressiveness knob?","workspace":"accuracy/needle","retrieval_query":"aggressiveness compression knob numeric value range","answers":["0.0 to 1.0","from 0.0 to 1.0","0.0 (lossless) to 1.0 (maximum)"]}
{"id":"longqa-anthropic-cache-discount","domain":"qa","prompt":"What discount does Anthropic prompt caching apply to cached input tokens?","workspace":"accuracy/longqa","retrieval_query":"anthropic prompt caching cached input token discount percent","answers":["90%","a 90% discount","90 percent"]}
{"id":"jsonqa-operator-clearance","domain":"qa","prompt":"What security clearance is assigned to the operator named Kestrel in the roster?","workspace":"accuracy/jsoncrush","retrieval_query":"operator Kestrel security clearance roster level","answers":["omega-7"]}
{"id":"code-factorial","domain":"code","prompt":"Implement the POSIX shell function factorial in solution.sh so that `factorial n` prints n! (and 1 for n<=1). Output only the file contents.","workspace":"accuracy/code","target_file":"solution.sh","test_cmd":"sh test.sh"}