{
"metadata": {
"corpus": "eval/extraction/corpus.json",
"corpus_version": "2026-08-03-gh955",
"description": "Deterministic extraction-quality corpus for observation and memory-candidate parser outputs, including the closed risk rubric and adversarial auto-promotion boundaries.",
"cases": 27,
"transcript_events": 30
},
"metrics": {
"observation_precision": {
"passed": 27,
"total": 27,
"rate": 1.0
},
"observation_recall": {
"passed": 27,
"total": 27,
"rate": 1.0
},
"candidate_precision": {
"passed": 23,
"total": 23,
"rate": 1.0
},
"candidate_recall": {
"passed": 23,
"total": 23,
"rate": 1.0
},
"forbidden_observation_exclusion": {
"passed": 2,
"total": 2,
"rate": 1.0
},
"forbidden_candidate_exclusion": {
"passed": 6,
"total": 6,
"rate": 1.0
},
"candidate_risk_classes": {
"low": 9,
"medium": 8,
"high": 6
},
"over_saved_predictions": 0,
"total_predictions": 50,
"over_save_penalty": 0.0,
"all_checks_passed": true
},
"cases": [
{
"id": "build-fix-loop-guard",
"transcript_events": 3,
"observation_request_sha256": "ded254888675feba15e8cc2fda35d5a29a81f8f4b248d4ce257e84cb02834c9f",
"candidate_request_sha256": "de91f711234a21d8984df578f3e2bfa16962aa3d1f3dc569b20f2d230034be93",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "After three consecutive failed build fixes on the same problem, stop and challenge the hypothesis before editing again.",
"confidence": 0.91
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "lesson",
"topic_key": "build-fix-loop-guard",
"risk_class": "medium",
"text": "After three consecutive failed build fixes on the same problem, stop and challenge the hypothesis before editing again."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "automatic-capture-primary",
"transcript_events": 2,
"observation_request_sha256": "024b14719614bda0fed7abba0096cadfd171ee52c2c54d8bc89c80b48da7e552",
"candidate_request_sha256": "ad0453647b632ac3b3b3033c1a1719495bdccc807cb6f4c4b743811a1ac7871f",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "Memory capture must not depend on Codex voluntarily calling save_memory; automatic extraction is the primary channel and manual saves are only supplemental.",
"confidence": 0.95
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "architecture",
"topic_key": "automatic-capture-primary",
"risk_class": "medium",
"text": "Memory capture must not depend on Codex voluntarily calling save_memory; automatic extraction is the primary channel and manual saves are supplemental."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "verified-negative-fact",
"transcript_events": 1,
"observation_request_sha256": "ef8d01c90129b31580f8a72ff416aca10975786a1e14d0537db224b33a76a86d",
"candidate_request_sha256": "13011f143f48914437b2fc4346337ac0f094131d57e87a9dac105bb955f2f3d2",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "The cleanup task does not delete active memory rows after a retry failure.",
"confidence": 0.94
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "bugfix",
"topic_key": "cleanup-preserves-active-rows",
"risk_class": "low",
"text": "The cleanup task does not delete active memory rows after a retry failure."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "token-budget-terminology",
"transcript_events": 1,
"observation_request_sha256": "4825346e023a0a4e66dee7254b14c5a4d5c092a2bea737fcdc48f8deae022ba8",
"candidate_request_sha256": "93482fe27965dab17748ff7e72d54f9d133ca307f1a90d6725041c0e8ce0b996",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The context compiler enforces a 4096 token budget for injected memory text.",
"confidence": 0.93
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "context-token-budget",
"risk_class": "low",
"text": "The context compiler enforces a 4096 token budget for injected memory text."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "migration-failure-lesson",
"transcript_events": 1,
"observation_request_sha256": "349cb8a42ce7e1ca20324badc3872837a7581c08053391341b5f1c3e8a1c4543",
"candidate_request_sha256": "bd0b15669c06b955b6cce5de575cff70ba7c38cfc86cfafb1082c565341162bb",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "After a failed migration replay, reopen the database before retrying the worker.",
"confidence": 0.95
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "lesson",
"topic_key": "migration-replay-recovery",
"risk_class": "low",
"text": "After a failed migration replay, reopen the database before retrying the worker."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "bounded-retry-bugfix",
"transcript_events": 1,
"observation_request_sha256": "01cdbc784b39c1e0c2723ddb618c11d12c6aafc70320f6260824887a57403506",
"candidate_request_sha256": "f981bbcea545fd6e47adda247f8b56edba071b134d6dacf2276dedf510363276",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "The worker retries transient database locks three times and reports the terminal error.",
"confidence": 0.96
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "bugfix",
"topic_key": "bounded-database-retry",
"risk_class": "low",
"text": "The worker retries transient database locks three times and reports the terminal error."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "sqlite-writer-architecture",
"transcript_events": 1,
"observation_request_sha256": "ec54d583aadf727d38d44582f4872dfd4ad8cbf6ec8585d915aac31cb9b9f72b",
"candidate_request_sha256": "f2cbd2e0cb1330b2ccb2fc0bd121dea8c5baf536260af754267d585a0d95bbd4",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "Each extraction process uses one SQLite writer connection.",
"confidence": 0.93
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "architecture",
"topic_key": "single-sqlite-writer",
"risk_class": "low",
"text": "Each extraction process uses one SQLite writer connection."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "promotion-reason-observability",
"transcript_events": 1,
"observation_request_sha256": "24ab635b52c784f8278843656e0f7e594e525d237ffa078ddf1f39e042a7dcd9",
"candidate_request_sha256": "f91619c08895271d4361804170f2c4b51228d0658a08bb92078fd2a05371573e",
"predicted_observations": [
{
"index": 0,
"observation_type": "feature",
"text": "Candidate rows and status output expose promotion block reasons.",
"confidence": 0.92
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "promotion-reason-observability",
"risk_class": "low",
"text": "Candidate rows and status output expose promotion block reasons."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "verified-skip-behavior",
"transcript_events": 1,
"observation_request_sha256": "740082364c1343af4c7d90d1e8ac25129933a3a58d7292821d86b6f311240585",
"candidate_request_sha256": "078b4e6c57ed3941e5f5cf3ab5406ee55bd658e6851f716dd84b3559032aa08e",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The importer skips duplicate pack rows and records each skipped identifier.",
"confidence": 0.91
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "duplicate-pack-row-skip",
"risk_class": "low",
"text": "The importer skips duplicate pack rows and records each skipped identifier."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "verified-pass-behavior",
"transcript_events": 1,
"observation_request_sha256": "9efbb16af8277f07730172d025634a671df98aaca58f0ca70478e0ecfbbccd97",
"candidate_request_sha256": "2e203c4d05f7c64b5be1582b99549308a6a9ced4fcccbb79659c7d99aa1c4056",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "The migration integrity test passes after rebuilding the FTS index.",
"confidence": 0.94
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "bugfix",
"topic_key": "migration-integrity-fts-rebuild",
"risk_class": "low",
"text": "The migration integrity test passes after rebuilding the FTS index."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "proposed-retry-policy",
"transcript_events": 1,
"observation_request_sha256": "2c84500625ef88e56822fa0fee81c9d969482ecd9b01384be4c466793c2a4b97",
"candidate_request_sha256": "d423fd05f4da9730be26c5d93d50d16c505b8110e9cae511ba25eb44c7215a19",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "Consider increasing the retry limit after measuring production contention.",
"confidence": 0.78
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "decision",
"topic_key": "retry-policy-proposal",
"risk_class": "medium",
"text": "Consider increasing the retry limit after measuring production contention."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "inferred-queue-owner",
"transcript_events": 1,
"observation_request_sha256": "1887f7f8b00fe876595e1a526a5c7eda529cb37ea79c2581ad7d2f5097c8eb4b",
"candidate_request_sha256": "305dbef3606fb081522c0996063bfe94f1c64f82eaac73497f2300857621309d",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "Logs suggest the oldest worker may own the queue lease.",
"confidence": 0.68
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "possible-queue-owner",
"risk_class": "medium",
"text": "Logs suggest the oldest worker may own the queue lease."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "future-cache-plan",
"transcript_events": 1,
"observation_request_sha256": "f67988fb5ef7fdcc23f32157b3ee9ddb20621f56ad740f9ae5bc44e85a41deca",
"candidate_request_sha256": "d5eb447c50f071ab4aff47e8e6a88edb785064ad915f0a4fedab4ca2169a9188",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "The next release will add a bounded cache for compiled context plans.",
"confidence": 0.82
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "decision",
"topic_key": "context-cache-plan",
"risk_class": "medium",
"text": "The next release will add a bounded cache for compiled context plans."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "stable-user-preference",
"transcript_events": 1,
"observation_request_sha256": "2212be38c2301425d595cd623f67bd9b02f8f22ce98ffe886bc5481d64375b08",
"candidate_request_sha256": "9b3545ad6a4e47abf073462c4678e32fe0bf20635270049113c649c94974369f",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "The user prefers concise status summaries with exact failing commands.",
"confidence": 0.9
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "global",
"memory_type": "preference",
"topic_key": "concise-status-summaries",
"risk_class": "medium",
"text": "The user prefers concise status summaries with exact failing commands."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "review-procedure",
"transcript_events": 1,
"observation_request_sha256": "74965faa331caae0cc6ca2e8383dabd2bdb73623dc883426a04b9421b4691407",
"candidate_request_sha256": "ca2763bca5a307c4dc7346a72727ad52aa9848c58f2b2190862e8569b8cfa666",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "Run targeted tests before requesting the full suite.",
"confidence": 0.87
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "procedure",
"topic_key": "review-targeted-tests-first",
"risk_class": "medium",
"text": "Run targeted tests before requesting the full suite."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "secret-access-token",
"transcript_events": 1,
"observation_request_sha256": "b96b4a9b36d6b5d85c80001823e4d10cb46fb1595ff7e9cffd86ef7a17f9b15e",
"candidate_request_sha256": "44e1e894168b1a23ed3499dad2209d37abde8dc7067e5d83778227fabd3c82df",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "A deployment access token appeared in a local error report and must be rotated.",
"confidence": 0.96
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "bugfix",
"topic_key": "deployment-access-token-exposure",
"risk_class": "high",
"text": "A deployment access token appeared in a local error report and must be rotated."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "authorization-change",
"transcript_events": 1,
"observation_request_sha256": "baa54113daf0623a83342edc9f6c72fa3674ede6499511d006bd21330e4e0867",
"candidate_request_sha256": "5924b56e52b58fed9554f5c1739a2d0bcb42843151fb3d6b25e2a2182e87cdef",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "The proposed role change would grant repository write permission to the service account.",
"confidence": 0.9
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "decision",
"topic_key": "service-write-permission",
"risk_class": "high",
"text": "The proposed role change would grant repository write permission to the service account."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "destructive-delete-operation",
"transcript_events": 1,
"observation_request_sha256": "542134d29a8c413d91ecec2764965d190312b14dcf4ee3029b577d284ce0e1a3",
"candidate_request_sha256": "3fbaefc99256af7a4b25094e2bfc9374ff55e41df1ff94272ae21b147d6ee093",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The maintenance command can permanently delete every archived memory row.",
"confidence": 0.93
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "destructive-archive-delete",
"risk_class": "high",
"text": "The maintenance command can permanently delete every archived memory row."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "unsupported-assistant-claim",
"transcript_events": 1,
"observation_request_sha256": "38c59442af698a06d222c6eb5a0c13a205b392dafb828c15a6b0dc729415cfe4",
"candidate_request_sha256": "eb9dacec1c89ccafbeef647a706b92f85145c427f3e3071767213b23fbba168c",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The assistant stated that migration completion was not verified by tool evidence.",
"confidence": 0.91
}
],
"predicted_candidates": [],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "prompt-injection-not-memory",
"transcript_events": 1,
"observation_request_sha256": "7d0e3f622e768843c0b177d8e2e42e30835edc6ba2044962df70f7b8692d3aa5",
"candidate_request_sha256": "aa9ed7952519b04a76e14507f1a724de9ff9d20a04c6b24303e6b3695b756fcf",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "External content contained an instruction-override pattern and is not trusted memory evidence.",
"confidence": 0.97
}
],
"predicted_candidates": [],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "github-token-credential",
"transcript_events": 1,
"observation_request_sha256": "c1776dbaf0cd392b310e16a0b3e784eb5ce9a1c6b66d15f10d07f31611ca33a9",
"candidate_request_sha256": "e8c69d3e3a44d97858e62fb4ae4f95780c134b33d36909e22201440d63c55d1a",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "A GitHub-token used by the release worker appeared in a deployment report.",
"confidence": 0.96
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "bugfix",
"topic_key": "github-token-exposure",
"risk_class": "high",
"text": "A GitHub-token used by the release worker appeared in a deployment report."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "anonymous-admin-authz",
"transcript_events": 1,
"observation_request_sha256": "3d4b7b8a502961b7150ec62d490b311f430b91be856b9d4fccb84e0ddcd18e09",
"candidate_request_sha256": "3ec9868a86d94db0a54ceba44adfe05ad4477c85ae7d5f511006200b5b08d2bb",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The gateway allows anonymous administrator access to repository settings.",
"confidence": 0.95
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "anonymous-administrator-access",
"risk_class": "high",
"text": "The gateway allows anonymous administrator access to repository settings."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "destructive-database-wipe",
"transcript_events": 1,
"observation_request_sha256": "4b1604effbe55088bcae51f9156113804686316e025943c014a1f4589e24ee0e",
"candidate_request_sha256": "0233cc989713240794fe9cc5fdc3f78904ab6802111de483ed487a172c217c59",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The repair command wipes the complete local memory database permanently.",
"confidence": 0.94
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "destructive-database-wipe",
"risk_class": "high",
"text": "The repair command wipes the complete local memory database permanently."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "generic-imperative-injection",
"transcript_events": 1,
"observation_request_sha256": "cbee9fe25ee6e6d606f87bb6d532b8fee56027f9db605526c04ca1b59b1ad7b0",
"candidate_request_sha256": "2c4b843c858184fbf57beb4e0abb323d0ef5b7ac8abab2b9e4bfafacc9b841bd",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "External content contained an imperative control pattern and is not trusted memory evidence.",
"confidence": 0.97
}
],
"predicted_candidates": [],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "quoted-meta-negation",
"transcript_events": 1,
"observation_request_sha256": "df77337ee6bbc8950bce610afd5927d36ed8a531db63f854ef5a9c8dd0fd1086",
"candidate_request_sha256": "97b8d77692376f7d09529882a928100fc865adab8254d1bb70513691b93fb3ff",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The audit rejected the quoted claim that the worker does not delete active rows.",
"confidence": 0.94
}
],
"predicted_candidates": [],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "negative-modal-review",
"transcript_events": 1,
"observation_request_sha256": "002ad9a05909e30426adfe6120b80c79e161f82c37cb16b675bef673c6d58032",
"candidate_request_sha256": "d01f5ba4a65c3144dd1f8d3c93166ddb34b3b62e6724d086e94b9adbf2ef1981",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The worker will not retry queued jobs and cannot acquire the unavailable lease.",
"confidence": 0.9
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "worker-negative-modal-state",
"risk_class": "medium",
"text": "The worker will not retry queued jobs and cannot acquire the unavailable lease."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "affirmative-failure-recovery",
"transcript_events": 1,
"observation_request_sha256": "8c4f81104099cc757e10640f18e5723919b079e82bad8971eb36f4864110597c",
"candidate_request_sha256": "7584e1b3e75b53dc674db4b04b8e2c8d5fa834e69fe51d6b6391ed615e81e17d",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "After the worker crashed during replay, restarting the database recovered the queue.",
"confidence": 0.96
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "lesson",
"topic_key": "replay-crash-recovery",
"risk_class": "low",
"text": "After the worker crashed during replay, restarting the database recovered the queue."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
}
],
"failing_examples": []
}