{
"metadata": {
"corpus": "eval/extraction/corpus.json",
"corpus_version": "2026-08-03-gh955",
"description": "Deterministic extraction-quality corpus for observation and memory-candidate parser outputs, including the closed risk rubric and adversarial auto-promotion boundaries.",
"cases": 27,
"transcript_events": 30
},
"metrics": {
"observation_precision": {
"passed": 27,
"total": 27,
"rate": 1.0
},
"observation_recall": {
"passed": 27,
"total": 27,
"rate": 1.0
},
"candidate_precision": {
"passed": 23,
"total": 23,
"rate": 1.0
},
"candidate_recall": {
"passed": 23,
"total": 23,
"rate": 1.0
},
"forbidden_observation_exclusion": {
"passed": 2,
"total": 2,
"rate": 1.0
},
"forbidden_candidate_exclusion": {
"passed": 6,
"total": 6,
"rate": 1.0
},
"candidate_risk_classes": {
"low": 9,
"medium": 8,
"high": 6
},
"over_saved_predictions": 0,
"total_predictions": 50,
"over_save_penalty": 0.0,
"all_checks_passed": true
},
"cases": [
{
"id": "build-fix-loop-guard",
"transcript_events": 3,
"observation_request_sha256": "cc47030422213d9f0ca512a1425ae7625f9510b9916d67060d5d36fea453d1e3",
"candidate_request_sha256": "de91f711234a21d8984df578f3e2bfa16962aa3d1f3dc569b20f2d230034be93",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "After three consecutive failed build fixes on the same problem, stop and challenge the hypothesis before editing again.",
"confidence": 0.91
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "lesson",
"topic_key": "build-fix-loop-guard",
"risk_class": "medium",
"text": "After three consecutive failed build fixes on the same problem, stop and challenge the hypothesis before editing again."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "automatic-capture-primary",
"transcript_events": 2,
"observation_request_sha256": "8935a5480c7310f4d6d5240052687fa050b07f2ef1837b82afeb5bcbc958c4b0",
"candidate_request_sha256": "ad0453647b632ac3b3b3033c1a1719495bdccc807cb6f4c4b743811a1ac7871f",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "Memory capture must not depend on Codex voluntarily calling save_memory; automatic extraction is the primary channel and manual saves are only supplemental.",
"confidence": 0.95
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "architecture",
"topic_key": "automatic-capture-primary",
"risk_class": "medium",
"text": "Memory capture must not depend on Codex voluntarily calling save_memory; automatic extraction is the primary channel and manual saves are supplemental."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "verified-negative-fact",
"transcript_events": 1,
"observation_request_sha256": "de868beb761c5bde7469842d8c7d5fcdf4da8569af58a97c8bea230b87478233",
"candidate_request_sha256": "13011f143f48914437b2fc4346337ac0f094131d57e87a9dac105bb955f2f3d2",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "The cleanup task does not delete active memory rows after a retry failure.",
"confidence": 0.94
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "bugfix",
"topic_key": "cleanup-preserves-active-rows",
"risk_class": "low",
"text": "The cleanup task does not delete active memory rows after a retry failure."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "token-budget-terminology",
"transcript_events": 1,
"observation_request_sha256": "d6943aa5865a72f6b84ac901410adc3bd52343c04a511c11231b4652e9db6705",
"candidate_request_sha256": "93482fe27965dab17748ff7e72d54f9d133ca307f1a90d6725041c0e8ce0b996",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The context compiler enforces a 4096 token budget for injected memory text.",
"confidence": 0.93
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "context-token-budget",
"risk_class": "low",
"text": "The context compiler enforces a 4096 token budget for injected memory text."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "migration-failure-lesson",
"transcript_events": 1,
"observation_request_sha256": "e9b856c9f4134774d40e496554caf92740006f3bc883a9c9bbd134b14e1d298c",
"candidate_request_sha256": "bd0b15669c06b955b6cce5de575cff70ba7c38cfc86cfafb1082c565341162bb",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "After a failed migration replay, reopen the database before retrying the worker.",
"confidence": 0.95
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "lesson",
"topic_key": "migration-replay-recovery",
"risk_class": "low",
"text": "After a failed migration replay, reopen the database before retrying the worker."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "bounded-retry-bugfix",
"transcript_events": 1,
"observation_request_sha256": "614b7e49ff450bbbb0a6f38e98af025afe2057557ab5d49da507791b1287f78f",
"candidate_request_sha256": "f981bbcea545fd6e47adda247f8b56edba071b134d6dacf2276dedf510363276",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "The worker retries transient database locks three times and reports the terminal error.",
"confidence": 0.96
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "bugfix",
"topic_key": "bounded-database-retry",
"risk_class": "low",
"text": "The worker retries transient database locks three times and reports the terminal error."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "sqlite-writer-architecture",
"transcript_events": 1,
"observation_request_sha256": "32cc4dbf733c989921f8b7515144c54b0a37a3a5d32f280be6b0893cf102397e",
"candidate_request_sha256": "f2cbd2e0cb1330b2ccb2fc0bd121dea8c5baf536260af754267d585a0d95bbd4",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "Each extraction process uses one SQLite writer connection.",
"confidence": 0.93
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "architecture",
"topic_key": "single-sqlite-writer",
"risk_class": "low",
"text": "Each extraction process uses one SQLite writer connection."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "promotion-reason-observability",
"transcript_events": 1,
"observation_request_sha256": "9968e56a42614fcb5b6275bde9bbc55f4b2fbf49309058e10f75bbda51596457",
"candidate_request_sha256": "f91619c08895271d4361804170f2c4b51228d0658a08bb92078fd2a05371573e",
"predicted_observations": [
{
"index": 0,
"observation_type": "feature",
"text": "Candidate rows and status output expose promotion block reasons.",
"confidence": 0.92
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "promotion-reason-observability",
"risk_class": "low",
"text": "Candidate rows and status output expose promotion block reasons."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "verified-skip-behavior",
"transcript_events": 1,
"observation_request_sha256": "e18186f70042d8ffc7f9696ab8876985a808e06c68377da458c3649f67a410f8",
"candidate_request_sha256": "078b4e6c57ed3941e5f5cf3ab5406ee55bd658e6851f716dd84b3559032aa08e",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The importer skips duplicate pack rows and records each skipped identifier.",
"confidence": 0.91
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "duplicate-pack-row-skip",
"risk_class": "low",
"text": "The importer skips duplicate pack rows and records each skipped identifier."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "verified-pass-behavior",
"transcript_events": 1,
"observation_request_sha256": "56d103c768820cce699d5c3133a5d1adb55ff126a0812dd8a620d2c392c74266",
"candidate_request_sha256": "2e203c4d05f7c64b5be1582b99549308a6a9ced4fcccbb79659c7d99aa1c4056",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "The migration integrity test passes after rebuilding the FTS index.",
"confidence": 0.94
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "bugfix",
"topic_key": "migration-integrity-fts-rebuild",
"risk_class": "low",
"text": "The migration integrity test passes after rebuilding the FTS index."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "proposed-retry-policy",
"transcript_events": 1,
"observation_request_sha256": "d5c6dd3436f4903c889989384c1da9e9c27623fbd4ffa98822d0bdfb54a38139",
"candidate_request_sha256": "d423fd05f4da9730be26c5d93d50d16c505b8110e9cae511ba25eb44c7215a19",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "Consider increasing the retry limit after measuring production contention.",
"confidence": 0.78
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "decision",
"topic_key": "retry-policy-proposal",
"risk_class": "medium",
"text": "Consider increasing the retry limit after measuring production contention."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "inferred-queue-owner",
"transcript_events": 1,
"observation_request_sha256": "f3001fa018b220549a57926c66bf1fc86e851310604067a7824edc564d08a2e2",
"candidate_request_sha256": "305dbef3606fb081522c0996063bfe94f1c64f82eaac73497f2300857621309d",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "Logs suggest the oldest worker may own the queue lease.",
"confidence": 0.68
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "possible-queue-owner",
"risk_class": "medium",
"text": "Logs suggest the oldest worker may own the queue lease."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "future-cache-plan",
"transcript_events": 1,
"observation_request_sha256": "6309ab3fbba3d4ee0e5b1fc3f37d72f93553483e22d29d53007ef2dc4b1541d1",
"candidate_request_sha256": "d5eb447c50f071ab4aff47e8e6a88edb785064ad915f0a4fedab4ca2169a9188",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "The next release will add a bounded cache for compiled context plans.",
"confidence": 0.82
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "decision",
"topic_key": "context-cache-plan",
"risk_class": "medium",
"text": "The next release will add a bounded cache for compiled context plans."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "stable-user-preference",
"transcript_events": 1,
"observation_request_sha256": "f0c279f9a687ab65f9dc1d574608405309ee73181fe10ca8538b18598a093375",
"candidate_request_sha256": "9b3545ad6a4e47abf073462c4678e32fe0bf20635270049113c649c94974369f",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "The user prefers concise status summaries with exact failing commands.",
"confidence": 0.9
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "global",
"memory_type": "preference",
"topic_key": "concise-status-summaries",
"risk_class": "medium",
"text": "The user prefers concise status summaries with exact failing commands."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "review-procedure",
"transcript_events": 1,
"observation_request_sha256": "f0c39e55679d462c0aef10c697ce921d014b6db153da13c9a9847b5dbe1d5ad8",
"candidate_request_sha256": "ca2763bca5a307c4dc7346a72727ad52aa9848c58f2b2190862e8569b8cfa666",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "Run targeted tests before requesting the full suite.",
"confidence": 0.87
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "procedure",
"topic_key": "review-targeted-tests-first",
"risk_class": "medium",
"text": "Run targeted tests before requesting the full suite."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "secret-access-token",
"transcript_events": 1,
"observation_request_sha256": "4be217a49196267c3e42f78367d4e879aa1bd04a223141a5ad5bbec3acac05a9",
"candidate_request_sha256": "44e1e894168b1a23ed3499dad2209d37abde8dc7067e5d83778227fabd3c82df",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "A deployment access token appeared in a local error report and must be rotated.",
"confidence": 0.96
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "bugfix",
"topic_key": "deployment-access-token-exposure",
"risk_class": "high",
"text": "A deployment access token appeared in a local error report and must be rotated."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "authorization-change",
"transcript_events": 1,
"observation_request_sha256": "78e864541c23481efc33823631d4961a35772868a6763b9c86430ebd33e5451e",
"candidate_request_sha256": "5924b56e52b58fed9554f5c1739a2d0bcb42843151fb3d6b25e2a2182e87cdef",
"predicted_observations": [
{
"index": 0,
"observation_type": "decision",
"text": "The proposed role change would grant repository write permission to the service account.",
"confidence": 0.9
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "decision",
"topic_key": "service-write-permission",
"risk_class": "high",
"text": "The proposed role change would grant repository write permission to the service account."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "destructive-delete-operation",
"transcript_events": 1,
"observation_request_sha256": "e71e8a6946a26b5c09d5cb65189381023b6fc980319e17129414811fb53c9cee",
"candidate_request_sha256": "3fbaefc99256af7a4b25094e2bfc9374ff55e41df1ff94272ae21b147d6ee093",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The maintenance command can permanently delete every archived memory row.",
"confidence": 0.93
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "destructive-archive-delete",
"risk_class": "high",
"text": "The maintenance command can permanently delete every archived memory row."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "unsupported-assistant-claim",
"transcript_events": 1,
"observation_request_sha256": "32a36e2107bacb750d471efe8a83cbc713609f3fd873972e6d75fd8917d5c836",
"candidate_request_sha256": "eb9dacec1c89ccafbeef647a706b92f85145c427f3e3071767213b23fbba168c",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The assistant stated that migration completion was not verified by tool evidence.",
"confidence": 0.91
}
],
"predicted_candidates": [],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "prompt-injection-not-memory",
"transcript_events": 1,
"observation_request_sha256": "5be15aef9903fbbb493660ee8e1dd2b4ad57f491a30522529a52f30f30243d82",
"candidate_request_sha256": "aa9ed7952519b04a76e14507f1a724de9ff9d20a04c6b24303e6b3695b756fcf",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "External content contained an instruction-override pattern and is not trusted memory evidence.",
"confidence": 0.97
}
],
"predicted_candidates": [],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "github-token-credential",
"transcript_events": 1,
"observation_request_sha256": "098d92bcb3952a5ec5b217630aff3911d6ad598bec9b1735a3faa22b6da6c498",
"candidate_request_sha256": "e8c69d3e3a44d97858e62fb4ae4f95780c134b33d36909e22201440d63c55d1a",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "A GitHub-token used by the release worker appeared in a deployment report.",
"confidence": 0.96
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "bugfix",
"topic_key": "github-token-exposure",
"risk_class": "high",
"text": "A GitHub-token used by the release worker appeared in a deployment report."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "anonymous-admin-authz",
"transcript_events": 1,
"observation_request_sha256": "5e9da8bfa0f5d9dc73e763d11271549851fafa35367d641fab4d88bc079beae1",
"candidate_request_sha256": "3ec9868a86d94db0a54ceba44adfe05ad4477c85ae7d5f511006200b5b08d2bb",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The gateway allows anonymous administrator access to repository settings.",
"confidence": 0.95
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "anonymous-administrator-access",
"risk_class": "high",
"text": "The gateway allows anonymous administrator access to repository settings."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "destructive-database-wipe",
"transcript_events": 1,
"observation_request_sha256": "08db3510c2a695fa8a08610368e4351d7cb9dbc948c8088d74fcbfdb9cd2030f",
"candidate_request_sha256": "0233cc989713240794fe9cc5fdc3f78904ab6802111de483ed487a172c217c59",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The repair command wipes the complete local memory database permanently.",
"confidence": 0.94
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "destructive-database-wipe",
"risk_class": "high",
"text": "The repair command wipes the complete local memory database permanently."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "generic-imperative-injection",
"transcript_events": 1,
"observation_request_sha256": "63f1983c6f12479fa3e1c04f5bb837fcb21e0a81413c24ba5dd3314e398293ad",
"candidate_request_sha256": "2c4b843c858184fbf57beb4e0abb323d0ef5b7ac8abab2b9e4bfafacc9b841bd",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "External content contained an imperative control pattern and is not trusted memory evidence.",
"confidence": 0.97
}
],
"predicted_candidates": [],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "quoted-meta-negation",
"transcript_events": 1,
"observation_request_sha256": "78790f66ad256653d7b4e53d57d8481c0e405cc9f89907ccc16d047c497ab792",
"candidate_request_sha256": "97b8d77692376f7d09529882a928100fc865adab8254d1bb70513691b93fb3ff",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The audit rejected the quoted claim that the worker does not delete active rows.",
"confidence": 0.94
}
],
"predicted_candidates": [],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "negative-modal-review",
"transcript_events": 1,
"observation_request_sha256": "d73ea86a2c57b5663b73910ac436b3d3cce9d25941e0977f49676c0f58ff447a",
"candidate_request_sha256": "d01f5ba4a65c3144dd1f8d3c93166ddb34b3b62e6724d086e94b9adbf2ef1981",
"predicted_observations": [
{
"index": 0,
"observation_type": "discovery",
"text": "The worker will not retry queued jobs and cannot acquire the unavailable lease.",
"confidence": 0.9
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "discovery",
"topic_key": "worker-negative-modal-state",
"risk_class": "medium",
"text": "The worker will not retry queued jobs and cannot acquire the unavailable lease."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
},
{
"id": "affirmative-failure-recovery",
"transcript_events": 1,
"observation_request_sha256": "4e97c863d3cf496412e23a5e3b2973ea765f7531b0e690c1f6f2219a07dbb34a",
"candidate_request_sha256": "7584e1b3e75b53dc674db4b04b8e2c8d5fa834e69fe51d6b6391ed615e81e17d",
"predicted_observations": [
{
"index": 0,
"observation_type": "bugfix",
"text": "After the worker crashed during replay, restarting the database recovered the queue.",
"confidence": 0.96
}
],
"predicted_candidates": [
{
"index": 0,
"scope": "project",
"memory_type": "lesson",
"topic_key": "replay-crash-recovery",
"risk_class": "low",
"text": "After the worker crashed during replay, restarting the database recovered the queue."
}
],
"missing_expected_observations": [],
"unexpected_observations": [],
"forbidden_observations": [],
"missing_expected_candidates": [],
"unexpected_candidates": [],
"forbidden_candidates": [],
"over_saved_predictions": 0,
"pass": true
}
],
"failing_examples": []
}