1use serde::{Deserialize, Serialize, Serializer};
2use serde_json::Value;
3use std::collections::BTreeMap;
4use std::path::PathBuf;
5
6use crate::eval::coding_bench::{RememContextAuditSnapshot, RememContextAuditStatus};
7use crate::eval::memory_bench::types::{
8 MemoryBenchPolicyOutcome, MemoryBenchPolicySummary, MemoryBenchSuiteFixture,
9};
10
11#[derive(Debug, Clone)]
12pub struct BenchVerifyOptions {
13 pub root: PathBuf,
14 pub claim_registry_path: PathBuf,
15}
16
17impl BenchVerifyOptions {
18 pub fn new(root: impl Into<PathBuf>, claim_registry_path: impl Into<PathBuf>) -> Self {
19 Self {
20 root: root.into(),
21 claim_registry_path: claim_registry_path.into(),
22 }
23 }
24}
25
26#[derive(Debug, Clone)]
27pub struct VerifiedArtifact<T> {
28 pub path: String,
29 pub sha256: String,
30 pub value: T,
31}
32
33#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
34pub struct BenchVerifyFailure {
35 pub path: String,
36 pub message: String,
37}
38
39#[derive(Debug, Clone, Serialize)]
40pub struct BenchVerifyReport {
41 pub schema_version: u32,
42 pub root: String,
43 pub passed: bool,
44 pub manifests_checked: usize,
45 pub reports_checked: usize,
46 pub run_artifacts_checked: usize,
47 pub artifact_files_checked: usize,
48 pub failures: Vec<BenchVerifyFailure>,
49 pub authority_verdict: AuthorityVerdict,
50 #[serde(skip)]
51 pub(crate) verified_artifacts: VerifiedBenchmarkArtifacts,
52}
53
54#[derive(Serialize)]
55struct PersistedBenchVerifyReport<'a> {
56 schema_version: u32,
57 root: &'a str,
58 passed: bool,
59 manifests_checked: usize,
60 reports_checked: usize,
61 run_artifacts_checked: usize,
62 artifact_files_checked: usize,
63 failures: &'a [BenchVerifyFailure],
64}
65
66pub(crate) fn serialize_persisted_bench_verify_report<S>(
67 report: &BenchVerifyReport,
68 serializer: S,
69) -> Result<S::Ok, S::Error>
70where
71 S: Serializer,
72{
73 PersistedBenchVerifyReport {
74 schema_version: report.schema_version,
75 root: &report.root,
76 passed: report.passed,
77 manifests_checked: report.manifests_checked,
78 reports_checked: report.reports_checked,
79 run_artifacts_checked: report.run_artifacts_checked,
80 artifact_files_checked: report.artifact_files_checked,
81 failures: &report.failures,
82 }
83 .serialize(serializer)
84}
85
86#[derive(Debug, Clone, Default)]
87pub(crate) struct VerifiedBenchmarkArtifacts {
88 pub memory_suites: Vec<VerifiedArtifact<MemoryBenchSuiteFixture>>,
89 pub manifests: Vec<VerifiedArtifact<PublicBenchmarkManifest>>,
90 pub reports: Vec<VerifiedArtifact<PublicBenchmarkReport>>,
91 pub memory_runs: Vec<VerifiedArtifact<MemoryRunArtifact>>,
92 pub coding_runs: Vec<VerifiedArtifact<CodingRunArtifact>>,
93 pub security_policy_outcomes: BTreeMap<String, MemoryBenchPolicyOutcome>,
94 pub claim_registry: Option<VerifiedArtifact<ClaimRegistryPolicy>>,
95 pub curator_logs: BTreeMap<String, VerifiedArtifact<CuratorLogArtifact>>,
96 pub official_coding_tests: BTreeMap<String, VerifiedArtifact<OfficialCodingTestEvidence>>,
97 pub treatment_maintenance:
98 BTreeMap<String, VerifiedArtifact<OfficialCodingMaintenanceEvidence>>,
99 pub official_evidence_authenticated: bool,
100}
101
102#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq)]
103pub enum AuthorityStatus {
104 #[serde(rename = "PASS")]
105 Pass,
106 #[serde(rename = "FAIL")]
107 Fail,
108 #[serde(rename = "INSUFFICIENT")]
109 Insufficient,
110}
111
112#[derive(Debug, Clone, Serialize)]
113pub struct AuthorityVerdict {
114 pub schema_version: u32,
115 pub status: AuthorityStatus,
116 pub consumed_bytes: BTreeMap<String, String>,
117 pub implementation: ImplementationAuthorityBinding,
118 pub security: SecurityAuthorityVerdict,
119 pub gh931: Gh931AuthorityVerdict,
120 pub release: ReleaseAuthorityVerdict,
121 pub diagnostics: Vec<String>,
122}
123
124#[derive(Debug, Clone, Serialize)]
125pub struct ImplementationAuthorityBinding {
126 pub build_git_sha: Option<String>,
127 pub checkout_git_sha: Option<String>,
128 pub build_source_dirty: Option<bool>,
129 pub checkout_source_dirty: Option<bool>,
130 pub build_production_input_tree_sha256: Option<String>,
131 pub checkout_production_input_tree_sha256: Option<String>,
132 pub production_pathspec_sha256: Option<String>,
133 pub executable_source_equivalent: bool,
134 pub diagnostics: Vec<String>,
135}
136
137#[derive(Debug, Clone, Serialize)]
138pub struct Gh931AuthorityVerdict {
139 pub status: AuthorityStatus,
140 pub measurement_ready: bool,
141 pub registry: Gh931RegistryBinding,
142 pub report: Option<Gh931ReportBinding>,
143 pub completeness: Gh931Completeness,
144 pub condition_completion: Vec<Gh931ConditionCompletion>,
145 pub paired_statistics: Vec<super::report::CodingPairedStatistic>,
146 pub maintenance: Gh931MaintenanceVerdict,
147 pub stop_loss: Gh931StopLossVerdict,
148 pub claims: Vec<Gh931ClaimVerdict>,
149 pub diagnostics: Vec<String>,
150}
151
152#[derive(Debug, Clone, Serialize)]
153pub struct Gh931ConditionCompletion {
154 pub condition: String,
155 pub eligible_started: usize,
156 pub resolved: usize,
157}
158
159#[derive(Debug, Clone, Serialize)]
160pub struct Gh931MaintenanceVerdict {
161 pub status: AuthorityStatus,
162 pub curator_tasks: usize,
163 pub curator_sessions: usize,
164 pub curator_minutes: Option<f64>,
165 pub curated_minutes_per_100_sessions: Option<f64>,
166 pub remem_sessions: Option<usize>,
167 pub remem_minutes_per_100_sessions: Option<f64>,
168 pub reduction_pct: Option<f64>,
169 pub diagnostics: Vec<String>,
170}
171
172#[derive(Debug, Clone, Serialize)]
173pub struct Gh931RegistryBinding {
174 pub path: Option<String>,
175 pub sha256: Option<String>,
176 pub schema_version: Option<u32>,
177 pub issue: Option<String>,
178 pub locked: bool,
179 pub policy_valid: bool,
180 pub declared_statuses: Vec<AuthorityStatus>,
181}
182
183#[derive(Debug, Clone, Serialize)]
184pub struct Gh931ReportBinding {
185 pub path: String,
186 pub sha256: String,
187 pub conditions: Vec<String>,
188 pub models_by_condition: BTreeMap<String, Vec<Value>>,
189 pub platforms: Vec<String>,
190 pub producing_shas: Vec<String>,
191 pub production_input_trees: Vec<String>,
192 pub source_dirty_attestations: Vec<Option<bool>>,
193}
194
195#[derive(Debug, Clone, Default, Serialize)]
196pub struct Gh931Completeness {
197 pub expected_tasks: usize,
198 pub expected_conditions: usize,
199 pub expected_runs_per_task: usize,
200 pub expected_runs: usize,
201 pub observed_runs: usize,
202 pub complete: bool,
203 pub attempts_ready: bool,
204 pub machine_outcomes_ready: bool,
205}
206
207#[derive(Debug, Clone, Serialize)]
208pub struct Gh931StopLossVerdict {
209 pub status: AuthorityStatus,
210 pub eligible_runs: usize,
211 pub memory_hurt_rate_pct: Option<f64>,
212 pub stale_memory_followed_rate_pct: Option<f64>,
213 pub diagnostics: Vec<String>,
214}
215
216#[derive(Debug, Clone, Serialize)]
217pub struct Gh931ClaimVerdict {
218 pub id: String,
219 pub status: AuthorityStatus,
220 pub declared_registry_status: AuthorityStatus,
221 pub treatment: String,
222 pub control: String,
223 pub metric: String,
224 pub allowed_wording: Vec<String>,
225 pub forbidden_wording: Vec<String>,
226 pub diagnostics: Vec<String>,
227}
228
229#[derive(Debug, Clone, Deserialize)]
230#[serde(deny_unknown_fields)]
231pub(crate) struct ClaimRegistryPolicy {
232 pub schema_version: u32,
233 pub issue: String,
234 pub locked: bool,
235 pub claims: Vec<ClaimRegistryClaimPolicy>,
236}
237
238#[derive(Debug, Clone, Deserialize)]
239#[serde(deny_unknown_fields)]
240pub(crate) struct ClaimRegistryClaimPolicy {
241 pub id: String,
242 pub comparison: ClaimRegistryComparison,
243 pub metric: String,
244 pub gate: ClaimRegistryGate,
245 pub status: AuthorityStatus,
246 pub allowed_wording: Vec<String>,
247 pub forbidden_wording: Vec<String>,
248 pub supporting_report: Value,
249}
250
251#[derive(Debug, Clone, Deserialize)]
252#[serde(deny_unknown_fields)]
253pub(crate) struct ClaimRegistryComparison {
254 pub treatment: String,
255 pub control: String,
256}
257
258#[derive(Debug, Clone, Deserialize)]
259#[serde(untagged)]
260pub(crate) enum ClaimRegistryGate {
261 Superiority(ClaimSuperiorityGate),
262 NonInferiority(ClaimNonInferiorityGate),
263 StopLoss(ClaimStopLossGate),
264}
265
266#[derive(Debug, Clone, Deserialize)]
267#[serde(deny_unknown_fields)]
268pub(crate) struct ClaimSuperiorityGate {
269 pub min_effect_pp: f64,
270 pub ci_lower_bound_pp_gt: f64,
271 pub ci_level: f64,
272 pub statistical_unit: String,
273 pub method: String,
274}
275
276#[derive(Debug, Clone, Deserialize)]
277#[serde(deny_unknown_fields)]
278pub(crate) struct ClaimNonInferiorityGate {
279 pub non_inferiority_margin_pp: f64,
280 pub human_maintenance_reduction_min_pct: f64,
281 pub ci_level: f64,
282 pub statistical_unit: String,
283 pub method: String,
284}
285
286#[derive(Debug, Clone, Deserialize)]
287#[serde(deny_unknown_fields)]
288pub(crate) struct ClaimStopLossGate {
289 pub memory_hurt_max_pct: f64,
290 pub stale_memory_followed_max_pct: f64,
291}
292
293#[derive(Debug, Clone, Deserialize)]
294#[serde(deny_unknown_fields)]
295pub(crate) struct CuratorLogArtifact {
296 pub schema_version: u32,
297 pub condition: String,
298 pub task_id: String,
299 pub target_blind: bool,
300 pub budget: CuratorBudget,
301 pub sessions: Vec<CuratorSession>,
302 pub totals: CuratorTotals,
303 pub final_char_count: usize,
304 pub final_file_sha256: String,
305}
306
307#[derive(Debug, Clone, Deserialize)]
308#[serde(deny_unknown_fields)]
309pub(crate) struct CuratorBudget {
310 pub minutes_per_session: f64,
311 pub max_chars: usize,
312}
313
314#[derive(Debug, Clone, Deserialize)]
315#[serde(deny_unknown_fields)]
316pub(crate) struct CuratorSession {
317 pub episode_id: String,
318 pub minutes_spent: f64,
319 pub edit_count: u64,
320 pub deletion_count: u64,
321 pub conflict_resolution_count: u64,
322 pub chars_after: usize,
323}
324
325#[derive(Debug, Clone, Deserialize)]
326#[serde(deny_unknown_fields)]
327pub(crate) struct CuratorTotals {
328 pub maintenance_minutes: f64,
329 pub update_count: u64,
330 pub deletion_count: u64,
331 pub conflict_resolution_count: u64,
332}
333
334#[derive(Debug, Clone, Deserialize)]
335#[serde(deny_unknown_fields)]
336pub(crate) struct OfficialCodingTestEvidence {
337 pub schema_version: u32,
338 pub task_id: String,
339 pub condition: String,
340 pub run_index: u32,
341 pub attempt_id: String,
342 pub commands: Vec<OfficialCodingCommandResult>,
343}
344
345#[derive(Debug, Clone, Deserialize)]
346#[serde(deny_unknown_fields)]
347pub(crate) struct OfficialCodingCommandResult {
348 pub command: String,
349 pub exit_code: Option<i32>,
350 pub timed_out: bool,
351}
352
353impl OfficialCodingTestEvidence {
354 pub(crate) fn command_validation_error(&self) -> Option<&'static str> {
355 if self.commands.is_empty() {
356 return Some("official coding test evidence requires commands");
357 }
358 self.commands
359 .iter()
360 .any(|result| {
361 result.command.trim().is_empty() || result.timed_out != result.exit_code.is_none()
362 })
363 .then_some("official coding command result is internally inconsistent")
364 }
365
366 pub(crate) fn matches_registered_scorer_commands(&self, task_id: &str) -> bool {
367 let Ok(fixture): Result<Value, _> = serde_json::from_str(include_str!(
368 "../../../eval/coding-bench/fixtures/tasks.json"
369 )) else {
370 return false;
371 };
372 let Some(task) = fixture["tasks"].as_array().and_then(|tasks| {
373 tasks
374 .iter()
375 .find(|task| task["id"].as_str() == Some(task_id))
376 }) else {
377 return false;
378 };
379 let Some(expected) = task["score"]["commands"].as_array().and_then(|commands| {
380 commands
381 .iter()
382 .map(|command| {
383 command
384 .as_array()?
385 .iter()
386 .map(Value::as_str)
387 .collect::<Option<Vec<_>>>()
388 .map(|parts| parts.join(" "))
389 })
390 .collect::<Option<Vec<_>>>()
391 }) else {
392 return false;
393 };
394 self.commands
395 .iter()
396 .map(|result| result.command.as_str())
397 .eq(expected.iter().map(String::as_str))
398 }
399
400 pub(crate) fn resolved(&self) -> bool {
401 !self.commands.is_empty()
402 && self
403 .commands
404 .iter()
405 .all(|result| !result.timed_out && result.exit_code == Some(0))
406 }
407
408 pub(crate) fn failure_reason(&self) -> Option<&'static str> {
409 if self.commands.iter().any(|result| result.timed_out) {
410 Some("timeout")
411 } else if self
412 .commands
413 .iter()
414 .any(|result| result.exit_code != Some(0))
415 {
416 Some("test_failure")
417 } else {
418 None
419 }
420 }
421}
422
423#[derive(Debug, Clone, Deserialize)]
424#[serde(deny_unknown_fields)]
425pub(crate) struct OfficialCodingMaintenanceEvidence {
426 pub schema_version: u32,
427 pub task_id: String,
428 pub condition: String,
429 pub run_index: u32,
430 pub attempt_id: String,
431 pub measurement: OfficialCodingMaintenanceMeasurement,
432}
433
434#[derive(Debug, Clone, Deserialize)]
435#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
436pub(crate) enum OfficialCodingMaintenanceMeasurement {
437 SupervisorTimed {
438 minutes: f64,
439 session_count: usize,
440 },
441 ZeroWork {
442 minutes: f64,
443 work_events: u64,
444 session_count: usize,
445 },
446}
447
448impl OfficialCodingMaintenanceMeasurement {
449 pub(crate) fn is_valid(&self) -> bool {
450 match self {
451 Self::SupervisorTimed {
452 minutes,
453 session_count,
454 } => minutes.is_finite() && *minutes > 0.0 && *session_count > 0,
455 Self::ZeroWork {
456 minutes,
457 work_events,
458 session_count,
459 } => minutes.is_finite() && *minutes == 0.0 && *work_events == 0 && *session_count > 0,
460 }
461 }
462}
463
464impl OfficialCodingMaintenanceEvidence {
465 pub(crate) fn minutes(&self) -> f64 {
466 match &self.measurement {
467 OfficialCodingMaintenanceMeasurement::SupervisorTimed { minutes, .. }
468 | OfficialCodingMaintenanceMeasurement::ZeroWork { minutes, .. } => *minutes,
469 }
470 }
471
472 pub(crate) fn session_count(&self) -> usize {
473 match &self.measurement {
474 OfficialCodingMaintenanceMeasurement::SupervisorTimed { session_count, .. }
475 | OfficialCodingMaintenanceMeasurement::ZeroWork { session_count, .. } => {
476 *session_count
477 }
478 }
479 }
480}
481
482#[derive(Debug, Clone, Serialize)]
483pub struct SecurityAuthorityVerdict {
484 pub status: AuthorityStatus,
485 pub runs_recomputed: usize,
486 pub policy_failure_count: usize,
487 pub reports: Vec<SecurityReportAuthorityVerdict>,
488 pub diagnostics: Vec<String>,
489}
490
491#[derive(Debug, Clone, Serialize)]
492pub struct SecurityReportAuthorityVerdict {
493 pub report_path: String,
494 pub report_sha256: String,
495 pub status: AuthorityStatus,
496 pub target: Option<String>,
497 pub models: Vec<Value>,
498 pub platforms: Vec<String>,
499 pub producing_shas: Vec<String>,
500 pub production_input_trees: Vec<String>,
501 pub source_dirty_attestations: Vec<Option<bool>>,
502 pub runs_recomputed: usize,
503 pub policy_failure_count: usize,
504 pub policy_summary: Option<MemoryBenchPolicySummary>,
505 pub diagnostics: Vec<String>,
506}
507
508#[derive(Debug, Clone, Serialize)]
509pub struct ReleaseAuthorityVerdict {
510 pub status: AuthorityStatus,
511 pub ready: bool,
512 pub required_targets: Vec<String>,
513 pub current_targets: Vec<String>,
514 pub missing_targets: Vec<String>,
515 pub stale_targets: Vec<String>,
516 pub diagnostics: Vec<String>,
517}
518
519#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq)]
520#[serde(rename_all = "snake_case")]
521pub enum BenchmarkLayer {
522 MemorySystemCapability,
523 CodingAgentOutcome,
524}
525
526#[derive(Debug, Clone, Deserialize, Serialize)]
527pub struct PublicBenchmarkManifest {
528 pub schema_version: u32,
529 pub benchmark_id: String,
530 pub layer: BenchmarkLayer,
531 pub version: String,
532 pub created_at_epoch: i64,
533 pub source_policy: SourcePolicy,
534 #[serde(default)]
535 pub conditions: Vec<String>,
536 #[serde(default)]
537 pub reports: Vec<String>,
538}
539
540#[derive(Debug, Clone, Deserialize, Serialize)]
541pub struct SourcePolicy {
542 pub private_user_memory_allowed: bool,
543 pub requires_temp_remem_data_dir: bool,
544 pub external_dataset_revision: Option<String>,
545}
546
547#[derive(Debug, Clone, Deserialize, Serialize)]
548pub struct PublicBenchmarkReport {
549 pub schema_version: u32,
550 pub benchmark_id: String,
551 pub benchmark_version: String,
552 #[serde(default, skip_serializing_if = "Option::is_none")]
553 pub suite: Option<String>,
554 #[serde(default)]
555 pub run_phase: Option<String>,
556 #[serde(default)]
557 pub matrix_namespace: Option<String>,
558 pub layer: BenchmarkLayer,
559 #[serde(default)]
560 pub conditions: Vec<String>,
561 #[serde(default)]
562 pub schema_refs: Vec<String>,
563 #[serde(default)]
564 pub run_artifacts: Vec<String>,
565 #[serde(default)]
566 pub aggregate_metrics: Value,
567 pub claim_level: String,
568 pub verifier: ReportVerifierMetadata,
569}
570
571#[derive(Debug, Clone, Deserialize, Serialize)]
572pub struct ReportVerifierMetadata {
573 pub required: bool,
574 pub schema_version: u32,
575}
576
577#[derive(Debug, Clone, Deserialize, Serialize)]
578pub struct RunEnvironment {
579 pub os: String,
580 pub arch: String,
581 pub remem_commit: String,
582 pub remem_data_dir: String,
583 #[serde(default)]
584 pub docker_image_digest: Option<String>,
585 #[serde(default)]
586 pub fixture_revision: Option<String>,
587 #[serde(default)]
588 pub repo_base_commit: Option<String>,
589 #[serde(default)]
590 pub source_dirty: Option<bool>,
591 #[serde(default)]
592 pub production_input_tree_sha256: Option<String>,
593}
594
595#[derive(Debug, Clone, Deserialize, Serialize)]
596pub struct MemoryRunArtifact {
597 pub schema_version: u32,
598 pub benchmark_id: String,
599 pub benchmark_version: String,
600 pub layer: BenchmarkLayer,
601 pub suite: String,
602 pub condition: String,
603 pub task_id: String,
604 pub run_index: u32,
605 pub reference_time_epoch: i64,
606 #[serde(default)]
607 pub reader_model: Value,
608 pub environment: RunEnvironment,
609 #[serde(default)]
610 pub answer: Value,
611 pub retrieval: MemoryRetrievalEvidence,
612 pub evidence: MemoryCitationEvidence,
613 #[serde(default)]
614 pub metrics: Value,
615 pub diagnosis: MemoryDiagnosis,
616 #[serde(default)]
617 pub artifacts: BTreeMap<String, String>,
618 #[serde(default)]
619 pub artifact_sha256: BTreeMap<String, String>,
620 #[serde(default)]
621 pub suite_content_identity: Option<String>,
622}
623
624#[derive(Debug, Clone, Deserialize, Serialize)]
625pub struct MemoryRetrievalEvidence {
626 #[serde(default)]
627 pub retrieved_memory_ids: Vec<i64>,
628 #[serde(default)]
629 pub retrieved_supporting_evidence_ids: Vec<String>,
630 #[serde(default)]
631 pub gold_supporting_event_ids: Vec<String>,
632 #[serde(default)]
633 pub missing_supporting_evidence_ids: Vec<String>,
634}
635
636#[derive(Debug, Clone, Deserialize, Serialize)]
637pub struct MemoryCitationEvidence {
638 #[serde(default)]
639 pub cited_memory_ids: Vec<i64>,
640 #[serde(default)]
641 pub cited_event_ids: Vec<String>,
642}
643
644#[derive(Debug, Clone, Deserialize, Serialize)]
645pub struct MemoryDiagnosis {
646 pub write_side_gap: bool,
647 pub retrieval_side_gap: bool,
648 pub reader_gap: bool,
649 pub policy_abstention: bool,
650 #[serde(default)]
651 pub notes: Vec<String>,
652}
653
654#[derive(Debug, Clone, Deserialize, Serialize)]
655pub struct CodingRunArtifact {
656 pub schema_version: u32,
657 pub benchmark_id: String,
658 pub benchmark_version: String,
659 pub run_phase: String,
660 pub matrix_namespace: String,
661 pub layer: BenchmarkLayer,
662 pub condition: String,
663 pub task_id: String,
664 pub run_index: u32,
665 #[serde(default)]
666 pub attempt_id: Option<String>,
667 #[serde(default)]
668 pub target_started: Option<bool>,
669 #[serde(default)]
670 pub model: Value,
671 pub environment: RunEnvironment,
672 pub resolved: bool,
673 pub failure_reason: Option<String>,
674 pub metrics: CodingRunMetrics,
675 #[serde(default)]
676 pub memory_contract: Option<CodingMemoryContract>,
677 #[serde(default)]
678 pub context_audit_status: Option<RememContextAuditStatus>,
679 #[serde(default)]
680 pub context_audit_failure_reason: Option<String>,
681 #[serde(default)]
682 pub remem_context_audit: Option<RememContextAuditSnapshot>,
683 #[serde(default)]
684 pub injected_context_sha256: Option<String>,
685 #[serde(default)]
686 pub artifacts: BTreeMap<String, String>,
687}
688
689#[derive(Debug, Clone, Deserialize, Serialize)]
690pub struct CodingRunMetrics {
691 pub tokens_input: Option<u64>,
692 pub tokens_output: Option<u64>,
693 pub tokens_total: Option<u64>,
694 pub turns: Option<u64>,
695 pub wall_time_ms: Option<u64>,
696 pub tool_calls: Option<u64>,
697 pub commands_run: Option<u64>,
698}
699
700#[derive(Debug, Clone, Deserialize, Serialize)]
701pub struct CodingMemoryContract {
702 #[serde(default)]
703 pub injected_memory_ids: Vec<i64>,
704 #[serde(default)]
705 pub used_memory_ids: Vec<i64>,
706 pub citation_precision: f64,
707 pub citation_recall: f64,
708 pub stale_used_count: u64,
709 pub irrelevant_injection_count: u64,
710 pub missing_relevant_memory_count: u64,
711 pub memory_helped: bool,
712 pub memory_hurt: bool,
713}