1use anyhow::{bail, Result};
2use serde::Serialize;
3use std::collections::{BTreeMap, BTreeSet};
4
5use crate::eval::current_memory_contracts::{
6 CurrentMemoryContractEvalReport, CurrentMemoryContractRateMetric,
7};
8
9use super::audit_contract::{
10 verify_context_audit_snapshot, RememContextAuditSnapshot, RememContextAuditStatus,
11};
12use super::types::{CodingBenchFailureReason, CodingMemoryAttribution};
13
14pub const CODING_AGENT_AB_SPEC_PATH: &str = "docs/specs/issue385-coding-agent-ab/TECH.md";
15pub const CURRENT_MEMORY_CONTRACT_SPEC_PATH: &str = "docs/specs/current-memory-contracts/TECH.md";
16pub const MIN_RUNS_PER_CONDITION: usize = 3;
17const REQUIRED_CONDITIONS: [CodingBenchCondition; 3] = [
18 CodingBenchCondition::RememSeededSessionStart,
19 CodingBenchCondition::NoMemory,
20 CodingBenchCondition::CuratedFileExpert,
21];
22
23#[derive(Debug, Clone, Serialize, PartialEq)]
24pub struct CodingBenchReport {
25 pub schema_version: u32,
26 pub benchmark_spec_path: &'static str,
27 pub current_memory_contract_spec_path: &'static str,
28 pub runs_per_condition: usize,
29 pub conditions: Vec<CodingBenchConditionReport>,
30}
31
32#[derive(Debug, Clone, Serialize, PartialEq)]
33pub struct CodingBenchConditionReport {
34 pub name: CodingBenchCondition,
35 pub runs: Vec<CodingBenchRunReport>,
36}
37
38#[derive(Debug, Clone, Serialize, PartialEq)]
39pub struct CodingBenchRunReport {
40 pub condition: CodingBenchCondition,
41 pub task_id: String,
42 pub run_index: usize,
43 #[serde(rename = "resolved")]
44 pub task_success: bool,
45 #[serde(rename = "failure_reason")]
46 pub task_failure_reason: Option<CodingBenchFailureReason>,
47 pub memory_contract_status: CodingBenchMemoryContractStatus,
48 pub runtime_contract_failure: bool,
49 pub runtime_contract_failure_reason: Option<String>,
50 pub context_audit_status: RememContextAuditStatus,
51 pub context_audit_failure_reason: Option<String>,
52 pub score: CodingBenchRunScoreEvidence,
53 pub metrics: CodingBenchRunMetrics,
54 pub final_head_sha: Option<String>,
55 pub patch_artifact_path: Option<String>,
56 pub unauthorized_path_changes: Vec<String>,
57 #[serde(skip_serializing_if = "Option::is_none")]
58 pub memory_contract: Option<CodingMemoryAttribution>,
59 pub remem_contract_snapshot: Option<RememContractSnapshot>,
60}
61
62#[derive(Debug, Clone, Serialize, PartialEq)]
63pub struct CodingBenchRunScoreEvidence {
64 pub commands: Vec<CodingBenchScoreCommandEvidence>,
65}
66
67#[derive(Debug, Clone, Serialize, PartialEq)]
68pub struct CodingBenchScoreCommandEvidence {
69 pub command: Vec<String>,
70 pub exit_code: i32,
71 pub stdout: Option<String>,
72 pub stderr: Option<String>,
73 pub output_artifact_path: Option<String>,
74}
75
76#[derive(Debug, Clone, Serialize, PartialEq)]
77pub struct CodingBenchRunMetrics {
78 pub tokens_input: Option<u64>,
79 pub tokens_output: Option<u64>,
80 pub tokens_total: Option<u64>,
81 pub token_accounting_unsupported_reason: Option<String>,
82 pub turns: Option<u64>,
83 pub wall_time_ms: Option<u64>,
84}
85
86#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq, PartialOrd, Ord)]
87#[serde(rename_all = "snake_case")]
88pub enum CodingBenchCondition {
89 NoMemory,
90 #[serde(rename = "remem_seeded_sessionstart")]
91 RememSeededSessionStart,
92 CuratedFileExpert,
93}
94
95#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)]
96#[serde(rename_all = "snake_case")]
97pub enum CodingBenchMemoryContractStatus {
98 Passed,
99 Failed,
100 NotApplicable,
101}
102
103#[derive(Debug, Clone, Serialize, PartialEq)]
104pub struct RememContractSnapshot {
105 pub schema_version: u32,
106 pub source: &'static str,
107 pub spec_path: &'static str,
108 pub captured_at_epoch: i64,
109 pub contract_health: RememContractHealth,
110 pub context_audit: Option<RememContextAuditSnapshot>,
111 pub citation_precision: CurrentMemoryContractRateMetric,
112 pub staleness_handling: RememStalenessHandlingSnapshot,
113 pub temporal_fact_eligibility: RememTemporalFactEligibilitySnapshot,
114 pub injected_memory_audit: RememInjectedMemoryAuditSnapshot,
115 pub usage_feedback_coverage: RememUsageFeedbackCoverageSnapshot,
116 pub current_memory_contracts: CurrentMemoryContractEvalReport,
117}
118
119#[derive(Debug, Clone, Serialize, PartialEq)]
120pub struct RememContractHealth {
121 pub all_checks_passed: bool,
122 pub failing_examples: Vec<String>,
123 pub warning_count: usize,
124 pub warnings: Vec<RememContractWarning>,
125}
126
127#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
128pub struct RememContractWarning {
129 pub code: &'static str,
130 pub message: String,
131}
132
133#[derive(Debug, Clone, Serialize, PartialEq)]
134pub struct RememStalenessHandlingSnapshot {
135 pub tracked: CurrentMemoryContractRateMetric,
136 pub untracked: CurrentMemoryContractRateMetric,
137 pub history_tracked: CurrentMemoryContractRateMetric,
138 pub verify_before_trust: CurrentMemoryContractRateMetric,
139 pub error: CurrentMemoryContractRateMetric,
140}
141
142#[derive(Debug, Clone, Serialize, PartialEq)]
143pub struct RememTemporalFactEligibilitySnapshot {
144 pub invalidated_fact_exclusion: CurrentMemoryContractRateMetric,
145 pub expired_fact_exclusion: CurrentMemoryContractRateMetric,
146 pub as_of_fact_retrieval: CurrentMemoryContractRateMetric,
147}
148
149#[derive(Debug, Clone, Serialize, PartialEq)]
150pub struct RememInjectedMemoryAuditSnapshot {
151 pub injected: CurrentMemoryContractRateMetric,
152 pub dropped: CurrentMemoryContractRateMetric,
153 pub abstained: CurrentMemoryContractRateMetric,
154}
155
156#[derive(Debug, Clone, Serialize, PartialEq)]
157pub struct RememUsageFeedbackCoverageSnapshot {
158 pub citation_event_matched: CurrentMemoryContractRateMetric,
159 pub usage_event_linked_to_injection_item: CurrentMemoryContractRateMetric,
160}
161
162pub fn build_remem_contract_snapshot(
163 contract_report: CurrentMemoryContractEvalReport,
164 captured_at_epoch: i64,
165 context_audit: Option<RememContextAuditSnapshot>,
166) -> RememContractSnapshot {
167 let mut warnings = Vec::new();
168 let mut failing_examples = contract_report.failing_examples.clone();
169 if contract_report.metadata.real_db_touched {
170 warnings.push(RememContractWarning {
171 code: "current_memory_contract_real_db_touched",
172 message: "current-memory-contract eval touched the real runtime database".to_string(),
173 });
174 failing_examples.push("current-memory-contract eval touched real runtime database".into());
175 }
176 let all_checks_passed =
177 contract_report.metrics.all_checks_passed && failing_examples.is_empty();
178
179 RememContractSnapshot {
180 schema_version: 1,
181 source: "current_memory_contracts",
182 spec_path: CURRENT_MEMORY_CONTRACT_SPEC_PATH,
183 captured_at_epoch,
184 contract_health: RememContractHealth {
185 all_checks_passed,
186 failing_examples,
187 warning_count: warnings.len(),
188 warnings,
189 },
190 context_audit,
191 citation_precision: contract_report.metrics.usage.citation_event_matched.clone(),
192 staleness_handling: RememStalenessHandlingSnapshot {
193 tracked: contract_report.metrics.staleness.tracked.clone(),
194 untracked: contract_report.metrics.staleness.untracked.clone(),
195 history_tracked: contract_report.metrics.staleness.history_tracked.clone(),
196 verify_before_trust: contract_report
197 .metrics
198 .staleness
199 .verify_before_trust
200 .clone(),
201 error: contract_report.metrics.staleness.error.clone(),
202 },
203 temporal_fact_eligibility: RememTemporalFactEligibilitySnapshot {
204 invalidated_fact_exclusion: contract_report
205 .metrics
206 .temporal
207 .invalidated_fact_exclusion
208 .clone(),
209 expired_fact_exclusion: contract_report
210 .metrics
211 .temporal
212 .expired_fact_exclusion
213 .clone(),
214 as_of_fact_retrieval: contract_report
215 .metrics
216 .temporal
217 .as_of_fact_retrieval
218 .clone(),
219 },
220 injected_memory_audit: RememInjectedMemoryAuditSnapshot {
221 injected: contract_report.metrics.injection.audit_injected.clone(),
222 dropped: contract_report.metrics.injection.audit_dropped.clone(),
223 abstained: contract_report.metrics.injection.audit_abstained.clone(),
224 },
225 usage_feedback_coverage: RememUsageFeedbackCoverageSnapshot {
226 citation_event_matched: contract_report.metrics.usage.citation_event_matched.clone(),
227 usage_event_linked_to_injection_item: contract_report
228 .metrics
229 .usage
230 .usage_event_linked_to_injection_item
231 .clone(),
232 },
233 current_memory_contracts: contract_report,
234 }
235}
236
237pub fn validate_contract_snapshots(report: &CodingBenchReport) -> Result<()> {
238 validate_condition_matrix(report)?;
239
240 for condition in &report.conditions {
241 for run in &condition.runs {
242 if run.condition != condition.name {
243 bail!(
244 "coding bench run {}#{} condition {:?} is nested under {:?}",
245 run.task_id,
246 run.run_index,
247 run.condition,
248 condition.name
249 );
250 }
251 if run.task_success && run.task_failure_reason.is_some() {
252 bail!(
253 "coding bench run {}#{} has task_success=true with stale task_failure_reason",
254 run.task_id,
255 run.run_index
256 );
257 }
258 if !run.task_success && run.task_failure_reason.is_none() {
259 bail!(
260 "coding bench run {}#{} failed task without task_failure_reason",
261 run.task_id,
262 run.run_index
263 );
264 }
265 if run.runtime_contract_failure
266 && run
267 .runtime_contract_failure_reason
268 .as_deref()
269 .is_none_or(|reason| reason.trim().is_empty())
270 {
271 bail!(
272 "coding bench run {}#{} has runtime contract failure without reason",
273 run.task_id,
274 run.run_index
275 );
276 }
277 if !run.runtime_contract_failure && run.runtime_contract_failure_reason.is_some() {
278 bail!(
279 "coding bench run {}#{} has runtime_contract_failure=false with stale runtime_contract_failure_reason",
280 run.task_id,
281 run.run_index
282 );
283 }
284 validate_score_and_patch_evidence(run)?;
285 validate_token_accounting(run)?;
286 validate_required_run_metrics(run)?;
287
288 match run.condition {
289 CodingBenchCondition::RememSeededSessionStart => validate_remem_run_contract(run)?,
290 CodingBenchCondition::NoMemory | CodingBenchCondition::CuratedFileExpert => {
291 if run.memory_contract_status != CodingBenchMemoryContractStatus::NotApplicable
292 {
293 bail!(
294 "{:?} run {}#{} must mark memory_contract_status as not_applicable",
295 run.condition,
296 run.task_id,
297 run.run_index
298 );
299 }
300 if run.remem_contract_snapshot.is_some() {
301 bail!(
302 "{:?} run {}#{} must not carry a remem contract snapshot",
303 run.condition,
304 run.task_id,
305 run.run_index
306 );
307 }
308 if run.runtime_contract_failure || run.runtime_contract_failure_reason.is_some()
309 {
310 bail!(
311 "{:?} run {}#{} must not report remem runtime contract failure",
312 run.condition,
313 run.task_id,
314 run.run_index
315 );
316 }
317 if run.context_audit_status != RememContextAuditStatus::NotApplicable
318 || run.context_audit_failure_reason.is_some()
319 {
320 bail!(
321 "{:?} run {}#{} must mark ContextAudit not_applicable without a failure reason",
322 run.condition,
323 run.task_id,
324 run.run_index
325 );
326 }
327 if run.memory_contract.is_some() {
328 bail!(
329 "{:?} run {}#{} must not carry memory_contract attribution",
330 run.condition,
331 run.task_id,
332 run.run_index
333 );
334 }
335 }
336 }
337 }
338 }
339 Ok(())
340}
341
342fn validate_condition_matrix(report: &CodingBenchReport) -> Result<()> {
343 if report.runs_per_condition < MIN_RUNS_PER_CONDITION {
344 bail!(
345 "coding bench report runs_per_condition={} is below required minimum {MIN_RUNS_PER_CONDITION}",
346 report.runs_per_condition
347 );
348 }
349
350 let mut reports_by_condition = BTreeMap::new();
351 for condition in &report.conditions {
352 if reports_by_condition
353 .insert(condition.name, condition)
354 .is_some()
355 {
356 bail!(
357 "coding bench report contains duplicate {:?} condition",
358 condition.name
359 );
360 }
361 }
362 for required in REQUIRED_CONDITIONS {
363 if !reports_by_condition.contains_key(&required) {
364 bail!(
365 "coding bench report missing required {:?} condition",
366 required
367 );
368 }
369 }
370 if reports_by_condition.len() != REQUIRED_CONDITIONS.len() {
371 bail!(
372 "coding bench report condition count={} does not match required condition count={}",
373 reports_by_condition.len(),
374 REQUIRED_CONDITIONS.len()
375 );
376 }
377
378 let mut task_ids = BTreeSet::new();
379 for condition in reports_by_condition.values() {
380 for run in &condition.runs {
381 task_ids.insert(run.task_id.clone());
382 }
383 }
384 if task_ids.is_empty() {
385 bail!("coding bench report has no task runs");
386 }
387
388 for required in REQUIRED_CONDITIONS {
389 let Some(condition) = reports_by_condition.get(&required) else {
390 bail!(
391 "coding bench report missing required {:?} condition",
392 required
393 );
394 };
395 let mut seen = BTreeSet::new();
396 for run in &condition.runs {
397 if run.run_index >= report.runs_per_condition {
398 bail!(
399 "{:?} run {}#{} is outside runs_per_condition={}",
400 required,
401 run.task_id,
402 run.run_index,
403 report.runs_per_condition
404 );
405 }
406 if !seen.insert((run.task_id.clone(), run.run_index)) {
407 bail!(
408 "{:?} condition repeats run {}#{}",
409 required,
410 run.task_id,
411 run.run_index
412 );
413 }
414 }
415 for task_id in &task_ids {
416 for run_index in 0..report.runs_per_condition {
417 if !seen.contains(&(task_id.clone(), run_index)) {
418 bail!(
419 "{:?} condition missing run {}#{}",
420 required,
421 task_id,
422 run_index
423 );
424 }
425 }
426 }
427 }
428
429 Ok(())
430}
431
432fn validate_score_and_patch_evidence(run: &CodingBenchRunReport) -> Result<()> {
433 if run.score.commands.is_empty() {
434 bail!(
435 "coding bench run {}#{} is missing score command evidence",
436 run.task_id,
437 run.run_index
438 );
439 }
440 for (command_index, command) in run.score.commands.iter().enumerate() {
441 if command.command.is_empty() || command.command.iter().any(|part| part.trim().is_empty()) {
442 bail!(
443 "coding bench run {}#{} score command {command_index} has blank command argv",
444 run.task_id,
445 run.run_index
446 );
447 }
448 let has_inline_output = command
449 .stdout
450 .as_deref()
451 .is_some_and(|stdout| !stdout.trim().is_empty())
452 || command
453 .stderr
454 .as_deref()
455 .is_some_and(|stderr| !stderr.trim().is_empty());
456 let has_output_artifact = command
457 .output_artifact_path
458 .as_deref()
459 .is_some_and(|path| !path.trim().is_empty());
460 if !has_inline_output && !has_output_artifact {
461 bail!(
462 "coding bench run {}#{} score command {command_index} has no output evidence",
463 run.task_id,
464 run.run_index
465 );
466 }
467 }
468
469 let has_final_head_sha = run
470 .final_head_sha
471 .as_deref()
472 .is_some_and(|sha| !sha.trim().is_empty());
473 let has_patch_artifact = run
474 .patch_artifact_path
475 .as_deref()
476 .is_some_and(|path| !path.trim().is_empty());
477 if !has_final_head_sha && !has_patch_artifact {
478 bail!(
479 "coding bench run {}#{} is missing final_head_sha or patch_artifact_path",
480 run.task_id,
481 run.run_index
482 );
483 }
484 if let Some(sha) = run.final_head_sha.as_deref() {
485 let sha = sha.trim();
486 if sha.len() != 40 || !sha.chars().all(|ch| ch.is_ascii_hexdigit()) {
487 bail!(
488 "coding bench run {}#{} final_head_sha is not a full git SHA",
489 run.task_id,
490 run.run_index
491 );
492 }
493 }
494 if run
495 .unauthorized_path_changes
496 .iter()
497 .any(|path| path.trim().is_empty())
498 {
499 bail!(
500 "coding bench run {}#{} has blank unauthorized path change",
501 run.task_id,
502 run.run_index
503 );
504 }
505
506 Ok(())
507}
508
509fn validate_token_accounting(run: &CodingBenchRunReport) -> Result<()> {
510 let token_fields = [
511 run.metrics.tokens_input,
512 run.metrics.tokens_output,
513 run.metrics.tokens_total,
514 ];
515 let token_fields_present = token_fields.iter().filter(|value| value.is_some()).count();
516 let unsupported_reason = run
517 .metrics
518 .token_accounting_unsupported_reason
519 .as_deref()
520 .map(str::trim);
521 let has_unsupported_reason = unsupported_reason.is_some_and(|reason| !reason.is_empty());
522
523 match (token_fields_present, has_unsupported_reason) {
524 (3, false) => {
525 let input = run.metrics.tokens_input.unwrap_or_default();
526 let output = run.metrics.tokens_output.unwrap_or_default();
527 let total = run.metrics.tokens_total.unwrap_or_default();
528 if input.saturating_add(output) != total {
529 bail!(
530 "coding bench run {}#{} tokens_total={} does not equal tokens_input + tokens_output ({input} + {output})",
531 run.task_id,
532 run.run_index,
533 total
534 );
535 }
536 Ok(())
537 }
538 (3, true) => bail!(
539 "coding bench run {}#{} has token metrics and token_accounting_unsupported_reason",
540 run.task_id,
541 run.run_index
542 ),
543 (0, true) => Ok(()),
544 (0, false) => bail!(
545 "coding bench run {}#{} is missing token accounting without token_accounting_unsupported_reason",
546 run.task_id,
547 run.run_index
548 ),
549 (_, _) => bail!(
550 "coding bench run {}#{} must record complete token accounting or token_accounting_unsupported_reason",
551 run.task_id,
552 run.run_index
553 ),
554 }
555}
556
557fn validate_required_run_metrics(run: &CodingBenchRunReport) -> Result<()> {
558 if run.metrics.turns.is_none() {
559 bail!(
560 "coding bench run {}#{} is missing turns",
561 run.task_id,
562 run.run_index
563 );
564 }
565 if run.metrics.wall_time_ms.is_none() {
566 bail!(
567 "coding bench run {}#{} is missing wall_time_ms",
568 run.task_id,
569 run.run_index
570 );
571 }
572 Ok(())
573}
574
575fn validate_remem_run_contract(run: &CodingBenchRunReport) -> Result<()> {
576 validate_memory_attribution(run)?;
577 let snapshot = run.remem_contract_snapshot.as_ref().ok_or_else(|| {
578 anyhow::anyhow!(
579 "remem run {}#{} is missing current memory contract snapshot",
580 run.task_id,
581 run.run_index
582 )
583 })?;
584 let embedded_contract_passed = snapshot.current_memory_contracts.metrics.all_checks_passed
585 && snapshot
586 .current_memory_contracts
587 .failing_examples
588 .is_empty()
589 && !snapshot.current_memory_contracts.metadata.real_db_touched;
590 if snapshot.contract_health.all_checks_passed != embedded_contract_passed {
591 bail!(
592 "remem run {}#{} contract_health does not match embedded current_memory_contracts",
593 run.task_id,
594 run.run_index
595 );
596 }
597 let context_audit_failed = match run.context_audit_status {
598 RememContextAuditStatus::Verified => {
599 if run.context_audit_failure_reason.is_some() {
600 bail!(
601 "remem run {}#{} has verified ContextAudit with a failure reason",
602 run.task_id,
603 run.run_index
604 );
605 }
606 let context_audit = snapshot.context_audit.as_ref().ok_or_else(|| {
607 anyhow::anyhow!(
608 "remem run {}#{} is missing ContextAudit snapshot",
609 run.task_id,
610 run.run_index
611 )
612 })?;
613 verify_context_audit_snapshot(context_audit)?;
614 false
615 }
616 RememContextAuditStatus::ContractFailure => {
617 if run
618 .context_audit_failure_reason
619 .as_deref()
620 .is_none_or(|reason| reason.trim().is_empty())
621 {
622 bail!(
623 "remem run {}#{} has ContextAudit contract failure without reason",
624 run.task_id,
625 run.run_index
626 );
627 }
628 if snapshot.context_audit.is_some() {
629 bail!(
630 "remem run {}#{} has ContextAudit contract failure with a stale snapshot",
631 run.task_id,
632 run.run_index
633 );
634 }
635 true
636 }
637 RememContextAuditStatus::NotApplicable => {
638 bail!(
639 "remem run {}#{} must not mark ContextAudit not_applicable",
640 run.task_id,
641 run.run_index
642 );
643 }
644 };
645 let contract_failed = !embedded_contract_passed || context_audit_failed;
646 let expected_status = if contract_failed {
647 CodingBenchMemoryContractStatus::Failed
648 } else {
649 CodingBenchMemoryContractStatus::Passed
650 };
651 if run.memory_contract_status != expected_status {
652 bail!(
653 "remem run {}#{} memory_contract_status={:?} does not match contract health={}",
654 run.task_id,
655 run.run_index,
656 run.memory_contract_status,
657 snapshot.contract_health.all_checks_passed
658 );
659 }
660 if contract_failed != run.runtime_contract_failure {
661 bail!(
662 "remem run {}#{} runtime_contract_failure={} does not match contract health={}",
663 run.task_id,
664 run.run_index,
665 run.runtime_contract_failure,
666 snapshot.contract_health.all_checks_passed
667 );
668 }
669 Ok(())
670}
671
672fn validate_memory_attribution(run: &CodingBenchRunReport) -> Result<()> {
673 let attribution = run.memory_contract.as_ref().ok_or_else(|| {
674 anyhow::anyhow!(
675 "remem run {}#{} is missing memory_contract attribution",
676 run.task_id,
677 run.run_index
678 )
679 })?;
680 validate_rate(attribution.citation_precision, "citation_precision", run)?;
681 validate_rate(attribution.citation_recall, "citation_recall", run)?;
682 ensure_unique_positive_ids(&attribution.injected_memory_ids, "injected_memory_ids", run)?;
683 ensure_unique_positive_ids(&attribution.used_memory_ids, "used_memory_ids", run)?;
684 if attribution.memory_helped && attribution.memory_hurt {
685 bail!(
686 "remem run {}#{} memory_contract cannot mark both memory_helped and memory_hurt",
687 run.task_id,
688 run.run_index
689 );
690 }
691 if run
692 .task_failure_reason
693 .is_some_and(CodingBenchFailureReason::is_memory_specific)
694 && !attribution.memory_hurt
695 {
696 bail!(
697 "remem run {}#{} has memory-specific failure without memory_hurt=true",
698 run.task_id,
699 run.run_index
700 );
701 }
702 Ok(())
703}
704
705fn validate_rate(value: f64, field: &str, run: &CodingBenchRunReport) -> Result<()> {
706 if !(0.0..=1.0).contains(&value) || !value.is_finite() {
707 bail!(
708 "remem run {}#{} memory_contract {field} must be a finite rate between 0 and 1",
709 run.task_id,
710 run.run_index
711 );
712 }
713 Ok(())
714}
715
716fn ensure_unique_positive_ids(ids: &[i64], field: &str, run: &CodingBenchRunReport) -> Result<()> {
717 let mut seen = BTreeSet::new();
718 for id in ids {
719 if *id <= 0 {
720 bail!(
721 "remem run {}#{} memory_contract {field} contains non-positive id",
722 run.task_id,
723 run.run_index
724 );
725 }
726 if !seen.insert(*id) {
727 bail!(
728 "remem run {}#{} memory_contract {field} contains duplicate id",
729 run.task_id,
730 run.run_index
731 );
732 }
733 }
734 Ok(())
735}