1use std::collections::{BTreeMap, BTreeSet};
9use std::path::{Path, PathBuf};
10
11use serde::{Deserialize, Serialize};
12use serde_json::Value as JsonValue;
13use sha2::{Digest, Sha256};
14
15use super::{
16 assemble_context, estimate_chunk_tokens, render_assembled_chunks, ArtifactRecord,
17 AssembleDedup, AssembleOptions, AssembleStrategy,
18};
19use crate::value::VmError;
20
21pub const CONTEXT_EVAL_SCHEMA_VERSION: u32 = 1;
22pub const CONTEXT_EVAL_MANIFEST_TYPE: &str = "harn.context_eval.manifest.v1";
23pub const CONTEXT_EVAL_REPORT_TYPE: &str = "harn.context_eval.report.v1";
24
25#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
26#[serde(default)]
27pub struct ContextEvalManifest {
28 #[serde(rename = "_type")]
29 pub type_name: String,
30 pub version: u32,
31 pub id: String,
32 pub name: Option<String>,
33 pub description: Option<String>,
34 pub modes: Vec<ContextEvalMode>,
35 pub tasks: Vec<ContextEvalTask>,
36 pub metadata: BTreeMap<String, JsonValue>,
37}
38
39#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
40#[serde(default)]
41pub struct ContextEvalMode {
42 pub id: String,
43 pub name: Option<String>,
44 pub kind: String,
45 pub description: Option<String>,
46 #[serde(default, alias = "artifact-ids")]
47 pub artifact_ids: Vec<String>,
48 #[serde(default, alias = "include-artifact-kinds")]
49 pub include_artifact_kinds: Vec<String>,
50 #[serde(default, alias = "exclude-artifact-kinds")]
51 pub exclude_artifact_kinds: Vec<String>,
52 #[serde(default, alias = "budget-tokens")]
53 pub budget_tokens: Option<usize>,
54 #[serde(default, alias = "assemble-strategy")]
55 pub assemble_strategy: Option<String>,
56 pub dedup: Option<String>,
57 #[serde(default, alias = "microcompact-threshold")]
58 pub microcompact_threshold: Option<usize>,
59 #[serde(default, alias = "semantic-overlap")]
60 pub semantic_overlap: Option<f64>,
61 #[serde(default, alias = "projection-policy")]
62 pub projection_policy: Option<String>,
63 #[serde(default, alias = "transcript-keep-last")]
64 pub transcript_keep_last: Option<usize>,
65 #[serde(default, alias = "tool-disclosure")]
66 pub tool_disclosure: Option<String>,
67 #[serde(default, alias = "tool-allowlist")]
68 pub tool_allowlist: Vec<String>,
69 #[serde(default, alias = "expected-cache-hit")]
70 pub expected_cache_hit: Option<bool>,
71 #[serde(default, alias = "cache-namespace")]
72 pub cache_namespace: Option<String>,
73 #[serde(default, alias = "compaction-policy")]
74 pub compaction_policy: Option<JsonValue>,
75 pub preprocessing: Option<String>,
76 pub metadata: BTreeMap<String, JsonValue>,
77}
78
79#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
80#[serde(default)]
81pub struct ContextEvalTask {
82 pub id: String,
83 pub name: Option<String>,
84 pub objective: String,
85 #[serde(default, alias = "reference-answer")]
86 pub reference_answer: Option<String>,
87 pub artifacts: Vec<ArtifactRecord>,
88 pub transcript: Vec<ContextEvalTranscriptMessage>,
89 pub tools: Vec<ContextEvalTool>,
90 #[serde(default, alias = "tool-events")]
91 pub tool_events: Vec<ContextEvalToolEvent>,
92 pub expected: ContextEvalExpected,
93 pub observed: ContextEvalObserved,
94 #[serde(default, alias = "mode-observations")]
95 pub mode_observations: BTreeMap<String, ContextEvalObserved>,
96 pub metadata: BTreeMap<String, JsonValue>,
97}
98
99#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
100#[serde(default)]
101pub struct ContextEvalTranscriptMessage {
102 pub role: String,
103 pub content: String,
104 #[serde(default, alias = "estimated-tokens")]
105 pub estimated_tokens: Option<usize>,
106}
107
108#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
109#[serde(default)]
110pub struct ContextEvalTool {
111 pub name: String,
112 pub description: Option<String>,
113 pub capability: Option<String>,
114 pub deterministic: Option<bool>,
115}
116
117#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
118#[serde(default)]
119pub struct ContextEvalToolEvent {
120 pub order: Option<usize>,
121 pub name: String,
122 pub phase: Option<String>,
123 pub success: Option<bool>,
124 pub quality: Option<String>,
125 pub recovery: Option<bool>,
126}
127
128#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
129#[serde(default)]
130pub struct ContextEvalExpected {
131 #[serde(default, alias = "required-terms")]
132 pub required_terms: Vec<String>,
133 #[serde(default, alias = "expected-artifact-ids")]
134 pub expected_artifact_ids: Vec<String>,
135 #[serde(default, alias = "expected-tools")]
136 pub expected_tools: Vec<String>,
137 #[serde(default, alias = "max-input-tokens")]
138 pub max_input_tokens: Option<usize>,
139}
140
141#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
142#[serde(default)]
143pub struct ContextEvalObserved {
144 #[serde(default, alias = "final-response")]
145 pub final_response: Option<String>,
146 #[serde(default, alias = "latency-ms")]
147 pub latency_ms: Option<u64>,
148 #[serde(default, alias = "input-tokens")]
149 pub input_tokens: Option<usize>,
150 #[serde(default, alias = "output-tokens")]
151 pub output_tokens: Option<usize>,
152 #[serde(default, alias = "cost-usd")]
153 pub cost_usd: Option<f64>,
154 #[serde(default, alias = "cache-hit")]
155 pub cache_hit: Option<bool>,
156 #[serde(default, alias = "compaction-count")]
157 pub compaction_count: Option<usize>,
158 pub metadata: BTreeMap<String, JsonValue>,
159}
160
161#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
162#[serde(default)]
163pub struct ContextEvalReport {
164 #[serde(rename = "_type")]
165 pub type_name: String,
166 pub schema_version: u32,
167 pub manifest_id: String,
168 pub manifest_name: Option<String>,
169 pub pass: bool,
170 pub total_runs: usize,
171 pub passed_runs: usize,
172 pub failed_runs: usize,
173 pub total_tasks: usize,
174 pub total_modes: usize,
175 pub aggregate: ContextEvalAggregate,
176 pub modes: Vec<ContextEvalModeSummary>,
177 pub tasks: Vec<ContextEvalTaskSummary>,
178 pub runs: Vec<ContextEvalRunReport>,
179 pub metadata: BTreeMap<String, JsonValue>,
180}
181
182#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
183#[serde(default)]
184pub struct ContextEvalAggregate {
185 pub mean_final_correctness: f64,
186 pub mean_tool_call_quality: f64,
187 pub total_latency_ms: u64,
188 pub total_input_tokens: usize,
189 pub total_output_tokens: usize,
190 pub total_cost_usd: f64,
191 pub total_compaction_count: usize,
192 pub total_error_recovery_count: usize,
193}
194
195#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
196#[serde(default)]
197pub struct ContextEvalModeSummary {
198 pub id: String,
199 pub kind: String,
200 pub projection_policy: String,
201 pub tool_disclosure: String,
202 pub preprocessing: ContextEvalPreprocessing,
203}
204
205#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
206#[serde(default)]
207pub struct ContextEvalPreprocessing {
208 pub mode: String,
209 pub llm_enabled: bool,
210}
211
212#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
213#[serde(default)]
214pub struct ContextEvalTaskSummary {
215 pub id: String,
216 pub name: Option<String>,
217 pub required_terms: Vec<String>,
218 pub expected_artifact_ids: Vec<String>,
219 pub expected_tools: Vec<String>,
220}
221
222#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
223#[serde(default)]
224pub struct ContextEvalRunReport {
225 pub run_id: String,
226 pub task_id: String,
227 pub mode_id: String,
228 pub mode_kind: String,
229 pub status: String,
230 pub passed: bool,
231 pub final_correctness: ContextEvalCorrectness,
232 pub reads_before_first_edit: usize,
233 pub tool_call_quality: ContextEvalToolQuality,
234 pub latency_ms: u64,
235 pub input_tokens: usize,
236 pub output_tokens: usize,
237 pub cost_usd: f64,
238 pub compaction_count: usize,
239 pub projection: ContextEvalProjectionReport,
240 pub context: ContextEvalContextReport,
241 pub cache: ContextEvalCacheReport,
242 pub error_recovery_count: usize,
243 pub preprocessing: ContextEvalPreprocessing,
244 pub failures: Vec<String>,
245}
246
247#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
248#[serde(default)]
249pub struct ContextEvalCorrectness {
250 pub passed: bool,
251 pub score: f64,
252 pub required_terms_present: Vec<String>,
253 pub required_terms_missing: Vec<String>,
254 pub expected_artifact_ids_present: Vec<String>,
255 pub expected_artifact_ids_missing: Vec<String>,
256 pub source: String,
257}
258
259#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
260#[serde(default)]
261pub struct ContextEvalToolQuality {
262 pub score: f64,
263 pub expected_tools: Vec<String>,
264 pub observed_tools: Vec<String>,
265 pub matched_tools: Vec<String>,
266 pub missing_tools: Vec<String>,
267 pub unnecessary_tools: Vec<String>,
268}
269
270#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
271#[serde(default)]
272pub struct ContextEvalProjectionReport {
273 pub policy: String,
274 pub source_message_count: usize,
275 pub retained_message_count: usize,
276 pub retained_tokens: usize,
277}
278
279#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
280#[serde(default)]
281pub struct ContextEvalContextReport {
282 pub projection_policy: String,
283 pub tool_disclosure: String,
284 pub artifact_count: usize,
285 pub selected_artifact_ids: Vec<String>,
286 pub dropped_artifact_ids: Vec<String>,
287 pub rendered_bytes: usize,
288 pub rendered_tokens: usize,
289 pub budget_tokens: usize,
290 pub assemble_strategy: String,
291 pub dedup: String,
292 pub exposed_tools: Vec<String>,
293}
294
295#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
296#[serde(default)]
297pub struct ContextEvalCacheReport {
298 pub namespace: String,
299 pub key: String,
300 pub stable_input_hash: String,
301 pub deterministic_order: bool,
302 pub hit: Option<bool>,
303}
304
305struct PreparedModeRun {
306 artifacts: Vec<ArtifactRecord>,
307 rendered_context: String,
308 selected_artifact_ids: Vec<String>,
309 dropped_artifact_ids: Vec<String>,
310 projection: ContextEvalProjectionReport,
311 transcript_text: String,
312 exposed_tools: Vec<String>,
313 visible_tool_events: Vec<ContextEvalToolEvent>,
314}
315
316pub fn load_context_eval_manifest(path: &Path) -> Result<ContextEvalManifest, VmError> {
317 let content = std::fs::read_to_string(path).map_err(|error| {
318 VmError::Runtime(format!("failed to read context eval manifest: {error}"))
319 })?;
320 let mut manifest: ContextEvalManifest =
321 if path.extension().and_then(|ext| ext.to_str()) == Some("toml") {
322 toml::from_str(&content).map_err(|error| {
323 VmError::Runtime(format!("failed to parse context eval TOML: {error}"))
324 })?
325 } else {
326 serde_json::from_str(&content).map_err(|error| {
327 VmError::Runtime(format!("failed to parse context eval JSON: {error}"))
328 })?
329 };
330 normalize_context_eval_manifest(&mut manifest)?;
331 Ok(manifest)
332}
333
334pub fn evaluate_context_eval_manifest(
335 manifest: &ContextEvalManifest,
336) -> Result<ContextEvalReport, VmError> {
337 let mut manifest = manifest.clone();
338 normalize_context_eval_manifest(&mut manifest)?;
339
340 let modes = manifest
341 .modes
342 .iter()
343 .map(|mode| ContextEvalModeSummary {
344 id: mode.id.clone(),
345 kind: mode_kind(mode),
346 projection_policy: projection_policy(mode),
347 tool_disclosure: tool_disclosure(mode),
348 preprocessing: preprocessing_report(mode),
349 })
350 .collect::<Vec<_>>();
351 let tasks = manifest
352 .tasks
353 .iter()
354 .map(|task| ContextEvalTaskSummary {
355 id: task.id.clone(),
356 name: task.name.clone(),
357 required_terms: sorted_strings(&task.expected.required_terms),
358 expected_artifact_ids: sorted_strings(&task.expected.expected_artifact_ids),
359 expected_tools: sorted_strings(&task.expected.expected_tools),
360 })
361 .collect::<Vec<_>>();
362
363 let mut runs = Vec::new();
364 for task in &manifest.tasks {
365 for mode in &manifest.modes {
366 runs.push(evaluate_task_mode(task, mode)?);
367 }
368 }
369
370 let total_runs = runs.len();
371 let passed_runs = runs.iter().filter(|run| run.passed).count();
372 let failed_runs = total_runs.saturating_sub(passed_runs);
373 let aggregate = aggregate_runs(&runs);
374 Ok(ContextEvalReport {
375 type_name: CONTEXT_EVAL_REPORT_TYPE.to_string(),
376 schema_version: CONTEXT_EVAL_SCHEMA_VERSION,
377 manifest_id: manifest.id,
378 manifest_name: manifest.name,
379 pass: failed_runs == 0,
380 total_runs,
381 passed_runs,
382 failed_runs,
383 total_tasks: tasks.len(),
384 total_modes: modes.len(),
385 aggregate,
386 modes,
387 tasks,
388 runs,
389 metadata: manifest.metadata,
390 })
391}
392
393fn normalize_context_eval_manifest(manifest: &mut ContextEvalManifest) -> Result<(), VmError> {
394 if manifest.type_name.is_empty() {
395 manifest.type_name = CONTEXT_EVAL_MANIFEST_TYPE.to_string();
396 }
397 if manifest.type_name != CONTEXT_EVAL_MANIFEST_TYPE {
398 return Err(VmError::Runtime(format!(
399 "context eval manifest _type must be {CONTEXT_EVAL_MANIFEST_TYPE}"
400 )));
401 }
402 if manifest.version == 0 {
403 manifest.version = CONTEXT_EVAL_SCHEMA_VERSION;
404 }
405 if manifest.version != CONTEXT_EVAL_SCHEMA_VERSION {
406 return Err(VmError::Runtime(format!(
407 "context eval manifest version must be {CONTEXT_EVAL_SCHEMA_VERSION}"
408 )));
409 }
410 if manifest.id.trim().is_empty() {
411 manifest.id = "context-eval".to_string();
412 }
413 if manifest.modes.is_empty() {
414 return Err(VmError::Runtime(
415 "context eval manifest must declare at least one mode".to_string(),
416 ));
417 }
418 if manifest.tasks.is_empty() {
419 return Err(VmError::Runtime(
420 "context eval manifest must declare at least one task".to_string(),
421 ));
422 }
423 let mut mode_ids = BTreeSet::new();
424 for (index, mode) in manifest.modes.iter_mut().enumerate() {
425 if mode.id.trim().is_empty() {
426 mode.id = format!("mode_{}", index + 1);
427 }
428 if !mode_ids.insert(mode.id.clone()) {
429 return Err(VmError::Runtime(format!(
430 "context eval manifest has duplicate mode id '{}'",
431 mode.id
432 )));
433 }
434 if mode.kind.trim().is_empty() {
435 mode.kind = mode.id.clone();
436 }
437 }
438 let mut task_ids = BTreeSet::new();
439 for (index, task) in manifest.tasks.iter_mut().enumerate() {
440 if task.id.trim().is_empty() {
441 task.id = format!("task_{}", index + 1);
442 }
443 if !task_ids.insert(task.id.clone()) {
444 return Err(VmError::Runtime(format!(
445 "context eval manifest has duplicate task id '{}'",
446 task.id
447 )));
448 }
449 if task.objective.trim().is_empty() {
450 return Err(VmError::Runtime(format!(
451 "context eval task '{}' must declare objective",
452 task.id
453 )));
454 }
455 for (artifact_index, artifact) in task.artifacts.iter_mut().enumerate() {
456 normalize_eval_artifact(artifact, &task.id, artifact_index);
457 }
458 task.tools.sort_by(|left, right| left.name.cmp(&right.name));
459 task.tool_events.sort_by(|left, right| {
460 left.order
461 .unwrap_or(usize::MAX)
462 .cmp(&right.order.unwrap_or(usize::MAX))
463 .then_with(|| left.name.cmp(&right.name))
464 });
465 }
466 Ok(())
467}
468
469fn normalize_eval_artifact(artifact: &mut ArtifactRecord, task_id: &str, index: usize) {
470 if artifact.type_name.is_empty() {
471 artifact.type_name = "artifact".to_string();
472 }
473 if artifact.id.trim().is_empty() {
474 artifact.id = format!("{task_id}_artifact_{}", index + 1);
475 }
476 if artifact.kind.trim().is_empty() {
477 artifact.kind = "artifact".to_string();
478 }
479 if artifact.created_at.trim().is_empty() {
480 artifact.created_at = "1970-01-01T00:00:00Z".to_string();
481 }
482 if artifact.estimated_tokens.is_none() {
483 artifact.estimated_tokens = artifact
484 .text
485 .as_ref()
486 .map(|text| ((text.len() as f64) / 4.0).ceil() as usize);
487 }
488 if artifact.priority.is_none() {
489 artifact.priority = Some(40);
490 }
491}
492
493fn evaluate_task_mode(
494 task: &ContextEvalTask,
495 mode: &ContextEvalMode,
496) -> Result<ContextEvalRunReport, VmError> {
497 let prepared = prepare_mode_run(task, mode)?;
498 let mode_id = mode.id.clone();
499 let mode_kind = mode_kind(mode);
500 let observed = task
501 .mode_observations
502 .get(&mode_id)
503 .unwrap_or(&task.observed);
504 let visible_input = visible_input(task, &prepared);
505 let (final_surface, correctness_source) = observed
506 .final_response
507 .as_ref()
508 .map(|response| (response.as_str(), "final_response"))
509 .unwrap_or_else(|| (visible_input.as_str(), "context_projection"));
510 let final_correctness = score_correctness(
511 &task.expected,
512 final_surface,
513 &prepared.selected_artifact_ids,
514 correctness_source,
515 );
516 let tool_call_quality =
517 score_tools(&task.expected.expected_tools, &prepared.visible_tool_events);
518 let reads_before_first_edit = reads_before_first_edit(&prepared.visible_tool_events);
519 let error_recovery_count = error_recovery_count(&prepared.visible_tool_events);
520 let input_tokens = observed
521 .input_tokens
522 .unwrap_or_else(|| estimate_chunk_tokens(&visible_input));
523 let output_tokens = observed
524 .output_tokens
525 .or_else(|| {
526 task.reference_answer
527 .as_ref()
528 .map(|text| estimate_chunk_tokens(text))
529 })
530 .unwrap_or(0);
531 let compaction_count = observed.compaction_count.unwrap_or(0) + mode_compaction_count(mode);
532 let latency_ms = observed.latency_ms.unwrap_or(0);
533 let cost_usd = observed.cost_usd.unwrap_or(0.0);
534 let mut failures = Vec::new();
535 if !final_correctness.required_terms_missing.is_empty() {
536 failures.push(format!(
537 "missing required terms: {}",
538 final_correctness.required_terms_missing.join(", ")
539 ));
540 }
541 if !final_correctness.expected_artifact_ids_missing.is_empty() {
542 failures.push(format!(
543 "missing expected artifacts: {}",
544 final_correctness.expected_artifact_ids_missing.join(", ")
545 ));
546 }
547 if !tool_call_quality.missing_tools.is_empty() {
548 failures.push(format!(
549 "missing expected tools: {}",
550 tool_call_quality.missing_tools.join(", ")
551 ));
552 }
553 if let Some(max) = task.expected.max_input_tokens {
554 if input_tokens > max {
555 failures.push(format!("input tokens {input_tokens} exceed max {max}"));
556 }
557 }
558 let passed = failures.is_empty();
559 let stable_input_hash = stable_hash(&[
560 task.id.as_str(),
561 mode.id.as_str(),
562 &prepared.selected_artifact_ids.join("\n"),
563 prepared.rendered_context.as_str(),
564 prepared.transcript_text.as_str(),
565 &prepared.exposed_tools.join("\n"),
566 ]);
567 let cache_namespace = mode
568 .cache_namespace
569 .clone()
570 .unwrap_or_else(|| "harn.context_eval".to_string());
571 Ok(ContextEvalRunReport {
572 run_id: format!("{}__{}", task.id, mode.id),
573 task_id: task.id.clone(),
574 mode_id,
575 mode_kind,
576 status: if passed { "pass" } else { "fail" }.to_string(),
577 passed,
578 final_correctness,
579 reads_before_first_edit,
580 tool_call_quality,
581 latency_ms,
582 input_tokens,
583 output_tokens,
584 cost_usd,
585 compaction_count,
586 projection: prepared.projection,
587 context: ContextEvalContextReport {
588 projection_policy: projection_policy(mode),
589 tool_disclosure: tool_disclosure(mode),
590 artifact_count: prepared.artifacts.len(),
591 selected_artifact_ids: prepared.selected_artifact_ids,
592 dropped_artifact_ids: prepared.dropped_artifact_ids,
593 rendered_bytes: prepared.rendered_context.len(),
594 rendered_tokens: estimate_chunk_tokens(&prepared.rendered_context),
595 budget_tokens: mode_budget_tokens(mode),
596 assemble_strategy: assemble_strategy(mode)?.as_str().to_string(),
597 dedup: assemble_dedup(mode)?.as_str().to_string(),
598 exposed_tools: prepared.exposed_tools,
599 },
600 cache: ContextEvalCacheReport {
601 namespace: cache_namespace.clone(),
602 key: {
603 #[expect(clippy::string_slice, reason = "stable_hash yields 64 ASCII hex chars")]
604 let prefix = &stable_input_hash[..32];
605 format!("{cache_namespace}:{prefix}")
606 },
607 stable_input_hash,
608 deterministic_order: true,
609 hit: mode.expected_cache_hit.or(observed.cache_hit),
610 },
611 error_recovery_count,
612 preprocessing: preprocessing_report(mode),
613 failures,
614 })
615}
616
617fn prepare_mode_run(
618 task: &ContextEvalTask,
619 mode: &ContextEvalMode,
620) -> Result<PreparedModeRun, VmError> {
621 let filtered = filter_artifacts(task, mode);
622 let options = AssembleOptions {
623 budget_tokens: mode_budget_tokens(mode),
624 dedup: assemble_dedup(mode)?,
625 strategy: assemble_strategy(mode)?,
626 query: Some(task.objective.clone()),
627 microcompact_threshold: mode.microcompact_threshold.unwrap_or(2_000),
628 semantic_overlap: mode.semantic_overlap.unwrap_or(0.85),
629 };
630 let assembled = assemble_context(&filtered, &options, None);
631 let selected_artifact_ids = sorted_strings(
632 &assembled
633 .included
634 .iter()
635 .map(|item| item.artifact_id.clone())
636 .collect::<Vec<_>>(),
637 );
638 let dropped_artifact_ids = sorted_strings(
639 &assembled
640 .dropped
641 .iter()
642 .map(|item| item.artifact_id.clone())
643 .collect::<Vec<_>>(),
644 );
645 let rendered_context = if assembled.chunks.is_empty() {
646 String::new()
647 } else {
648 render_assembled_chunks(&assembled)
649 };
650 let (projection, transcript_text) = project_transcript(task, mode);
651 let exposed_tools = exposed_tools(task, mode);
652 let visible_tool_events = visible_tool_events(task, &exposed_tools, mode);
653 Ok(PreparedModeRun {
654 artifacts: filtered,
655 rendered_context,
656 selected_artifact_ids,
657 dropped_artifact_ids,
658 projection,
659 transcript_text,
660 exposed_tools,
661 visible_tool_events,
662 })
663}
664
665fn filter_artifacts(task: &ContextEvalTask, mode: &ContextEvalMode) -> Vec<ArtifactRecord> {
666 if mode_kind(mode) == "cold" || mode_budget_tokens(mode) == 0 {
667 return Vec::new();
668 }
669 let include_ids: BTreeSet<&str> = mode.artifact_ids.iter().map(String::as_str).collect();
670 let include_kinds: BTreeSet<&str> = mode
671 .include_artifact_kinds
672 .iter()
673 .map(String::as_str)
674 .collect();
675 let exclude_kinds: BTreeSet<&str> = mode
676 .exclude_artifact_kinds
677 .iter()
678 .map(String::as_str)
679 .collect();
680 let kind = mode_kind(mode);
681 task.artifacts
682 .iter()
683 .filter(|artifact| include_ids.is_empty() || include_ids.contains(artifact.id.as_str()))
684 .filter(|artifact| {
685 include_kinds.is_empty()
686 || include_kinds.contains(artifact.kind.as_str())
687 || include_kinds.contains(
688 artifact
689 .metadata
690 .get("context_tier")
691 .and_then(JsonValue::as_str)
692 .unwrap_or(""),
693 )
694 })
695 .filter(|artifact| !exclude_kinds.contains(artifact.kind.as_str()))
696 .filter(|artifact| {
697 default_mode_allows_artifact(
698 &kind,
699 artifact,
700 include_ids.is_empty() && include_kinds.is_empty(),
701 )
702 })
703 .cloned()
704 .collect()
705}
706
707fn default_mode_allows_artifact(
708 kind: &str,
709 artifact: &ArtifactRecord,
710 using_default_filter: bool,
711) -> bool {
712 if !using_default_filter {
713 return true;
714 }
715 match kind {
716 "cold" => false,
717 "scanned" => artifact_matches_tier(artifact, &["scan", "scanned", "tier1_scan"]),
718 "enriched" => artifact_matches_tier(
719 artifact,
720 &[
721 "scan",
722 "scanned",
723 "tier1_scan",
724 "enrichment",
725 "enriched",
726 "tier2_enrichment",
727 ],
728 ),
729 _ => true,
730 }
731}
732
733fn artifact_matches_tier(artifact: &ArtifactRecord, labels: &[&str]) -> bool {
734 labels.iter().any(|label| {
735 artifact.kind == *label
736 || artifact
737 .metadata
738 .get("context_tier")
739 .and_then(JsonValue::as_str)
740 == Some(*label)
741 })
742}
743
744fn project_transcript(
745 task: &ContextEvalTask,
746 mode: &ContextEvalMode,
747) -> (ContextEvalProjectionReport, String) {
748 let policy = projection_policy(mode);
749 let keep_last = mode.transcript_keep_last.unwrap_or(match policy.as_str() {
750 "none" => 0,
751 "summary" | "compacted" => 1,
752 "last_n" | "projected" => 2,
753 _ => task.transcript.len(),
754 });
755 let retained: Vec<&ContextEvalTranscriptMessage> = match policy.as_str() {
756 "none" => Vec::new(),
757 "full" => task.transcript.iter().collect(),
758 "summary" | "compacted" | "last_n" | "projected" => task
759 .transcript
760 .iter()
761 .rev()
762 .take(keep_last)
763 .collect::<Vec<_>>()
764 .into_iter()
765 .rev()
766 .collect(),
767 _ => task.transcript.iter().collect(),
768 };
769 let transcript_text = retained
770 .iter()
771 .map(|message| format!("{}: {}", message.role, message.content))
772 .collect::<Vec<_>>()
773 .join("\n");
774 let retained_tokens = retained
775 .iter()
776 .map(|message| {
777 message
778 .estimated_tokens
779 .unwrap_or_else(|| estimate_chunk_tokens(&message.content))
780 })
781 .sum();
782 (
783 ContextEvalProjectionReport {
784 policy,
785 source_message_count: task.transcript.len(),
786 retained_message_count: retained.len(),
787 retained_tokens,
788 },
789 transcript_text,
790 )
791}
792
793fn exposed_tools(task: &ContextEvalTask, mode: &ContextEvalMode) -> Vec<String> {
794 let disclosure = tool_disclosure(mode);
795 let allowlist: BTreeSet<&str> = mode.tool_allowlist.iter().map(String::as_str).collect();
796 let mut names = match disclosure.as_str() {
797 "none" => Vec::new(),
798 "full" => task.tools.iter().map(|tool| tool.name.clone()).collect(),
799 "limited" | "tool_search_limited" => task
800 .tools
801 .iter()
802 .filter(|tool| allowlist.contains(tool.name.as_str()))
803 .map(|tool| tool.name.clone())
804 .collect(),
805 _ => task.tools.iter().map(|tool| tool.name.clone()).collect(),
806 };
807 if disclosure == "tool_search_limited" && !names.iter().any(|name| name == "tool_search") {
808 names.push("tool_search".to_string());
809 }
810 sorted_strings(&names)
811}
812
813fn visible_tool_events(
814 task: &ContextEvalTask,
815 exposed_tools: &[String],
816 mode: &ContextEvalMode,
817) -> Vec<ContextEvalToolEvent> {
818 if tool_disclosure(mode) == "full" {
819 return task.tool_events.clone();
820 }
821 let exposed: BTreeSet<&str> = exposed_tools.iter().map(String::as_str).collect();
822 task.tool_events
823 .iter()
824 .filter(|event| exposed.contains(event.name.as_str()))
825 .cloned()
826 .collect()
827}
828
829fn visible_input(task: &ContextEvalTask, prepared: &PreparedModeRun) -> String {
830 [
831 task.objective.as_str(),
832 prepared.rendered_context.as_str(),
833 prepared.transcript_text.as_str(),
834 &prepared.exposed_tools.join("\n"),
835 ]
836 .into_iter()
837 .filter(|part| !part.trim().is_empty())
838 .collect::<Vec<_>>()
839 .join("\n\n")
840}
841
842fn score_correctness(
843 expected: &ContextEvalExpected,
844 surface: &str,
845 selected_artifact_ids: &[String],
846 source: &str,
847) -> ContextEvalCorrectness {
848 let lower_surface = surface.to_ascii_lowercase();
849 let mut present_terms = Vec::new();
850 let mut missing_terms = Vec::new();
851 for term in sorted_strings(&expected.required_terms) {
852 if lower_surface.contains(&term.to_ascii_lowercase()) {
853 present_terms.push(term);
854 } else {
855 missing_terms.push(term);
856 }
857 }
858 let selected: BTreeSet<&str> = selected_artifact_ids.iter().map(String::as_str).collect();
859 let mut present_artifacts = Vec::new();
860 let mut missing_artifacts = Vec::new();
861 for id in sorted_strings(&expected.expected_artifact_ids) {
862 if selected.contains(id.as_str()) {
863 present_artifacts.push(id);
864 } else {
865 missing_artifacts.push(id);
866 }
867 }
868 let term_score = fraction(
869 present_terms.len(),
870 present_terms.len() + missing_terms.len(),
871 );
872 let artifact_score = fraction(
873 present_artifacts.len(),
874 present_artifacts.len() + missing_artifacts.len(),
875 );
876 let score = if expected.required_terms.is_empty() && expected.expected_artifact_ids.is_empty() {
877 1.0
878 } else if expected.required_terms.is_empty() || expected.expected_artifact_ids.is_empty() {
879 term_score.max(artifact_score)
880 } else {
881 f64::midpoint(term_score, artifact_score)
882 };
883 ContextEvalCorrectness {
884 passed: missing_terms.is_empty() && missing_artifacts.is_empty(),
885 score: round4(score),
886 required_terms_present: present_terms,
887 required_terms_missing: missing_terms,
888 expected_artifact_ids_present: present_artifacts,
889 expected_artifact_ids_missing: missing_artifacts,
890 source: source.to_string(),
891 }
892}
893
894fn score_tools(
895 expected_tools: &[String],
896 events: &[ContextEvalToolEvent],
897) -> ContextEvalToolQuality {
898 let expected = sorted_strings(expected_tools);
899 let expected_set: BTreeSet<&str> = expected.iter().map(String::as_str).collect();
900 let observed = sorted_strings(
901 &events
902 .iter()
903 .map(|event| event.name.clone())
904 .collect::<Vec<_>>(),
905 );
906 let observed_set: BTreeSet<&str> = observed.iter().map(String::as_str).collect();
907 let matched_tools = expected
908 .iter()
909 .filter(|tool| observed_set.contains(tool.as_str()))
910 .cloned()
911 .collect::<Vec<_>>();
912 let missing_tools = expected
913 .iter()
914 .filter(|tool| !observed_set.contains(tool.as_str()))
915 .cloned()
916 .collect::<Vec<_>>();
917 let unnecessary_tools = observed
918 .iter()
919 .filter(|tool| !expected_set.contains(tool.as_str()) && !is_edit_tool(tool))
920 .cloned()
921 .collect::<Vec<_>>();
922 let denominator = expected.len() + unnecessary_tools.len();
923 let score = if denominator == 0 {
924 1.0
925 } else {
926 matched_tools.len() as f64 / denominator as f64
927 };
928 ContextEvalToolQuality {
929 score: round4(score),
930 expected_tools: expected,
931 observed_tools: observed,
932 matched_tools,
933 missing_tools,
934 unnecessary_tools,
935 }
936}
937
938fn reads_before_first_edit(events: &[ContextEvalToolEvent]) -> usize {
939 let mut reads = 0;
940 for event in events {
941 if is_edit_event(event) {
942 break;
943 }
944 if is_read_event(event) {
945 reads += 1;
946 }
947 }
948 reads
949}
950
951fn error_recovery_count(events: &[ContextEvalToolEvent]) -> usize {
952 events
953 .iter()
954 .filter(|event| {
955 event.recovery == Some(true)
956 || event
957 .phase
958 .as_deref()
959 .is_some_and(|phase| phase.contains("recovery") || phase.contains("error"))
960 || event.quality.as_deref() == Some("recovery")
961 })
962 .count()
963}
964
965fn aggregate_runs(runs: &[ContextEvalRunReport]) -> ContextEvalAggregate {
966 let total = runs.len();
967 let mean_final_correctness = mean(total, runs.iter().map(|run| run.final_correctness.score));
968 let mean_tool_call_quality = mean(total, runs.iter().map(|run| run.tool_call_quality.score));
969 ContextEvalAggregate {
970 mean_final_correctness,
971 mean_tool_call_quality,
972 total_latency_ms: runs.iter().map(|run| run.latency_ms).sum(),
973 total_input_tokens: runs.iter().map(|run| run.input_tokens).sum(),
974 total_output_tokens: runs.iter().map(|run| run.output_tokens).sum(),
975 total_cost_usd: round6(runs.iter().map(|run| run.cost_usd).sum()),
976 total_compaction_count: runs.iter().map(|run| run.compaction_count).sum(),
977 total_error_recovery_count: runs.iter().map(|run| run.error_recovery_count).sum(),
978 }
979}
980
981fn mode_kind(mode: &ContextEvalMode) -> String {
982 let value = mode.kind.trim();
983 if value.is_empty() {
984 mode.id.clone()
985 } else {
986 value.to_string()
987 }
988}
989
990fn mode_budget_tokens(mode: &ContextEvalMode) -> usize {
991 mode.budget_tokens
992 .unwrap_or_else(|| match mode_kind(mode).as_str() {
993 "cold" => 0,
994 "scanned" => 800,
995 "enriched" => 1_200,
996 "hud_pack" | "projected" | "compacted" | "tool_search_limited" => 1_600,
997 "full" => 64_000,
998 _ => 8_000,
999 })
1000}
1001
1002fn assemble_strategy(mode: &ContextEvalMode) -> Result<AssembleStrategy, VmError> {
1003 mode.assemble_strategy
1004 .as_deref()
1005 .map(AssembleStrategy::parse)
1006 .transpose()
1007 .map_err(VmError::Runtime)
1008 .map(|value| value.unwrap_or(AssembleStrategy::Relevance))
1009}
1010
1011fn assemble_dedup(mode: &ContextEvalMode) -> Result<AssembleDedup, VmError> {
1012 mode.dedup
1013 .as_deref()
1014 .map(AssembleDedup::parse)
1015 .transpose()
1016 .map_err(VmError::Runtime)
1017 .map(|value| value.unwrap_or(AssembleDedup::Chunked))
1018}
1019
1020fn projection_policy(mode: &ContextEvalMode) -> String {
1021 mode.projection_policy
1022 .clone()
1023 .unwrap_or_else(|| match mode_kind(mode).as_str() {
1024 "cold" | "scanned" | "enriched" | "hud_pack" | "tool_search_limited" => {
1025 "none".to_string()
1026 }
1027 "projected" => "last_n".to_string(),
1028 "compacted" => "compacted".to_string(),
1029 "full" => "full".to_string(),
1030 _ => "none".to_string(),
1031 })
1032}
1033
1034fn tool_disclosure(mode: &ContextEvalMode) -> String {
1035 mode.tool_disclosure
1036 .clone()
1037 .unwrap_or_else(|| match mode_kind(mode).as_str() {
1038 "cold" | "scanned" | "enriched" | "hud_pack" => "none".to_string(),
1039 "tool_search_limited" => "tool_search_limited".to_string(),
1040 "full" => "full".to_string(),
1041 _ => "limited".to_string(),
1042 })
1043}
1044
1045fn preprocessing_report(mode: &ContextEvalMode) -> ContextEvalPreprocessing {
1046 let preprocessing = mode
1047 .preprocessing
1048 .clone()
1049 .unwrap_or_else(|| "deterministic".to_string());
1050 ContextEvalPreprocessing {
1051 llm_enabled: preprocessing == "llm",
1052 mode: preprocessing,
1053 }
1054}
1055
1056fn mode_compaction_count(mode: &ContextEvalMode) -> usize {
1057 usize::from(mode_kind(mode) == "compacted" || mode.compaction_policy.is_some())
1058}
1059
1060fn is_read_event(event: &ContextEvalToolEvent) -> bool {
1061 event
1062 .phase
1063 .as_deref()
1064 .is_some_and(|phase| phase == "read" || phase == "scan")
1065 || event.name.starts_with("read")
1066 || event.name.starts_with("search")
1067 || event.name.starts_with("list")
1068}
1069
1070fn is_edit_event(event: &ContextEvalToolEvent) -> bool {
1071 event
1072 .phase
1073 .as_deref()
1074 .is_some_and(|phase| phase == "edit" || phase == "write" || phase == "mutation")
1075 || is_edit_tool(&event.name)
1076}
1077
1078fn is_edit_tool(name: &str) -> bool {
1079 name.starts_with("edit")
1080 || name.starts_with("write")
1081 || name.starts_with("apply")
1082 || name.contains("patch")
1083}
1084
1085fn fraction(numerator: usize, denominator: usize) -> f64 {
1086 if denominator == 0 {
1087 1.0
1088 } else {
1089 numerator as f64 / denominator as f64
1090 }
1091}
1092
1093fn mean(total: usize, values: impl Iterator<Item = f64>) -> f64 {
1094 if total == 0 {
1095 0.0
1096 } else {
1097 round4(values.sum::<f64>() / total as f64)
1098 }
1099}
1100
1101fn sorted_strings(values: &[String]) -> Vec<String> {
1102 values
1103 .iter()
1104 .map(|value| value.trim())
1105 .filter(|value| !value.is_empty())
1106 .map(ToOwned::to_owned)
1107 .collect::<BTreeSet<_>>()
1108 .into_iter()
1109 .collect()
1110}
1111
1112fn stable_hash(parts: &[&str]) -> String {
1113 let mut hasher = Sha256::new();
1114 for part in parts {
1115 hasher.update((part.len() as u64).to_le_bytes());
1116 hasher.update(part.as_bytes());
1117 }
1118 hasher
1119 .finalize()
1120 .iter()
1121 .map(|byte| format!("{byte:02x}"))
1122 .collect()
1123}
1124
1125fn round4(value: f64) -> f64 {
1126 (value * 10_000.0).round() / 10_000.0
1127}
1128
1129fn round6(value: f64) -> f64 {
1130 (value * 1_000_000.0).round() / 1_000_000.0
1131}
1132
1133pub fn context_eval_default_output_dir() -> PathBuf {
1134 PathBuf::from(".harn-runs/context-eval/latest")
1135}
1136
1137#[cfg(test)]
1138mod tests {
1139 use super::*;
1140
1141 fn artifact(id: &str, kind: &str, text: &str) -> ArtifactRecord {
1142 ArtifactRecord {
1143 type_name: "artifact".to_string(),
1144 id: id.to_string(),
1145 kind: kind.to_string(),
1146 title: Some(id.to_string()),
1147 text: Some(text.to_string()),
1148 data: None,
1149 source: Some("fixture".to_string()),
1150 created_at: "2026-05-23T00:00:00Z".to_string(),
1151 freshness: Some("fresh".to_string()),
1152 priority: Some(80),
1153 lineage: Vec::new(),
1154 relevance: Some(1.0),
1155 estimated_tokens: None,
1156 stage: None,
1157 metadata: BTreeMap::new(),
1158 }
1159 }
1160
1161 #[test]
1162 fn context_eval_scores_modes_deterministically() {
1163 let manifest = ContextEvalManifest {
1164 type_name: CONTEXT_EVAL_MANIFEST_TYPE.to_string(),
1165 version: 1,
1166 id: "smoke".to_string(),
1167 modes: vec![
1168 ContextEvalMode {
1169 id: "cold".to_string(),
1170 kind: "cold".to_string(),
1171 ..Default::default()
1172 },
1173 ContextEvalMode {
1174 id: "pack".to_string(),
1175 kind: "hud_pack".to_string(),
1176 artifact_ids: vec!["runbook".to_string()],
1177 tool_disclosure: Some("limited".to_string()),
1178 tool_allowlist: vec!["read_file".to_string()],
1179 ..Default::default()
1180 },
1181 ],
1182 tasks: vec![ContextEvalTask {
1183 id: "task".to_string(),
1184 objective: "Find the rollback command".to_string(),
1185 artifacts: vec![artifact(
1186 "runbook",
1187 "context_pack",
1188 "Use deploy rollback now.",
1189 )],
1190 tools: vec![ContextEvalTool {
1191 name: "read_file".to_string(),
1192 ..Default::default()
1193 }],
1194 tool_events: vec![ContextEvalToolEvent {
1195 order: Some(1),
1196 name: "read_file".to_string(),
1197 phase: Some("read".to_string()),
1198 success: Some(true),
1199 quality: Some("useful".to_string()),
1200 recovery: None,
1201 }],
1202 expected: ContextEvalExpected {
1203 required_terms: vec!["deploy rollback".to_string()],
1204 expected_artifact_ids: vec!["runbook".to_string()],
1205 expected_tools: vec!["read_file".to_string()],
1206 ..Default::default()
1207 },
1208 ..Default::default()
1209 }],
1210 ..Default::default()
1211 };
1212
1213 let report = evaluate_context_eval_manifest(&manifest).expect("context eval succeeds");
1214 assert_eq!(report.total_runs, 2);
1215 assert_eq!(report.passed_runs, 1);
1216 assert!(!report.runs[0].passed);
1217 assert!(report.runs[1].passed);
1218 assert_eq!(report.runs[1].reads_before_first_edit, 1);
1219 assert_eq!(report.runs[1].tool_call_quality.score, 1.0);
1220 assert_eq!(report.runs[1].cache.stable_input_hash.len(), 64);
1221 }
1222}