1use std::collections::{BTreeMap, BTreeSet};
2
3use serde::Serialize;
4use serde_json::json;
5
6use crate::graph_index::GraphIndex;
7use crate::retrieval::{GroundIndex, Hit, RelationMatch, ground_subgraph, ground_with};
8use crate::schema::{Edge, Graph, Kind};
9use crate::workflow::{
10 TaskContextItem, TaskEvent, TaskOperator, TaskState, apply_task_event, draft_task_dag,
11 initial_run, ready_tasks, task_context_slice, validate_dag,
12};
13
14use super::{
15 ContextEdgeExpectation, GoldenCase, RelationMatchExpectation, kind_label,
16 missing_relation_matches, receipt_coverage,
17};
18
19#[derive(Serialize, Clone, Debug, Default)]
21pub struct DogfoodDifficultyStat {
22 pub total: usize,
23 pub passed: usize,
24 pub mean_score: f64,
25}
26
27#[derive(Serialize, Clone, Debug)]
29pub struct DogfoodCaseResult {
30 pub query: String,
31 pub difficulty: String,
32 #[serde(skip_serializing_if = "Option::is_none")]
33 pub expected_profile: Option<String>,
34 #[serde(skip_serializing_if = "Option::is_none")]
35 pub profile: Option<String>,
36 #[serde(skip_serializing_if = "Option::is_none")]
37 pub profile_ok: Option<bool>,
38 pub route_top: Option<String>,
39 #[serde(skip_serializing_if = "Option::is_none")]
40 pub expected_partition: Option<String>,
41 #[serde(skip_serializing_if = "Option::is_none")]
42 pub route_partition: Option<String>,
43 #[serde(skip_serializing_if = "Option::is_none")]
44 pub partition_ok: Option<bool>,
45 pub route_rank: Option<usize>,
46 pub route_ok: bool,
47 #[serde(skip_serializing_if = "Option::is_none")]
48 pub expected_focus_route: Option<String>,
49 #[serde(default, skip_serializing_if = "Vec::is_empty")]
50 pub expected_focus_routes: Vec<String>,
51 pub focus_route: String,
52 #[serde(skip_serializing_if = "Option::is_none")]
53 pub focus_route_ok: Option<bool>,
54 #[serde(skip_serializing_if = "Vec::is_empty")]
55 pub route_relation_matches: Vec<RelationMatch>,
56 #[serde(skip_serializing_if = "Option::is_none")]
57 pub route_relation_ok: Option<bool>,
58 pub route_relation_expected: usize,
59 pub route_relation_missing: usize,
60 pub context_recall: Option<f64>,
61 pub context_noise: Option<f64>,
62 pub edge_recall: Option<f64>,
63 pub receipt_coverage: f64,
64 pub requires_code_ok: bool,
65 pub requires_docs_ok: bool,
66 #[serde(rename = "brief_valid")]
67 pub workflow_valid: bool,
68 #[serde(rename = "brief_bounded")]
69 pub workflow_bounded: bool,
70 pub expected_operators_ok: bool,
71 pub task_context_ok: bool,
72 pub run_advancement_ok: bool,
73 pub score: f64,
74 pub passed: bool,
75 pub pack_size: usize,
76 pub node_count: usize,
77 pub missing_context: Vec<String>,
78 pub leaked_context: Vec<String>,
79 pub missing_edges: Vec<ContextEdgeExpectation>,
80 pub missing_route_relation_matches: Vec<RelationMatchExpectation>,
81 pub missing_operators: Vec<String>,
82 pub missing_task_context: Vec<String>,
83 pub ready_after_context: Vec<String>,
84 pub protocol: DogfoodProtocolComparison,
85}
86
87#[derive(Serialize, Clone, Debug, Default)]
89pub struct DogfoodProtocolRun {
90 pub request_tokens: usize,
91 pub response_tokens: usize,
92 pub total_tokens: usize,
93 pub route_ok: bool,
94 #[serde(skip_serializing_if = "Option::is_none")]
95 pub confidence_band: Option<String>,
96 #[serde(skip_serializing_if = "Option::is_none")]
97 pub trust_tier: Option<String>,
98 #[serde(skip_serializing_if = "Option::is_none")]
99 pub trust_verdict: Option<String>,
100 pub context_recall: Option<f64>,
101 pub receipt_coverage: f64,
102}
103
104#[derive(Serialize, Clone, Debug, Default)]
106pub struct DogfoodProtocolComparison {
107 pub legacy: DogfoodProtocolRun,
108 pub ekf_compact: DogfoodProtocolRun,
109 pub request_token_overhead: isize,
110 pub response_token_delta: isize,
111 pub total_token_delta: isize,
112 pub response_token_savings_rate: f64,
113 pub total_token_savings_rate: f64,
114 pub quality_preserved: bool,
115}
116
117#[derive(Serialize, Clone, Debug)]
119pub struct DogfoodReport {
120 pub dogfood_cases: usize,
121 pub pass_rate: f64,
122 pub mean_score: f64,
123 pub route_hit_rate: f64,
124 #[serde(skip_serializing_if = "Option::is_none")]
125 pub focus_route_match_rate: Option<f64>,
126 #[serde(skip_serializing_if = "Option::is_none")]
127 pub partition_match_rate: Option<f64>,
128 #[serde(skip_serializing_if = "Option::is_none")]
129 pub relation_evidence_recall: Option<f64>,
130 pub relation_evidence_expected: usize,
131 pub relation_evidence_missing: usize,
132 pub context_recall: f64,
133 pub context_noise: f64,
134 pub edge_recall: f64,
135 pub receipt_coverage: f64,
136 pub code_context_rate: f64,
137 pub docs_context_rate: f64,
138 #[serde(rename = "brief_valid_rate")]
139 pub workflow_valid_rate: f64,
140 #[serde(rename = "brief_bounded_rate")]
141 pub workflow_bounded_rate: f64,
142 pub operator_coverage_rate: f64,
143 pub task_context_rate: f64,
144 pub run_advancement_rate: f64,
145 #[serde(skip_serializing_if = "Option::is_none")]
146 pub profile_match_rate: Option<f64>,
147 pub protocol: DogfoodProtocolSummary,
148 pub by_difficulty: BTreeMap<String, DogfoodDifficultyStat>,
149 pub cases: Vec<DogfoodCaseResult>,
150}
151
152#[derive(Serialize, Clone, Debug, Default)]
154pub struct DogfoodProtocolSummary {
155 pub legacy_request_tokens: usize,
156 pub legacy_response_tokens: usize,
157 pub legacy_total_tokens: usize,
158 pub ekf_request_tokens: usize,
159 pub ekf_response_tokens: usize,
160 pub ekf_total_tokens: usize,
161 pub request_token_overhead: isize,
162 pub response_token_delta: isize,
163 pub total_token_delta: isize,
164 pub response_token_savings_rate: f64,
165 pub total_token_savings_rate: f64,
166 pub ekf_quality_preservation_rate: f64,
167 pub ekf_route_hit_rate: f64,
168 pub ekf_context_recall: f64,
169 pub ekf_confidence_coverage_rate: f64,
170 pub ekf_trust_tier_coverage_rate: f64,
171 pub confidence_bands: BTreeMap<String, usize>,
172 pub trust_tiers: BTreeMap<String, usize>,
173 pub trust_verdicts: BTreeMap<String, usize>,
174}
175
176pub struct DogfoodProtocolInput<'a> {
178 pub case: &'a GoldenCase,
179 pub hits: &'a [Hit],
180 pub route_ok: bool,
181 pub context_order: &'a [String],
182 pub edges: &'a [Edge],
183 pub context_judgment: &'a DogfoodContextJudgment,
184 pub receipt_coverage: f64,
185 pub route_trust_tier: Option<String>,
186 pub route_trust_verdict: Option<String>,
187 pub limit: usize,
188 pub depth: usize,
189 pub width: usize,
190}
191
192#[derive(Clone, Debug)]
193pub struct DogfoodContextJudgment {
194 pub recall: Option<f64>,
195 pub noise: Option<f64>,
196 pub edge_recall: Option<f64>,
197 pub missing: Vec<String>,
198 pub leaked: Vec<String>,
199 pub missing_edges: Vec<ContextEdgeExpectation>,
200}
201
202pub fn evaluate_dogfood(
205 graph: &Graph,
206 cases: &[GoldenCase],
207 limit: usize,
208 depth: usize,
209 width: usize,
210 max_nodes: usize,
211 min_case_score: f64,
212) -> DogfoodReport {
213 let ground_index = GroundIndex::build(graph);
214 let graph_index = GraphIndex::build(graph);
215 let mut results = Vec::new();
216
217 for c in cases.iter().filter(|case| !case.garbage) {
218 let hits = ground_with(graph, &ground_index, &c.query, limit);
219 let route_top = hits.first().map(|hit| hit.id.clone());
220 let route_partition = route_top
221 .as_deref()
222 .and_then(|id| graph_index.node(id))
223 .and_then(|node| node.partition.clone());
224 let partition_ok = c
225 .expected_partition
226 .as_ref()
227 .map(|expected| route_partition.as_deref() == Some(expected.as_str()));
228 let route_rank = hits
229 .iter()
230 .position(|hit| c.expect.iter().any(|expected| expected == &hit.id))
231 .map(|idx| idx + 1);
232 let route_ok =
233 (c.expect.is_empty() || route_rank.is_some()) && partition_ok.unwrap_or(true);
234 let route_relation_matches = hits
235 .first()
236 .map(|hit| hit.relation_matches.clone())
237 .unwrap_or_default();
238 let missing_route_relation_matches =
239 missing_relation_matches(&c.route_relation_must, &route_relation_matches);
240 let route_relation_ok = (!c.route_relation_must.is_empty())
241 .then_some(missing_route_relation_matches.is_empty());
242 let route_relation_expected = c.route_relation_must.len();
243 let route_relation_missing = missing_route_relation_matches.len();
244
245 let sg = ground_subgraph(graph, &c.query, limit, depth, width);
246 let focus_route_ok =
247 acceptable_focus_routes(c).map(|expected| expected.contains(&sg.route));
248 let context_ids = sg
249 .context_order
250 .iter()
251 .map(String::as_str)
252 .collect::<BTreeSet<_>>();
253 let context_judgment = judge_dogfood_context(c, &context_ids, &sg.edges);
254 let pack_size = sg.context_order.len();
255 let receipt_coverage = receipt_coverage(&graph_index, &sg.context_order);
256 let protocol = compare_protocol_economics(DogfoodProtocolInput {
257 case: c,
258 hits: &hits,
259 route_ok,
260 context_order: &sg.context_order,
261 edges: &sg.edges,
262 context_judgment: &context_judgment,
263 receipt_coverage,
264 route_trust_tier: None,
265 route_trust_verdict: None,
266 limit,
267 depth,
268 width,
269 });
270 let has_code_context = sg.context_order.iter().any(|id| {
271 graph_index
272 .node(id)
273 .is_some_and(|node| is_dogfood_code_kind(node.kind))
274 });
275 let has_docs_context = sg.context_order.iter().any(|id| {
276 graph_index
277 .node(id)
278 .is_some_and(|node| is_dogfood_docs_kind(node.kind))
279 });
280 let requires_code_ok = !c.requires_code || has_code_context;
281 let requires_docs_ok = !c.requires_docs || has_docs_context;
282
283 let context = sg
284 .context_order
285 .iter()
286 .filter_map(|id| {
287 let node = graph_index.node(id)?;
288 Some(TaskContextItem {
289 id: node.id.clone(),
290 title: node.title.clone(),
291 kind: kind_label(node.kind),
292 source: graph_index.source_of(&node.id).unwrap_or_default(),
293 })
294 })
295 .collect::<Vec<_>>();
296 let dag = draft_task_dag(&c.query, &context);
297 let workflow_valid = validate_dag(&dag).is_ok();
298 let workflow_bounded = dag.nodes.len() <= max_nodes;
299 let operators = dag
300 .nodes
301 .iter()
302 .map(|node| dogfood_operator_label(node.operator))
303 .collect::<BTreeSet<_>>();
304 let missing_operators = c
305 .expected_operators
306 .iter()
307 .filter(|operator| !operators.contains(operator.as_str()))
308 .cloned()
309 .collect::<Vec<_>>();
310 let expected_operators_ok = missing_operators.is_empty();
311 let (run_advancement_ok, ready_after_context) = advance_dogfood_context_step(&dag);
312 let missing_task_context = ready_after_context
313 .iter()
314 .filter(|task_id| !dogfood_task_context_is_useful(&dag, task_id, &context))
315 .cloned()
316 .collect::<Vec<_>>();
317 let task_context_ok = missing_task_context.is_empty();
318
319 let mut checks = Vec::new();
320 checks.push(route_ok);
321 if let Some(ok) = focus_route_ok {
322 checks.push(ok);
323 }
324 if let Some(recall) = context_judgment.recall {
325 checks.push(recall >= 1.0);
326 }
327 if let Some(noise) = context_judgment.noise {
328 checks.push(noise == 0.0);
329 }
330 if let Some(edge_recall) = context_judgment.edge_recall {
331 checks.push(edge_recall >= 1.0);
332 }
333 if let Some(ok) = route_relation_ok {
334 checks.push(ok);
335 }
336 checks.push(receipt_coverage >= 1.0);
337 checks.push(requires_code_ok);
338 checks.push(requires_docs_ok);
339 checks.push(workflow_valid);
340 checks.push(workflow_bounded);
341 checks.push(expected_operators_ok);
342 checks.push(task_context_ok);
343 checks.push(run_advancement_ok);
344
345 let score = score_dogfood_checks(&checks);
346
347 results.push(DogfoodCaseResult {
348 query: c.query.clone(),
349 difficulty: dogfood_case_difficulty(c),
350 expected_profile: None,
351 profile: None,
352 profile_ok: None,
353 route_top,
354 expected_partition: c.expected_partition.clone(),
355 route_partition,
356 partition_ok,
357 route_rank,
358 route_ok,
359 expected_focus_route: c.expected_focus_route.clone(),
360 expected_focus_routes: c.expected_focus_routes.clone(),
361 focus_route: sg.route,
362 focus_route_ok,
363 route_relation_matches,
364 route_relation_ok,
365 route_relation_expected,
366 route_relation_missing,
367 context_recall: context_judgment.recall,
368 context_noise: context_judgment.noise,
369 edge_recall: context_judgment.edge_recall,
370 receipt_coverage,
371 requires_code_ok,
372 requires_docs_ok,
373 workflow_valid,
374 workflow_bounded,
375 expected_operators_ok,
376 task_context_ok,
377 run_advancement_ok,
378 score,
379 passed: score >= min_case_score,
380 pack_size,
381 node_count: dag.nodes.len(),
382 missing_context: context_judgment.missing,
383 leaked_context: context_judgment.leaked,
384 missing_edges: context_judgment.missing_edges,
385 missing_route_relation_matches,
386 missing_operators,
387 missing_task_context,
388 ready_after_context,
389 protocol,
390 });
391 }
392
393 summarize_dogfood(results)
394}
395
396pub fn dogfood_case_difficulty(case: &GoldenCase) -> String {
397 case.difficulty
398 .clone()
399 .or_else(|| case.class.clone())
400 .unwrap_or_else(|| "unclassified".to_string())
401}
402
403pub fn judge_dogfood_context(
404 case: &GoldenCase,
405 context_ids: &BTreeSet<&str>,
406 edges: &[Edge],
407) -> DogfoodContextJudgment {
408 let missing = case
409 .context_must
410 .iter()
411 .filter(|id| !context_ids.contains(id.as_str()))
412 .cloned()
413 .collect::<Vec<_>>();
414 let leaked = case
415 .context_must_not
416 .iter()
417 .filter(|id| context_ids.contains(id.as_str()))
418 .cloned()
419 .collect::<Vec<_>>();
420 let recall = if case.context_must.is_empty() {
421 None
422 } else {
423 Some((case.context_must.len() - missing.len()) as f64 / case.context_must.len() as f64)
424 };
425 let noise = if case.context_must_not.is_empty() {
426 None
427 } else {
428 Some(leaked.len() as f64 / case.context_must_not.len() as f64)
429 };
430 let missing_edges = case
431 .context_edges_must
432 .iter()
433 .filter(|expected| !edge_present(expected, edges))
434 .cloned()
435 .collect::<Vec<_>>();
436 let edge_recall = if case.context_edges_must.is_empty() {
437 None
438 } else {
439 Some(
440 (case.context_edges_must.len() - missing_edges.len()) as f64
441 / case.context_edges_must.len() as f64,
442 )
443 };
444 DogfoodContextJudgment {
445 recall,
446 noise,
447 edge_recall,
448 missing,
449 leaked,
450 missing_edges,
451 }
452}
453
454fn edge_present(expected: &ContextEdgeExpectation, edges: &[Edge]) -> bool {
455 edges.iter().any(|edge| {
456 edge.from == expected.from && edge.to == expected.to && edge.relation == expected.relation
457 })
458}
459
460pub fn score_dogfood_checks(checks: &[bool]) -> f64 {
461 if checks.is_empty() {
462 0.0
463 } else {
464 checks.iter().filter(|ok| **ok).count() as f64 / checks.len() as f64
465 }
466}
467
468pub fn summarize_dogfood(cases: Vec<DogfoodCaseResult>) -> DogfoodReport {
469 let n = cases.len();
470 let frac = |count: usize| if n == 0 { 0.0 } else { count as f64 / n as f64 };
471 let mean = |sum: f64| if n == 0 { 0.0 } else { sum / n as f64 };
472 let mean_optional = |values: Vec<Option<f64>>, default: f64| {
473 let present = values.into_iter().flatten().collect::<Vec<_>>();
474 if present.is_empty() {
475 default
476 } else {
477 present.iter().sum::<f64>() / present.len() as f64
478 }
479 };
480
481 let mut by_difficulty = BTreeMap::<String, DogfoodDifficultyStat>::new();
482 let mut score_by_difficulty = BTreeMap::<String, f64>::new();
483 for case in &cases {
484 let stat = by_difficulty.entry(case.difficulty.clone()).or_default();
485 stat.total += 1;
486 stat.passed += case.passed as usize;
487 *score_by_difficulty
488 .entry(case.difficulty.clone())
489 .or_default() += case.score;
490 }
491 for (difficulty, stat) in &mut by_difficulty {
492 stat.mean_score = score_by_difficulty
493 .get(difficulty)
494 .copied()
495 .unwrap_or_default()
496 / stat.total as f64;
497 }
498 let relation_evidence_expected = cases
499 .iter()
500 .map(|case| case.route_relation_expected)
501 .sum::<usize>();
502 let relation_evidence_missing = cases
503 .iter()
504 .map(|case| case.route_relation_missing)
505 .sum::<usize>();
506 let relation_evidence_recall = if relation_evidence_expected == 0 {
507 None
508 } else {
509 Some(
510 (relation_evidence_expected - relation_evidence_missing) as f64
511 / relation_evidence_expected as f64,
512 )
513 };
514
515 DogfoodReport {
516 dogfood_cases: n,
517 pass_rate: frac(cases.iter().filter(|case| case.passed).count()),
518 mean_score: mean(cases.iter().map(|case| case.score).sum()),
519 route_hit_rate: frac(cases.iter().filter(|case| case.route_ok).count()),
520 focus_route_match_rate: focus_route_match_rate(&cases),
521 partition_match_rate: partition_match_rate(&cases),
522 relation_evidence_recall,
523 relation_evidence_expected,
524 relation_evidence_missing,
525 context_recall: mean_optional(cases.iter().map(|case| case.context_recall).collect(), 0.0),
526 context_noise: mean_optional(cases.iter().map(|case| case.context_noise).collect(), 0.0),
527 edge_recall: mean_optional(cases.iter().map(|case| case.edge_recall).collect(), 0.0),
528 receipt_coverage: mean(cases.iter().map(|case| case.receipt_coverage).sum()),
529 code_context_rate: frac(cases.iter().filter(|case| case.requires_code_ok).count()),
530 docs_context_rate: frac(cases.iter().filter(|case| case.requires_docs_ok).count()),
531 workflow_valid_rate: frac(cases.iter().filter(|case| case.workflow_valid).count()),
532 workflow_bounded_rate: frac(cases.iter().filter(|case| case.workflow_bounded).count()),
533 operator_coverage_rate: frac(
534 cases
535 .iter()
536 .filter(|case| case.expected_operators_ok)
537 .count(),
538 ),
539 task_context_rate: frac(cases.iter().filter(|case| case.task_context_ok).count()),
540 run_advancement_rate: frac(cases.iter().filter(|case| case.run_advancement_ok).count()),
541 profile_match_rate: profile_match_rate(&cases),
542 protocol: summarize_protocol(&cases),
543 by_difficulty,
544 cases,
545 }
546}
547
548pub fn compare_protocol_economics(input: DogfoodProtocolInput<'_>) -> DogfoodProtocolComparison {
549 const EKF_MAX_ITEMS: usize = 3;
550 const EKF_MAX_CONTEXT: usize = 40;
551 const EKF_MAX_RECEIPTS: usize = 3;
552
553 let legacy_request = json!({
554 "query": input.case.query.as_str(),
555 "limit": input.limit,
556 "depth": input.depth,
557 "width": input.width
558 });
559 let edge_refs = input
560 .edges
561 .iter()
562 .map(|edge| {
563 json!({
564 "from": edge.from.as_str(),
565 "to": edge.to.as_str(),
566 "relation": edge.relation.as_str()
567 })
568 })
569 .collect::<Vec<_>>();
570 let hit_ids = input
571 .hits
572 .iter()
573 .map(|hit| hit.id.as_str())
574 .collect::<Vec<_>>();
575 let legacy_response = json!({
576 "route_top": input.hits.first().map(|hit| hit.id.as_str()),
577 "hits": hit_ids,
578 "context": input.context_order,
579 "edges": edge_refs,
580 "receipt_coverage": input.receipt_coverage,
581 "context_recall": input.context_judgment.recall
582 });
583
584 let partitions = input
585 .case
586 .expected_partition
587 .as_ref()
588 .map(|partition| vec![partition.as_str()])
589 .unwrap_or_default();
590 let ekf_request = json!({
591 "ekf": {
592 "ekf_version": "0.1",
593 "intent": {
594 "query": input.case.query.as_str(),
595 "task": input.case.class.as_deref()
596 },
597 "scope": {
598 "partitions": partitions
599 },
600 "evidence": {
601 "require_receipts": true,
602 "include_relation_evidence": !input.case.route_relation_must.is_empty()
603 || !input.case.context_edges_must.is_empty()
604 },
605 "output": {
606 "format": "evidence_packet",
607 "verbosity": "compact",
608 "max_items": EKF_MAX_ITEMS,
609 "max_context": EKF_MAX_CONTEXT,
610 "max_receipts": EKF_MAX_RECEIPTS,
611 "include_next_tools": true
612 }
613 }
614 });
615 let ekf_context = input
616 .context_order
617 .iter()
618 .take(EKF_MAX_CONTEXT)
619 .cloned()
620 .collect::<Vec<_>>();
621 let ekf_context_ids = ekf_context
622 .iter()
623 .map(String::as_str)
624 .collect::<BTreeSet<_>>();
625 let ekf_context_judgment = judge_dogfood_context(input.case, &ekf_context_ids, input.edges);
626 let ekf_evidence = input
627 .hits
628 .iter()
629 .take(EKF_MAX_ITEMS)
630 .map(|hit| hit.id.as_str())
631 .collect::<Vec<_>>();
632 let ekf_response = json!({
633 "kind": "eidos.evidence_packet",
634 "route_top": input.hits.first().map(|hit| hit.id.as_str()),
635 "evidence": ekf_evidence,
636 "context": ekf_context,
637 "omitted_context": input.context_order.len().saturating_sub(EKF_MAX_CONTEXT),
638 "receipts_retained": input.context_order.len().min(EKF_MAX_RECEIPTS),
639 "receipt_coverage": input.receipt_coverage,
640 "context_recall": ekf_context_judgment.recall,
641 "ekf_request": ekf_request["ekf"]
642 });
643
644 let legacy_request_tokens = estimate_tokens(&legacy_request);
645 let legacy_response_tokens = estimate_tokens(&legacy_response);
646 let ekf_request_tokens = estimate_tokens(&ekf_request);
647 let ekf_response_tokens = estimate_tokens(&ekf_response);
648 let legacy_total_tokens = legacy_request_tokens + legacy_response_tokens;
649 let ekf_total_tokens = ekf_request_tokens + ekf_response_tokens;
650 let confidence_band = input
651 .hits
652 .first()
653 .map(|hit| format!("{:?}", hit.confidence).to_lowercase());
654 let legacy = DogfoodProtocolRun {
655 request_tokens: legacy_request_tokens,
656 response_tokens: legacy_response_tokens,
657 total_tokens: legacy_total_tokens,
658 route_ok: input.route_ok,
659 confidence_band: confidence_band.clone(),
660 trust_tier: input.route_trust_tier.clone(),
661 trust_verdict: input.route_trust_verdict.clone(),
662 context_recall: input.context_judgment.recall,
663 receipt_coverage: input.receipt_coverage,
664 };
665 let ekf_compact = DogfoodProtocolRun {
666 request_tokens: ekf_request_tokens,
667 response_tokens: ekf_response_tokens,
668 total_tokens: ekf_total_tokens,
669 route_ok: input.route_ok,
670 confidence_band,
671 trust_tier: input.route_trust_tier.clone(),
672 trust_verdict: input.route_trust_verdict.clone(),
673 context_recall: ekf_context_judgment.recall,
674 receipt_coverage: input.receipt_coverage,
675 };
676 let request_token_overhead = ekf_request_tokens as isize - legacy_request_tokens as isize;
677 let response_token_delta = legacy_response_tokens as isize - ekf_response_tokens as isize;
678 let total_token_delta = legacy_total_tokens as isize - ekf_total_tokens as isize;
679 let quality_preserved = input.route_ok
680 && ekf_context_judgment
681 .recall
682 .zip(input.context_judgment.recall)
683 .is_none_or(|(ekf, legacy)| ekf >= legacy);
684
685 DogfoodProtocolComparison {
686 legacy,
687 ekf_compact,
688 request_token_overhead,
689 response_token_delta,
690 total_token_delta,
691 response_token_savings_rate: savings_rate(response_token_delta, legacy_response_tokens),
692 total_token_savings_rate: savings_rate(total_token_delta, legacy_total_tokens),
693 quality_preserved,
694 }
695}
696
697fn estimate_tokens(value: &serde_json::Value) -> usize {
698 serde_json::to_string(value)
699 .map(|text| text.len().div_ceil(4).max(1))
700 .unwrap_or(1)
701}
702
703fn savings_rate(delta: isize, baseline_tokens: usize) -> f64 {
704 if baseline_tokens == 0 {
705 0.0
706 } else {
707 delta as f64 / baseline_tokens as f64
708 }
709}
710
711fn summarize_protocol(cases: &[DogfoodCaseResult]) -> DogfoodProtocolSummary {
712 let mut summary = DogfoodProtocolSummary::default();
713 for case in cases {
714 summary.legacy_request_tokens += case.protocol.legacy.request_tokens;
715 summary.legacy_response_tokens += case.protocol.legacy.response_tokens;
716 summary.legacy_total_tokens += case.protocol.legacy.total_tokens;
717 summary.ekf_request_tokens += case.protocol.ekf_compact.request_tokens;
718 summary.ekf_response_tokens += case.protocol.ekf_compact.response_tokens;
719 summary.ekf_total_tokens += case.protocol.ekf_compact.total_tokens;
720 }
721 summary.request_token_overhead =
722 summary.ekf_request_tokens as isize - summary.legacy_request_tokens as isize;
723 summary.response_token_delta =
724 summary.legacy_response_tokens as isize - summary.ekf_response_tokens as isize;
725 summary.total_token_delta =
726 summary.legacy_total_tokens as isize - summary.ekf_total_tokens as isize;
727 summary.response_token_savings_rate =
728 savings_rate(summary.response_token_delta, summary.legacy_response_tokens);
729 summary.total_token_savings_rate =
730 savings_rate(summary.total_token_delta, summary.legacy_total_tokens);
731 let n = cases.len();
732 if n > 0 {
733 summary.ekf_quality_preservation_rate = cases
734 .iter()
735 .filter(|case| case.protocol.quality_preserved)
736 .count() as f64
737 / n as f64;
738 summary.ekf_route_hit_rate = cases
739 .iter()
740 .filter(|case| case.protocol.ekf_compact.route_ok)
741 .count() as f64
742 / n as f64;
743 summary.ekf_confidence_coverage_rate = cases
744 .iter()
745 .filter(|case| case.protocol.ekf_compact.confidence_band.is_some())
746 .count() as f64
747 / n as f64;
748 summary.ekf_trust_tier_coverage_rate = cases
749 .iter()
750 .filter(|case| case.protocol.ekf_compact.trust_tier.is_some())
751 .count() as f64
752 / n as f64;
753 let recalls = cases
754 .iter()
755 .filter_map(|case| case.protocol.ekf_compact.context_recall)
756 .collect::<Vec<_>>();
757 if !recalls.is_empty() {
758 summary.ekf_context_recall = recalls.iter().sum::<f64>() / recalls.len() as f64;
759 }
760 }
761 for case in cases {
762 if let Some(confidence) = &case.protocol.ekf_compact.confidence_band {
763 *summary
764 .confidence_bands
765 .entry(confidence.clone())
766 .or_default() += 1;
767 }
768 if let Some(tier) = &case.protocol.ekf_compact.trust_tier {
769 *summary.trust_tiers.entry(tier.clone()).or_default() += 1;
770 }
771 if let Some(verdict) = &case.protocol.ekf_compact.trust_verdict {
772 *summary.trust_verdicts.entry(verdict.clone()).or_default() += 1;
773 }
774 }
775 summary
776}
777
778fn focus_route_match_rate(cases: &[DogfoodCaseResult]) -> Option<f64> {
779 let judged = cases
780 .iter()
781 .filter_map(|case| case.focus_route_ok)
782 .collect::<Vec<_>>();
783 if judged.is_empty() {
784 None
785 } else {
786 Some(judged.iter().filter(|ok| **ok).count() as f64 / judged.len() as f64)
787 }
788}
789
790fn acceptable_focus_routes(c: &GoldenCase) -> Option<Vec<String>> {
791 let mut expected = c.expected_focus_routes.clone();
792 if let Some(route) = &c.expected_focus_route {
793 expected.push(route.clone());
794 }
795 expected.sort();
796 expected.dedup();
797 (!expected.is_empty()).then_some(expected)
798}
799
800fn partition_match_rate(cases: &[DogfoodCaseResult]) -> Option<f64> {
801 let judged = cases
802 .iter()
803 .filter_map(|case| case.partition_ok)
804 .collect::<Vec<_>>();
805 (!judged.is_empty())
806 .then(|| judged.iter().filter(|ok| **ok).count() as f64 / judged.len() as f64)
807}
808
809fn profile_match_rate(cases: &[DogfoodCaseResult]) -> Option<f64> {
810 let judged = cases
811 .iter()
812 .filter_map(|case| case.profile_ok)
813 .collect::<Vec<_>>();
814 if judged.is_empty() {
815 None
816 } else {
817 Some(judged.iter().filter(|ok| **ok).count() as f64 / judged.len() as f64)
818 }
819}
820
821pub fn dogfood_task_context_is_useful(
822 dag: &crate::workflow::TaskDag,
823 task_id: &str,
824 context: &[TaskContextItem],
825) -> bool {
826 let Ok(slice) = task_context_slice(dag, task_id, context) else {
827 return false;
828 };
829 if slice.tools.is_empty() || slice.required_checks.is_empty() {
830 return false;
831 }
832 if slice.operator == TaskOperator::ReadContext {
833 !slice.context.is_empty() && !slice.receipts.is_empty()
834 } else {
835 !slice.context.is_empty() || !slice.artifact_inputs.is_empty() || !slice.receipts.is_empty()
836 }
837}
838
839pub fn is_dogfood_code_kind(kind: Kind) -> bool {
840 matches!(
841 kind,
842 Kind::Function | Kind::Type | Kind::Trait | Kind::Module
843 )
844}
845
846pub fn is_dogfood_docs_kind(kind: Kind) -> bool {
847 matches!(kind, Kind::Doc | Kind::Skill | Kind::Section | Kind::Agent)
848}
849
850pub fn dogfood_operator_label(operator: TaskOperator) -> &'static str {
851 match operator {
852 TaskOperator::ReadContext => "read_context",
853 TaskOperator::Decompose => "decompose",
854 TaskOperator::Edit => "edit",
855 TaskOperator::Check => "check",
856 TaskOperator::Review => "review",
857 TaskOperator::ProposeDoc => "propose_doc",
858 TaskOperator::HumanApproval => "human_approval",
859 }
860}
861
862pub fn advance_dogfood_context_step(dag: &crate::workflow::TaskDag) -> (bool, Vec<String>) {
863 let Ok(run) = initial_run(dag) else {
864 return (false, Vec::new());
865 };
866 if run.states.get("context") != Some(&TaskState::Ready) {
867 return (false, Vec::new());
868 }
869 let Ok(run) = apply_task_event(
870 dag,
871 &run,
872 TaskEvent {
873 task_id: "context".to_string(),
874 to: TaskState::Running,
875 evidence: vec!["dogfood:focus-context".to_string()],
876 note: None,
877 },
878 ) else {
879 return (false, Vec::new());
880 };
881 let Ok(run) = apply_task_event(
882 dag,
883 &run,
884 TaskEvent {
885 task_id: "context".to_string(),
886 to: TaskState::Passed,
887 evidence: vec!["dogfood:context-read".to_string()],
888 note: None,
889 },
890 ) else {
891 return (false, Vec::new());
892 };
893 let Ok(ready) = ready_tasks(dag, &run) else {
894 return (false, Vec::new());
895 };
896 let ok = !ready.is_empty() && !ready.iter().any(|id| id == "context");
897 (ok, ready)
898}
899
900#[cfg(test)]
901mod tests {
902 use super::*;
903
904 #[test]
905 fn summarize_dogfood_reports_profile_match_rate_for_judged_cases() {
906 let report = summarize_dogfood(vec![
907 dogfood_case(Some(true)),
908 dogfood_case(Some(false)),
909 dogfood_case(None),
910 ]);
911
912 assert_eq!(report.dogfood_cases, 3);
913 assert_eq!(report.profile_match_rate, Some(0.5));
914 }
915
916 #[test]
917 fn summarize_dogfood_reports_relation_evidence_recall_for_judged_cases() {
918 let mut pass = dogfood_case(None);
919 pass.route_relation_expected = 1;
920 pass.route_relation_missing = 0;
921 let mut fail = dogfood_case(None);
922 fail.route_relation_expected = 1;
923 fail.route_relation_missing = 1;
924
925 let report = summarize_dogfood(vec![pass, fail, dogfood_case(None)]);
926
927 assert_eq!(report.relation_evidence_recall, Some(0.5));
928 assert_eq!(report.relation_evidence_expected, 2);
929 assert_eq!(report.relation_evidence_missing, 1);
930 }
931
932 fn dogfood_case(profile_ok: Option<bool>) -> DogfoodCaseResult {
933 DogfoodCaseResult {
934 query: "task".to_string(),
935 difficulty: "easy".to_string(),
936 expected_profile: profile_ok.map(|_| "safe-change".to_string()),
937 profile: profile_ok.map(|ok| {
938 if ok {
939 "safe-change".to_string()
940 } else {
941 "other".to_string()
942 }
943 }),
944 profile_ok,
945 route_top: Some("doc.task".to_string()),
946 expected_partition: None,
947 route_partition: None,
948 partition_ok: None,
949 route_rank: Some(1),
950 route_ok: true,
951 expected_focus_route: None,
952 expected_focus_routes: Vec::new(),
953 focus_route: "doc.task".to_string(),
954 focus_route_ok: None,
955 route_relation_matches: Vec::new(),
956 route_relation_ok: None,
957 route_relation_expected: 0,
958 route_relation_missing: 0,
959 context_recall: Some(1.0),
960 context_noise: Some(0.0),
961 edge_recall: Some(1.0),
962 receipt_coverage: 1.0,
963 requires_code_ok: true,
964 requires_docs_ok: true,
965 workflow_valid: true,
966 workflow_bounded: true,
967 expected_operators_ok: true,
968 task_context_ok: true,
969 run_advancement_ok: true,
970 score: 1.0,
971 passed: true,
972 pack_size: 1,
973 node_count: 1,
974 missing_context: Vec::new(),
975 leaked_context: Vec::new(),
976 missing_edges: Vec::new(),
977 missing_route_relation_matches: Vec::new(),
978 missing_operators: Vec::new(),
979 missing_task_context: Vec::new(),
980 ready_after_context: Vec::new(),
981 protocol: DogfoodProtocolComparison::default(),
982 }
983 }
984}