lean_ctx/core/triage/
validation.rs

1//! Gold-set evaluation for the deterministic triage baseline.
2
3use std::collections::HashMap;
4
5use serde::Deserialize;
6
7use super::{TaskAnalysisInput, TaskAnalyzer, rules::RuleTriageBackend};
8
9const GOLD_SET: &str = include_str!("../../../data/triage_gold_set.jsonl");
10
11#[derive(Debug, Clone, PartialEq)]
12pub struct ValidationResult {
13    pub total: usize,
14    pub correct: usize,
15    pub accuracy: f64,
16    /// Per intent: `(true_positives, gold_examples)`; derive recall as `tp/gold`.
17    pub per_intent: HashMap<String, (usize, usize)>,
18}
19
20#[derive(Deserialize)]
21struct GoldTask {
22    query: String,
23    labels: GoldLabels,
24}
25
26#[derive(Deserialize)]
27struct GoldLabels {
28    intent: String,
29}
30
31pub fn validate_against_gold_set() -> ValidationResult {
32    let tasks: Vec<GoldTask> = GOLD_SET
33        .lines()
34        .map(|line| serde_json::from_str(line).expect("gold set must be valid JSONL"))
35        .collect();
36    let mut gold_counts = HashMap::<String, usize>::new();
37    let mut predicted_counts = HashMap::<String, usize>::new();
38    let mut true_positives = HashMap::<String, usize>::new();
39    for task in &tasks {
40        *gold_counts.entry(task.labels.intent.clone()).or_default() += 1;
41        let predicted = RuleTriageBackend
42            .analyze(&TaskAnalysisInput {
43                query: task.query.clone(),
44                ..Default::default()
45            })
46            .expect("rule backend must classify gold tasks")
47            .profile
48            .intent;
49        *predicted_counts.entry(predicted.clone()).or_default() += 1;
50        if predicted == task.labels.intent {
51            *true_positives.entry(predicted).or_default() += 1;
52        }
53    }
54    let correct = true_positives.values().sum();
55    let per_intent = gold_counts
56        .iter()
57        .map(|(intent, total)| {
58            (
59                intent.clone(),
60                (*true_positives.get(intent).unwrap_or(&0), *total),
61            )
62        })
63        .collect();
64    for (intent, predicted) in predicted_counts {
65        let tp = *true_positives.get(&intent).unwrap_or(&0) as f64;
66        let precision = tp / predicted as f64;
67        let recall = tp / *gold_counts.get(&intent).unwrap_or(&0) as f64;
68        let _ = (precision, recall);
69    }
70    ValidationResult {
71        total: tasks.len(),
72        correct,
73        accuracy: correct as f64 / tasks.len() as f64,
74        per_intent,
75    }
76}
lean_ctx/core/triage/validation.rs

lean_ctx/core/triage/
validation.rs
