remem/eval/extraction/
types.rs

1use std::fmt::{self, Display};
2
3use serde::{Deserialize, Serialize};
4
5pub type ExtractionRateMetric = crate::eval::governance::RateMetric;
6
7pub const DEFAULT_CORPUS_PATH: &str = "eval/extraction/corpus.json";
8pub const DEFAULT_BASELINE_PATH: &str = "eval/extraction/baseline.json";
9
10#[derive(Debug, Clone)]
11pub struct ExtractionEvalOptions {
12    pub corpus_path: String,
13}
14
15impl Default for ExtractionEvalOptions {
16    fn default() -> Self {
17        Self {
18            corpus_path: DEFAULT_CORPUS_PATH.to_string(),
19        }
20    }
21}
22
23#[derive(Debug, Clone, Deserialize)]
24pub(crate) struct ExtractionCorpus {
25    pub version: String,
26    #[serde(default)]
27    pub description: String,
28    pub cases: Vec<ExtractionCase>,
29}
30
31#[derive(Debug, Clone, Deserialize)]
32pub(crate) struct ExtractionCase {
33    pub id: String,
34    #[serde(default)]
35    pub transcript: Vec<TranscriptEvent>,
36    pub observation_output: String,
37    pub candidate_output: String,
38    #[serde(default)]
39    pub expected_observations: Vec<ObservationExpectation>,
40    #[serde(default)]
41    pub forbidden_observations: Vec<ObservationExpectation>,
42    #[serde(default)]
43    pub expected_candidates: Vec<CandidateExpectation>,
44    #[serde(default)]
45    pub forbidden_candidates: Vec<CandidateExpectation>,
46}
47
48#[derive(Debug, Clone, Deserialize)]
49pub(crate) struct TranscriptEvent {
50    pub id: String,
51    pub role: String,
52    pub content: String,
53    #[serde(default)]
54    pub tool_name: Option<String>,
55    #[serde(default)]
56    pub event_type: Option<String>,
57    #[serde(default)]
58    pub token_estimate: Option<i64>,
59    #[serde(default)]
60    pub created_at_epoch: Option<i64>,
61}
62
63#[derive(Debug, Clone, Deserialize)]
64pub(crate) struct ObservationExpectation {
65    pub id: String,
66    #[serde(default)]
67    pub observation_type: Option<String>,
68    #[serde(default)]
69    pub text_contains: Vec<String>,
70}
71
72#[derive(Debug, Clone, Deserialize)]
73pub(crate) struct CandidateExpectation {
74    pub id: String,
75    #[serde(default)]
76    pub scope: Option<String>,
77    #[serde(default)]
78    pub memory_type: Option<String>,
79    #[serde(default)]
80    pub topic_key: Option<String>,
81    #[serde(default)]
82    pub risk_class: Option<String>,
83    #[serde(default)]
84    pub text_contains: Vec<String>,
85}
86
87#[derive(Debug, Clone, Serialize, PartialEq)]
88pub struct ExtractionEvalReport {
89    pub metadata: ExtractionEvalMetadata,
90    pub metrics: ExtractionMetricSummary,
91    pub cases: Vec<ExtractionCaseReport>,
92    pub failing_examples: Vec<String>,
93}
94
95#[derive(Debug, Clone, Serialize, PartialEq)]
96pub struct ExtractionEvalMetadata {
97    pub corpus: String,
98    pub corpus_version: String,
99    pub description: String,
100    pub cases: usize,
101    pub transcript_events: usize,
102}
103
104#[derive(Debug, Clone, Serialize, PartialEq)]
105pub struct ExtractionMetricSummary {
106    pub observation_precision: ExtractionRateMetric,
107    pub observation_recall: ExtractionRateMetric,
108    pub candidate_precision: ExtractionRateMetric,
109    pub candidate_recall: ExtractionRateMetric,
110    pub forbidden_observation_exclusion: ExtractionRateMetric,
111    pub forbidden_candidate_exclusion: ExtractionRateMetric,
112    pub over_saved_predictions: usize,
113    pub total_predictions: usize,
114    pub over_save_penalty: f64,
115    pub all_checks_passed: bool,
116}
117
118#[derive(Debug, Clone, Serialize, PartialEq)]
119pub struct ExtractionCaseReport {
120    pub id: String,
121    pub transcript_events: usize,
122    pub observation_request_sha256: String,
123    pub candidate_request_sha256: String,
124    pub predicted_observations: Vec<ObservationPrediction>,
125    pub predicted_candidates: Vec<CandidatePrediction>,
126    pub missing_expected_observations: Vec<String>,
127    pub unexpected_observations: Vec<usize>,
128    pub forbidden_observations: Vec<String>,
129    pub missing_expected_candidates: Vec<String>,
130    pub unexpected_candidates: Vec<usize>,
131    pub forbidden_candidates: Vec<String>,
132    pub over_saved_predictions: usize,
133    pub pass: bool,
134}
135
136#[derive(Debug, Clone, Serialize, PartialEq)]
137pub struct ObservationPrediction {
138    pub index: usize,
139    pub observation_type: String,
140    pub text: String,
141    pub confidence: Option<f64>,
142}
143
144#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
145pub struct CandidatePrediction {
146    pub index: usize,
147    pub scope: String,
148    pub memory_type: String,
149    pub topic_key: String,
150    pub risk_class: String,
151    pub text: String,
152}
153
154impl Display for ExtractionEvalReport {
155    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
156        writeln!(
157            f,
158            "Extraction eval: {} cases from {}",
159            self.metadata.cases, self.metadata.corpus
160        )?;
161        writeln!(
162            f,
163            "Observations: precision {} recall {} forbidden exclusion {}",
164            format_rate(&self.metrics.observation_precision),
165            format_rate(&self.metrics.observation_recall),
166            format_rate(&self.metrics.forbidden_observation_exclusion)
167        )?;
168        writeln!(
169            f,
170            "Candidates: precision {} recall {} forbidden exclusion {}",
171            format_rate(&self.metrics.candidate_precision),
172            format_rate(&self.metrics.candidate_recall),
173            format_rate(&self.metrics.forbidden_candidate_exclusion)
174        )?;
175        writeln!(
176            f,
177            "Over-save penalty: {:.4} ({}/{})",
178            self.metrics.over_save_penalty,
179            self.metrics.over_saved_predictions,
180            self.metrics.total_predictions
181        )?;
182        writeln!(f, "All checks passed: {}", self.metrics.all_checks_passed)?;
183        if !self.failing_examples.is_empty() {
184            writeln!(f, "Failures:")?;
185            for failure in &self.failing_examples {
186                writeln!(f, "- {failure}")?;
187            }
188        }
189        Ok(())
190    }
191}
192
193fn format_rate(metric: &ExtractionRateMetric) -> String {
194    format!("{}/{} ({:.4})", metric.passed, metric.total, metric.rate)
195}
remem/eval/extraction/types.rs

remem/eval/extraction/
types.rs
