lc_evaluation/
criteria.rs

1//! Core evaluation types and traits: EvalError, Score, Example, Dataset,
2//! plus the Evaluator / Predictor traits.
3
4use async_trait::async_trait;
5use serde::{Deserialize, Serialize};
6
7/// Evaluation error
8#[derive(Debug, thiserror::Error)]
9#[non_exhaustive]
10pub enum EvalError {
11    /// Underlying IO error (e.g. file read failure).
12    #[error("IO error: {0}")]
13    IoError(String),
14    /// Data parse error (e.g. JSON/JSONL parse failure).
15    #[error("parse error: {0}")]
16    ParseError(String),
17    /// Embedding computation error.
18    #[error("embedding error: {0}")]
19    EmbeddingError(String),
20    /// Predictor execution error.
21    #[error("prediction error: {0}")]
22    PredictorError(String),
23    /// Prediction/reference sample counts differ in corpus-level evaluation (one-to-one).
24    #[error(
25        "length mismatch: {predictions} predictions vs {references} references; \
26         sample counts must match"
27    )]
28    LengthMismatch {
29        /// Prediction sample count
30        predictions: usize,
31        /// Reference sample count
32        references: usize,
33    },
34}
35
36/// P2-6: errors from the shared judge core (lc-core::judge) map into the evaluation error domain,
37/// so `structured_call(...).await?` works directly in a `Result<_, EvalError>` context.
38impl From<lc_core::judge::StructuredJudgeError> for EvalError {
39    fn from(e: lc_core::judge::StructuredJudgeError) -> Self {
40        match e {
41            lc_core::judge::StructuredJudgeError::Call(s) => EvalError::PredictorError(s),
42            lc_core::judge::StructuredJudgeError::Parse(s) => EvalError::ParseError(s),
43            // `StructuredJudgeError` is `#[non_exhaustive]`; forward any future
44            // variants to the generic predictor-error slot.
45            _ => EvalError::PredictorError(e.to_string()),
46        }
47    }
48}
49
50/// Evaluation score (0.0–1.0, 1.0 is best)
51#[derive(Debug, Clone, Serialize, Deserialize)]
52pub struct Score {
53    /// Score value (0.0–1.0)
54    pub value: f64,
55    /// Optional score label
56    #[serde(skip_serializing_if = "Option::is_none")]
57    pub label: Option<String>,
58}
59
60impl Score {
61    /// Constructs a score between 0.0 and 1.0.
62    ///
63    /// P2-8: Rust's `f64::clamp(0.0, 1.0)` returns NaN for NaN input, polluting
64    /// the summary mean/std. A NaN pre-check is done here, treating it as 0.0
65    /// (negative/positive infinity are left to `clamp` to converge to the bounds).
66    pub fn new(value: f64) -> Self {
67        let value = if value.is_nan() {
68            log::warn!("Score::new received NaN, treating as 0.0");
69            0.0
70        } else {
71            value
72        };
73        Self {
74            value: value.clamp(0.0, 1.0),
75            label: None,
76        }
77    }
78
79    /// Attaches a score label (builder style).
80    pub fn with_label(mut self, label: impl Into<String>) -> Self {
81        self.label = Some(label.into());
82        self
83    }
84}
85
86/// Evaluation example
87#[derive(Debug, Clone, Serialize, Deserialize)]
88pub struct Example {
89    /// Evaluation input
90    pub input: String,
91    /// Reference answer
92    pub reference: String,
93    /// Retrieved RAG contexts, in retrieval rank order (top-ranked first).
94    ///
95    /// B9 (v0.22.4): consumed by [`RagEvaluator`]s (context precision/recall).
96    /// Old JSONL rows without this field deserialize to an empty vec.
97    #[serde(default)]
98    pub contexts: Vec<String>,
99}
100
101impl Example {
102    /// Constructs an evaluation example.
103    pub fn new(input: impl Into<String>, reference: impl Into<String>) -> Self {
104        Self {
105            input: input.into(),
106            reference: reference.into(),
107            contexts: Vec::new(),
108        }
109    }
110
111    /// Constructs a RAG evaluation example with retrieved contexts (rank order preserved).
112    pub fn with_contexts(
113        input: impl Into<String>,
114        reference: impl Into<String>,
115        contexts: Vec<String>,
116    ) -> Self {
117        Self {
118            input: input.into(),
119            reference: reference.into(),
120            contexts,
121        }
122    }
123}
124
125/// Dataset
126#[derive(Debug, Clone, Serialize, Deserialize)]
127pub struct Dataset {
128    /// Evaluation examples in the dataset
129    pub examples: Vec<Example>,
130}
131
132impl Dataset {
133    /// Constructs a dataset.
134    pub fn new(examples: Vec<Example>) -> Self {
135        Self { examples }
136    }
137
138    /// Loads from a JSONL file (one `{input, reference}` per line).
139    ///
140    /// P2-2: async I/O (`tokio::fs`), avoiding synchronous blocking in the async evaluation pipeline.
141    pub async fn from_jsonl(path: &str) -> Result<Self, EvalError> {
142        let content = tokio::fs::read_to_string(path)
143            .await
144            .map_err(|e| EvalError::IoError(e.to_string()))?;
145        let mut examples = Vec::new();
146        for (i, line) in content.lines().enumerate() {
147            let line = line.trim();
148            if line.is_empty() {
149                continue;
150            }
151            let ex: Example = serde_json::from_str(line)
152                .map_err(|e| EvalError::ParseError(format!("line {}: {}", i + 1, e)))?;
153            examples.push(ex);
154        }
155        Ok(Self { examples })
156    }
157
158    /// Returns the number of examples.
159    pub fn len(&self) -> usize {
160        self.examples.len()
161    }
162
163    /// Whether the dataset is empty.
164    pub fn is_empty(&self) -> bool {
165        self.examples.is_empty()
166    }
167}
168
169/// Evaluator trait
170#[async_trait]
171pub trait Evaluator: Send + Sync {
172    /// Scores a single prediction
173    async fn eval(
174        &self,
175        input: &str,
176        prediction: &str,
177        reference: &str,
178    ) -> Result<Score, EvalError>;
179
180    /// Evaluator name (used in report summaries)
181    fn name(&self) -> &str;
182}
183
184/// Pairwise-comparison evaluator trait (arena mode): judges which of two answers (A/B) for the same input is better.
185///
186/// P1-1: a first-class citizen alongside the pointwise `Evaluator`; `EvalRunner` accepts both,
187/// so arena evaluation also enters the unified report. Scoring: 1.0 = A wins, 0.5 = tie, 0.0 = B wins.
188#[async_trait]
189pub trait PairwiseEvaluator: Send + Sync {
190    /// Compares answers A and B, returning a 0-1 score
191    /// (1.0 = A wins, 0.5 = tie, 0.0 = B wins).
192    async fn eval_pair(&self, input: &str, a: &str, b: &str) -> Result<Score, EvalError>;
193
194    /// Evaluator name (used in report summaries)
195    fn name(&self) -> &str;
196}
197
198/// RAG evaluator trait (RAGAS-style): scores a prediction together with the retrieved
199/// contexts, which a plain [`Evaluator`] has no slot for.
200///
201/// Contexts are passed in **retrieval rank order** (top-ranked chunk first) — rank-aware
202/// metrics such as context precision depend on it. B9 (v0.22.4): accepted by `EvalRunner`
203/// alongside [`Evaluator`] / [`PairwiseEvaluator`]; the runner takes contexts from
204/// [`Example::contexts`].
205#[async_trait]
206pub trait RagEvaluator: Send + Sync {
207    /// Scores a single prediction given the question, retrieved contexts, and reference answer.
208    async fn eval_rag(
209        &self,
210        input: &str,
211        prediction: &str,
212        contexts: &[String],
213        reference: &str,
214    ) -> Result<Score, EvalError>;
215
216    /// Evaluator name (used in report summaries)
217    fn name(&self) -> &str;
218}
219
220/// Predictor trait (the object under evaluation: LLMChain / Agent, etc.)
221#[async_trait]
222pub trait Predictor: Send + Sync {
223    /// Predicts on a single input, returning the text result.
224    async fn predict(&self, input: &str) -> Result<String, EvalError>;
225
226    /// Reports the token usage of the most recent [`predict`](Self::predict) call, if the
227    /// predictor meters it (E1). Defaults to `None` — evaluators with no token metering keep
228    /// the report's cost ledger at zero (zero behavior change).
229    async fn report_token_usage(&self) -> Option<crate::TokenUsage> {
230        None
231    }
232
233    /// Called once before the dataset loop with the evaluation run id (B9).
234    ///
235    /// Default no-op. A system under test that runs chains/agents can store the id and put it
236    /// into `RunnableConfig.metadata["trace_id"]`; the agent executor then propagates it to
237    /// callback/OTel spans, making the eval report's run id the join key to traces.
238    async fn begin_run(&self, _run_id: &str) {}
239}
240
241#[cfg(test)]
242mod tests {
243    use super::*;
244
245    #[test]
246    fn test_score_new_normal() {
247        assert!((Score::new(0.5).value - 0.5).abs() < 1e-9);
248    }
249
250    #[test]
251    fn test_score_new_clamps_overflow() {
252        assert_eq!(Score::new(2.0).value, 1.0);
253        assert_eq!(Score::new(-1.0).value, 0.0);
254        assert_eq!(Score::new(f64::INFINITY).value, 1.0);
255        assert_eq!(Score::new(f64::NEG_INFINITY).value, 0.0);
256    }
257
258    /// P2-8: NaN no longer passes through `.clamp(0.0, 1.0)` to pollute the summary statistics.
259    #[test]
260    fn test_score_new_nan_guarded() {
261        assert_eq!(Score::new(f64::NAN).value, 0.0);
262        // ensure NaN is cleaned up rather than lingering in the statistics
263        assert!(Score::new(f64::NAN).value.is_finite());
264    }
265
266    /// P2-2: from_jsonl reads the file asynchronously; a per-line parse failure carries the line number.
267    #[tokio::test]
268    async fn test_from_jsonl_async() {
269        let dir = tempfile::tempdir().unwrap();
270        let path = dir.path().join("data.jsonl");
271        std::fs::write(
272            &path,
273            "{\"input\":\"q1\",\"reference\":\"a1\"}\n\n{\"input\":\"q2\",\"reference\":\"a2\"}\n",
274        )
275        .unwrap();
276        let dataset = Dataset::from_jsonl(path.to_str().unwrap()).await.unwrap();
277        assert_eq!(dataset.len(), 2);
278        assert_eq!(dataset.examples[1].input, "q2");
279        assert_eq!(dataset.examples[1].reference, "a2");
280    }
281
282    #[tokio::test]
283    async fn test_from_jsonl_missing_file() {
284        let err = Dataset::from_jsonl("不存在-的文件.jsonl")
285            .await
286            .unwrap_err();
287        assert!(matches!(err, EvalError::IoError(_)));
288    }
289
290    #[tokio::test]
291    async fn test_contexts_round_trip_and_old_row_default() {
292        // old row without contexts still parses, contexts empty
293        let old: Example = serde_json::from_str(r#"{"input":"q","reference":"r"}"#).unwrap();
294        assert!(old.contexts.is_empty());
295
296        // new row keeps contexts in their declared (rank) order
297        let with_ctx = Example::with_contexts("q", "r", vec!["top".into(), "second".into()]);
298        let json = serde_json::to_string(&with_ctx).unwrap();
299        let back: Example = serde_json::from_str(&json).unwrap();
300        assert_eq!(back.contexts, vec!["top", "second"]);
301    }
302
303    #[tokio::test]
304    async fn test_from_jsonl_bad_line() {
305        let dir = tempfile::tempdir().unwrap();
306        let path = dir.path().join("bad.jsonl");
307        std::fs::write(&path, "{\"input\":\"q\"}\n").unwrap();
308        let err = Dataset::from_jsonl(path.to_str().unwrap())
309            .await
310            .unwrap_err();
311        assert!(matches!(err, EvalError::ParseError(_)));
312    }
313}
lc_evaluation/criteria.rs

lc_evaluation/
criteria.rs
