lc_evaluation/
criteria.rs

1//! 评测核心类型与 trait:EvalError、Score、Example、Dataset,
2//! 以及 Evaluator / Predictor trait。
3
4use async_trait::async_trait;
5use serde::{Deserialize, Serialize};
6
7/// 评测错误
8#[derive(Debug, thiserror::Error)]
9pub enum EvalError {
10    #[error("IO 错误: {0}")]
11    IoError(String),
12    #[error("解析错误: {0}")]
13    ParseError(String),
14    #[error("嵌入错误: {0}")]
15    EmbeddingError(String),
16    #[error("预测错误: {0}")]
17    PredictorError(String),
18}
19
20/// 评测分数(0.0–1.0,1.0 为最佳)
21#[derive(Debug, Clone, Serialize, Deserialize)]
22pub struct Score {
23    pub value: f64,
24    #[serde(skip_serializing_if = "Option::is_none")]
25    pub label: Option<String>,
26}
27
28impl Score {
29    pub fn new(value: f64) -> Self {
30        Self {
31            value: value.clamp(0.0, 1.0),
32            label: None,
33        }
34    }
35
36    pub fn with_label(mut self, label: impl Into<String>) -> Self {
37        self.label = Some(label.into());
38        self
39    }
40}
41
42/// 评测样例
43#[derive(Debug, Clone, Serialize, Deserialize)]
44pub struct Example {
45    pub input: String,
46    pub reference: String,
47}
48
49impl Example {
50    pub fn new(input: impl Into<String>, reference: impl Into<String>) -> Self {
51        Self {
52            input: input.into(),
53            reference: reference.into(),
54        }
55    }
56}
57
58/// 数据集
59#[derive(Debug, Clone, Serialize, Deserialize)]
60pub struct Dataset {
61    pub examples: Vec<Example>,
62}
63
64impl Dataset {
65    pub fn new(examples: Vec<Example>) -> Self {
66        Self { examples }
67    }
68
69    /// 从 JSONL 文件加载(每行一个 ``{input, reference}``)
70    pub fn from_jsonl(path: &str) -> Result<Self, EvalError> {
71        let content =
72            std::fs::read_to_string(path).map_err(|e| EvalError::IoError(e.to_string()))?;
73        let mut examples = Vec::new();
74        for (i, line) in content.lines().enumerate() {
75            let line = line.trim();
76            if line.is_empty() {
77                continue;
78            }
79            let ex: Example = serde_json::from_str(line)
80                .map_err(|e| EvalError::ParseError(format!("第 {} 行: {}", i + 1, e)))?;
81            examples.push(ex);
82        }
83        Ok(Self { examples })
84    }
85
86    pub fn len(&self) -> usize {
87        self.examples.len()
88    }
89
90    pub fn is_empty(&self) -> bool {
91        self.examples.is_empty()
92    }
93}
94
95/// 评测器 trait
96#[async_trait]
97pub trait Evaluator: Send + Sync {
98    /// 对单条预测打分
99    async fn eval(
100        &self,
101        input: &str,
102        prediction: &str,
103        reference: &str,
104    ) -> Result<Score, EvalError>;
105
106    /// 评测器名称(用于报告汇总)
107    fn name(&self) -> &str;
108}
109
110/// 预测器 trait(待评测的对象:LLMChain / Agent 等)
111#[async_trait]
112pub trait Predictor: Send + Sync {
113    async fn predict(&self, input: &str) -> Result<String, EvalError>;
114}
lc_evaluation/criteria.rs

lc_evaluation/
criteria.rs
