lc_evaluation/
faithfulness.rs

1//! 忠实度评测器:检测回答是否忠于参考上下文(幻觉检测)。
2//!
3//! 思路来自 Ragas 的 faithfulness:把回答拆成原子陈述,
4//! 逐条判断能否从参考上下文推导出来,通过率即为忠实度。
5//! 这里 reference 充当"上下文/检索内容",prediction 是待检测的回答。
6
7use async_trait::async_trait;
8use futures_util::stream::{self, StreamExt};
9use serde::Deserialize;
10
11use lc_core::judge::{structured_call, truncate, StructuredJudgeError};
12use lc_core::tools::ToolDefinition;
13use lc_core::BaseChatModel;
14use lc_schema::Message;
15
16use super::{EvalError, Evaluator, Score};
17
18/// P1-5: 单次评测最多同时打给 judge 的陈述验证并发数(防限流 N 路全挂)。
19const MAX_CONCURRENT_VERIFY: usize = 4;
20
21/// P2-5: 单条陈述的裁判 prompt 里参考上下文的字符上限。
22/// 完整长参考只截一次、N 条陈述复用,避免每条重复传输整段上下文。
23const DEFAULT_MAX_CONTEXT_CHARS: usize = 2000;
24
25/// 忠实度评测器(幻觉检测):回答有多忠于参考上下文。
26///
27/// 把 prediction 拆成原子陈述,逐条问裁判能否从 reference 推导,
28/// 通过率 = 能推导的陈述数 / 总陈述数。
29pub struct Faithfulness<M: BaseChatModel> {
30    judge: M,
31    /// 用 LLM 拆原子陈述(默认 false,用规则按标点拆)
32    llm_split: bool,
33    /// 空预测(无可验证陈述)的得分,默认 0.0(没回答=不忠实)。
34    empty_score: f64,
35    /// 参考上下文单条传输上限(字符,默认 [`DEFAULT_MAX_CONTEXT_CHARS`])。
36    max_context_chars: usize,
37}
38
39/// 把回答拆成原子陈述(按句号、问号、感叹号、分号、换行切分)。
40fn split_claims(prediction: &str) -> Vec<String> {
41    prediction
42        .split(['。', '.', '!', '?', '；', ';', '\n'])
43        .map(|s| s.trim().to_string())
44        .filter(|s| !s.is_empty())
45        .collect()
46}
47
48impl<M: BaseChatModel> Faithfulness<M> {
49    pub fn new(judge: M) -> Self {
50        Self {
51            judge,
52            llm_split: false,
53            empty_score: 0.0, // P0-2: 空预测默认 0 分(没回答=不忠实)
54            max_context_chars: DEFAULT_MAX_CONTEXT_CHARS,
55        }
56    }
57
58    /// 用 LLM 拆原子陈述(默认按标点规则拆;LLM 拆能处理逗号复合句)
59    pub fn with_llm_split(mut self, v: bool) -> Self {
60        self.llm_split = v;
61        self
62    }
63
64    /// 空预测的得分:默认 0.0(没回答=不忠实),可设 1.0 表示"没编造即忠实"
65    pub fn with_empty_score(mut self, score: f64) -> Self {
66        self.empty_score = score;
67        self
68    }
69
70    /// 参考上下文单条传输上限(字符)。P2-5: 默认 2000,防止长参考被每条陈述重复完整塞进 prompt。
71    pub fn with_max_context_chars(mut self, max: usize) -> Self {
72        self.max_context_chars = max;
73        self
74    }
75
76    /// 问裁判:单条陈述能否从上下文推导。
77    async fn verify_claim(&self, context: &str, claim: &str) -> Result<bool, EvalError> {
78        let system =
79            "你是事实核查员。判断给定的陈述能否从参考上下文中推导出来。调用 check_claim 工具提交判定。"
80                .to_string();
81        let user =
82            format!("参考上下文:\n{context}\n\n陈述:\n{claim}\n\n这条陈述能从上下文推导出来吗?");
83        let messages = vec![Message::system(system), Message::human(user)];
84
85        // P0-1: 优先结构化输出(verdict 布尔);不支持工具绑定的模型走文本解析回落。
86        let args: VerdictArgs = structured_call(&self.judge, verdict_tool(), messages, |raw| {
87            let verdict = parse_yes_no(raw).ok_or_else(|| {
88                StructuredJudgeError::Parse(format!(
89                    "无法从裁判回复解析是/否: {}",
90                    truncate(raw, 200)
91                ))
92            })?;
93            Ok(VerdictArgs {
94                verdict,
95                reason: String::new(),
96            })
97        })
98        .await?;
99        Ok(args.verdict)
100    }
101
102    /// 用 LLM 把回答拆成原子陈述(每行一条),处理规则拆不动的复合句。
103    async fn split_claims_llm(&self, prediction: &str) -> Result<Vec<String>, EvalError> {
104        let system =
105            "你是文本分析助手。把回答拆成原子陈述,每条一行,只输出陈述本身,不要编号不要解释。"
106                .to_string();
107        let user = format!("回答:\n{prediction}\n\n把它拆成原子陈述,每行一条:");
108        let result = self
109            .judge
110            .chat_with_system(system, vec![Message::human(user)])
111            .await
112            .map_err(|e| EvalError::PredictorError(e.to_string()))?;
113        Ok(result
114            .content
115            .lines()
116            .map(|s| s.trim().to_string())
117            .filter(|s| !s.is_empty())
118            .collect())
119    }
120}
121
122#[async_trait]
123impl<M: BaseChatModel> Evaluator for Faithfulness<M> {
124    async fn eval(
125        &self,
126        _input: &str,
127        prediction: &str,
128        reference: &str,
129    ) -> Result<Score, EvalError> {
130        let claims = if self.llm_split {
131            self.split_claims_llm(prediction).await?
132        } else {
133            split_claims(prediction)
134        };
135        if claims.is_empty() {
136            return Ok(Score::new(self.empty_score).with_label("no_claims"));
137        }
138        // P2-5: 参考上下文只截取一次,各陈述复用同一份(避免完整长参考被 N 条重复传输)。
139        let context = truncate(reference, self.max_context_chars);
140        // 并发验证各陈述(每条一次 LLM 调用),但用 buffer_unordered 限流:
141        // P1-5——join_all 无上限并发打同一个 judge,命中限流会 N 路全挂。
142        // `ctx` 是 Copy 的引用,闭包可反复捕获;若直接捕获 `context` 会被 async move
143        // 逐条 move 出去,map(FnMut) 编译不过。
144        let ctx = &context;
145        let total = claims.len();
146        let results: Vec<Result<bool, EvalError>> = stream::iter(claims)
147            .map(|claim| async move { self.verify_claim(ctx, &claim).await })
148            .buffer_unordered(MAX_CONCURRENT_VERIFY)
149            .collect()
150            .await;
151        let mut supported = 0usize;
152        for r in results {
153            if r? {
154                supported += 1;
155            }
156        }
157        let value = supported as f64 / total as f64;
158        Ok(Score::new(value).with_label("faithfulness"))
159    }
160
161    fn name(&self) -> &str {
162        "faithfulness"
163    }
164}
165
166/// 结构化判定参数(经 tool_calls 返回)。
167#[derive(Debug, Deserialize)]
168struct VerdictArgs {
169    verdict: bool,
170    /// 让 LLM 附上简短理由(改善判定质量),当前不消费。
171    #[serde(default)]
172    #[allow(dead_code)]
173    reason: String,
174}
175
176/// 构建判定工具:让 LLM 以 `{"verdict": bool, "reason": "..."}` 提交判定。
177fn verdict_tool() -> ToolDefinition {
178    ToolDefinition::new(
179        "check_claim",
180        "判断陈述能否从参考上下文推导出来,提交布尔判定。",
181    )
182    .with_parameters(serde_json::json!({
183        "type": "object",
184        "properties": {
185            "verdict": { "type": "boolean", "description": "能否从上下文推导" },
186            "reason": { "type": "string", "description": "简短依据" }
187        },
188        "required": ["verdict", "reason"]
189    }))
190}
191
192/// 解析"是/否"。无任何是/否标记时返回 `None`(解析失败,由调用方报错),
193/// 而非静默默认 false——避免 LLM 跑题回复被当成"不忠实"。
194fn parse_yes_no(raw: &str) -> Option<bool> {
195    let lower = raw.to_lowercase();
196    // 先判否定(避免"不是""不能"被"是""能"误判;否定词优先于肯定词)
197    if lower.contains("否")
198        || lower.contains("no")
199        || lower.contains("不能")
200        || lower.contains("不是")
201        || lower.contains("false")
202    {
203        return Some(false);
204    }
205    if lower.contains("是")
206        || lower.contains("yes")
207        || lower.contains("能")
208        || lower.contains("true")
209    {
210        return Some(true);
211    }
212    None
213}
214
215#[cfg(test)]
216mod tests {
217    use super::*;
218    use futures_util::Stream;
219    use lc_core::language_models::LLMResult;
220    use lc_core::{BaseLanguageModel, Runnable, RunnableConfig};
221    use lc_schema::MessageType;
222    use std::pin::Pin;
223    use std::sync::atomic::{AtomicUsize, Ordering};
224    use std::sync::{Arc, Mutex};
225
226    #[derive(Debug)]
227    struct JudgeError(String);
228    impl std::fmt::Display for JudgeError {
229        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
230            write!(f, "{}", self.0)
231        }
232    }
233    impl std::error::Error for JudgeError {}
234
235    struct SeqMockJudge {
236        replies: Vec<String>,
237        call: Arc<AtomicUsize>,
238        last_user: Arc<Mutex<Option<String>>>,
239    }
240    impl SeqMockJudge {
241        fn new(replies: Vec<String>) -> Self {
242            Self {
243                replies,
244                call: Arc::new(AtomicUsize::new(0)),
245                last_user: Arc::new(Mutex::new(None)),
246            }
247        }
248        fn last_user_content(&self) -> String {
249            self.last_user.lock().unwrap().clone().unwrap_or_default()
250        }
251    }
252
253    #[async_trait]
254    impl Runnable<Vec<Message>, LLMResult> for SeqMockJudge {
255        type Error = JudgeError;
256        async fn invoke(
257            &self,
258            _input: Vec<Message>,
259            _config: Option<RunnableConfig>,
260        ) -> Result<LLMResult, Self::Error> {
261            Err(JudgeError("use chat".into()))
262        }
263    }
264    #[async_trait]
265    impl BaseLanguageModel<Vec<Message>, LLMResult> for SeqMockJudge {
266        fn model_name(&self) -> &str {
267            "seq-mock"
268        }
269        fn get_num_tokens(&self, t: &str) -> usize {
270            t.len()
271        }
272        fn with_temperature(self, _: f32) -> Self {
273            self
274        }
275        fn with_max_tokens(self, _: usize) -> Self {
276            self
277        }
278    }
279    #[async_trait]
280    impl BaseChatModel for SeqMockJudge {
281        async fn chat(
282            &self,
283            messages: Vec<Message>,
284            _config: Option<RunnableConfig>,
285        ) -> Result<LLMResult, Self::Error> {
286            let idx = self.call.fetch_add(1, Ordering::SeqCst);
287            let reply = self.replies.get(idx).cloned().unwrap_or_default();
288            if let Some(human) = messages
289                .iter()
290                .find(|m| m.message_type == MessageType::Human)
291            {
292                *self.last_user.lock().unwrap() = Some(human.content.clone());
293            }
294            Ok(LLMResult {
295                content: reply,
296                model: "seq-mock".to_string(),
297                token_usage: None,
298                tool_calls: None,
299                thinking_content: None,
300            })
301        }
302        async fn stream_chat(
303            &self,
304            _messages: Vec<Message>,
305            _config: Option<RunnableConfig>,
306        ) -> Result<Pin<Box<dyn Stream<Item = Result<String, Self::Error>> + Send>>, Self::Error>
307        {
308            Err(JudgeError("not supported".into()))
309        }
310    }
311
312    #[test]
313    fn test_split_claims() {
314        let claims = split_claims("巴黎是法国首都。伦敦是英国首都。");
315        assert_eq!(claims.len(), 2);
316        assert_eq!(claims[0], "巴黎是法国首都");
317        assert_eq!(claims[1], "伦敦是英国首都");
318    }
319
320    #[test]
321    fn test_split_claims_empty() {
322        assert!(split_claims("").is_empty());
323        assert!(split_claims("。。。").is_empty());
324    }
325
326    #[tokio::test]
327    async fn test_faithfulness_all_supported() {
328        let judge = Faithfulness::new(SeqMockJudge::new(vec!["是".into(), "是".into()]));
329        let s = judge
330            .eval("", "巴黎是法国首都。伦敦是英国首都。", "ctx")
331            .await
332            .unwrap();
333        assert!((s.value - 1.0).abs() < 1e-9);
334    }
335
336    #[tokio::test]
337    async fn test_faithfulness_half_supported() {
338        let judge = Faithfulness::new(SeqMockJudge::new(vec!["是".into(), "否".into()]));
339        let s = judge
340            .eval("", "巴黎是法国首都。伦敦是英国首都。", "ctx")
341            .await
342            .unwrap();
343        assert!((s.value - 0.5).abs() < 1e-9);
344    }
345
346    #[tokio::test]
347    async fn test_faithfulness_none_supported() {
348        let judge = Faithfulness::new(SeqMockJudge::new(vec!["否".into(), "否".into()]));
349        let s = judge
350            .eval("", "巴黎是法国首都。伦敦是英国首都。", "ctx")
351            .await
352            .unwrap();
353        assert!((s.value - 0.0).abs() < 1e-9);
354    }
355
356    #[tokio::test]
357    async fn test_faithfulness_empty_prediction() {
358        // P0-2: 空预测默认 0 分(没回答=不忠实)
359        let judge = Faithfulness::new(SeqMockJudge::new(vec![]));
360        let s = judge.eval("", "", "ctx").await.unwrap();
361        assert!((s.value - 0.0).abs() < 1e-9);
362        assert_eq!(s.label.as_deref(), Some("no_claims"));
363    }
364
365    #[tokio::test]
366    async fn test_faithfulness_empty_score_configurable() {
367        // 可显式配成 1.0(没编造即忠实)
368        let judge = Faithfulness::new(SeqMockJudge::new(vec![])).with_empty_score(1.0);
369        let s = judge.eval("", "", "ctx").await.unwrap();
370        assert!((s.value - 1.0).abs() < 1e-9);
371    }
372
373    #[tokio::test]
374    async fn test_faithfulness_llm_split() {
375        // 逗号复合句规则拆只算 1 条;LLM 拆能拆成 2 条分别验证
376        let judge = Faithfulness::new(SeqMockJudge::new(vec![
377            "巴黎是法国首都\n伦敦是英国首都".into(),
378            "是".into(),
379            "是".into(),
380        ]))
381        .with_llm_split(true);
382        let s = judge
383            .eval("", "巴黎是法国首都,伦敦是英国首都。", "ctx")
384            .await
385            .unwrap();
386        assert!((s.value - 1.0).abs() < 1e-9);
387    }
388
389    #[test]
390    fn test_parse_yes_no() {
391        assert_eq!(parse_yes_no("是"), Some(true));
392        assert_eq!(parse_yes_no("yes"), Some(true));
393        assert_eq!(parse_yes_no("否"), Some(false));
394        assert_eq!(parse_yes_no("no"), Some(false));
395        assert_eq!(parse_yes_no("不是"), Some(false));
396        assert_eq!(parse_yes_no("不能"), Some(false));
397        // 无任何是/否标记 = 解析失败,不应静默默认
398        assert_eq!(parse_yes_no("我不会告诉你"), None);
399    }
400
401    /// P0-1: 支持 bind_tools 的模型走结构化输出(verdict 布尔),不再依赖文本解析。
402    #[tokio::test]
403    async fn test_faithfulness_structured_verdict() {
404        use crate::test_support::ToolJudge;
405        // 两条陈述:一条支持、一条不支持 → 忠实度 0.5
406        let judge = Faithfulness::new(ToolJudge::sequence(vec![
407            r#"{"verdict": true, "reason": "能从上下文推导"}"#.into(),
408            r#"{"verdict": false, "reason": "无法推导"}"#.into(),
409        ]));
410        let s = judge
411            .eval("", "巴黎是法国首都。伦敦是英国首都。", "巴黎是法国首都")
412            .await
413            .unwrap();
414        assert!((s.value - 0.5).abs() < 1e-9);
415    }
416
417    /// P0-1: 全部不支持 → 0 分。
418    #[tokio::test]
419    async fn test_faithfulness_structured_all_false() {
420        use crate::test_support::ToolJudge;
421        let judge = Faithfulness::new(ToolJudge::new(
422            r#"{"verdict": false, "reason": "均无法推导"}"#,
423        ));
424        let s = judge
425            .eval("", "巴黎是法国首都。伦敦是英国首都。", "巴黎是法国首都")
426            .await
427            .unwrap();
428        assert!((s.value - 0.0).abs() < 1e-9);
429    }
430
431    /// P2-5: 长参考上下文只截取一次,N 条陈述复用同一份,不重复整段发送。
432    #[tokio::test]
433    async fn test_faithfulness_reference_truncated_once() {
434        let judge = SeqMockJudge::new(vec!["是".into(), "是".into()]);
435        let f = Faithfulness::new(judge).with_max_context_chars(10);
436        let long_ref =
437            "这是一段非常长的参考上下文,远超默认的单条传输上限,里面藏了一个不该被完整发送的尾巴"
438                .to_string();
439        let s = f
440            .eval("", "巴黎是首都。伦敦是首都。", &long_ref)
441            .await
442            .unwrap();
443        assert!((s.value - 1.0).abs() < 1e-9);
444        let sent = f.judge.last_user_content();
445        // 参考上下文被截到预算内:首部仍在、远在预算外的尾巴不会被发出去
446        assert!(sent.contains("这是一段非常长"), "实际发送: {sent}");
447        assert!(!sent.contains("不该被完整发送"), "完整长参考被重复发送");
448    }
449}
lc_evaluation/faithfulness.rs

lc_evaluation/
faithfulness.rs
