lc_evaluation/
faithfulness.rs

1//! 忠实度评测器:检测回答是否忠于参考上下文(幻觉检测)。
2//!
3//! 思路来自 Ragas 的 faithfulness:把回答拆成原子陈述,
4//! 逐条判断能否从参考上下文推导出来,通过率即为忠实度。
5//! 这里 reference 充当"上下文/检索内容",prediction 是待检测的回答。
6
7use async_trait::async_trait;
8use futures_util::stream::{self, StreamExt};
9use serde::Deserialize;
10
11use lc_core::judge::{structured_call, truncate, StructuredJudgeError};
12use lc_core::tools::ToolDefinition;
13use lc_core::BaseChatModel;
14use lc_schema::Message;
15
16use super::{EvalError, Evaluator, Score};
17
18/// P1-5: 单次评测最多同时打给 judge 的陈述验证并发数(防限流 N 路全挂)。
19const MAX_CONCURRENT_VERIFY: usize = 4;
20
21/// P2-5: 单条陈述的裁判 prompt 里参考上下文的字符上限。
22/// 完整长参考只截一次、N 条陈述复用,避免每条重复传输整段上下文。
23const DEFAULT_MAX_CONTEXT_CHARS: usize = 2000;
24
25/// 忠实度评测器(幻觉检测):回答有多忠于参考上下文。
26///
27/// 把 prediction 拆成原子陈述,逐条问裁判能否从 reference 推导,
28/// 通过率 = 能推导的陈述数 / 总陈述数。
29pub struct Faithfulness<M: BaseChatModel> {
30    judge: M,
31    /// 用 LLM 拆原子陈述(默认 false,用规则按标点拆)
32    llm_split: bool,
33    /// 空预测(无可验证陈述)的得分,默认 0.0(没回答=不忠实)。
34    empty_score: f64,
35    /// 参考上下文单条传输上限(字符,默认 [`DEFAULT_MAX_CONTEXT_CHARS`])。
36    max_context_chars: usize,
37}
38
39/// 把回答拆成原子陈述(按句号、问号、感叹号、分号、换行切分)。
40fn split_claims(prediction: &str) -> Vec<String> {
41    prediction
42        .split(['。', '.', '!', '?', '；', ';', '\n'])
43        .map(|s| s.trim().to_string())
44        .filter(|s| !s.is_empty())
45        .collect()
46}
47
48impl<M: BaseChatModel> Faithfulness<M> {
49    /// 创建忠实度评测器。
50    pub fn new(judge: M) -> Self {
51        Self {
52            judge,
53            llm_split: false,
54            empty_score: 0.0, // P0-2: 空预测默认 0 分(没回答=不忠实)
55            max_context_chars: DEFAULT_MAX_CONTEXT_CHARS,
56        }
57    }
58
59    /// 用 LLM 拆原子陈述(默认按标点规则拆;LLM 拆能处理逗号复合句)
60    pub fn with_llm_split(mut self, v: bool) -> Self {
61        self.llm_split = v;
62        self
63    }
64
65    /// 空预测的得分:默认 0.0(没回答=不忠实),可设 1.0 表示"没编造即忠实"
66    pub fn with_empty_score(mut self, score: f64) -> Self {
67        self.empty_score = score;
68        self
69    }
70
71    /// 参考上下文单条传输上限(字符)。P2-5: 默认 2000,防止长参考被每条陈述重复完整塞进 prompt。
72    pub fn with_max_context_chars(mut self, max: usize) -> Self {
73        self.max_context_chars = max;
74        self
75    }
76
77    /// 问裁判:单条陈述能否从上下文推导。
78    async fn verify_claim(&self, context: &str, claim: &str) -> Result<bool, EvalError> {
79        let system =
80            "你是事实核查员。判断给定的陈述能否从参考上下文中推导出来。调用 check_claim 工具提交判定。"
81                .to_string();
82        let user =
83            format!("参考上下文:\n{context}\n\n陈述:\n{claim}\n\n这条陈述能从上下文推导出来吗?");
84        let messages = vec![Message::system(system), Message::human(user)];
85
86        // P0-1: 优先结构化输出(verdict 布尔);不支持工具绑定的模型走文本解析回落。
87        let args: VerdictArgs = structured_call(&self.judge, verdict_tool(), messages, |raw| {
88            let verdict = parse_yes_no(raw).ok_or_else(|| {
89                StructuredJudgeError::Parse(format!(
90                    "failed to parse yes/no from judge reply: {}",
91                    truncate(raw, 200)
92                ))
93            })?;
94            Ok(VerdictArgs {
95                verdict,
96                reason: String::new(),
97            })
98        })
99        .await?;
100        Ok(args.verdict)
101    }
102
103    /// 用 LLM 把回答拆成原子陈述(每行一条),处理规则拆不动的复合句。
104    async fn split_claims_llm(&self, prediction: &str) -> Result<Vec<String>, EvalError> {
105        let system =
106            "你是文本分析助手。把回答拆成原子陈述,每条一行,只输出陈述本身,不要编号不要解释。"
107                .to_string();
108        let user = format!("回答:\n{prediction}\n\n把它拆成原子陈述,每行一条:");
109        let result = self
110            .judge
111            .chat_with_system(system, vec![Message::human(user)])
112            .await
113            .map_err(|e| EvalError::PredictorError(e.to_string()))?;
114        Ok(result
115            .content
116            .lines()
117            .map(|s| s.trim().to_string())
118            .filter(|s| !s.is_empty())
119            .collect())
120    }
121}
122
123#[async_trait]
124impl<M: BaseChatModel> Evaluator for Faithfulness<M> {
125    async fn eval(
126        &self,
127        _input: &str,
128        prediction: &str,
129        reference: &str,
130    ) -> Result<Score, EvalError> {
131        let claims = if self.llm_split {
132            self.split_claims_llm(prediction).await?
133        } else {
134            split_claims(prediction)
135        };
136        if claims.is_empty() {
137            return Ok(Score::new(self.empty_score).with_label("no_claims"));
138        }
139        // P2-5: 参考上下文只截取一次,各陈述复用同一份(避免完整长参考被 N 条重复传输)。
140        let context = truncate(reference, self.max_context_chars);
141        // 并发验证各陈述(每条一次 LLM 调用),但用 buffer_unordered 限流:
142        // P1-5——join_all 无上限并发打同一个 judge,命中限流会 N 路全挂。
143        // `ctx` 是 Copy 的引用,闭包可反复捕获;若直接捕获 `context` 会被 async move
144        // 逐条 move 出去,map(FnMut) 编译不过。
145        let ctx = &context;
146        let total = claims.len();
147        let results: Vec<Result<bool, EvalError>> = stream::iter(claims)
148            .map(|claim| async move { self.verify_claim(ctx, &claim).await })
149            .buffer_unordered(MAX_CONCURRENT_VERIFY)
150            .collect()
151            .await;
152        let mut supported = 0usize;
153        for r in results {
154            if r? {
155                supported += 1;
156            }
157        }
158        let value = supported as f64 / total as f64;
159        Ok(Score::new(value).with_label("faithfulness"))
160    }
161
162    fn name(&self) -> &str {
163        "faithfulness"
164    }
165}
166
167/// 结构化判定参数(经 tool_calls 返回)。
168#[derive(Debug, Deserialize)]
169struct VerdictArgs {
170    verdict: bool,
171    /// 让 LLM 附上简短理由(改善判定质量),当前不消费。
172    #[serde(default)]
173    #[allow(dead_code)]
174    reason: String,
175}
176
177/// 构建判定工具:让 LLM 以 `{"verdict": bool, "reason": "..."}` 提交判定。
178fn verdict_tool() -> ToolDefinition {
179    ToolDefinition::new(
180        "check_claim",
181        "判断陈述能否从参考上下文推导出来,提交布尔判定。",
182    )
183    .with_parameters(serde_json::json!({
184        "type": "object",
185        "properties": {
186            "verdict": { "type": "boolean", "description": "能否从上下文推导" },
187            "reason": { "type": "string", "description": "简短依据" }
188        },
189        "required": ["verdict", "reason"]
190    }))
191}
192
193/// 解析"是/否"。无任何是/否标记时返回 `None`(解析失败,由调用方报错),
194/// 而非静默默认 false——避免 LLM 跑题回复被当成"不忠实"。
195fn parse_yes_no(raw: &str) -> Option<bool> {
196    let lower = raw.to_lowercase();
197    // 先判否定(避免"不是""不能"被"是""能"误判;否定词优先于肯定词)
198    if lower.contains("否")
199        || lower.contains("no")
200        || lower.contains("不能")
201        || lower.contains("不是")
202        || lower.contains("false")
203    {
204        return Some(false);
205    }
206    if lower.contains("是")
207        || lower.contains("yes")
208        || lower.contains("能")
209        || lower.contains("true")
210    {
211        return Some(true);
212    }
213    None
214}
215
216#[cfg(test)]
217mod tests {
218    use super::*;
219    use futures_util::Stream;
220    use lc_core::language_models::LLMResult;
221    use lc_core::{BaseLanguageModel, Runnable, RunnableConfig};
222    use lc_schema::MessageType;
223    use std::pin::Pin;
224    use std::sync::atomic::{AtomicUsize, Ordering};
225    use std::sync::{Arc, Mutex};
226
227    #[derive(Debug)]
228    struct JudgeError(String);
229    impl std::fmt::Display for JudgeError {
230        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
231            write!(f, "{}", self.0)
232        }
233    }
234    impl std::error::Error for JudgeError {}
235
236    struct SeqMockJudge {
237        replies: Vec<String>,
238        call: Arc<AtomicUsize>,
239        last_user: Arc<Mutex<Option<String>>>,
240    }
241    impl SeqMockJudge {
242        fn new(replies: Vec<String>) -> Self {
243            Self {
244                replies,
245                call: Arc::new(AtomicUsize::new(0)),
246                last_user: Arc::new(Mutex::new(None)),
247            }
248        }
249        fn last_user_content(&self) -> String {
250            self.last_user
251                .lock()
252                .unwrap_or_else(|e| e.into_inner())
253                .clone()
254                .unwrap_or_default()
255        }
256    }
257
258    #[async_trait]
259    impl Runnable<Vec<Message>, LLMResult> for SeqMockJudge {
260        type Error = JudgeError;
261        async fn invoke(
262            &self,
263            _input: Vec<Message>,
264            _config: Option<RunnableConfig>,
265        ) -> Result<LLMResult, Self::Error> {
266            Err(JudgeError("use chat".into()))
267        }
268    }
269    #[async_trait]
270    impl BaseLanguageModel<Vec<Message>, LLMResult> for SeqMockJudge {
271        fn model_name(&self) -> &str {
272            "seq-mock"
273        }
274        fn get_num_tokens(&self, t: &str) -> usize {
275            t.len()
276        }
277        fn with_temperature(self, _: f32) -> Self {
278            self
279        }
280        fn with_max_tokens(self, _: usize) -> Self {
281            self
282        }
283    }
284    #[async_trait]
285    impl BaseChatModel for SeqMockJudge {
286        async fn chat(
287            &self,
288            messages: Vec<Message>,
289            _config: Option<RunnableConfig>,
290        ) -> Result<LLMResult, Self::Error> {
291            let idx = self.call.fetch_add(1, Ordering::SeqCst);
292            let reply = self.replies.get(idx).cloned().unwrap_or_default();
293            if let Some(human) = messages
294                .iter()
295                .find(|m| m.message_type == MessageType::Human)
296            {
297                *self.last_user.lock().unwrap_or_else(|e| e.into_inner()) =
298                    Some(human.content.clone());
299            }
300            Ok(LLMResult {
301                content: reply,
302                model: "seq-mock".to_string(),
303                token_usage: None,
304                tool_calls: None,
305                thinking_content: None,
306            })
307        }
308        async fn stream_chat(
309            &self,
310            _messages: Vec<Message>,
311            _config: Option<RunnableConfig>,
312        ) -> Result<Pin<Box<dyn Stream<Item = Result<String, Self::Error>> + Send>>, Self::Error>
313        {
314            Err(JudgeError("not supported".into()))
315        }
316    }
317
318    #[test]
319    fn test_split_claims() {
320        let claims = split_claims("巴黎是法国首都。伦敦是英国首都。");
321        assert_eq!(claims.len(), 2);
322        assert_eq!(claims[0], "巴黎是法国首都");
323        assert_eq!(claims[1], "伦敦是英国首都");
324    }
325
326    #[test]
327    fn test_split_claims_empty() {
328        assert!(split_claims("").is_empty());
329        assert!(split_claims("。。。").is_empty());
330    }
331
332    #[tokio::test]
333    async fn test_faithfulness_all_supported() {
334        let judge = Faithfulness::new(SeqMockJudge::new(vec!["是".into(), "是".into()]));
335        let s = judge
336            .eval("", "巴黎是法国首都。伦敦是英国首都。", "ctx")
337            .await
338            .unwrap();
339        assert!((s.value - 1.0).abs() < 1e-9);
340    }
341
342    #[tokio::test]
343    async fn test_faithfulness_half_supported() {
344        let judge = Faithfulness::new(SeqMockJudge::new(vec!["是".into(), "否".into()]));
345        let s = judge
346            .eval("", "巴黎是法国首都。伦敦是英国首都。", "ctx")
347            .await
348            .unwrap();
349        assert!((s.value - 0.5).abs() < 1e-9);
350    }
351
352    #[tokio::test]
353    async fn test_faithfulness_none_supported() {
354        let judge = Faithfulness::new(SeqMockJudge::new(vec!["否".into(), "否".into()]));
355        let s = judge
356            .eval("", "巴黎是法国首都。伦敦是英国首都。", "ctx")
357            .await
358            .unwrap();
359        assert!((s.value - 0.0).abs() < 1e-9);
360    }
361
362    #[tokio::test]
363    async fn test_faithfulness_empty_prediction() {
364        // P0-2: 空预测默认 0 分(没回答=不忠实)
365        let judge = Faithfulness::new(SeqMockJudge::new(vec![]));
366        let s = judge.eval("", "", "ctx").await.unwrap();
367        assert!((s.value - 0.0).abs() < 1e-9);
368        assert_eq!(s.label.as_deref(), Some("no_claims"));
369    }
370
371    #[tokio::test]
372    async fn test_faithfulness_empty_score_configurable() {
373        // 可显式配成 1.0(没编造即忠实)
374        let judge = Faithfulness::new(SeqMockJudge::new(vec![])).with_empty_score(1.0);
375        let s = judge.eval("", "", "ctx").await.unwrap();
376        assert!((s.value - 1.0).abs() < 1e-9);
377    }
378
379    #[tokio::test]
380    async fn test_faithfulness_llm_split() {
381        // 逗号复合句规则拆只算 1 条;LLM 拆能拆成 2 条分别验证
382        let judge = Faithfulness::new(SeqMockJudge::new(vec![
383            "巴黎是法国首都\n伦敦是英国首都".into(),
384            "是".into(),
385            "是".into(),
386        ]))
387        .with_llm_split(true);
388        let s = judge
389            .eval("", "巴黎是法国首都,伦敦是英国首都。", "ctx")
390            .await
391            .unwrap();
392        assert!((s.value - 1.0).abs() < 1e-9);
393    }
394
395    #[test]
396    fn test_parse_yes_no() {
397        assert_eq!(parse_yes_no("是"), Some(true));
398        assert_eq!(parse_yes_no("yes"), Some(true));
399        assert_eq!(parse_yes_no("否"), Some(false));
400        assert_eq!(parse_yes_no("no"), Some(false));
401        assert_eq!(parse_yes_no("不是"), Some(false));
402        assert_eq!(parse_yes_no("不能"), Some(false));
403        // 无任何是/否标记 = 解析失败,不应静默默认
404        assert_eq!(parse_yes_no("我不会告诉你"), None);
405    }
406
407    /// P0-1: 支持 bind_tools 的模型走结构化输出(verdict 布尔),不再依赖文本解析。
408    #[tokio::test]
409    async fn test_faithfulness_structured_verdict() {
410        use crate::test_support::ToolJudge;
411        // 两条陈述:一条支持、一条不支持 → 忠实度 0.5
412        let judge = Faithfulness::new(ToolJudge::sequence(vec![
413            r#"{"verdict": true, "reason": "能从上下文推导"}"#.into(),
414            r#"{"verdict": false, "reason": "无法推导"}"#.into(),
415        ]));
416        let s = judge
417            .eval("", "巴黎是法国首都。伦敦是英国首都。", "巴黎是法国首都")
418            .await
419            .unwrap();
420        assert!((s.value - 0.5).abs() < 1e-9);
421    }
422
423    /// P0-1: 全部不支持 → 0 分。
424    #[tokio::test]
425    async fn test_faithfulness_structured_all_false() {
426        use crate::test_support::ToolJudge;
427        let judge = Faithfulness::new(ToolJudge::new(
428            r#"{"verdict": false, "reason": "均无法推导"}"#,
429        ));
430        let s = judge
431            .eval("", "巴黎是法国首都。伦敦是英国首都。", "巴黎是法国首都")
432            .await
433            .unwrap();
434        assert!((s.value - 0.0).abs() < 1e-9);
435    }
436
437    /// P2-5: 长参考上下文只截取一次,N 条陈述复用同一份,不重复整段发送。
438    #[tokio::test]
439    async fn test_faithfulness_reference_truncated_once() {
440        let judge = SeqMockJudge::new(vec!["是".into(), "是".into()]);
441        let f = Faithfulness::new(judge).with_max_context_chars(10);
442        let long_ref =
443            "这是一段非常长的参考上下文,远超默认的单条传输上限,里面藏了一个不该被完整发送的尾巴"
444                .to_string();
445        let s = f
446            .eval("", "巴黎是首都。伦敦是首都。", &long_ref)
447            .await
448            .unwrap();
449        assert!((s.value - 1.0).abs() < 1e-9);
450        let sent = f.judge.last_user_content();
451        // 参考上下文被截到预算内:首部仍在、远在预算外的尾巴不会被发出去
452        assert!(sent.contains("这是一段非常长"), "actual sent: {sent}");
453        assert!(
454            !sent.contains("不该被完整发送"),
455            "full long reference was sent repeatedly"
456        );
457    }
458}
lc_evaluation/faithfulness.rs

lc_evaluation/
faithfulness.rs
