lc_evaluation/
pairwise.rs

1//! 成对比较评测器:让 LLM 裁判在两个回答中二选一(竞技场模式)。
2//!
3//! 带位置偏差缓解:交换 A/B 顺序跑两次,两次都选同一个才算真赢,否则判平局。
4
5use async_trait::async_trait;
6use futures_util::future;
7use serde::Deserialize;
8
9use lc_core::judge::{structured_call, truncate, StructuredJudgeError};
10use lc_core::tools::ToolDefinition;
11use lc_core::BaseChatModel;
12use lc_schema::Message;
13
14use super::{EvalError, PairwiseEvaluator, Score};
15
16/// 成对比较结果
17#[derive(Debug, Clone, PartialEq, Eq)]
18pub enum Verdict {
19    AWins,
20    BWins,
21    Tie,
22}
23
24/// 裁判选了哪个位置
25#[derive(Debug, Clone, PartialEq, Eq)]
26enum Pick {
27    First,
28    Second,
29    Tie,
30}
31
32/// 成对比较评测器(用 LLM 当裁判,二选一)。
33///
34/// P1-1: 实现 `PairwiseEvaluator` trait,可与单点 `Evaluator` 一起进 `EvalRunner`
35/// 统一报告;直接调用仍可走 `compare` 拿细粒度 `Verdict`(A 赢 / B 赢 / 平局)。
36pub struct PairwiseJudge<M: BaseChatModel> {
37    judge: M,
38    rubric: String,
39}
40
41const DEFAULT_PAIRWISE_RUBRIC: &str = "\
42正确性:回答是否事实准确、是否切题。
43完整性:是否完整回答了问题。
44清晰性:表达是否清晰、简洁。";
45
46impl<M: BaseChatModel> PairwiseJudge<M> {
47    pub fn new(judge: M) -> Self {
48        Self {
49            judge,
50            rubric: DEFAULT_PAIRWISE_RUBRIC.to_string(),
51        }
52    }
53
54    pub fn with_rubric(mut self, rubric: impl Into<String>) -> Self {
55        self.rubric = rubric.into();
56        self
57    }
58
59    /// 比较 A、B 两个回答,返回谁更好。
60    ///
61    /// 交换 A/B 顺序跑两次,消除位置偏差:两次都选同一个才算真赢,否则判平局。
62    /// P2-4: 两次 ask 相互独立,用 `future::join` 并发发起(消除 N+1 串行往返)。
63    pub async fn compare(&self, input: &str, a: &str, b: &str) -> Result<Verdict, EvalError> {
64        let (v1, v2) = future::join(self.ask(input, a, b), self.ask(input, b, a)).await;
65        let v1 = v1?; // A 在前
66        let v2 = v2?; // 交换,B 在前
67
68        Ok(match (v1, v2) {
69            (Pick::Tie, _) | (_, Pick::Tie) => Verdict::Tie,
70            (Pick::First, Pick::Second) => Verdict::AWins, // v1 选 A(前),v2 选 A(后)
71            (Pick::Second, Pick::First) => Verdict::BWins, // v1 选 B(后),v2 选 B(前)
72            _ => Verdict::Tie, // 位置偏差:两次选的位置一致但映射回不同答案
73        })
74    }
75
76    async fn ask(&self, input: &str, first: &str, second: &str) -> Result<Pick, EvalError> {
77        let system = format!(
78            "你是裁判。根据评分标准,判断两个回答哪个更好。调用 pick_better 工具提交判定。\n\n\
79             评分标准:\n{rubric}\n\n\
80             判定的 verdict 取三者之一:\"a\"(第一个更好) / \"b\"(第二个更好) / \"tie\"(平局)",
81            rubric = self.rubric
82        );
83        let user =
84            format!("题目:\n{input}\n\n第一个回答:\n{first}\n\n第二个回答:\n{second}\n\n哪个更好?");
85        let messages = vec![Message::system(system), Message::human(user)];
86
87        // P0-1: 优先结构化输出(verdict: a/b/tie);不支持工具绑定的模型走文本解析回落。
88        let args: PickArgs = structured_call(&self.judge, pick_tool(), messages, |raw| {
89            let pick = parse_pick(raw).ok_or_else(|| {
90                StructuredJudgeError::Parse(format!(
91                    "无法从裁判回复解析胜负: {}",
92                    truncate(raw, 200)
93                ))
94            })?;
95            Ok(PickArgs {
96                verdict: pick_to_str(pick).to_string(),
97                reason: String::new(),
98            })
99        })
100        .await?;
101        str_to_pick(&args.verdict)
102    }
103}
104
105/// P1-1: 作为 `PairwiseEvaluator` 进 `EvalRunner`,judge 以
106/// (a=prediction, b=reference) 为两个候选。得分映射:1.0 = A 优、
107/// 0.5 = 平局、0.0 = B 优,label 保留裁决含义(a_wins / tie / b_wins)。
108#[async_trait]
109impl<M: BaseChatModel> PairwiseEvaluator for PairwiseJudge<M> {
110    async fn eval_pair(&self, input: &str, a: &str, b: &str) -> Result<Score, EvalError> {
111        let (value, label) = match self.compare(input, a, b).await? {
112            Verdict::AWins => (1.0, "a_wins"),
113            Verdict::Tie => (0.5, "tie"),
114            Verdict::BWins => (0.0, "b_wins"),
115        };
116        Ok(Score::new(value).with_label(label))
117    }
118
119    fn name(&self) -> &str {
120        "pairwise"
121    }
122}
123
124/// 结构化判定参数(经 tool_calls 返回)。
125#[derive(Debug, Deserialize)]
126struct PickArgs {
127    verdict: String, // "a" | "b" | "tie"
128    /// 让 LLM 附上简短理由(改善判定质量),当前不消费。
129    #[serde(default)]
130    #[allow(dead_code)]
131    reason: String,
132}
133
134/// 构建二选一工具:让 LLM 以 `{"verdict": "a"|"b"|"tie", "reason": "..."}` 提交判定。
135fn pick_tool() -> ToolDefinition {
136    ToolDefinition::new(
137        "pick_better",
138        "判断两个回答哪个更好。verdict 取 \"a\"(第一个更好)、\"b\"(第二个更好)、\"tie\"(平局)。",
139    )
140    .with_parameters(serde_json::json!({
141        "type": "object",
142        "properties": {
143            "verdict": {
144                "type": "string",
145                "enum": ["a", "b", "tie"],
146                "description": "a=第一个更好, b=第二个更好, tie=平局"
147            },
148            "reason": { "type": "string", "description": "简短理由" }
149        },
150        "required": ["verdict", "reason"]
151    }))
152}
153
154fn pick_to_str(pick: Pick) -> &'static str {
155    match pick {
156        Pick::First => "a",
157        Pick::Second => "b",
158        Pick::Tie => "tie",
159    }
160}
161
162/// 把结构化 verdict 字符串映射回 `Pick`;非法值报解析错误。
163fn str_to_pick(verdict: &str) -> Result<Pick, EvalError> {
164    match verdict {
165        "a" => Ok(Pick::First),
166        "b" => Ok(Pick::Second),
167        "tie" => Ok(Pick::Tie),
168        other => Err(EvalError::ParseError(format!(
169            "裁判返回了非法 verdict: {}",
170            other
171        ))),
172    }
173}
174
175/// 解析裁判回复为 Pick。无任何有效标记时返回 `None`(解析失败,由调用方报错),
176/// 而非静默默认为平局——避免 LLM 跑题回复被当成"无偏好"。
177fn parse_pick(raw: &str) -> Option<Pick> {
178    let lower = raw.to_lowercase();
179    if lower.contains("平局") || lower.contains("tie") || lower.contains("一样") {
180        return Some(Pick::Tie);
181    }
182    // 第一个 / 前者 / former:任一措辞,取最早出现位置
183    let first_pos = ["第一个", "first", "前者", "former"]
184        .into_iter()
185        .filter_map(|kw| lower.find(kw))
186        .min();
187    // 第二个 / 后者 / latter
188    let second_pos = ["第二个", "second", "后者", "latter"]
189        .into_iter()
190        .filter_map(|kw| lower.find(kw))
191        .min();
192    match (first_pos, second_pos) {
193        (Some(f), Some(s)) if f < s => Some(Pick::First),
194        (Some(_), Some(_)) => Some(Pick::Second),
195        (Some(_), None) => Some(Pick::First),
196        (None, Some(_)) => Some(Pick::Second),
197        (None, None) => None,
198    }
199}
200
201#[cfg(test)]
202mod tests {
203    use super::*;
204    use async_trait::async_trait;
205    use futures_util::Stream;
206    use lc_core::language_models::LLMResult;
207    use lc_core::{BaseLanguageModel, Runnable, RunnableConfig};
208    use std::pin::Pin;
209    use std::sync::atomic::{AtomicUsize, Ordering};
210    use std::sync::Arc;
211
212    #[derive(Debug)]
213    struct JudgeError(String);
214    impl std::fmt::Display for JudgeError {
215        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
216            write!(f, "{}", self.0)
217        }
218    }
219    impl std::error::Error for JudgeError {}
220
221    /// 依次返回预设回复的 mock 裁判
222    struct SeqMockJudge {
223        replies: Vec<String>,
224        call: Arc<AtomicUsize>,
225    }
226    impl SeqMockJudge {
227        fn new(replies: Vec<String>) -> Self {
228            Self {
229                replies,
230                call: Arc::new(AtomicUsize::new(0)),
231            }
232        }
233    }
234
235    #[async_trait]
236    impl Runnable<Vec<Message>, LLMResult> for SeqMockJudge {
237        type Error = JudgeError;
238        async fn invoke(
239            &self,
240            _input: Vec<Message>,
241            _config: Option<RunnableConfig>,
242        ) -> Result<LLMResult, Self::Error> {
243            Err(JudgeError("use chat".into()))
244        }
245    }
246
247    #[async_trait]
248    impl BaseLanguageModel<Vec<Message>, LLMResult> for SeqMockJudge {
249        fn model_name(&self) -> &str {
250            "seq-mock"
251        }
252        fn get_num_tokens(&self, t: &str) -> usize {
253            t.len()
254        }
255        fn with_temperature(self, _: f32) -> Self {
256            self
257        }
258        fn with_max_tokens(self, _: usize) -> Self {
259            self
260        }
261    }
262
263    #[async_trait]
264    impl BaseChatModel for SeqMockJudge {
265        async fn chat(
266            &self,
267            _messages: Vec<Message>,
268            _config: Option<RunnableConfig>,
269        ) -> Result<LLMResult, Self::Error> {
270            let idx = self.call.fetch_add(1, Ordering::SeqCst);
271            let reply = self.replies.get(idx).cloned().unwrap_or_default();
272            Ok(LLMResult {
273                content: reply,
274                model: "seq-mock".to_string(),
275                token_usage: None,
276                tool_calls: None,
277                thinking_content: None,
278            })
279        }
280        async fn stream_chat(
281            &self,
282            _messages: Vec<Message>,
283            _config: Option<RunnableConfig>,
284        ) -> Result<Pin<Box<dyn Stream<Item = Result<String, Self::Error>> + Send>>, Self::Error>
285        {
286            Err(JudgeError("not supported".into()))
287        }
288    }
289
290    #[tokio::test]
291    async fn test_pairwise_a_wins() {
292        // 第一次(A在前)选第一个=A;第二次(B在前)选第二个=A => A赢
293        let judge = PairwiseJudge::new(SeqMockJudge::new(vec![
294            "第一个更好".into(),
295            "第二个更好".into(),
296        ]));
297        assert_eq!(judge.compare("q", "A", "B").await.unwrap(), Verdict::AWins);
298    }
299
300    #[tokio::test]
301    async fn test_pairwise_b_wins() {
302        // 第一次(A在前)选第二个=B;第二次(B在前)选第一个=B => B赢
303        let judge = PairwiseJudge::new(SeqMockJudge::new(vec![
304            "第二个更好".into(),
305            "第一个更好".into(),
306        ]));
307        assert_eq!(judge.compare("q", "A", "B").await.unwrap(), Verdict::BWins);
308    }
309
310    #[tokio::test]
311    async fn test_pairwise_position_bias_tie() {
312        // 裁判总选第一个(位置偏差):两次都选 first => 映射回不同答案 => 平局
313        let judge = PairwiseJudge::new(SeqMockJudge::new(vec![
314            "第一个更好".into(),
315            "第一个更好".into(),
316        ]));
317        assert_eq!(judge.compare("q", "A", "B").await.unwrap(), Verdict::Tie);
318    }
319
320    #[tokio::test]
321    async fn test_pairwise_explicit_tie() {
322        let judge = PairwiseJudge::new(SeqMockJudge::new(vec!["平局".into(), "平局".into()]));
323        assert_eq!(judge.compare("q", "A", "B").await.unwrap(), Verdict::Tie);
324    }
325
326    #[test]
327    fn test_parse_pick() {
328        assert_eq!(parse_pick("第一个更好"), Some(Pick::First));
329        assert_eq!(parse_pick("第二个更好"), Some(Pick::Second));
330        assert_eq!(parse_pick("平局"), Some(Pick::Tie));
331        assert_eq!(parse_pick("两个一样好"), Some(Pick::Tie));
332        assert_eq!(parse_pick("第二个比第一个好"), Some(Pick::Second));
333        // 前者/后者、former/latter:LLM 不一定按"第一个"格式回
334        assert_eq!(parse_pick("前者更好"), Some(Pick::First));
335        assert_eq!(parse_pick("后者更准确"), Some(Pick::Second));
336        assert_eq!(parse_pick("the former is better"), Some(Pick::First));
337        assert_eq!(parse_pick("the latter wins"), Some(Pick::Second));
338        // 无任何有效标记 = 解析失败,不应静默默认为平局
339        assert_eq!(parse_pick("我无法判断"), None);
340    }
341
342    /// P0-1: 支持 bind_tools 的模型走结构化输出(verdict: a/b/tie)。
343    #[tokio::test]
344    async fn test_pairwise_structured_verdict() {
345        use crate::test_support::ToolJudge;
346        // A 赢:第一轮(A 在前)选 "a"(第一个=A),第二轮(B 在前)选 "b"(第二个=A)
347        let judge = PairwiseJudge::new(ToolJudge::sequence(vec![
348            r#"{"verdict": "a", "reason": "第一个更完整"}"#.into(),
349            r#"{"verdict": "b", "reason": "第二个更完整"}"#.into(),
350        ]));
351        assert_eq!(judge.compare("q", "A", "B").await.unwrap(), Verdict::AWins);
352    }
353
354    #[tokio::test]
355    async fn test_pairwise_structured_verdict_b() {
356        use crate::test_support::ToolJudge;
357        // B 赢:第一轮(A 在前)选 "b"(第二个=B),第二轮(B 在前)选 "a"(第一个=B)
358        let judge = PairwiseJudge::new(ToolJudge::sequence(vec![
359            r#"{"verdict": "b", "reason": "第二个更准确"}"#.into(),
360            r#"{"verdict": "a", "reason": "第一个更准确"}"#.into(),
361        ]));
362        assert_eq!(judge.compare("q", "A", "B").await.unwrap(), Verdict::BWins);
363    }
364
365    #[tokio::test]
366    async fn test_pairwise_structured_verdict_tie() {
367        use crate::test_support::ToolJudge;
368        let judge = PairwiseJudge::new(ToolJudge::new(
369            r#"{"verdict": "tie", "reason": "难分高下"}"#,
370        ));
371        assert_eq!(judge.compare("q", "A", "B").await.unwrap(), Verdict::Tie);
372    }
373}
lc_evaluation/pairwise.rs

lc_evaluation/
pairwise.rs
