lc-testkit 0.22.0

Testing harness for langchainrust — record real LLM exchanges and replay them offline
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

//! RecordingProvider → JSONL → ReplayProvider roundtrip: the same response comes back.
//!
//! This is the harness's core closed loop: one real call lands on disk, then replay from the file
//! yields the same result.

mod common;

use common::FakeModel;
use lc_core::language_models::BaseChatModel;
use lc_core::tools::ToolDefinition;
use lc_schema::Message;
use lc_testkit::{RecordingProvider, ReplayProvider};

#[tokio::test]
async fn record_then_replay_roundtrip() {
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("capture.jsonl");

    // 1. Make one real call (fake model), the response is written to the recording file
    let recorded =
        RecordingProvider::new(FakeModel::new("Rust 是一门系统编程语言。"), &path).unwrap();
    let result = recorded
        .chat(
            vec![Message::system("测试"), Message::human("什么是 Rust?")],
            None,
        )
        .await
        .unwrap();
    assert_eq!(result.content, "Rust 是一门系统编程语言。");

    // 2. The file has exactly 1 valid JSONL line
    let raw = std::fs::read_to_string(&path).unwrap();
    assert_eq!(raw.lines().count(), 1);

    // 3. Replay from the file: content and token counts match
    let replay = ReplayProvider::from_file(&path).unwrap();
    assert_eq!(replay.len(), 1);

    let replayed = replay
        .chat(vec![Message::human("什么是 Rust?")], None)
        .await
        .unwrap();
    assert_eq!(replayed.content, result.content);
    assert_eq!(replayed.model, result.model);
    // The fake model's token counts are deterministic: 2 prompt / 1 completion / 3 total
    let tokens = replayed.token_usage.as_ref().expect("回放应带 token 计数");
    assert_eq!(tokens.prompt_tokens, 2);
    assert_eq!(tokens.completion_tokens, 1);
    assert_eq!(tokens.total_tokens, 3);
}

#[tokio::test]
async fn record_with_bound_tools_then_replay() {
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("tools.jsonl");

    // 1. Record after binding tools: the exchange's tools field should land on disk
    let recorded = RecordingProvider::new(FakeModel::new("计算结果是 5。"), &path).unwrap();
    let bound = recorded.bind_tools(vec![ToolDefinition::new("calculator", "数学计算")]);
    let result = bound
        .chat(vec![Message::system("测试"), Message::human("2+3=?")], None)
        .await
        .unwrap();
    assert_eq!(result.content, "计算结果是 5。");

    // 2. The file should contain the tool name
    let raw = std::fs::read_to_string(&path).unwrap();
    assert!(
        raw.contains("calculator"),
        "录播文件应包含绑定的工具名: {raw}"
    );

    // 3. Replay: tools preserved, response identical
    let replay = ReplayProvider::from_file(&path).unwrap();
    assert_eq!(replay.len(), 1);
    let replayed = replay
        .chat(vec![Message::human("2+3=?")], None)
        .await
        .unwrap();
    assert_eq!(replayed.content, result.content);
}

#[test]
fn old_fixture_without_tools_still_deserializes() {
    // Old-format fixture (llm_chain_f01.jsonl) has no tools field → reads as None, zero changes.
    let line = r#"{"messages":[{"content":"q","type":"human"}],"response":{"content":"a","model":"m","token_usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}}"#;
    let exchange: lc_testkit::RecordedExchange = serde_json::from_str(line).unwrap();
    assert!(exchange.tools.is_none());
}