webshift 0.2.5

Denoised web search library — fetch, clean, and rerank web content for AI agents.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

//! Live integration tests for LLM-augmented pipeline.
//!
//! These tests hit real LLM + search services configured in `test.toml`.
//! Run with: `cargo test -p webshift --features "backends llm" -- --ignored`

mod common;

use common::TestConfig;

fn load_or_skip() -> TestConfig {
    TestConfig::load().expect(
        "test.toml not found at workspace root — copy test.toml.example and configure",
    )
}

/// Requires: LLM enabled + at least one backend enabled.
fn require_llm_and_backend(tc: &TestConfig) -> Option<String> {
    if !tc.llm.enabled {
        println!("SKIP: LLM not enabled in test.toml");
        return None;
    }
    let backends = tc.enabled_backends();
    if backends.is_empty() {
        println!("SKIP: no backend enabled in test.toml");
        return None;
    }
    Some(backends[0].to_string())
}

#[tokio::test]
#[ignore]
async fn llm_expansion_live() {
    let tc = load_or_skip();
    let backend = match require_llm_and_backend(&tc) {
        Some(b) => b,
        None => return,
    };
    let mut config = tc.to_webshift_config(&backend);
    config.llm.expansion_enabled = true;
    config.llm.summarization_enabled = false;
    config.llm.llm_rerank_enabled = false;

    let result = webshift::query(&["quantum computing"], &config)
        .await
        .expect("query failed");

    println!("Expanded queries: {:?}", result.queries);
    assert!(
        result.queries.len() > 1,
        "expected LLM to expand queries, got: {:?}",
        result.queries
    );
    assert!(!result.sources.is_empty(), "expected sources");
}

#[tokio::test]
#[ignore]
async fn llm_summarization_live() {
    let tc = load_or_skip();
    let backend = match require_llm_and_backend(&tc) {
        Some(b) => b,
        None => return,
    };
    let mut config = tc.to_webshift_config(&backend);
    config.llm.expansion_enabled = false;
    config.llm.summarization_enabled = true;
    config.llm.llm_rerank_enabled = false;

    let result = webshift::query(&["rust async runtime"], &config)
        .await
        .expect("query failed");

    assert!(
        result.summary.is_some(),
        "expected LLM summary, got None (error: {:?})",
        result.llm_summary_error
    );
    let summary = result.summary.as_ref().unwrap();
    println!(
        "Summary ({} words, {} chars):\n{}",
        summary.split_whitespace().count(),
        summary.len(),
        summary
    );
    assert!(summary.len() > 50, "summary too short: {}", summary.len());
}

#[tokio::test]
#[ignore]
async fn llm_full_pipeline_live() {
    let tc = load_or_skip();
    let backend = match require_llm_and_backend(&tc) {
        Some(b) => b,
        None => return,
    };
    let mut config = tc.to_webshift_config(&backend);
    config.llm.expansion_enabled = true;
    config.llm.summarization_enabled = true;
    config.llm.llm_rerank_enabled = true;

    let result = webshift::query(&["benefits of functional programming"], &config)
        .await
        .expect("query failed");

    println!("Queries: {:?}", result.queries);
    println!(
        "Sources: {} fetched, {} failed",
        result.stats.fetched, result.stats.failed
    );
    for s in &result.sources {
        println!(
            "  [{}] {} ({} chars, truncated={})",
            s.id,
            s.title,
            s.content.len(),
            s.truncated
        );
    }
    if let Some(ref summary) = result.summary {
        println!(
            "Summary: {} words",
            summary.split_whitespace().count()
        );
    }
    if let Some(ref err) = result.llm_summary_error {
        println!("Summary error: {err}");
    }

    assert!(result.queries.len() > 1, "expansion should produce >1 query");
    assert!(!result.sources.is_empty(), "should have sources");
}