lingshu-tools 0.10.0

Tool registry, ToolHandler trait, and 50+ tool implementations
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

//! Bot-challenge HTML detection — **diagnostics only** (not used in metasearch hot path).
//!
//! Python `ddgs` gates on HTTP status (`200` → parse, `403/429/…` → raise). It does not
//! inspect HTML for CAPTCHA strings. Metasearch follows that contract; these helpers remain
//! for tests, `/doctor`, and future observability.

/// True when the response looks like a DDG CAPTCHA interstitial (diagnostic).
pub fn is_bot_challenge(html: &str) -> bool {
    let lower = html.to_ascii_lowercase();
    lower.contains("anomaly-modal")
        || lower.contains("bots use duckduckgo")
        || lower.contains("sorry, you have been blocked")
        || lower.contains("if this error persists") && lower.contains("duckduckgo.com")
}

/// True when HTML looks blocked **and** contains no parseable SERP markers (diagnostic).
pub fn is_engine_blocked(html: &str) -> bool {
    if html.contains("b_algo") || html.contains("result__a") {
        return false;
    }
    if is_bot_challenge(html) {
        return true;
    }
    let lower = html.to_ascii_lowercase();
    lower.contains("captchachallenge")
        || lower.contains("turnstile")
        || (lower.contains("captcha") && lower.contains("bing.com"))
        || (lower.contains("challenge") && lower.contains("verify you are human"))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn detects_anomaly_modal() {
        assert!(is_bot_challenge(
            r#"<div class="anomaly-modal__title">...</div>"#
        ));
    }

    #[test]
    fn real_results_page_is_not_challenge() {
        let html = r#"<a class="result__a" href="https://example.com">Example</a>"#;
        assert!(!is_bot_challenge(html));
    }

    #[test]
    fn detects_bing_captcha_page_without_serp() {
        assert!(is_engine_blocked(
            r#"<html><body>captcha challenge from bing.com</body></html>"#
        ));
    }

    #[test]
    fn bing_serp_with_turnstile_js_is_not_blocked() {
        let html = r#"
            <script>turnstile challenge widget</script>
            <li class="b_algo"><h2><a href="https://example.com">Example</a></h2><p>snippet</p></li>
        "#;
        assert!(!is_engine_blocked(html));
    }
}