websearch/
extract.rs

1//! Parse DuckDuckGo Lite's table layout into structured results, and tell a
2//! real results page apart from a bot challenge.
3
4use once_cell::sync::Lazy;
5use scraper::{Html, Selector};
6use url::Url;
7
8use super::types::{SearchResult, SearchStatus};
9use crate::compress::compress_text;
10
11/// One selector list matching result links *and* snippets, so `select` walks
12/// them in document order and each snippet lands on the link that precedes it.
13static RESULT_PARTS: Lazy<Selector> = Lazy::new(|| {
14    Selector::parse("a.result-link, a.result__a, .result-snippet, .result__snippet")
15        .expect("static selector")
16});
17
18/// Markers that identify a results page even when it holds no results, so an
19/// empty search is not mistaken for a block.
20static RESULTS_PAGE: Lazy<Selector> =
21    Lazy::new(|| Selector::parse(".result-count, .no-results, .results").expect("static selector"));
22
23/// Phrases DuckDuckGo serves on its anomaly/challenge page — which comes back
24/// with HTTP 200, so the status line alone never reveals it.
25const CHALLENGE_MARKERS: [&str; 5] = [
26    "bots use duckduckgo",
27    "/anomaly",
28    "anomaly.js",
29    "unusual traffic",
30    "are you a robot",
31];
32
33/// Resolve the real destination URL from a DDG Lite result href.
34///
35/// DDG Lite wraps targets in a redirect like
36/// `//duckduckgo.com/l/?uddg=<percent-encoded-url>&rut=…`. We pull the
37/// `uddg` parameter back out (already percent-decoded by the URL parser).
38/// Protocol-relative hrefs (`//host/path`) get an `https:` scheme; anything
39/// already absolute is returned unchanged.
40pub fn resolve_result_url(href: &str) -> String {
41    let href = href.trim();
42    if href.is_empty() {
43        return String::new();
44    }
45
46    // Normalize protocol-relative URLs so they can be parsed.
47    let absolute = if let Some(stripped) = href.strip_prefix("//") {
48        format!("https://{stripped}")
49    } else {
50        href.to_string()
51    };
52
53    if let Ok(parsed) = Url::parse(&absolute) {
54        if let Some((_, target)) = parsed.query_pairs().find(|(k, _)| k == "uddg") {
55            return target.into_owned();
56        }
57        return parsed.to_string();
58    }
59
60    absolute
61}
62
63/// Parse a DDG Lite results page.
64///
65/// Links and snippets are read from a single document-order traversal and a
66/// snippet attaches to the most recent link. The previous version zipped two
67/// independent iterators, so one result without a snippet row — PDFs and some
68/// news rows have none — shifted every following snippet onto the wrong URL,
69/// handing the caller a description of a page it was not citing.
70pub fn parse_ddg_lite(html: &str, max_results: usize) -> Vec<SearchResult> {
71    let document = Html::parse_document(html);
72    let mut results: Vec<SearchResult> = Vec::new();
73
74    for el in document.select(&RESULT_PARTS) {
75        if el.value().name() == "a" {
76            let title = compress_text(&el.text().collect::<String>());
77            let url = resolve_result_url(el.value().attr("href").unwrap_or(""));
78            if title.is_empty() && url.is_empty() {
79                continue;
80            }
81            results.push(SearchResult {
82                title,
83                snippet: String::new(),
84                url,
85                ref_index: results.len() + 1,
86            });
87        } else if let Some(last) = results.last_mut() {
88            // A snippet belongs to the link above it; ignore a stray second one
89            // and any snippet appearing before the first result.
90            if last.snippet.is_empty() {
91                last.snippet = compress_text(&el.text().collect::<String>());
92            }
93        }
94    }
95
96    results.truncate(max_results);
97    results
98}
99
100/// Classify a fetched results page.
101///
102/// A challenge page comes back as HTTP 200 with no result rows, which the old
103/// parser reported as `result_count: 0` — indistinguishable from a query that
104/// genuinely has no hits, so an agent concluded "nothing exists" and moved on.
105/// An unrecognized page is treated as [`SearchStatus::Blocked`] rather than
106/// empty: a page we cannot parse is a failure, and reporting it as success is
107/// the bug this exists to fix.
108pub fn classify_page(html: &str, result_count: usize) -> SearchStatus {
109    if result_count > 0 {
110        return SearchStatus::Ok;
111    }
112    let haystack = html.to_ascii_lowercase();
113    if CHALLENGE_MARKERS.iter().any(|m| haystack.contains(m)) {
114        return SearchStatus::Blocked;
115    }
116    // No results, but the page is structurally a results page: a real "no hits"
117    // answer.
118    if Html::parse_document(html)
119        .select(&RESULTS_PAGE)
120        .next()
121        .is_some()
122    {
123        return SearchStatus::Empty;
124    }
125    SearchStatus::Blocked
126}
127
128#[cfg(test)]
129mod tests {
130    use super::*;
131
132    #[test]
133    fn snippet_stays_with_its_own_link() {
134        // The middle result has no snippet row.
135        let html = r#"<table>
136            <tr><td><a href="https://a.test/1" class="result-link">Alpha</a></td></tr>
137            <tr><td class="result-snippet">About ALPHA.</td></tr>
138            <tr><td><a href="https://b.test/2" class="result-link">Bravo</a></td></tr>
139            <tr><td><a href="https://c.test/3" class="result-link">Charlie</a></td></tr>
140            <tr><td class="result-snippet">About CHARLIE.</td></tr>
141        </table>"#;
142        let results = parse_ddg_lite(html, 10);
143        assert_eq!(results.len(), 3);
144        assert_eq!(results[0].snippet, "About ALPHA.");
145        assert_eq!(results[1].snippet, "", "Bravo must not inherit a snippet");
146        assert_eq!(results[2].snippet, "About CHARLIE.");
147    }
148
149    #[test]
150    fn a_stray_leading_snippet_is_ignored() {
151        let html = r#"<table>
152            <tr><td class="result-snippet">Orphan.</td></tr>
153            <tr><td><a href="https://a.test/1" class="result-link">Alpha</a></td></tr>
154            <tr><td class="result-snippet">About ALPHA.</td></tr>
155        </table>"#;
156        let results = parse_ddg_lite(html, 10);
157        assert_eq!(results.len(), 1);
158        assert_eq!(results[0].snippet, "About ALPHA.");
159    }
160
161    #[test]
162    fn alternate_class_names_are_recognized() {
163        // DDG has served both `result-link` and `result__a` across endpoints.
164        let html = r#"<a href="https://a.test/1" class="result__a">Alpha</a>
165                      <div class="result__snippet">About ALPHA.</div>"#;
166        let results = parse_ddg_lite(html, 10);
167        assert_eq!(results.len(), 1);
168        assert_eq!(results[0].snippet, "About ALPHA.");
169    }
170
171    #[test]
172    fn challenge_page_is_blocked_not_empty() {
173        let html = "<html><body><h1>Unfortunately, bots use DuckDuckGo too.</h1>\
174                    <form action=\"/anomaly\"></form></body></html>";
175        assert!(parse_ddg_lite(html, 10).is_empty());
176        assert_eq!(classify_page(html, 0), SearchStatus::Blocked);
177    }
178
179    #[test]
180    fn genuine_no_results_page_is_empty() {
181        let html = "<html><body><table><tr><td class=\"result-count\">No results.</td>\
182                    </tr></table></body></html>";
183        assert_eq!(classify_page(html, 0), SearchStatus::Empty);
184    }
185
186    #[test]
187    fn unrecognized_markup_is_blocked() {
188        // Silently reporting "no results" for a page we cannot parse is exactly
189        // the failure mode this guards against.
190        assert_eq!(
191            classify_page("<html><body>something else entirely</body></html>", 0),
192            SearchStatus::Blocked
193        );
194    }
195
196    #[test]
197    fn results_present_is_always_ok() {
198        assert_eq!(classify_page("<html>whatever</html>", 3), SearchStatus::Ok);
199    }
200}
websearch/extract.rs

websearch/
extract.rs
