web_capture/
search.rs

1//! Structured search-provider capture (issue #130).
2//!
3//! Turns a query + provider into a normalized, machine-readable result set so
4//! that browser, CLI, and server callers all consume one consistent contract
5//! instead of each reimplementing provider-specific scraping. Server-side and
6//! CLI callers fetch provider pages directly (no CORS restriction), so this
7//! module defaults to the `fetch` capture mode. Providers that expose a native
8//! CORS/JSON API (Wikipedia) are preferred; HTML search engines are parsed
9//! best-effort and report CAPTCHA/blocking through `diagnostics`.
10//!
11//! Normalized result shape (camelCase JSON):
12//! ```json
13//! {
14//!   "query": "...", "provider": "...", "captureMode": "fetch",
15//!   "capturedAt": "2026-05-18T20:30:00Z",
16//!   "results": [{ "rank": 1, "title": "...", "url": "...", "snippet": "..." }],
17//!   "diagnostics": { "status": 200, "blockedByCors": false,
18//!                    "blockedByCaptcha": false, "sourceUrl": "..." }
19//! }
20//! ```
21
22use scraper::{Html, Selector};
23use serde::{Deserialize, Serialize};
24use std::collections::BTreeMap;
25use url::form_urlencoded::byte_serialize;
26
27const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
28
29/// Providers understood by the search contract.
30pub const SEARCH_PROVIDERS: [&str; 5] = ["wikipedia", "duckduckgo", "google", "bing", "brave"];
31
32/// Default provider when none is supplied.
33pub const DEFAULT_PROVIDER: &str = "wikipedia";
34
35/// Default number of results requested/returned.
36pub const DEFAULT_LIMIT: usize = 10;
37
38/// A single normalized search result.
39#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
40pub struct SearchResultItem {
41    pub rank: usize,
42    pub title: String,
43    pub url: String,
44    pub snippet: String,
45}
46
47/// Structured diagnostics describing how the capture went.
48#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
49#[serde(rename_all = "camelCase")]
50pub struct SearchDiagnostics {
51    pub status: u16,
52    pub blocked_by_cors: bool,
53    pub blocked_by_captcha: bool,
54    pub source_url: String,
55    #[serde(skip_serializing_if = "Option::is_none")]
56    pub error: Option<String>,
57}
58
59/// The full normalized search capture result.
60#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
61#[serde(rename_all = "camelCase")]
62pub struct SearchResult {
63    pub query: String,
64    pub provider: String,
65    pub capture_mode: String,
66    pub captured_at: String,
67    pub results: Vec<SearchResultItem>,
68    pub diagnostics: SearchDiagnostics,
69}
70
71/// Parsed rankings paired with the exact provider response used to derive them.
72#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
73#[serde(rename_all = "camelCase")]
74pub struct SearchCapture {
75    pub result: SearchResult,
76    pub receipt: crate::transport::ResponseReceipt,
77}
78
79/// Returns true if `provider` is one of the supported providers.
80#[must_use]
81pub fn is_supported_provider(provider: &str) -> bool {
82    SEARCH_PROVIDERS.contains(&provider)
83}
84
85/// Normalize whitespace and decode basic HTML entities in extracted text.
86fn clean_text(text: &str) -> String {
87    // `scraper` already returns decoded text nodes, but snippets assembled from
88    // multiple nodes can carry stray whitespace; collapse it to a single line.
89    text.split_whitespace().collect::<Vec<_>>().join(" ")
90}
91
92/// Build the provider-native source URL for a query.
93///
94/// # Errors
95///
96/// Returns an error string when `provider` is not supported.
97pub fn build_search_url(provider: &str, query: &str, limit: usize) -> Result<String, String> {
98    let q: String = byte_serialize(query.as_bytes()).collect();
99    match provider {
100        "wikipedia" => Ok(format!(
101            "https://en.wikipedia.org/w/rest.php/v1/search/page?q={q}&limit={limit}"
102        )),
103        "duckduckgo" => Ok(format!("https://html.duckduckgo.com/html/?q={q}")),
104        "google" => Ok(format!("https://www.google.com/search?q={q}&num={limit}")),
105        "bing" => Ok(format!("https://www.bing.com/search?q={q}&count={limit}")),
106        "brave" => Ok(format!("https://search.brave.com/search?q={q}")),
107        other => Err(format!(
108            "Unknown search provider \"{other}\". Supported: {}",
109            SEARCH_PROVIDERS.join(", ")
110        )),
111    }
112}
113
114/// Detect provider CAPTCHA / bot-block interstitials in an HTML body.
115#[must_use]
116pub fn looks_like_captcha(html: &str) -> bool {
117    let lower = html.to_lowercase();
118    lower.contains("captcha")
119        || lower.contains("unusual traffic")
120        || lower.contains("are you a robot")
121        || lower.contains("/sorry/index")
122        || lower.contains("automated queries")
123}
124
125/// Decode a `DuckDuckGo` redirect href (`//duckduckgo.com/l/?uddg=...`).
126fn resolve_duckduckgo_href(href: &str) -> String {
127    if href.is_empty() {
128        return String::new();
129    }
130    let normalized = href
131        .strip_prefix("//")
132        .map_or_else(|| href.to_string(), |stripped| format!("https:{stripped}"));
133    if let Ok(parsed) = url::Url::parse(&normalized) {
134        if let Some((_, value)) = parsed.query_pairs().find(|(k, _)| k == "uddg") {
135            return value.into_owned();
136        }
137        return parsed.to_string();
138    }
139    href.to_string()
140}
141
142/// Wikipedia REST search page entry.
143#[derive(Debug, Deserialize)]
144struct WikiPage {
145    key: Option<String>,
146    title: Option<String>,
147    excerpt: Option<String>,
148    description: Option<String>,
149}
150
151#[derive(Debug, Deserialize)]
152struct WikiResponse {
153    pages: Option<Vec<WikiPage>>,
154}
155
156fn strip_tags(input: &str) -> String {
157    let mut out = String::with_capacity(input.len());
158    let mut in_tag = false;
159    for c in input.chars() {
160        match c {
161            '<' => in_tag = true,
162            '>' => in_tag = false,
163            _ if !in_tag => out.push(c),
164            _ => {}
165        }
166    }
167    out
168}
169
170fn parse_wikipedia(body: &str, limit: usize) -> Vec<SearchResultItem> {
171    let parsed: WikiResponse = match serde_json::from_str(body) {
172        Ok(value) => value,
173        Err(_) => return Vec::new(),
174    };
175    let pages = parsed.pages.unwrap_or_default();
176    pages
177        .into_iter()
178        .take(limit)
179        .enumerate()
180        .map(|(i, page)| {
181            let key = page
182                .key
183                .clone()
184                .or_else(|| page.title.clone())
185                .unwrap_or_default();
186            let title = clean_text(&page.title.or(page.key).unwrap_or_default());
187            let snippet_raw = page.excerpt.or(page.description).unwrap_or_default();
188            let snippet = clean_text(&strip_tags(&snippet_raw));
189            let encoded: String = byte_serialize(key.as_bytes()).collect();
190            SearchResultItem {
191                rank: i + 1,
192                title,
193                url: format!("https://en.wikipedia.org/wiki/{encoded}"),
194                snippet,
195            }
196        })
197        .collect()
198}
199
200/// Extract trimmed text content of the first element matching `selector`.
201fn first_text(element: &scraper::ElementRef, selector: &Selector) -> String {
202    element
203        .select(selector)
204        .next()
205        .map(|el| clean_text(&el.text().collect::<String>()))
206        .unwrap_or_default()
207}
208
209fn parse_duckduckgo(doc: &Html, limit: usize) -> Vec<SearchResultItem> {
210    let body_sel = Selector::parse(".result__body").unwrap();
211    let web_sel = Selector::parse(".web-result").unwrap();
212    let anchor_sel = Selector::parse("a.result__a").unwrap();
213    let snippet_sel = Selector::parse(".result__snippet").unwrap();
214
215    let mut containers: Vec<_> = doc.select(&body_sel).collect();
216    if containers.is_empty() {
217        containers = doc.select(&web_sel).collect();
218    }
219
220    let mut results = Vec::new();
221    for el in containers {
222        if results.len() >= limit {
223            break;
224        }
225        if let Some(anchor) = el.select(&anchor_sel).next() {
226            let title = clean_text(&anchor.text().collect::<String>());
227            let url = resolve_duckduckgo_href(anchor.value().attr("href").unwrap_or_default());
228            let snippet = first_text(&el, &snippet_sel);
229            if !title.is_empty() && !url.is_empty() {
230                results.push(SearchResultItem {
231                    rank: results.len() + 1,
232                    title,
233                    url,
234                    snippet,
235                });
236            }
237        }
238    }
239    results
240}
241
242fn parse_google(doc: &Html, limit: usize) -> Vec<SearchResultItem> {
243    let block_sel = Selector::parse("div.g, div.tF2Cxc, div.MjjYud").unwrap();
244    let anchor_sel = Selector::parse("a[href^=\"http\"]").unwrap();
245    let title_sel = Selector::parse("h3").unwrap();
246    let snippet_sel = Selector::parse("div[data-sncf], .VwiC3b, .IsZvec").unwrap();
247
248    let mut results = Vec::new();
249    for el in doc.select(&block_sel) {
250        if results.len() >= limit {
251            break;
252        }
253        let url = el
254            .select(&anchor_sel)
255            .next()
256            .and_then(|a| a.value().attr("href"))
257            .unwrap_or_default()
258            .to_string();
259        let title = first_text(&el, &title_sel);
260        let snippet = first_text(&el, &snippet_sel);
261        if !title.is_empty() && !url.is_empty() {
262            results.push(SearchResultItem {
263                rank: results.len() + 1,
264                title,
265                url,
266                snippet,
267            });
268        }
269    }
270    results
271}
272
273fn parse_bing(doc: &Html, limit: usize) -> Vec<SearchResultItem> {
274    let block_sel = Selector::parse("li.b_algo").unwrap();
275    let anchor_sel = Selector::parse("h2 a").unwrap();
276    let snippet_sel = Selector::parse(".b_caption p, p").unwrap();
277
278    let mut results = Vec::new();
279    for el in doc.select(&block_sel) {
280        if results.len() >= limit {
281            break;
282        }
283        if let Some(anchor) = el.select(&anchor_sel).next() {
284            let title = clean_text(&anchor.text().collect::<String>());
285            let url = anchor.value().attr("href").unwrap_or_default().to_string();
286            let snippet = first_text(&el, &snippet_sel);
287            if !title.is_empty() && !url.is_empty() {
288                results.push(SearchResultItem {
289                    rank: results.len() + 1,
290                    title,
291                    url,
292                    snippet,
293                });
294            }
295        }
296    }
297    results
298}
299
300fn parse_brave(doc: &Html, limit: usize) -> Vec<SearchResultItem> {
301    let block_sel = Selector::parse("div.snippet").unwrap();
302    let anchor_sel = Selector::parse("a[href^=\"http\"]").unwrap();
303    let title_sel = Selector::parse(".snippet-title, .title").unwrap();
304    let snippet_sel = Selector::parse(".snippet-description, .snippet-content").unwrap();
305
306    let mut results = Vec::new();
307    for el in doc.select(&block_sel) {
308        if results.len() >= limit {
309            break;
310        }
311        let anchor = el.select(&anchor_sel).next();
312        let url = anchor
313            .and_then(|a| a.value().attr("href"))
314            .unwrap_or_default()
315            .to_string();
316        let mut title = first_text(&el, &title_sel);
317        if title.is_empty() {
318            if let Some(a) = anchor {
319                title = clean_text(&a.text().collect::<String>());
320            }
321        }
322        let snippet = first_text(&el, &snippet_sel);
323        if !title.is_empty() && !url.is_empty() {
324            results.push(SearchResultItem {
325                rank: results.len() + 1,
326                title,
327                url,
328                snippet,
329            });
330        }
331    }
332    results
333}
334
335/// Parse a provider response body into normalized result rows.
336///
337/// Pure function (no network) so it can be unit-tested against fixtures.
338/// Returns the parsed rows and whether the body looked like a CAPTCHA wall.
339#[must_use]
340pub fn parse_search_results(
341    provider: &str,
342    body: &str,
343    limit: usize,
344) -> (Vec<SearchResultItem>, bool) {
345    if provider == "wikipedia" {
346        return (parse_wikipedia(body, limit), false);
347    }
348    let blocked = looks_like_captcha(body);
349    let doc = Html::parse_document(body);
350    let results = match provider {
351        "duckduckgo" => parse_duckduckgo(&doc, limit),
352        "google" => parse_google(&doc, limit),
353        "bing" => parse_bing(&doc, limit),
354        "brave" => parse_brave(&doc, limit),
355        _ => Vec::new(),
356    };
357    (results, blocked)
358}
359
360/// Render a normalized search result as Markdown.
361#[must_use]
362pub fn format_search_as_markdown(result: &SearchResult) -> String {
363    let mut lines = Vec::new();
364    lines.push(format!("# Search results for \"{}\"", result.query));
365    lines.push(String::new());
366    lines.push(format!("- Provider: `{}`", result.provider));
367    lines.push(format!("- Capture mode: `{}`", result.capture_mode));
368    lines.push(format!("- Captured at: {}", result.captured_at));
369    lines.push(format!("- Source: {}", result.diagnostics.source_url));
370    if result.diagnostics.blocked_by_captcha {
371        lines.push("- ⚠️ Provider returned a CAPTCHA / bot-block page.".to_string());
372    }
373    lines.push(String::new());
374    if result.results.is_empty() {
375        lines.push("_No results._".to_string());
376        return lines.join("\n");
377    }
378    for item in &result.results {
379        lines.push(format!("{}. [{}]({})", item.rank, item.title, item.url));
380        if !item.snippet.is_empty() {
381            lines.push(format!("   {}", item.snippet));
382        }
383    }
384    lines.join("\n")
385}
386
387/// Capture structured search results for a query from a provider.
388///
389/// `captured_at` is injected (RFC 3339 timestamp) so the result is
390/// deterministic for callers and tests. A transport failure is recorded in
391/// `diagnostics` rather than returned as an error, mirroring the JS contract.
392///
393/// # Errors
394///
395/// Returns an error string for an empty query or unsupported provider.
396#[cfg(feature = "runtime")]
397pub async fn search(
398    query: &str,
399    provider: &str,
400    limit: usize,
401    capture_mode: &str,
402    captured_at: &str,
403) -> Result<SearchResult, String> {
404    if query.trim().is_empty() {
405        return Err("Missing `query` parameter".to_string());
406    }
407    if !is_supported_provider(provider) {
408        return Err(format!(
409            "Unknown search provider \"{provider}\". Supported: {}",
410            SEARCH_PROVIDERS.join(", ")
411        ));
412    }
413
414    let source_url = build_search_url(provider, query, limit)?;
415    let mut diagnostics = SearchDiagnostics {
416        status: 0,
417        blocked_by_cors: false,
418        blocked_by_captcha: false,
419        source_url: source_url.clone(),
420        error: None,
421    };
422    let transport = crate::transport::ReqwestTransport::default();
423    match search_with_transport(
424        query,
425        provider,
426        limit,
427        capture_mode,
428        captured_at,
429        &transport,
430    )
431    .await
432    {
433        Ok(capture) => return Ok(capture.result),
434        Err(error) => diagnostics.error = Some(error.to_string()),
435    }
436
437    Ok(SearchResult {
438        query: query.to_string(),
439        provider: provider.to_string(),
440        capture_mode: capture_mode.to_string(),
441        captured_at: captured_at.to_string(),
442        results: Vec::new(),
443        diagnostics,
444    })
445}
446
447/// Search through caller-owned transport and return the exact source receipt.
448///
449/// Dropping this future cancels the in-flight transport future. This provides
450/// cancellation without coupling the public API to a particular async runtime.
451pub async fn search_with_transport(
452    query: &str,
453    provider: &str,
454    limit: usize,
455    capture_mode: &str,
456    captured_at: &str,
457    transport: &dyn crate::transport::Transport,
458) -> std::result::Result<SearchCapture, crate::transport::TransportError> {
459    if query.trim().is_empty() {
460        return Err(crate::transport::TransportError {
461            kind: "invalid_request".to_string(),
462            message: "Missing `query` parameter".to_string(),
463            source_url: String::new(),
464        });
465    }
466    let source_url = build_search_url(provider, query, limit).map_err(|message| {
467        crate::transport::TransportError {
468            kind: "invalid_request".to_string(),
469            message,
470            source_url: String::new(),
471        }
472    })?;
473    let accept = if provider == "wikipedia" {
474        "application/json"
475    } else {
476        "text/html,application/xhtml+xml"
477    };
478    let request = crate::transport::TransportRequest {
479        url: source_url.clone(),
480        method: "GET".to_string(),
481        headers: BTreeMap::from([
482            ("user-agent".to_string(), USER_AGENT.to_string()),
483            ("accept".to_string(), accept.to_string()),
484            ("accept-language".to_string(), "en-US,en;q=0.9".to_string()),
485            ("accept-encoding".to_string(), "identity".to_string()),
486        ]),
487    };
488    let receipt = crate::transport::capture_response_with_transport(request, transport).await?;
489    let body = String::from_utf8_lossy(&receipt.body);
490    let (results, blocked_by_captcha) = parse_search_results(provider, &body, limit);
491    let result = SearchResult {
492        query: query.to_string(),
493        provider: provider.to_string(),
494        capture_mode: capture_mode.to_string(),
495        captured_at: captured_at.to_string(),
496        results,
497        diagnostics: SearchDiagnostics {
498            status: receipt.status,
499            blocked_by_cors: false,
500            blocked_by_captcha,
501            source_url,
502            error: None,
503        },
504    };
505    Ok(SearchCapture { result, receipt })
506}
507
508#[cfg(test)]
509mod tests {
510    use super::*;
511
512    const WIKI_JSON: &str = r#"{"pages":[
513        {"id":1,"key":"Formal_methods","title":"Formal methods","excerpt":"the <span>study</span> of <b>formal</b>","description":"rigorous"},
514        {"id":2,"key":"Formal_system","title":"Formal system","excerpt":"an abstract structure","description":""}
515    ]}"#;
516
517    const DDG_HTML: &str = r#"
518        <div class="result__body">
519          <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fa&rut=abc">First &amp; Best</a>
520          <div class="result__snippet">Snippet about the <b>first</b> result</div>
521        </div>
522        <div class="result__body">
523          <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.org%2Fb">Second result</a>
524          <div class="result__snippet">Snippet two</div>
525        </div>
526    "#;
527
528    const BING_HTML: &str = r#"
529        <ol id="b_results">
530          <li class="b_algo">
531            <h2><a href="https://bing-result.example/1">Bing One</a></h2>
532            <div class="b_caption"><p>Bing snippet one</p></div>
533          </li>
534        </ol>
535    "#;
536
537    #[test]
538    fn builds_wikipedia_url() {
539        assert_eq!(
540            build_search_url("wikipedia", "formal", 5).unwrap(),
541            "https://en.wikipedia.org/w/rest.php/v1/search/page?q=formal&limit=5"
542        );
543    }
544
545    #[test]
546    fn rejects_unknown_provider_url() {
547        assert!(build_search_url("yahoo", "x", 5).is_err());
548    }
549
550    #[test]
551    fn parses_wikipedia_json() {
552        let (results, blocked) = parse_search_results("wikipedia", WIKI_JSON, 10);
553        assert!(!blocked);
554        assert_eq!(results.len(), 2);
555        assert_eq!(results[0].title, "Formal methods");
556        assert_eq!(
557            results[0].url,
558            "https://en.wikipedia.org/wiki/Formal_methods"
559        );
560        assert_eq!(results[0].snippet, "the study of formal");
561        assert_eq!(
562            results[1].url,
563            "https://en.wikipedia.org/wiki/Formal_system"
564        );
565    }
566
567    #[test]
568    fn respects_limit() {
569        let (results, _) = parse_search_results("wikipedia", WIKI_JSON, 1);
570        assert_eq!(results.len(), 1);
571    }
572
573    #[test]
574    fn parses_duckduckgo_and_decodes_redirects() {
575        let (results, _) = parse_search_results("duckduckgo", DDG_HTML, 10);
576        assert_eq!(results.len(), 2);
577        assert_eq!(results[0].title, "First & Best");
578        assert_eq!(results[0].url, "https://example.com/a");
579        assert_eq!(results[0].snippet, "Snippet about the first result");
580        assert_eq!(results[1].url, "https://example.org/b");
581    }
582
583    #[test]
584    fn parses_bing() {
585        let (results, _) = parse_search_results("bing", BING_HTML, 10);
586        assert_eq!(results.len(), 1);
587        assert_eq!(results[0].title, "Bing One");
588        assert_eq!(results[0].url, "https://bing-result.example/1");
589        assert_eq!(results[0].snippet, "Bing snippet one");
590    }
591
592    #[test]
593    fn empty_json_yields_no_results() {
594        let (results, _) = parse_search_results("wikipedia", "not json", 10);
595        assert!(results.is_empty());
596    }
597
598    #[test]
599    fn detects_captcha() {
600        assert!(looks_like_captcha("Please solve the CAPTCHA"));
601        assert!(looks_like_captcha(
602            "Our systems have detected unusual traffic"
603        ));
604        assert!(!looks_like_captcha("normal results page"));
605    }
606
607    #[test]
608    fn formats_markdown() {
609        let result = SearchResult {
610            query: "formal-ai".to_string(),
611            provider: "wikipedia".to_string(),
612            capture_mode: "fetch".to_string(),
613            captured_at: "2026-05-30T00:00:00Z".to_string(),
614            results: vec![SearchResultItem {
615                rank: 1,
616                title: "Formal methods".to_string(),
617                url: "https://en.wikipedia.org/wiki/Formal_methods".to_string(),
618                snippet: "study of formal".to_string(),
619            }],
620            diagnostics: SearchDiagnostics {
621                status: 200,
622                blocked_by_cors: false,
623                blocked_by_captcha: false,
624                source_url: "https://example.com".to_string(),
625                error: None,
626            },
627        };
628        let md = format_search_as_markdown(&result);
629        assert!(md.contains("# Search results for \"formal-ai\""));
630        assert!(md.contains("1. [Formal methods](https://en.wikipedia.org/wiki/Formal_methods)"));
631        assert!(md.contains("study of formal"));
632    }
633
634    #[test]
635    fn serializes_camel_case_contract() {
636        let result = SearchResult {
637            query: "q".to_string(),
638            provider: "wikipedia".to_string(),
639            capture_mode: "fetch".to_string(),
640            captured_at: "t".to_string(),
641            results: vec![],
642            diagnostics: SearchDiagnostics {
643                status: 200,
644                blocked_by_cors: false,
645                blocked_by_captcha: false,
646                source_url: "u".to_string(),
647                error: None,
648            },
649        };
650        let json = serde_json::to_string(&result).unwrap();
651        assert!(json.contains("\"captureMode\""));
652        assert!(json.contains("\"capturedAt\""));
653        assert!(json.contains("\"blockedByCaptcha\""));
654        assert!(json.contains("\"sourceUrl\""));
655        assert!(!json.contains("\"error\""));
656    }
657}
web_capture/search.rs

web_capture/
search.rs
