web_search/providers/
web_capture.rs

1//! web-capture component-library provider.
2//!
3//! Issue #3 (R3/R4) asks `web-search` to use `link-assistant/web-capture` as a
4//! component library so this project can focus on search aggregation rather than
5//! re-implementing per-provider scraping. Mirrors the JavaScript
6//! `src/providers/web-capture.js`.
7//!
8//! The Rust provider delegates to the published `web-capture` crate and keeps
9//! the same graceful empty-result behavior the JavaScript provider uses for
10//! component errors.
11
12use async_trait::async_trait;
13use std::collections::BTreeMap;
14
15use super::base::{SearchOptions, SearchProvider, SearchResult};
16use crate::error::SearchError;
17use crate::transport::{ReqwestTransport, SearchTransport, TransportRequest};
18
19/// Providers exposed by web-capture's search contract.
20pub const SUPPORTED_PROVIDERS: [&str; 5] = web_capture::SEARCH_PROVIDERS;
21
22/// Provider that delegates to the web-capture component library.
23pub struct WebCaptureProvider {
24    name: String,
25    engine: String,
26    enabled: bool,
27    weight: f64,
28}
29
30impl WebCaptureProvider {
31    /// Create a provider bound to a web-capture engine (default `wikipedia`).
32    pub fn new(engine: impl Into<String>) -> Self {
33        let engine = engine.into();
34        Self {
35            name: format!("wc:{engine}"),
36            engine,
37            enabled: true,
38            weight: 1.0,
39        }
40    }
41
42    /// The web-capture engine this provider delegates to.
43    pub fn engine(&self) -> &str {
44        &self.engine
45    }
46
47    /// Adapt normalized web-capture items into the web-search result contract.
48    pub fn adapt_items(&self, items: Vec<web_capture::SearchResultItem>) -> Vec<SearchResult> {
49        items
50            .into_iter()
51            .enumerate()
52            .filter_map(|(index, item)| {
53                if item.url.trim().is_empty() {
54                    return None;
55                }
56
57                Some(SearchResult {
58                    title: if item.title.trim().is_empty() {
59                        "Untitled".to_string()
60                    } else {
61                        item.title
62                    },
63                    url: item.url,
64                    snippet: item.snippet,
65                    source: self.name.clone(),
66                    rank: if item.rank == 0 { index + 1 } else { item.rank },
67                    score: None,
68                    sources: None,
69                })
70            })
71            .collect()
72    }
73}
74
75impl Default for WebCaptureProvider {
76    fn default() -> Self {
77        Self::new("wikipedia")
78    }
79}
80
81#[async_trait]
82impl SearchProvider for WebCaptureProvider {
83    fn name(&self) -> &str {
84        &self.name
85    }
86
87    fn is_available(&self) -> bool {
88        self.enabled
89    }
90
91    fn weight(&self) -> f64 {
92        self.weight
93    }
94
95    fn set_weight(&mut self, weight: f64) {
96        self.weight = weight.clamp(0.0, 1.0);
97    }
98
99    fn set_enabled(&mut self, enabled: bool) {
100        self.enabled = enabled;
101    }
102
103    async fn search(
104        &self,
105        query: &str,
106        options: &SearchOptions,
107    ) -> Result<Vec<SearchResult>, SearchError> {
108        match self
109            .search_with_transport(query, options, &ReqwestTransport::default())
110            .await
111        {
112            Ok(results) => Ok(results),
113            Err(error) => {
114                tracing::warn!(
115                    provider = self.name(),
116                    error = %error,
117                    "WebCaptureProvider returned no results"
118                );
119                Ok(Vec::new())
120            }
121        }
122    }
123
124    async fn search_with_transport(
125        &self,
126        query: &str,
127        options: &SearchOptions,
128        transport: &dyn SearchTransport,
129    ) -> Result<Vec<SearchResult>, SearchError> {
130        if query.trim().is_empty() {
131            return Ok(Vec::new());
132        }
133
134        let limit = options.limit.unwrap_or(web_capture::DEFAULT_LIMIT);
135
136        let url = web_capture::search::build_search_url(&self.engine, query, limit).map_err(
137            |message| SearchError::ApiError {
138                provider: self.name.clone(),
139                message,
140            },
141        )?;
142        let response = transport
143            .execute(TransportRequest {
144                method: "GET".to_string(),
145                url,
146                headers: BTreeMap::from([(
147                    "User-Agent".to_string(),
148                    "Mozilla/5.0 (compatible; web-search/0.3)".to_string(),
149                )]),
150                body: None,
151            })
152            .await?;
153        if !(200..300).contains(&response.status) {
154            return Err(SearchError::ApiError {
155                provider: self.name.clone(),
156                message: format!("HTTP {}", response.status),
157            });
158        }
159        let body = String::from_utf8_lossy(&response.body);
160        let (items, blocked) =
161            web_capture::search::parse_search_results(&self.engine, &body, limit);
162        if blocked {
163            return Err(SearchError::ApiError {
164                provider: self.name.clone(),
165                message: "provider returned a CAPTCHA page".to_string(),
166            });
167        }
168        Ok(self.adapt_items(items))
169    }
170}
web_search/providers/web_capture.rs

web_search/providers/
web_capture.rs
