web_search/providers/
duckduckgo.rs

1//! DuckDuckGo search provider
2
3use async_trait::async_trait;
4use scraper::{Html, Selector};
5use std::collections::BTreeMap;
6
7use super::base::{SearchOptions, SearchProvider, SearchResult};
8use crate::error::SearchError;
9use crate::transport::{ReqwestTransport, SearchTransport, TransportRequest};
10
11/// DuckDuckGo search provider
12pub struct DuckDuckGoProvider {
13    name: String,
14    enabled: bool,
15    weight: f64,
16    base_url: String,
17}
18
19impl DuckDuckGoProvider {
20    /// Create a new DuckDuckGo provider
21    pub fn new() -> Self {
22        Self {
23            name: "duckduckgo".to_string(),
24            enabled: true,
25            weight: 1.0,
26            base_url: "https://html.duckduckgo.com/html/".to_string(),
27        }
28    }
29
30    fn parse_results(&self, html: &str, limit: usize) -> Vec<SearchResult> {
31        let document = Html::parse_document(html);
32        let mut results = Vec::new();
33
34        let result_selector =
35            Selector::parse(".result__a").unwrap_or_else(|_| Selector::parse("a").unwrap());
36        let snippet_selector = Selector::parse(".result__snippet")
37            .unwrap_or_else(|_| Selector::parse(".result__body").unwrap());
38
39        let links: Vec<_> = document.select(&result_selector).collect();
40        let snippets: Vec<_> = document.select(&snippet_selector).collect();
41
42        for (i, link) in links.iter().enumerate() {
43            if results.len() >= limit {
44                break;
45            }
46
47            let url = link.value().attr("href").unwrap_or_default();
48            if url.is_empty() || url.starts_with("//duckduckgo.com") || url.contains("ad_provider")
49            {
50                continue;
51            }
52
53            let decoded_url = urlencoding::decode(url).unwrap_or_else(|_| url.into());
54            let title = link.text().collect::<String>().trim().to_string();
55            let snippet = snippets
56                .get(i)
57                .map(|s| s.text().collect::<String>().trim().to_string())
58                .unwrap_or_default();
59
60            results.push(SearchResult {
61                title: if title.is_empty() {
62                    "Untitled".to_string()
63                } else {
64                    title
65                },
66                url: decoded_url.to_string(),
67                snippet,
68                source: self.name.clone(),
69                rank: results.len() + 1,
70                score: None,
71                sources: None,
72            });
73        }
74
75        results
76    }
77}
78
79impl Default for DuckDuckGoProvider {
80    fn default() -> Self {
81        Self::new()
82    }
83}
84
85#[async_trait]
86impl SearchProvider for DuckDuckGoProvider {
87    fn name(&self) -> &str {
88        &self.name
89    }
90
91    fn is_available(&self) -> bool {
92        self.enabled
93    }
94
95    fn weight(&self) -> f64 {
96        self.weight
97    }
98
99    fn set_weight(&mut self, weight: f64) {
100        self.weight = weight.clamp(0.0, 1.0);
101    }
102
103    fn set_enabled(&mut self, enabled: bool) {
104        self.enabled = enabled;
105    }
106
107    async fn search(
108        &self,
109        query: &str,
110        options: &SearchOptions,
111    ) -> Result<Vec<SearchResult>, SearchError> {
112        self.search_with_transport(query, options, &ReqwestTransport::default())
113            .await
114    }
115
116    async fn search_with_transport(
117        &self,
118        query: &str,
119        options: &SearchOptions,
120        transport: &dyn SearchTransport,
121    ) -> Result<Vec<SearchResult>, SearchError> {
122        if query.is_empty() {
123            return Ok(Vec::new());
124        }
125
126        let limit = options.limit.unwrap_or(10);
127        let mut params = vec![("q", query.to_string())];
128
129        if let Some(ref region) = options.region {
130            params.push(("kl", region.clone()));
131        } else {
132            params.push(("kl", "wt-wt".to_string()));
133        }
134
135        if let Some(safe) = options.safe_search {
136            params.push(("kp", if safe { "1" } else { "-2" }.to_string()));
137        }
138
139        let body = params
140            .into_iter()
141            .map(|(name, value)| {
142                format!(
143                    "{}={}",
144                    urlencoding::encode(name),
145                    urlencoding::encode(&value)
146                )
147            })
148            .collect::<Vec<_>>()
149            .join("&");
150        let response = transport
151            .execute(TransportRequest {
152                method: "POST".to_string(),
153                url: self.base_url.clone(),
154                headers: BTreeMap::from([
155                    (
156                        "Content-Type".to_string(),
157                        "application/x-www-form-urlencoded".to_string(),
158                    ),
159                    (
160                        "User-Agent".to_string(),
161                        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36".to_string(),
162                    ),
163                ]),
164                body: Some(body.into_bytes()),
165            })
166            .await?;
167
168        if !(200..300).contains(&response.status) {
169            return Err(SearchError::ApiError {
170                provider: self.name.clone(),
171                message: format!("HTTP {}", response.status),
172            });
173        }
174
175        let html = String::from_utf8_lossy(&response.body);
176        Ok(self.parse_results(&html, limit))
177    }
178}
web_search/providers/duckduckgo.rs

web_search/providers/
duckduckgo.rs
