1use async_trait::async_trait;
2use std::collections::HashMap;
3use std::future::Future;
4use std::pin::Pin;
5use std::sync::Arc;
6use std::time::{Duration, Instant};
7use url::Url;
8
9use crate::types::{SafeSearch, WebSearchResultItem, WebSearchTimeRange};
10
11#[derive(Clone)]
12pub struct WebSearchEngineInput {
13 pub backend_url: String,
14 pub query: String,
15 pub count: usize,
16 pub time_range: WebSearchTimeRange,
17 pub language: String,
18 pub safe_search: SafeSearch,
19 pub categories: Vec<String>,
20 pub timeout_ms: u64,
21 pub headers: HashMap<String, String>,
22 pub check_host: HostCheckFn,
25}
26
27pub type HostCheckFn = Arc<
28 dyn Fn(String) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>>
29 + Send
30 + Sync,
31>;
32
33pub struct WebSearchEngineResult {
34 pub results: Vec<WebSearchResultItem>,
35 pub backend_host: String,
36 pub elapsed_ms: u64,
37 pub engine: Option<String>,
40 pub engine_class: Option<EngineClass>,
42 pub engines: Option<Vec<String>>,
45 pub time_range_applied: Option<bool>,
48}
49
50#[async_trait]
51pub trait WebSearchEngine: Send + Sync {
52 async fn search(
53 &self,
54 input: WebSearchEngineInput,
55 ) -> Result<WebSearchEngineResult, SearchError>;
56
57 fn name(&self) -> &str {
60 "searxng"
61 }
62
63 fn engine_class(&self) -> EngineClass {
66 EngineClass::General
67 }
68}
69
70#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
75#[serde(rename_all = "snake_case")]
76pub enum EngineClass {
77 General,
78 Niche,
79 Vertical,
80}
81
82impl EngineClass {
83 pub fn label(self) -> &'static str {
85 match self {
86 Self::General => "general web",
87 Self::Niche => "indie/small-web index",
88 Self::Vertical => "encyclopedic",
89 }
90 }
91}
92
93#[derive(Debug, Clone, Copy, PartialEq, Eq)]
97pub enum SearchErrorCode {
98 InvalidParam,
99 ServerNotAvailable,
100 DnsError,
101 TlsError,
102 Timeout,
103 ConnectionReset,
104 SsrfBlocked,
105 IoError,
106}
107
108#[derive(Debug, Clone)]
109pub struct SearchError {
110 pub code: SearchErrorCode,
111 pub message: String,
112}
113
114impl SearchError {
115 pub fn new(code: SearchErrorCode, message: impl Into<String>) -> Self {
116 Self {
117 code,
118 message: message.into(),
119 }
120 }
121}
122
123pub struct ReqwestEngine {
124 client: reqwest::Client,
125}
126
127impl ReqwestEngine {
128 pub fn new() -> Self {
129 Self {
130 client: shared_client(),
131 }
132 }
133}
134
135pub(crate) fn shared_client() -> reqwest::Client {
138 reqwest::Client::builder()
139 .redirect(reqwest::redirect::Policy::none())
140 .build()
141 .expect("reqwest client build")
142}
143
144impl Default for ReqwestEngine {
145 fn default() -> Self {
146 Self::new()
147 }
148}
149
150pub fn default_engine() -> Arc<dyn WebSearchEngine> {
151 Arc::new(ReqwestEngine::new())
152}
153
154#[async_trait]
155impl WebSearchEngine for ReqwestEngine {
156 async fn search(
157 &self,
158 input: WebSearchEngineInput,
159 ) -> Result<WebSearchEngineResult, SearchError> {
160 let base = Url::parse(&input.backend_url).map_err(|_| {
161 SearchError::new(
162 SearchErrorCode::IoError,
163 format!("Invalid backend URL: {}", input.backend_url),
164 )
165 })?;
166 let host = base.host_str().unwrap_or("").to_string();
167
168 (input.check_host)(host.clone())
170 .await
171 .map_err(|msg| SearchError::new(SearchErrorCode::SsrfBlocked, msg))?;
172
173 let url = build_search_url(&base, &input);
174 let started = Instant::now();
175
176 let mut req = self
177 .client
178 .request(reqwest::Method::GET, url)
179 .timeout(Duration::from_millis(input.timeout_ms));
180 for (k, v) in &input.headers {
181 req = req.header(k, v);
182 }
183
184 let res = req.send().await.map_err(classify_reqwest_error)?;
185 let status = res.status().as_u16();
186
187 if status >= 400 {
188 if status >= 500 {
189 return Err(SearchError::new(
190 SearchErrorCode::ServerNotAvailable,
191 format!("Search backend returned HTTP {}", status),
192 ));
193 }
194 return Err(SearchError::new(
195 SearchErrorCode::InvalidParam,
196 format!("Search backend rejected the query with HTTP {}", status),
197 ));
198 }
199
200 let bytes = res.bytes().await.map_err(classify_reqwest_error)?;
201 let parsed: serde_json::Value = serde_json::from_slice(&bytes).map_err(|e| {
202 SearchError::new(
203 SearchErrorCode::IoError,
204 format!(
205 "Could not parse the search backend response as JSON: {}",
206 e
207 ),
208 )
209 })?;
210
211 let results = map_results(&parsed);
212 Ok(WebSearchEngineResult {
213 results,
214 backend_host: host,
215 elapsed_ms: started.elapsed().as_millis() as u64,
216 engine: Some("searxng".to_string()),
217 engine_class: None,
218 engines: None,
219 time_range_applied: if input.time_range == WebSearchTimeRange::All {
221 None
222 } else {
223 Some(true)
224 },
225 })
226 }
227}
228
229fn build_search_url(base: &Url, input: &WebSearchEngineInput) -> Url {
230 let mut url = base.clone();
232 let trimmed = url.path().trim_end_matches('/').to_string();
233 url.set_path(&format!("{}/search", trimmed));
234 {
235 let mut p = url.query_pairs_mut();
236 p.append_pair("q", &input.query);
237 p.append_pair("format", "json");
238 p.append_pair(
239 "safesearch",
240 &input.safe_search.to_numeric().to_string(),
241 );
242 if input.time_range != WebSearchTimeRange::All {
244 p.append_pair("time_range", input.time_range.as_str());
245 }
246 p.append_pair("language", &input.language);
247 p.append_pair("categories", &input.categories.join(","));
248 p.append_pair("pageno", "1");
249 }
250 url
251}
252
253fn map_results(parsed: &serde_json::Value) -> Vec<WebSearchResultItem> {
254 let raw = match parsed.get("results").and_then(|v| v.as_array()) {
255 Some(arr) => arr,
256 None => return Vec::new(),
257 };
258 let mut out: Vec<WebSearchResultItem> = Vec::new();
259 for entry in raw {
260 if !entry.is_object() {
261 continue;
262 }
263 let title = entry.get("title").and_then(|v| v.as_str()).unwrap_or("");
264 let url = entry.get("url").and_then(|v| v.as_str()).unwrap_or("");
265 if title.is_empty() || url.is_empty() {
267 continue;
268 }
269 let snippet = entry.get("content").and_then(|v| v.as_str()).unwrap_or("");
270 out.push(WebSearchResultItem {
271 title: title.to_string(),
272 url: url.to_string(),
273 snippet: snippet.to_string(),
274 age: None,
275 score: None,
276 source: None,
277 });
278 }
279 out
280}
281
282pub(crate) fn classify_reqwest_error(e: reqwest::Error) -> SearchError {
283 let msg = e.to_string();
284 let chain = error_chain(&e);
288 if e.is_timeout() {
289 return SearchError::new(SearchErrorCode::Timeout, msg);
290 }
291 if e.is_connect() {
292 let lower = chain.to_lowercase();
293 if lower.contains("dns")
294 || lower.contains("resolve")
295 || lower.contains("lookup")
296 || lower.contains("not known")
297 || lower.contains("no such host")
298 {
299 return SearchError::new(SearchErrorCode::DnsError, msg);
300 }
301 return SearchError::new(
305 SearchErrorCode::ServerNotAvailable,
306 format!("connection refused: {}", chain),
307 );
308 }
309 let lower = chain.to_lowercase();
310 if lower.contains("tls") || lower.contains("certificate") || lower.contains("ssl") {
311 return SearchError::new(SearchErrorCode::TlsError, msg);
312 }
313 if lower.contains("reset") {
314 return SearchError::new(SearchErrorCode::ConnectionReset, msg);
315 }
316 SearchError::new(SearchErrorCode::IoError, msg)
317}
318
319pub(crate) fn error_chain(e: &reqwest::Error) -> String {
321 let mut parts = vec![e.to_string()];
322 let mut src = std::error::Error::source(e);
323 while let Some(s) = src {
324 parts.push(s.to_string());
325 src = s.source();
326 }
327 parts.join("; ")
328}