1use std::collections::HashMap;
2use std::sync::{Mutex, OnceLock};
3
4use serde::{Deserialize, Serialize};
5
6use crate::internal::tool_runtime::web_html::{extract_ddg_results, SearchItem};
7
8fn http_client(timeout_secs: u64) -> reqwest::Client {
9 static CLIENTS: OnceLock<Mutex<HashMap<u64, reqwest::Client>>> = OnceLock::new();
10 let cache = CLIENTS.get_or_init(|| Mutex::new(HashMap::new()));
11 let mut cache = cache.lock().unwrap_or_else(|e| e.into_inner());
12 if let Some(client) = cache.get(&timeout_secs) {
13 return client.clone();
14 }
15 let client = reqwest::Client::builder()
16 .timeout(std::time::Duration::from_secs(timeout_secs))
17 .redirect(reqwest::redirect::Policy::limited(5))
18 .user_agent("MicroClaw/1.0")
19 .build()
20 .expect("failed to build HTTP client");
21 cache.insert(timeout_secs, client.clone());
22 client
23}
24
25#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
29#[serde(rename_all = "lowercase")]
30pub enum SearchBackend {
31 #[default]
32 Duckduckgo,
33 Searxng,
36 Brave,
38 Tavily,
40 Keenable,
43}
44
45fn default_search_max_results() -> usize {
46 8
47}
48
49#[derive(Debug, Clone, Serialize, Deserialize)]
53pub struct SearchProviderConfig {
54 #[serde(default)]
55 pub backend: SearchBackend,
56 #[serde(default)]
58 pub searxng_base_url: String,
59 #[serde(default)]
60 pub brave_api_key: String,
61 #[serde(default)]
62 pub tavily_api_key: String,
63 #[serde(default)]
65 pub keenable_api_key: String,
66 #[serde(default = "default_search_max_results")]
67 pub max_results: usize,
68}
69
70impl Default for SearchProviderConfig {
71 fn default() -> Self {
72 Self {
73 backend: SearchBackend::default(),
74 searxng_base_url: String::new(),
75 brave_api_key: String::new(),
76 tavily_api_key: String::new(),
77 keenable_api_key: String::new(),
78 max_results: default_search_max_results(),
79 }
80 }
81}
82
83impl SearchProviderConfig {
84 pub fn normalize(&mut self) {
85 self.searxng_base_url = self
86 .searxng_base_url
87 .trim()
88 .trim_end_matches('/')
89 .to_string();
90 self.brave_api_key = self.brave_api_key.trim().to_string();
91 self.tavily_api_key = self.tavily_api_key.trim().to_string();
92 self.keenable_api_key = self.keenable_api_key.trim().to_string();
93 if self.max_results == 0 {
94 self.max_results = default_search_max_results();
95 }
96 self.max_results = self.max_results.min(20);
97 }
98
99 pub fn effective_backend(&self) -> SearchBackend {
103 match self.backend {
104 SearchBackend::Searxng if self.searxng_base_url.trim().is_empty() => {
105 SearchBackend::Duckduckgo
106 }
107 SearchBackend::Brave if self.brave_api_key.trim().is_empty() => {
108 SearchBackend::Duckduckgo
109 }
110 SearchBackend::Tavily if self.tavily_api_key.trim().is_empty() => {
111 SearchBackend::Duckduckgo
112 }
113 other => other,
114 }
115 }
116}
117
118pub fn format_search_hits(hits: &[SearchItem]) -> String {
120 let mut output = String::new();
121 for (i, item) in hits.iter().enumerate() {
122 output.push_str(&format!(
123 "{}. {}\n {}\n {}\n\n",
124 i + 1,
125 item.title,
126 item.url,
127 item.snippet
128 ));
129 }
130 output
131}
132
133pub async fn search_with_provider(
136 query: &str,
137 config: &SearchProviderConfig,
138 timeout_secs: u64,
139) -> Result<Vec<SearchItem>, String> {
140 let max = config.max_results.clamp(1, 20);
141 let t = timeout_secs.max(1);
142 match config.effective_backend() {
143 SearchBackend::Duckduckgo => ddg_hits(query, t, max).await,
144 SearchBackend::Searxng => searxng_hits(query, &config.searxng_base_url, t, max).await,
145 SearchBackend::Brave => brave_hits(query, &config.brave_api_key, t, max).await,
146 SearchBackend::Tavily => tavily_hits(query, &config.tavily_api_key, t, max).await,
147 SearchBackend::Keenable => keenable_hits(query, &config.keenable_api_key, t, max).await,
148 }
149}
150
151async fn ddg_hits(query: &str, timeout_secs: u64, max: usize) -> Result<Vec<SearchItem>, String> {
152 let encoded = urlencoding::encode(query);
153 let url = format!("https://html.duckduckgo.com/html/?q={encoded}");
154 let client = http_client(timeout_secs);
155 let resp = client.get(&url).send().await.map_err(|e| e.to_string())?;
156 if !resp.status().is_success() {
157 return Err(format!("HTTP {}", resp.status()));
158 }
159 let body = resp.text().await.map_err(|e| e.to_string())?;
160 Ok(extract_ddg_results(&body, max))
161}
162
163#[derive(Deserialize)]
164struct SearxResponse {
165 #[serde(default)]
166 results: Vec<SearxResult>,
167}
168
169#[derive(Deserialize)]
170struct SearxResult {
171 #[serde(default)]
172 title: String,
173 #[serde(default)]
174 url: String,
175 #[serde(default)]
176 content: String,
177}
178
179async fn searxng_hits(
180 query: &str,
181 base_url: &str,
182 timeout_secs: u64,
183 max: usize,
184) -> Result<Vec<SearchItem>, String> {
185 let encoded = urlencoding::encode(query);
186 let url = format!("{base_url}/search?q={encoded}&format=json");
187 let client = http_client(timeout_secs);
188 let resp = client.get(&url).send().await.map_err(|e| e.to_string())?;
189 if !resp.status().is_success() {
190 return Err(format!("HTTP {}", resp.status()));
191 }
192 let parsed: SearxResponse = resp.json().await.map_err(|e| e.to_string())?;
193 Ok(parsed
194 .results
195 .into_iter()
196 .filter(|r| !r.url.is_empty())
197 .take(max)
198 .map(|r| SearchItem {
199 title: r.title,
200 url: r.url,
201 snippet: r.content,
202 })
203 .collect())
204}
205
206#[derive(Deserialize)]
207struct BraveResponse {
208 #[serde(default)]
209 web: Option<BraveWeb>,
210}
211
212#[derive(Deserialize)]
213struct BraveWeb {
214 #[serde(default)]
215 results: Vec<BraveResult>,
216}
217
218#[derive(Deserialize)]
219struct BraveResult {
220 #[serde(default)]
221 title: String,
222 #[serde(default)]
223 url: String,
224 #[serde(default)]
225 description: String,
226}
227
228async fn brave_hits(
229 query: &str,
230 api_key: &str,
231 timeout_secs: u64,
232 max: usize,
233) -> Result<Vec<SearchItem>, String> {
234 let encoded = urlencoding::encode(query);
235 let url = format!("https://api.search.brave.com/res/v1/web/search?q={encoded}&count={max}");
236 let client = http_client(timeout_secs);
237 let resp = client
238 .get(&url)
239 .header("Accept", "application/json")
240 .header("X-Subscription-Token", api_key)
241 .send()
242 .await
243 .map_err(|e| e.to_string())?;
244 if !resp.status().is_success() {
245 return Err(format!("HTTP {}", resp.status()));
246 }
247 let parsed: BraveResponse = resp.json().await.map_err(|e| e.to_string())?;
248 Ok(parsed
249 .web
250 .map(|w| w.results)
251 .unwrap_or_default()
252 .into_iter()
253 .filter(|r| !r.url.is_empty())
254 .take(max)
255 .map(|r| SearchItem {
256 title: r.title,
257 url: r.url,
258 snippet: r.description,
259 })
260 .collect())
261}
262
263#[derive(Serialize)]
264struct TavilyRequest<'a> {
265 api_key: &'a str,
266 query: &'a str,
267 max_results: usize,
268}
269
270#[derive(Deserialize)]
271struct TavilyResponse {
272 #[serde(default)]
273 results: Vec<TavilyResult>,
274}
275
276#[derive(Deserialize)]
277struct TavilyResult {
278 #[serde(default)]
279 title: String,
280 #[serde(default)]
281 url: String,
282 #[serde(default)]
283 content: String,
284}
285
286async fn tavily_hits(
287 query: &str,
288 api_key: &str,
289 timeout_secs: u64,
290 max: usize,
291) -> Result<Vec<SearchItem>, String> {
292 let client = http_client(timeout_secs);
293 let resp = client
294 .post("https://api.tavily.com/search")
295 .json(&TavilyRequest {
296 api_key,
297 query,
298 max_results: max,
299 })
300 .send()
301 .await
302 .map_err(|e| e.to_string())?;
303 if !resp.status().is_success() {
304 return Err(format!("HTTP {}", resp.status()));
305 }
306 let parsed: TavilyResponse = resp.json().await.map_err(|e| e.to_string())?;
307 Ok(parsed
308 .results
309 .into_iter()
310 .filter(|r| !r.url.is_empty())
311 .take(max)
312 .map(|r| SearchItem {
313 title: r.title,
314 url: r.url,
315 snippet: r.content,
316 })
317 .collect())
318}
319
320#[derive(Serialize)]
321struct KeenableRequest<'a> {
322 query: &'a str,
323 max_results: usize,
324}
325
326#[derive(Deserialize)]
327struct KeenableResponse {
328 #[serde(default)]
329 results: Vec<KeenableResult>,
330}
331
332#[derive(Deserialize)]
333struct KeenableResult {
334 #[serde(default)]
335 title: String,
336 #[serde(default)]
337 url: String,
338 #[serde(default)]
339 snippet: String,
340 #[serde(default)]
341 description: String,
342}
343
344fn keenable_request(
345 client: &reqwest::Client,
346 query: &str,
347 api_key: &str,
348 max: usize,
349) -> reqwest::RequestBuilder {
350 let api_key = api_key.trim();
351 let builder = if api_key.is_empty() {
352 client
353 .post("https://api.keenable.ai/v1/search/public")
354 .header("X-Keenable-Title", "MicroClaw")
355 } else {
356 client
357 .post("https://api.keenable.ai/v1/search")
358 .header("X-API-Key", api_key)
359 };
360 builder.json(&KeenableRequest {
361 query,
362 max_results: max,
363 })
364}
365
366async fn keenable_hits(
367 query: &str,
368 api_key: &str,
369 timeout_secs: u64,
370 max: usize,
371) -> Result<Vec<SearchItem>, String> {
372 let client = http_client(timeout_secs);
373 let resp = keenable_request(&client, query, api_key, max)
374 .send()
375 .await
376 .map_err(|e| e.to_string())?;
377 if !resp.status().is_success() {
378 return Err(format!("HTTP {}", resp.status()));
379 }
380 let parsed: KeenableResponse = resp.json().await.map_err(|e| e.to_string())?;
381 Ok(parsed
382 .results
383 .into_iter()
384 .filter(|r| !r.url.is_empty())
385 .take(max)
386 .map(|r| SearchItem {
387 title: r.title,
388 url: r.url,
389 snippet: if r.snippet.trim().is_empty() {
390 r.description
391 } else {
392 r.snippet
393 },
394 })
395 .collect())
396}
397
398pub async fn search_ddg_with_timeout(query: &str, timeout_secs: u64) -> Result<String, String> {
399 let hits = ddg_hits(query, timeout_secs.max(1), 8).await?;
400 Ok(format_search_hits(&hits))
401}
402
403pub async fn search_ddg(query: &str) -> Result<String, String> {
404 search_ddg_with_timeout(query, 15).await
405}
406
407#[cfg(test)]
408mod tests {
409 use super::*;
410
411 #[test]
412 fn default_backend_is_duckduckgo() {
413 let cfg = SearchProviderConfig::default();
414 assert_eq!(cfg.backend, SearchBackend::Duckduckgo);
415 assert_eq!(cfg.effective_backend(), SearchBackend::Duckduckgo);
416 assert_eq!(cfg.max_results, 8);
417 }
418
419 #[test]
420 fn effective_backend_downgrades_when_unconfigured() {
421 let cfg = SearchProviderConfig {
422 backend: SearchBackend::Searxng,
423 ..Default::default()
424 };
425 assert_eq!(cfg.effective_backend(), SearchBackend::Duckduckgo);
426
427 let cfg = SearchProviderConfig {
428 backend: SearchBackend::Brave,
429 ..Default::default()
430 };
431 assert_eq!(cfg.effective_backend(), SearchBackend::Duckduckgo);
432
433 let cfg = SearchProviderConfig {
434 backend: SearchBackend::Tavily,
435 tavily_api_key: "tvly-xxx".into(),
436 ..Default::default()
437 };
438 assert_eq!(cfg.effective_backend(), SearchBackend::Tavily);
439 }
440
441 #[test]
442 fn keenable_works_without_a_key() {
443 let cfg = SearchProviderConfig {
444 backend: SearchBackend::Keenable,
445 ..Default::default()
446 };
447 assert_eq!(cfg.effective_backend(), SearchBackend::Keenable);
448
449 let client = reqwest::Client::new();
450 let req = keenable_request(&client, "rust", "", 5).build().unwrap();
451 assert_eq!(
452 req.url().as_str(),
453 "https://api.keenable.ai/v1/search/public"
454 );
455 assert_eq!(req.headers().get("X-Keenable-Title").unwrap(), "MicroClaw");
456
457 let req = keenable_request(&client, "rust", " kn-test ", 5)
458 .build()
459 .unwrap();
460 assert_eq!(req.url().as_str(), "https://api.keenable.ai/v1/search");
461 assert_eq!(req.headers().get("X-API-Key").unwrap(), "kn-test");
462 }
463
464 #[test]
465 fn normalize_trims_and_bounds() {
466 let mut cfg = SearchProviderConfig {
467 backend: SearchBackend::Searxng,
468 searxng_base_url: " http://localhost:8888/ ".into(),
469 brave_api_key: " k ".into(),
470 max_results: 0,
471 ..Default::default()
472 };
473 cfg.normalize();
474 assert_eq!(cfg.searxng_base_url, "http://localhost:8888");
475 assert_eq!(cfg.brave_api_key, "k");
476 assert_eq!(cfg.max_results, 8);
477 assert_eq!(cfg.effective_backend(), SearchBackend::Searxng);
478 }
479
480 #[test]
481 fn format_hits_numbers_results() {
482 let hits = vec![
483 SearchItem {
484 title: "A".into(),
485 url: "https://a.example".into(),
486 snippet: "sa".into(),
487 },
488 SearchItem {
489 title: "B".into(),
490 url: "https://b.example".into(),
491 snippet: "sb".into(),
492 },
493 ];
494 let out = format_search_hits(&hits);
495 assert!(out.contains("1. A"));
496 assert!(out.contains("2. B"));
497 assert!(out.contains("https://a.example"));
498 }
499}