Skip to main content

microclaw_engine/internal/tool_runtime/
web_search.rs

1use std::collections::HashMap;
2use std::sync::{Mutex, OnceLock};
3
4use serde::{Deserialize, Serialize};
5
6use crate::internal::tool_runtime::web_html::{extract_ddg_results, SearchItem};
7
8fn http_client(timeout_secs: u64) -> reqwest::Client {
9    static CLIENTS: OnceLock<Mutex<HashMap<u64, reqwest::Client>>> = OnceLock::new();
10    let cache = CLIENTS.get_or_init(|| Mutex::new(HashMap::new()));
11    let mut cache = cache.lock().unwrap_or_else(|e| e.into_inner());
12    if let Some(client) = cache.get(&timeout_secs) {
13        return client.clone();
14    }
15    let client = reqwest::Client::builder()
16        .timeout(std::time::Duration::from_secs(timeout_secs))
17        .redirect(reqwest::redirect::Policy::limited(5))
18        .user_agent("MicroClaw/1.0")
19        .build()
20        .expect("failed to build HTTP client");
21    cache.insert(timeout_secs, client.clone());
22    client
23}
24
25/// Selectable web-search backend. DuckDuckGo (HTML scrape, no key) is the
26/// default and the fallback whenever a richer backend is selected but missing
27/// its credentials/endpoint.
28#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
29#[serde(rename_all = "lowercase")]
30pub enum SearchBackend {
31    #[default]
32    Duckduckgo,
33    /// Self-hosted SearXNG instance (`{base}/search?format=json`). Ideal for a
34    /// $5 VPS: no API key, no rate limit, fully local.
35    Searxng,
36    /// Brave Search API (`X-Subscription-Token`).
37    Brave,
38    /// Tavily research API (`POST /search`).
39    Tavily,
40    /// Keenable web search (`POST /v1/search`). Works without an API key on
41    /// the rate-limited public endpoint; `keenable_api_key` raises limits.
42    Keenable,
43}
44
45fn default_search_max_results() -> usize {
46    8
47}
48
49/// Configuration for the pluggable web-search provider. Defaults preserve the
50/// historical behavior exactly (DuckDuckGo, 8 results), so existing
51/// deployments see no change until they opt into a backend.
52#[derive(Debug, Clone, Serialize, Deserialize)]
53pub struct SearchProviderConfig {
54    #[serde(default)]
55    pub backend: SearchBackend,
56    /// Base URL of a SearXNG instance, e.g. `http://127.0.0.1:8888`.
57    #[serde(default)]
58    pub searxng_base_url: String,
59    #[serde(default)]
60    pub brave_api_key: String,
61    #[serde(default)]
62    pub tavily_api_key: String,
63    /// Optional: Keenable works key-free; a key switches to the keyed endpoint.
64    #[serde(default)]
65    pub keenable_api_key: String,
66    #[serde(default = "default_search_max_results")]
67    pub max_results: usize,
68}
69
70impl Default for SearchProviderConfig {
71    fn default() -> Self {
72        Self {
73            backend: SearchBackend::default(),
74            searxng_base_url: String::new(),
75            brave_api_key: String::new(),
76            tavily_api_key: String::new(),
77            keenable_api_key: String::new(),
78            max_results: default_search_max_results(),
79        }
80    }
81}
82
83impl SearchProviderConfig {
84    pub fn normalize(&mut self) {
85        self.searxng_base_url = self
86            .searxng_base_url
87            .trim()
88            .trim_end_matches('/')
89            .to_string();
90        self.brave_api_key = self.brave_api_key.trim().to_string();
91        self.tavily_api_key = self.tavily_api_key.trim().to_string();
92        self.keenable_api_key = self.keenable_api_key.trim().to_string();
93        if self.max_results == 0 {
94            self.max_results = default_search_max_results();
95        }
96        self.max_results = self.max_results.min(20);
97    }
98
99    /// The backend that will actually be used, downgrading to DuckDuckGo when
100    /// the selected backend lacks the credentials/endpoint it needs. This keeps
101    /// search working (degraded) instead of hard-failing on a misconfiguration.
102    pub fn effective_backend(&self) -> SearchBackend {
103        match self.backend {
104            SearchBackend::Searxng if self.searxng_base_url.trim().is_empty() => {
105                SearchBackend::Duckduckgo
106            }
107            SearchBackend::Brave if self.brave_api_key.trim().is_empty() => {
108                SearchBackend::Duckduckgo
109            }
110            SearchBackend::Tavily if self.tavily_api_key.trim().is_empty() => {
111                SearchBackend::Duckduckgo
112            }
113            other => other,
114        }
115    }
116}
117
118/// Render structured hits into the numbered text block the agent consumes.
119pub fn format_search_hits(hits: &[SearchItem]) -> String {
120    let mut output = String::new();
121    for (i, item) in hits.iter().enumerate() {
122        output.push_str(&format!(
123            "{}. {}\n   {}\n   {}\n\n",
124            i + 1,
125            item.title,
126            item.url,
127            item.snippet
128        ));
129    }
130    output
131}
132
133/// Run a search via the configured provider and return structured hits,
134/// transparently falling back to DuckDuckGo when the provider is misconfigured.
135pub async fn search_with_provider(
136    query: &str,
137    config: &SearchProviderConfig,
138    timeout_secs: u64,
139) -> Result<Vec<SearchItem>, String> {
140    let max = config.max_results.clamp(1, 20);
141    let t = timeout_secs.max(1);
142    match config.effective_backend() {
143        SearchBackend::Duckduckgo => ddg_hits(query, t, max).await,
144        SearchBackend::Searxng => searxng_hits(query, &config.searxng_base_url, t, max).await,
145        SearchBackend::Brave => brave_hits(query, &config.brave_api_key, t, max).await,
146        SearchBackend::Tavily => tavily_hits(query, &config.tavily_api_key, t, max).await,
147        SearchBackend::Keenable => keenable_hits(query, &config.keenable_api_key, t, max).await,
148    }
149}
150
151async fn ddg_hits(query: &str, timeout_secs: u64, max: usize) -> Result<Vec<SearchItem>, String> {
152    let encoded = urlencoding::encode(query);
153    let url = format!("https://html.duckduckgo.com/html/?q={encoded}");
154    let client = http_client(timeout_secs);
155    let resp = client.get(&url).send().await.map_err(|e| e.to_string())?;
156    if !resp.status().is_success() {
157        return Err(format!("HTTP {}", resp.status()));
158    }
159    let body = resp.text().await.map_err(|e| e.to_string())?;
160    Ok(extract_ddg_results(&body, max))
161}
162
163#[derive(Deserialize)]
164struct SearxResponse {
165    #[serde(default)]
166    results: Vec<SearxResult>,
167}
168
169#[derive(Deserialize)]
170struct SearxResult {
171    #[serde(default)]
172    title: String,
173    #[serde(default)]
174    url: String,
175    #[serde(default)]
176    content: String,
177}
178
179async fn searxng_hits(
180    query: &str,
181    base_url: &str,
182    timeout_secs: u64,
183    max: usize,
184) -> Result<Vec<SearchItem>, String> {
185    let encoded = urlencoding::encode(query);
186    let url = format!("{base_url}/search?q={encoded}&format=json");
187    let client = http_client(timeout_secs);
188    let resp = client.get(&url).send().await.map_err(|e| e.to_string())?;
189    if !resp.status().is_success() {
190        return Err(format!("HTTP {}", resp.status()));
191    }
192    let parsed: SearxResponse = resp.json().await.map_err(|e| e.to_string())?;
193    Ok(parsed
194        .results
195        .into_iter()
196        .filter(|r| !r.url.is_empty())
197        .take(max)
198        .map(|r| SearchItem {
199            title: r.title,
200            url: r.url,
201            snippet: r.content,
202        })
203        .collect())
204}
205
206#[derive(Deserialize)]
207struct BraveResponse {
208    #[serde(default)]
209    web: Option<BraveWeb>,
210}
211
212#[derive(Deserialize)]
213struct BraveWeb {
214    #[serde(default)]
215    results: Vec<BraveResult>,
216}
217
218#[derive(Deserialize)]
219struct BraveResult {
220    #[serde(default)]
221    title: String,
222    #[serde(default)]
223    url: String,
224    #[serde(default)]
225    description: String,
226}
227
228async fn brave_hits(
229    query: &str,
230    api_key: &str,
231    timeout_secs: u64,
232    max: usize,
233) -> Result<Vec<SearchItem>, String> {
234    let encoded = urlencoding::encode(query);
235    let url = format!("https://api.search.brave.com/res/v1/web/search?q={encoded}&count={max}");
236    let client = http_client(timeout_secs);
237    let resp = client
238        .get(&url)
239        .header("Accept", "application/json")
240        .header("X-Subscription-Token", api_key)
241        .send()
242        .await
243        .map_err(|e| e.to_string())?;
244    if !resp.status().is_success() {
245        return Err(format!("HTTP {}", resp.status()));
246    }
247    let parsed: BraveResponse = resp.json().await.map_err(|e| e.to_string())?;
248    Ok(parsed
249        .web
250        .map(|w| w.results)
251        .unwrap_or_default()
252        .into_iter()
253        .filter(|r| !r.url.is_empty())
254        .take(max)
255        .map(|r| SearchItem {
256            title: r.title,
257            url: r.url,
258            snippet: r.description,
259        })
260        .collect())
261}
262
263#[derive(Serialize)]
264struct TavilyRequest<'a> {
265    api_key: &'a str,
266    query: &'a str,
267    max_results: usize,
268}
269
270#[derive(Deserialize)]
271struct TavilyResponse {
272    #[serde(default)]
273    results: Vec<TavilyResult>,
274}
275
276#[derive(Deserialize)]
277struct TavilyResult {
278    #[serde(default)]
279    title: String,
280    #[serde(default)]
281    url: String,
282    #[serde(default)]
283    content: String,
284}
285
286async fn tavily_hits(
287    query: &str,
288    api_key: &str,
289    timeout_secs: u64,
290    max: usize,
291) -> Result<Vec<SearchItem>, String> {
292    let client = http_client(timeout_secs);
293    let resp = client
294        .post("https://api.tavily.com/search")
295        .json(&TavilyRequest {
296            api_key,
297            query,
298            max_results: max,
299        })
300        .send()
301        .await
302        .map_err(|e| e.to_string())?;
303    if !resp.status().is_success() {
304        return Err(format!("HTTP {}", resp.status()));
305    }
306    let parsed: TavilyResponse = resp.json().await.map_err(|e| e.to_string())?;
307    Ok(parsed
308        .results
309        .into_iter()
310        .filter(|r| !r.url.is_empty())
311        .take(max)
312        .map(|r| SearchItem {
313            title: r.title,
314            url: r.url,
315            snippet: r.content,
316        })
317        .collect())
318}
319
320#[derive(Serialize)]
321struct KeenableRequest<'a> {
322    query: &'a str,
323    max_results: usize,
324}
325
326#[derive(Deserialize)]
327struct KeenableResponse {
328    #[serde(default)]
329    results: Vec<KeenableResult>,
330}
331
332#[derive(Deserialize)]
333struct KeenableResult {
334    #[serde(default)]
335    title: String,
336    #[serde(default)]
337    url: String,
338    #[serde(default)]
339    snippet: String,
340    #[serde(default)]
341    description: String,
342}
343
344fn keenable_request(
345    client: &reqwest::Client,
346    query: &str,
347    api_key: &str,
348    max: usize,
349) -> reqwest::RequestBuilder {
350    let api_key = api_key.trim();
351    let builder = if api_key.is_empty() {
352        client
353            .post("https://api.keenable.ai/v1/search/public")
354            .header("X-Keenable-Title", "MicroClaw")
355    } else {
356        client
357            .post("https://api.keenable.ai/v1/search")
358            .header("X-API-Key", api_key)
359    };
360    builder.json(&KeenableRequest {
361        query,
362        max_results: max,
363    })
364}
365
366async fn keenable_hits(
367    query: &str,
368    api_key: &str,
369    timeout_secs: u64,
370    max: usize,
371) -> Result<Vec<SearchItem>, String> {
372    let client = http_client(timeout_secs);
373    let resp = keenable_request(&client, query, api_key, max)
374        .send()
375        .await
376        .map_err(|e| e.to_string())?;
377    if !resp.status().is_success() {
378        return Err(format!("HTTP {}", resp.status()));
379    }
380    let parsed: KeenableResponse = resp.json().await.map_err(|e| e.to_string())?;
381    Ok(parsed
382        .results
383        .into_iter()
384        .filter(|r| !r.url.is_empty())
385        .take(max)
386        .map(|r| SearchItem {
387            title: r.title,
388            url: r.url,
389            snippet: if r.snippet.trim().is_empty() {
390                r.description
391            } else {
392                r.snippet
393            },
394        })
395        .collect())
396}
397
398pub async fn search_ddg_with_timeout(query: &str, timeout_secs: u64) -> Result<String, String> {
399    let hits = ddg_hits(query, timeout_secs.max(1), 8).await?;
400    Ok(format_search_hits(&hits))
401}
402
403pub async fn search_ddg(query: &str) -> Result<String, String> {
404    search_ddg_with_timeout(query, 15).await
405}
406
407#[cfg(test)]
408mod tests {
409    use super::*;
410
411    #[test]
412    fn default_backend_is_duckduckgo() {
413        let cfg = SearchProviderConfig::default();
414        assert_eq!(cfg.backend, SearchBackend::Duckduckgo);
415        assert_eq!(cfg.effective_backend(), SearchBackend::Duckduckgo);
416        assert_eq!(cfg.max_results, 8);
417    }
418
419    #[test]
420    fn effective_backend_downgrades_when_unconfigured() {
421        let cfg = SearchProviderConfig {
422            backend: SearchBackend::Searxng,
423            ..Default::default()
424        };
425        assert_eq!(cfg.effective_backend(), SearchBackend::Duckduckgo);
426
427        let cfg = SearchProviderConfig {
428            backend: SearchBackend::Brave,
429            ..Default::default()
430        };
431        assert_eq!(cfg.effective_backend(), SearchBackend::Duckduckgo);
432
433        let cfg = SearchProviderConfig {
434            backend: SearchBackend::Tavily,
435            tavily_api_key: "tvly-xxx".into(),
436            ..Default::default()
437        };
438        assert_eq!(cfg.effective_backend(), SearchBackend::Tavily);
439    }
440
441    #[test]
442    fn keenable_works_without_a_key() {
443        let cfg = SearchProviderConfig {
444            backend: SearchBackend::Keenable,
445            ..Default::default()
446        };
447        assert_eq!(cfg.effective_backend(), SearchBackend::Keenable);
448
449        let client = reqwest::Client::new();
450        let req = keenable_request(&client, "rust", "", 5).build().unwrap();
451        assert_eq!(
452            req.url().as_str(),
453            "https://api.keenable.ai/v1/search/public"
454        );
455        assert_eq!(req.headers().get("X-Keenable-Title").unwrap(), "MicroClaw");
456
457        let req = keenable_request(&client, "rust", " kn-test ", 5)
458            .build()
459            .unwrap();
460        assert_eq!(req.url().as_str(), "https://api.keenable.ai/v1/search");
461        assert_eq!(req.headers().get("X-API-Key").unwrap(), "kn-test");
462    }
463
464    #[test]
465    fn normalize_trims_and_bounds() {
466        let mut cfg = SearchProviderConfig {
467            backend: SearchBackend::Searxng,
468            searxng_base_url: "  http://localhost:8888/  ".into(),
469            brave_api_key: "  k  ".into(),
470            max_results: 0,
471            ..Default::default()
472        };
473        cfg.normalize();
474        assert_eq!(cfg.searxng_base_url, "http://localhost:8888");
475        assert_eq!(cfg.brave_api_key, "k");
476        assert_eq!(cfg.max_results, 8);
477        assert_eq!(cfg.effective_backend(), SearchBackend::Searxng);
478    }
479
480    #[test]
481    fn format_hits_numbers_results() {
482        let hits = vec![
483            SearchItem {
484                title: "A".into(),
485                url: "https://a.example".into(),
486                snippet: "sa".into(),
487            },
488            SearchItem {
489                title: "B".into(),
490                url: "https://b.example".into(),
491                snippet: "sb".into(),
492            },
493        ];
494        let out = format_search_hits(&hits);
495        assert!(out.contains("1. A"));
496        assert!(out.contains("2. B"));
497        assert!(out.contains("https://a.example"));
498    }
499}