microclaw_engine/internal/tool_runtime/
web_search.rs

1use std::collections::HashMap;
2use std::sync::{Mutex, OnceLock};
3
4use serde::{Deserialize, Serialize};
5
6use crate::internal::tool_runtime::web_html::{extract_ddg_results, SearchItem};
7
8fn http_client(timeout_secs: u64) -> reqwest::Client {
9    static CLIENTS: OnceLock<Mutex<HashMap<u64, reqwest::Client>>> = OnceLock::new();
10    let cache = CLIENTS.get_or_init(|| Mutex::new(HashMap::new()));
11    let mut cache = cache.lock().unwrap_or_else(|e| e.into_inner());
12    if let Some(client) = cache.get(&timeout_secs) {
13        return client.clone();
14    }
15    let client = reqwest::Client::builder()
16        .timeout(std::time::Duration::from_secs(timeout_secs))
17        .redirect(reqwest::redirect::Policy::limited(5))
18        .user_agent("MicroClaw/1.0")
19        .build()
20        .expect("failed to build HTTP client");
21    cache.insert(timeout_secs, client.clone());
22    client
23}
24
25/// Selectable web-search backend. DuckDuckGo (HTML scrape, no key) is the
26/// default and the fallback whenever a richer backend is selected but missing
27/// its credentials/endpoint.
28#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
29#[serde(rename_all = "lowercase")]
30pub enum SearchBackend {
31    #[default]
32    Duckduckgo,
33    /// Self-hosted SearXNG instance (`{base}/search?format=json`). Ideal for a
34    /// $5 VPS: no API key, no rate limit, fully local.
35    Searxng,
36    /// Brave Search API (`X-Subscription-Token`).
37    Brave,
38    /// Tavily research API (`POST /search`).
39    Tavily,
40}
41
42fn default_search_max_results() -> usize {
43    8
44}
45
46/// Configuration for the pluggable web-search provider. Defaults preserve the
47/// historical behavior exactly (DuckDuckGo, 8 results), so existing
48/// deployments see no change until they opt into a backend.
49#[derive(Debug, Clone, Serialize, Deserialize)]
50pub struct SearchProviderConfig {
51    #[serde(default)]
52    pub backend: SearchBackend,
53    /// Base URL of a SearXNG instance, e.g. `http://127.0.0.1:8888`.
54    #[serde(default)]
55    pub searxng_base_url: String,
56    #[serde(default)]
57    pub brave_api_key: String,
58    #[serde(default)]
59    pub tavily_api_key: String,
60    #[serde(default = "default_search_max_results")]
61    pub max_results: usize,
62}
63
64impl Default for SearchProviderConfig {
65    fn default() -> Self {
66        Self {
67            backend: SearchBackend::default(),
68            searxng_base_url: String::new(),
69            brave_api_key: String::new(),
70            tavily_api_key: String::new(),
71            max_results: default_search_max_results(),
72        }
73    }
74}
75
76impl SearchProviderConfig {
77    pub fn normalize(&mut self) {
78        self.searxng_base_url = self
79            .searxng_base_url
80            .trim()
81            .trim_end_matches('/')
82            .to_string();
83        self.brave_api_key = self.brave_api_key.trim().to_string();
84        self.tavily_api_key = self.tavily_api_key.trim().to_string();
85        if self.max_results == 0 {
86            self.max_results = default_search_max_results();
87        }
88        self.max_results = self.max_results.min(20);
89    }
90
91    /// The backend that will actually be used, downgrading to DuckDuckGo when
92    /// the selected backend lacks the credentials/endpoint it needs. This keeps
93    /// search working (degraded) instead of hard-failing on a misconfiguration.
94    pub fn effective_backend(&self) -> SearchBackend {
95        match self.backend {
96            SearchBackend::Searxng if self.searxng_base_url.trim().is_empty() => {
97                SearchBackend::Duckduckgo
98            }
99            SearchBackend::Brave if self.brave_api_key.trim().is_empty() => {
100                SearchBackend::Duckduckgo
101            }
102            SearchBackend::Tavily if self.tavily_api_key.trim().is_empty() => {
103                SearchBackend::Duckduckgo
104            }
105            other => other,
106        }
107    }
108}
109
110/// Render structured hits into the numbered text block the agent consumes.
111pub fn format_search_hits(hits: &[SearchItem]) -> String {
112    let mut output = String::new();
113    for (i, item) in hits.iter().enumerate() {
114        output.push_str(&format!(
115            "{}. {}\n   {}\n   {}\n\n",
116            i + 1,
117            item.title,
118            item.url,
119            item.snippet
120        ));
121    }
122    output
123}
124
125/// Run a search via the configured provider and return structured hits,
126/// transparently falling back to DuckDuckGo when the provider is misconfigured.
127pub async fn search_with_provider(
128    query: &str,
129    config: &SearchProviderConfig,
130    timeout_secs: u64,
131) -> Result<Vec<SearchItem>, String> {
132    let max = config.max_results.clamp(1, 20);
133    let t = timeout_secs.max(1);
134    match config.effective_backend() {
135        SearchBackend::Duckduckgo => ddg_hits(query, t, max).await,
136        SearchBackend::Searxng => searxng_hits(query, &config.searxng_base_url, t, max).await,
137        SearchBackend::Brave => brave_hits(query, &config.brave_api_key, t, max).await,
138        SearchBackend::Tavily => tavily_hits(query, &config.tavily_api_key, t, max).await,
139    }
140}
141
142async fn ddg_hits(query: &str, timeout_secs: u64, max: usize) -> Result<Vec<SearchItem>, String> {
143    let encoded = urlencoding::encode(query);
144    let url = format!("https://html.duckduckgo.com/html/?q={encoded}");
145    let client = http_client(timeout_secs);
146    let resp = client.get(&url).send().await.map_err(|e| e.to_string())?;
147    if !resp.status().is_success() {
148        return Err(format!("HTTP {}", resp.status()));
149    }
150    let body = resp.text().await.map_err(|e| e.to_string())?;
151    Ok(extract_ddg_results(&body, max))
152}
153
154#[derive(Deserialize)]
155struct SearxResponse {
156    #[serde(default)]
157    results: Vec<SearxResult>,
158}
159
160#[derive(Deserialize)]
161struct SearxResult {
162    #[serde(default)]
163    title: String,
164    #[serde(default)]
165    url: String,
166    #[serde(default)]
167    content: String,
168}
169
170async fn searxng_hits(
171    query: &str,
172    base_url: &str,
173    timeout_secs: u64,
174    max: usize,
175) -> Result<Vec<SearchItem>, String> {
176    let encoded = urlencoding::encode(query);
177    let url = format!("{base_url}/search?q={encoded}&format=json");
178    let client = http_client(timeout_secs);
179    let resp = client.get(&url).send().await.map_err(|e| e.to_string())?;
180    if !resp.status().is_success() {
181        return Err(format!("HTTP {}", resp.status()));
182    }
183    let parsed: SearxResponse = resp.json().await.map_err(|e| e.to_string())?;
184    Ok(parsed
185        .results
186        .into_iter()
187        .filter(|r| !r.url.is_empty())
188        .take(max)
189        .map(|r| SearchItem {
190            title: r.title,
191            url: r.url,
192            snippet: r.content,
193        })
194        .collect())
195}
196
197#[derive(Deserialize)]
198struct BraveResponse {
199    #[serde(default)]
200    web: Option<BraveWeb>,
201}
202
203#[derive(Deserialize)]
204struct BraveWeb {
205    #[serde(default)]
206    results: Vec<BraveResult>,
207}
208
209#[derive(Deserialize)]
210struct BraveResult {
211    #[serde(default)]
212    title: String,
213    #[serde(default)]
214    url: String,
215    #[serde(default)]
216    description: String,
217}
218
219async fn brave_hits(
220    query: &str,
221    api_key: &str,
222    timeout_secs: u64,
223    max: usize,
224) -> Result<Vec<SearchItem>, String> {
225    let encoded = urlencoding::encode(query);
226    let url = format!("https://api.search.brave.com/res/v1/web/search?q={encoded}&count={max}");
227    let client = http_client(timeout_secs);
228    let resp = client
229        .get(&url)
230        .header("Accept", "application/json")
231        .header("X-Subscription-Token", api_key)
232        .send()
233        .await
234        .map_err(|e| e.to_string())?;
235    if !resp.status().is_success() {
236        return Err(format!("HTTP {}", resp.status()));
237    }
238    let parsed: BraveResponse = resp.json().await.map_err(|e| e.to_string())?;
239    Ok(parsed
240        .web
241        .map(|w| w.results)
242        .unwrap_or_default()
243        .into_iter()
244        .filter(|r| !r.url.is_empty())
245        .take(max)
246        .map(|r| SearchItem {
247            title: r.title,
248            url: r.url,
249            snippet: r.description,
250        })
251        .collect())
252}
253
254#[derive(Serialize)]
255struct TavilyRequest<'a> {
256    api_key: &'a str,
257    query: &'a str,
258    max_results: usize,
259}
260
261#[derive(Deserialize)]
262struct TavilyResponse {
263    #[serde(default)]
264    results: Vec<TavilyResult>,
265}
266
267#[derive(Deserialize)]
268struct TavilyResult {
269    #[serde(default)]
270    title: String,
271    #[serde(default)]
272    url: String,
273    #[serde(default)]
274    content: String,
275}
276
277async fn tavily_hits(
278    query: &str,
279    api_key: &str,
280    timeout_secs: u64,
281    max: usize,
282) -> Result<Vec<SearchItem>, String> {
283    let client = http_client(timeout_secs);
284    let resp = client
285        .post("https://api.tavily.com/search")
286        .json(&TavilyRequest {
287            api_key,
288            query,
289            max_results: max,
290        })
291        .send()
292        .await
293        .map_err(|e| e.to_string())?;
294    if !resp.status().is_success() {
295        return Err(format!("HTTP {}", resp.status()));
296    }
297    let parsed: TavilyResponse = resp.json().await.map_err(|e| e.to_string())?;
298    Ok(parsed
299        .results
300        .into_iter()
301        .filter(|r| !r.url.is_empty())
302        .take(max)
303        .map(|r| SearchItem {
304            title: r.title,
305            url: r.url,
306            snippet: r.content,
307        })
308        .collect())
309}
310
311pub async fn search_ddg_with_timeout(query: &str, timeout_secs: u64) -> Result<String, String> {
312    let hits = ddg_hits(query, timeout_secs.max(1), 8).await?;
313    Ok(format_search_hits(&hits))
314}
315
316pub async fn search_ddg(query: &str) -> Result<String, String> {
317    search_ddg_with_timeout(query, 15).await
318}
319
320#[cfg(test)]
321mod tests {
322    use super::*;
323
324    #[test]
325    fn default_backend_is_duckduckgo() {
326        let cfg = SearchProviderConfig::default();
327        assert_eq!(cfg.backend, SearchBackend::Duckduckgo);
328        assert_eq!(cfg.effective_backend(), SearchBackend::Duckduckgo);
329        assert_eq!(cfg.max_results, 8);
330    }
331
332    #[test]
333    fn effective_backend_downgrades_when_unconfigured() {
334        let cfg = SearchProviderConfig {
335            backend: SearchBackend::Searxng,
336            ..Default::default()
337        };
338        assert_eq!(cfg.effective_backend(), SearchBackend::Duckduckgo);
339
340        let cfg = SearchProviderConfig {
341            backend: SearchBackend::Brave,
342            ..Default::default()
343        };
344        assert_eq!(cfg.effective_backend(), SearchBackend::Duckduckgo);
345
346        let cfg = SearchProviderConfig {
347            backend: SearchBackend::Tavily,
348            tavily_api_key: "tvly-xxx".into(),
349            ..Default::default()
350        };
351        assert_eq!(cfg.effective_backend(), SearchBackend::Tavily);
352    }
353
354    #[test]
355    fn normalize_trims_and_bounds() {
356        let mut cfg = SearchProviderConfig {
357            backend: SearchBackend::Searxng,
358            searxng_base_url: "  http://localhost:8888/  ".into(),
359            brave_api_key: "  k  ".into(),
360            max_results: 0,
361            ..Default::default()
362        };
363        cfg.normalize();
364        assert_eq!(cfg.searxng_base_url, "http://localhost:8888");
365        assert_eq!(cfg.brave_api_key, "k");
366        assert_eq!(cfg.max_results, 8);
367        assert_eq!(cfg.effective_backend(), SearchBackend::Searxng);
368    }
369
370    #[test]
371    fn format_hits_numbers_results() {
372        let hits = vec![
373            SearchItem {
374                title: "A".into(),
375                url: "https://a.example".into(),
376                snippet: "sa".into(),
377            },
378            SearchItem {
379                title: "B".into(),
380                url: "https://b.example".into(),
381                snippet: "sb".into(),
382            },
383        ];
384        let out = format_search_hits(&hits);
385        assert!(out.contains("1. A"));
386        assert!(out.contains("2. B"));
387        assert!(out.contains("https://a.example"));
388    }
389}
microclaw_engine/internal/tool_runtime/web_search.rs

microclaw_engine/internal/tool_runtime/
web_search.rs
