apollo/tools/
web_search.rs

1//! Web search tool.
2//!
3//! Tries backends in order of result quality, using the first one configured:
4//!
5//! 1. **SearXNG** (`SEARXNG_URL`) — a self-hosted metasearch instance. Full web
6//!    results, no third-party key, nothing leaves the machine except the query.
7//! 2. **DuckDuckGo** — keyless, always available. Reads the lite result page
8//!    for a ranked list, and falls back to the Instant Answer API when that
9//!    page is unavailable.
10//! 3. **Perplexity** (`PERPLEXITY_API_KEY`) — paid, kept for existing setups.
11//!
12//! The lite endpoint answers 202 with a challenge stub to clients that do not
13//! look like browsers, so the request carries browser headers. That is a
14//! best-effort path by nature: when it is refused the Instant Answer API still
15//! answers "what is X", but only a SearXNG instance gives durable result lists.
16
17use async_trait::async_trait;
18use serde::Deserialize;
19
20use super::traits::*;
21use crate::text::truncate_chars;
22
23/// Cap on characters returned to the model from any one backend.
24const MAX_RESULT_CHARS: usize = 4000;
25
26/// DuckDuckGo's plainest result page — table markup, no scripting.
27const DDG_LITE_URL: &str = "https://lite.duckduckgo.com/lite/";
28
29/// Markers DuckDuckGo puts on an anti-bot challenge page instead of results.
30const DDG_BLOCK_MARKERS: &[&str] = &[
31    "anomaly-modal",
32    "challenge-platform",
33    "DDG.anomalyDetection",
34];
35
36const DDG_USER_AGENTS: &[&str] = &[
37    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
38    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
39    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
40    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0",
41    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
42];
43
44/// Search backends, most preferred first.
45#[derive(Debug, Clone, PartialEq, Eq)]
46pub enum Backend {
47    /// Self-hosted SearXNG instance at this base URL.
48    Searxng(String),
49    /// DuckDuckGo Instant Answer API — no configuration required.
50    DuckDuckGo,
51    /// Perplexity chat completions with the given API key.
52    Perplexity(String),
53}
54
55pub struct WebSearchTool {
56    backend: Backend,
57}
58
59impl WebSearchTool {
60    pub fn new() -> Self {
61        Self {
62            backend: Self::detect_backend(),
63        }
64    }
65
66    /// Pick a backend from the environment. DuckDuckGo is the floor, so this
67    /// always yields something usable.
68    fn detect_backend() -> Backend {
69        if let Ok(url) = std::env::var("SEARXNG_URL") {
70            let url = url.trim().trim_end_matches('/');
71            if !url.is_empty() {
72                return Backend::Searxng(url.to_string());
73            }
74        }
75        match std::env::var("PERPLEXITY_API_KEY") {
76            Ok(key) if !key.trim().is_empty() => Backend::Perplexity(key),
77            _ => Backend::DuckDuckGo,
78        }
79    }
80
81    /// Override the backend, for tests and explicit configuration.
82    pub fn with_backend(mut self, backend: Backend) -> Self {
83        self.backend = backend;
84        self
85    }
86
87    pub fn with_api_key(self, key: String) -> Self {
88        self.with_backend(Backend::Perplexity(key))
89    }
90
91    pub fn backend(&self) -> &Backend {
92        &self.backend
93    }
94
95    fn client() -> anyhow::Result<reqwest::Client> {
96        Ok(reqwest::Client::builder()
97            .timeout(std::time::Duration::from_secs(30))
98            .build()?)
99    }
100
101    async fn search_searxng(base_url: &str, query: &str) -> anyhow::Result<String> {
102        let resp = Self::client()?
103            .get(format!("{base_url}/search"))
104            .query(&[("q", query), ("format", "json")])
105            .send()
106            .await?;
107
108        if !resp.status().is_success() {
109            anyhow::bail!("searxng returned {}", resp.status());
110        }
111
112        let body: serde_json::Value = resp.json().await?;
113        let results = body
114            .get("results")
115            .and_then(|r| r.as_array())
116            .ok_or_else(|| anyhow::anyhow!("searxng response had no results array"))?;
117
118        Ok(format_results(results.iter().take(8).map(|r| SearchHit {
119            title: field(r, "title"),
120            url: field(r, "url"),
121            snippet: field(r, "content"),
122        })))
123    }
124
125    /// Ranked results when DuckDuckGo serves them, instant answers otherwise.
126    async fn search_duckduckgo(query: &str) -> anyhow::Result<String> {
127        match Self::scrape_duckduckgo(query).await {
128            Ok(text) => Ok(text),
129            Err(e) => {
130                tracing::debug!("duckduckgo result page unavailable ({e}), using instant answers");
131                Self::duckduckgo_instant(query).await
132            }
133        }
134    }
135
136    async fn scrape_duckduckgo(query: &str) -> anyhow::Result<String> {
137        let resp = Self::client()?
138            .post(DDG_LITE_URL)
139            .header(reqwest::header::USER_AGENT, pick_user_agent())
140            .header(
141                reqwest::header::ACCEPT,
142                "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
143            )
144            .header(reqwest::header::ACCEPT_LANGUAGE, "en-US,en;q=0.9")
145            .header(reqwest::header::REFERER, "https://lite.duckduckgo.com/")
146            .form(&[("q", query), ("kl", "us-en"), ("safe", "1")])
147            .send()
148            .await?;
149
150        // 202 is how the endpoint refuses a client it does not consider a browser.
151        if resp.status().as_u16() == 202 {
152            anyhow::bail!("soft-blocked");
153        }
154        if !resp.status().is_success() {
155            anyhow::bail!("duckduckgo returned {}", resp.status());
156        }
157
158        let body = resp.text().await?;
159        if looks_blocked(&body) {
160            anyhow::bail!("challenge page");
161        }
162
163        let hits = parse_lite_results(&body);
164        if hits.is_empty() {
165            anyhow::bail!("no results parsed");
166        }
167        Ok(format_results(hits.into_iter().take(8)))
168    }
169
170    async fn duckduckgo_instant(query: &str) -> anyhow::Result<String> {
171        let resp = Self::client()?
172            .get("https://api.duckduckgo.com/")
173            .query(&[
174                ("q", query),
175                ("format", "json"),
176                ("no_html", "1"),
177                ("no_redirect", "1"),
178            ])
179            .send()
180            .await?;
181
182        if !resp.status().is_success() {
183            anyhow::bail!("duckduckgo returned {}", resp.status());
184        }
185
186        // The endpoint answers with a text/javascript content type, so decode
187        // the body ourselves rather than relying on `.json()`.
188        let body: serde_json::Value = serde_json::from_str(&resp.text().await?)?;
189        Ok(format_duckduckgo(&body))
190    }
191
192    async fn search_perplexity(api_key: &str, query: &str) -> anyhow::Result<String> {
193        let resp = Self::client()?
194            .post("https://api.perplexity.ai/chat/completions")
195            .header("Authorization", format!("Bearer {api_key}"))
196            .header("Content-Type", "application/json")
197            .json(&serde_json::json!({
198                "model": "sonar",
199                "messages": [{"role": "user", "content": query}],
200            }))
201            .send()
202            .await?;
203
204        if !resp.status().is_success() {
205            let status = resp.status();
206            let text = resp.text().await.unwrap_or_default();
207            anyhow::bail!("perplexity {}: {}", status, truncate_chars(&text, 200));
208        }
209
210        let data: serde_json::Value = resp.json().await?;
211        Ok(data["choices"][0]["message"]["content"]
212            .as_str()
213            .unwrap_or("No results found")
214            .to_string())
215    }
216}
217
218struct SearchHit {
219    title: String,
220    url: String,
221    snippet: String,
222}
223
224/// Rotates the user agent so a long-running agent looks less uniform.
225fn pick_user_agent() -> &'static str {
226    let nanos = std::time::SystemTime::now()
227        .duration_since(std::time::UNIX_EPOCH)
228        .map(|d| d.subsec_nanos() as usize)
229        .unwrap_or(0);
230    DDG_USER_AGENTS[nanos % DDG_USER_AGENTS.len()]
231}
232
233fn looks_blocked(body: &str) -> bool {
234    DDG_BLOCK_MARKERS.iter().any(|m| body.contains(m))
235}
236
237/// Read one attribute out of an opening tag, tolerating either quote style.
238fn attr(tag: &str, name: &str) -> Option<String> {
239    let at = tag.find(&format!("{name}="))?;
240    let rest = &tag[at + name.len() + 1..];
241    let quote = rest.chars().next()?;
242    if quote != '"' && quote != '\'' {
243        return None;
244    }
245    let rest = &rest[quote.len_utf8()..];
246    let end = rest.find(quote)?;
247    Some(rest[..end].to_string())
248}
249
250/// Strip tags, decode the entities DuckDuckGo emits, and collapse whitespace.
251fn clean_text(fragment: &str) -> String {
252    let mut out = String::with_capacity(fragment.len());
253    let mut in_tag = false;
254    for ch in fragment.chars() {
255        match ch {
256            '<' => in_tag = true,
257            '>' => in_tag = false,
258            c if !in_tag => out.push(c),
259            _ => {}
260        }
261    }
262    // `&amp;` decodes last so an escaped entity is not decoded twice.
263    let decoded = out
264        .replace("&nbsp;", " ")
265        .replace("&lt;", "<")
266        .replace("&gt;", ">")
267        .replace("&quot;", "\"")
268        .replace("&#x27;", "'")
269        .replace("&#39;", "'")
270        .replace("&amp;", "&");
271    decoded.split_whitespace().collect::<Vec<_>>().join(" ")
272}
273
274/// Pull results out of the lite page, which lists each hit as an anchor of
275/// class `result-link` followed by an optional `result-snippet` cell.
276fn parse_lite_results(html: &str) -> Vec<SearchHit> {
277    let mut hits = Vec::new();
278    let mut rest = html;
279
280    while let Some(marker) = rest.find("result-link") {
281        let (before, after) = rest.split_at(marker);
282        let Some(gt) = after.find('>') else { break };
283        let body = &after[gt + 1..];
284        let Some(end) = body.find("</a>") else { break };
285        let tail = &body[end..];
286
287        let url = before
288            .rfind("<a ")
289            .and_then(|start| attr(&before[start..], "href"))
290            .unwrap_or_default();
291        let title = clean_text(&body[..end]);
292
293        // A snippet belongs to this hit only if it precedes the next one.
294        let next = tail.find("result-link").unwrap_or(tail.len());
295        let snippet = tail[..next]
296            .find("result-snippet")
297            .and_then(|at| {
298                let segment = &tail[at..next];
299                let open = segment.find('>')?;
300                let close = segment[open..].find("</td>")?;
301                Some(clean_text(&segment[open + 1..open + close]))
302            })
303            .unwrap_or_default();
304
305        if !url.is_empty() && !title.is_empty() {
306            hits.push(SearchHit {
307                title,
308                url,
309                snippet,
310            });
311        }
312        rest = tail;
313    }
314
315    hits
316}
317
318fn field(value: &serde_json::Value, key: &str) -> String {
319    value
320        .get(key)
321        .and_then(|v| v.as_str())
322        .unwrap_or_default()
323        .to_string()
324}
325
326fn format_results(hits: impl Iterator<Item = SearchHit>) -> String {
327    let mut out = String::new();
328    for (i, hit) in hits.enumerate() {
329        if !out.is_empty() {
330            out.push_str("\n\n");
331        }
332        out.push_str(&format!("{}. {}\n   {}", i + 1, hit.title, hit.url));
333        if !hit.snippet.trim().is_empty() {
334            out.push_str(&format!("\n   {}", hit.snippet.trim()));
335        }
336    }
337    if out.is_empty() {
338        "No results found".to_string()
339    } else {
340        truncate_chars(&out, MAX_RESULT_CHARS)
341    }
342}
343
344/// Render an Instant Answer payload: the abstract when there is one, then the
345/// related topics, which is all this endpoint offers in place of a result list.
346fn format_duckduckgo(body: &serde_json::Value) -> String {
347    let mut out = String::new();
348
349    let abstract_text = field(body, "AbstractText");
350    if !abstract_text.trim().is_empty() {
351        out.push_str(abstract_text.trim());
352        let source = field(body, "AbstractURL");
353        if !source.is_empty() {
354            out.push_str(&format!("\n\nSource: {source}"));
355        }
356    }
357
358    let answer = field(body, "Answer");
359    if !answer.trim().is_empty() {
360        if !out.is_empty() {
361            out.push_str("\n\n");
362        }
363        out.push_str(answer.trim());
364    }
365
366    // RelatedTopics nests one level for grouped results, so flatten it.
367    let mut topics: Vec<SearchHit> = Vec::new();
368    if let Some(list) = body.get("RelatedTopics").and_then(|t| t.as_array()) {
369        for entry in list {
370            match entry.get("Topics").and_then(|t| t.as_array()) {
371                Some(nested) => topics.extend(nested.iter().filter_map(related_topic)),
372                None => topics.extend(related_topic(entry)),
373            }
374        }
375    }
376
377    if !topics.is_empty() {
378        if !out.is_empty() {
379            out.push_str("\n\nRelated:\n");
380        }
381        out.push_str(&format_results(topics.into_iter().take(8)));
382    }
383
384    if out.trim().is_empty() {
385        "No results found".to_string()
386    } else {
387        truncate_chars(&out, MAX_RESULT_CHARS)
388    }
389}
390
391fn related_topic(entry: &serde_json::Value) -> Option<SearchHit> {
392    let text = field(entry, "Text");
393    if text.trim().is_empty() {
394        return None;
395    }
396    // Instant Answer topics have no separate title; the first clause reads as one.
397    let title = text.split(" - ").next().unwrap_or(&text).to_string();
398    Some(SearchHit {
399        title,
400        url: field(entry, "FirstURL"),
401        snippet: String::new(),
402    })
403}
404
405impl Default for WebSearchTool {
406    fn default() -> Self {
407        Self::new()
408    }
409}
410
411#[derive(Deserialize)]
412struct SearchArgs {
413    query: String,
414}
415
416#[async_trait]
417impl Tool for WebSearchTool {
418    fn name(&self) -> &str {
419        "web_search"
420    }
421
422    fn spec(&self) -> ToolSpec {
423        ToolSpec {
424            name: "web_search".to_string(),
425            description: "Search the web for information. Returns relevant results with snippets."
426                .to_string(),
427            parameters: serde_json::json!({
428                "type": "object",
429                "properties": {
430                    "query": {
431                        "type": "string",
432                        "description": "Search query"
433                    }
434                },
435                "required": ["query"]
436            }),
437        }
438    }
439
440    async fn execute(&self, arguments: &str) -> anyhow::Result<ToolResult> {
441        let args: SearchArgs = serde_json::from_str(arguments)?;
442        if args.query.trim().is_empty() {
443            return Ok(ToolResult::error("query must not be empty"));
444        }
445
446        let outcome = match &self.backend {
447            Backend::Searxng(url) => Self::search_searxng(url, &args.query).await,
448            Backend::DuckDuckGo => Self::search_duckduckgo(&args.query).await,
449            Backend::Perplexity(key) => Self::search_perplexity(key, &args.query).await,
450        };
451
452        match outcome {
453            Ok(text) => Ok(ToolResult::success(text)),
454            // A self-hosted instance that is down should not take search with
455            // it — fall back to the keyless backend before giving up.
456            Err(e) if matches!(self.backend, Backend::Searxng(_)) => {
457                tracing::warn!("searxng search failed, falling back to duckduckgo: {e}");
458                match Self::search_duckduckgo(&args.query).await {
459                    Ok(text) => Ok(ToolResult::success(text)),
460                    Err(e) => Ok(ToolResult::error(format!("search failed: {e}"))),
461                }
462            }
463            Err(e) => Ok(ToolResult::error(format!("search failed: {e}"))),
464        }
465    }
466}
467
468#[cfg(test)]
469mod tests {
470    use super::*;
471
472    #[test]
473    fn duckduckgo_is_the_keyless_default() {
474        // Nothing configured should still leave search usable.
475        assert_eq!(
476            WebSearchTool::new().backend(),
477            &Backend::DuckDuckGo,
478            "expected the keyless backend when no env vars are set"
479        );
480    }
481
482    #[test]
483    fn formats_an_abstract_with_its_source() {
484        let body = serde_json::json!({
485            "AbstractText": "Rust is a general-purpose programming language.",
486            "AbstractURL": "https://en.wikipedia.org/wiki/Rust",
487            "RelatedTopics": [],
488        });
489        let out = format_duckduckgo(&body);
490        assert!(out.contains("general-purpose programming language"));
491        assert!(out.contains("Source: https://en.wikipedia.org/wiki/Rust"));
492    }
493
494    #[test]
495    fn flattens_grouped_related_topics() {
496        let body = serde_json::json!({
497            "AbstractText": "",
498            "RelatedTopics": [
499                {"Text": "Top level - a description", "FirstURL": "https://example.com/a"},
500                {"Topics": [
501                    {"Text": "Nested one - detail", "FirstURL": "https://example.com/b"},
502                    {"Text": "Nested two - detail", "FirstURL": "https://example.com/c"},
503                ]},
504            ],
505        });
506        let out = format_duckduckgo(&body);
507        assert!(out.contains("Top level"));
508        assert!(out.contains("https://example.com/b"));
509        assert!(out.contains("https://example.com/c"));
510    }
511
512    #[test]
513    fn empty_payload_reports_no_results() {
514        let body = serde_json::json!({"AbstractText": "", "RelatedTopics": []});
515        assert_eq!(format_duckduckgo(&body), "No results found");
516    }
517
518    #[test]
519    fn skips_related_topics_without_text() {
520        let body = serde_json::json!({
521            "AbstractText": "",
522            "RelatedTopics": [
523                {"FirstURL": "https://example.com/no-text"},
524                {"Text": "Has text", "FirstURL": "https://example.com/ok"},
525            ],
526        });
527        let out = format_duckduckgo(&body);
528        assert!(out.contains("Has text"));
529        assert!(!out.contains("no-text"));
530    }
531
532    const LITE_PAGE: &str = r#"
533      <table border="0">
534        <tr><td valign="top">1.&nbsp;</td>
535            <td><a rel="nofollow" href="https://tokio.rs/" class='result-link'>Tokio - An asynchronous <b>Rust</b> runtime</a></td></tr>
536        <tr><td>&nbsp;</td>
537            <td class='result-snippet'><b>Tokio</b> is a library for fast &amp; reliable apps.</td></tr>
538        <tr><td valign="top">2.&nbsp;</td>
539            <td><a rel="nofollow" href="https://docs.rs/tokio" class='result-link'>tokio - Rust</a></td></tr>
540      </table>"#;
541
542    #[test]
543    fn parses_lite_result_page() {
544        let hits = parse_lite_results(LITE_PAGE);
545        assert_eq!(hits.len(), 2);
546        assert_eq!(hits[0].url, "https://tokio.rs/");
547        assert_eq!(hits[0].title, "Tokio - An asynchronous Rust runtime");
548        assert_eq!(
549            hits[0].snippet,
550            "Tokio is a library for fast & reliable apps."
551        );
552        assert_eq!(hits[1].url, "https://docs.rs/tokio");
553    }
554
555    #[test]
556    fn does_not_borrow_a_later_hits_snippet() {
557        // The second hit has no snippet of its own and must not inherit one.
558        let hits = parse_lite_results(LITE_PAGE);
559        assert!(hits[1].snippet.is_empty());
560    }
561
562    #[test]
563    fn parsing_a_page_without_results_yields_nothing() {
564        assert!(parse_lite_results("<html><body>nothing here</body></html>").is_empty());
565    }
566
567    #[test]
568    fn detects_challenge_pages() {
569        assert!(looks_blocked("<div id=\"anomaly-modal\">"));
570        assert!(looks_blocked("window.DDG.anomalyDetection = {}"));
571        assert!(!looks_blocked(LITE_PAGE));
572    }
573
574    #[test]
575    fn reads_attributes_in_either_quote_style() {
576        assert_eq!(attr("<a href=\"x\">", "href").as_deref(), Some("x"));
577        assert_eq!(attr("<a href='y'>", "href").as_deref(), Some("y"));
578        assert_eq!(attr("<a rel=nofollow>", "href"), None);
579    }
580
581    #[test]
582    fn user_agent_comes_from_the_pool() {
583        assert!(DDG_USER_AGENTS.contains(&pick_user_agent()));
584    }
585
586    /// Canary for the scraped path: DuckDuckGo can change its markup or start
587    /// refusing this client at any time. Ignored by default — it needs network.
588    /// Run with: `cargo test --lib duckduckgo_still_serves -- --ignored`
589    #[tokio::test]
590    #[ignore]
591    async fn duckduckgo_still_serves_a_result_page() {
592        let out = WebSearchTool::scrape_duckduckgo("rust tokio runtime")
593            .await
594            .expect("lite endpoint refused or markup changed");
595        assert!(out.contains("http"), "no links in: {out}");
596    }
597
598    #[test]
599    fn formats_searxng_style_hits() {
600        let hits = vec![
601            SearchHit {
602                title: "First".into(),
603                url: "https://example.com/1".into(),
604                snippet: "  a snippet  ".into(),
605            },
606            SearchHit {
607                title: "Second".into(),
608                url: "https://example.com/2".into(),
609                snippet: String::new(),
610            },
611        ];
612        let out = format_results(hits.into_iter());
613        assert!(out.starts_with("1. First"));
614        assert!(out.contains("   a snippet"));
615        assert!(out.contains("2. Second"));
616    }
617}
apollo/tools/web_search.rs

apollo/tools/
web_search.rs
