apollo/tools/
web_search.rs

1//! Web search tool.
2//!
3//! Tries backends in order of result quality, using the first one configured:
4//!
5//! 1. **SearXNG** (`SEARXNG_URL`) — a self-hosted metasearch instance. Full web
6//!    results, no third-party key, nothing leaves the machine except the query.
7//! 2. **DuckDuckGo** — keyless, always available. Reads the lite result page
8//!    for a ranked list, and falls back to the Instant Answer API when that
9//!    page is unavailable.
10//! 3. **Perplexity** (`PERPLEXITY_API_KEY`) — paid, kept for existing setups.
11//!
12//! The lite endpoint answers 202 with a challenge stub to clients that do not
13//! look like browsers, so the request carries browser headers. That is a
14//! best-effort path by nature: when it is refused the Instant Answer API still
15//! answers "what is X", but only a SearXNG instance gives durable result lists.
16
17use async_trait::async_trait;
18use serde::Deserialize;
19
20use super::traits::*;
21use crate::text::truncate_chars;
22
23/// Cap on characters returned to the model from any one backend.
24const MAX_RESULT_CHARS: usize = 4000;
25
26/// DuckDuckGo's plainest result page — table markup, no scripting.
27const DDG_LITE_URL: &str = "https://lite.duckduckgo.com/lite/";
28
29/// Markers DuckDuckGo puts on an anti-bot challenge page instead of results.
30const DDG_BLOCK_MARKERS: &[&str] = &[
31    "anomaly-modal",
32    "challenge-platform",
33    "DDG.anomalyDetection",
34];
35
36const DDG_USER_AGENTS: &[&str] = &[
37    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
38    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
39    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
40    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0",
41    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
42];
43
44/// Search backends, most preferred first.
45#[derive(Debug, Clone, PartialEq, Eq)]
46pub enum Backend {
47    /// Self-hosted SearXNG instance at this base URL.
48    Searxng(String),
49    /// DuckDuckGo Instant Answer API — no configuration required.
50    DuckDuckGo,
51    /// Perplexity chat completions with the given API key.
52    Perplexity(String),
53}
54
55pub struct WebSearchTool {
56    backend: Backend,
57}
58
59impl WebSearchTool {
60    pub fn new() -> Self {
61        Self {
62            backend: Self::detect_backend(),
63        }
64    }
65
66    /// Pick a backend from the environment. DuckDuckGo is the floor, so this
67    /// always yields something usable.
68    fn detect_backend() -> Backend {
69        if let Ok(url) = std::env::var("SEARXNG_URL") {
70            let url = url.trim().trim_end_matches('/');
71            if !url.is_empty() {
72                return Backend::Searxng(url.to_string());
73            }
74        }
75        match std::env::var("PERPLEXITY_API_KEY") {
76            Ok(key) if !key.trim().is_empty() => Backend::Perplexity(key),
77            _ => Backend::DuckDuckGo,
78        }
79    }
80
81    /// Override the backend, for tests and explicit configuration.
82    pub fn with_backend(mut self, backend: Backend) -> Self {
83        self.backend = backend;
84        self
85    }
86
87    pub fn with_api_key(self, key: String) -> Self {
88        self.with_backend(Backend::Perplexity(key))
89    }
90
91    pub fn backend(&self) -> &Backend {
92        &self.backend
93    }
94
95    fn client() -> anyhow::Result<reqwest::Client> {
96        Ok(crate::http::standard())
97    }
98
99    async fn search_searxng(base_url: &str, query: &str) -> anyhow::Result<String> {
100        let resp = Self::client()?
101            .get(format!("{base_url}/search"))
102            .query(&[("q", query), ("format", "json")])
103            .send()
104            .await?;
105
106        if !resp.status().is_success() {
107            anyhow::bail!("searxng returned {}", resp.status());
108        }
109
110        let body: serde_json::Value = resp.json().await?;
111        let results = body
112            .get("results")
113            .and_then(|r| r.as_array())
114            .ok_or_else(|| anyhow::anyhow!("searxng response had no results array"))?;
115
116        Ok(format_results(results.iter().take(8).map(|r| SearchHit {
117            title: field(r, "title"),
118            url: field(r, "url"),
119            snippet: field(r, "content"),
120        })))
121    }
122
123    /// Ranked results when DuckDuckGo serves them, instant answers otherwise.
124    async fn search_duckduckgo(query: &str) -> anyhow::Result<String> {
125        match Self::scrape_duckduckgo(query).await {
126            Ok(text) => Ok(text),
127            Err(e) => {
128                tracing::debug!("duckduckgo result page unavailable ({e}), using instant answers");
129                Self::duckduckgo_instant(query).await
130            }
131        }
132    }
133
134    async fn scrape_duckduckgo(query: &str) -> anyhow::Result<String> {
135        let resp = Self::client()?
136            .post(DDG_LITE_URL)
137            .header(reqwest::header::USER_AGENT, pick_user_agent())
138            .header(
139                reqwest::header::ACCEPT,
140                "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
141            )
142            .header(reqwest::header::ACCEPT_LANGUAGE, "en-US,en;q=0.9")
143            .header(reqwest::header::REFERER, "https://lite.duckduckgo.com/")
144            .form(&[("q", query), ("kl", "us-en"), ("safe", "1")])
145            .send()
146            .await?;
147
148        // 202 is how the endpoint refuses a client it does not consider a browser.
149        if resp.status().as_u16() == 202 {
150            anyhow::bail!("soft-blocked");
151        }
152        if !resp.status().is_success() {
153            anyhow::bail!("duckduckgo returned {}", resp.status());
154        }
155
156        let body = resp.text().await?;
157        if looks_blocked(&body) {
158            anyhow::bail!("challenge page");
159        }
160
161        let hits = parse_lite_results(&body);
162        if hits.is_empty() {
163            anyhow::bail!("no results parsed");
164        }
165        Ok(format_results(hits.into_iter().take(8)))
166    }
167
168    async fn duckduckgo_instant(query: &str) -> anyhow::Result<String> {
169        let resp = Self::client()?
170            .get("https://api.duckduckgo.com/")
171            .query(&[
172                ("q", query),
173                ("format", "json"),
174                ("no_html", "1"),
175                ("no_redirect", "1"),
176            ])
177            .send()
178            .await?;
179
180        if !resp.status().is_success() {
181            anyhow::bail!("duckduckgo returned {}", resp.status());
182        }
183
184        // The endpoint answers with a text/javascript content type, so decode
185        // the body ourselves rather than relying on `.json()`.
186        let body: serde_json::Value = serde_json::from_str(&resp.text().await?)?;
187        Ok(format_duckduckgo(&body))
188    }
189
190    async fn search_perplexity(api_key: &str, query: &str) -> anyhow::Result<String> {
191        let resp = Self::client()?
192            .post("https://api.perplexity.ai/chat/completions")
193            .header("Authorization", format!("Bearer {api_key}"))
194            .header("Content-Type", "application/json")
195            .json(&serde_json::json!({
196                "model": "sonar",
197                "messages": [{"role": "user", "content": query}],
198            }))
199            .send()
200            .await?;
201
202        if !resp.status().is_success() {
203            let status = resp.status();
204            let text = resp.text().await.unwrap_or_default();
205            anyhow::bail!("perplexity {}: {}", status, truncate_chars(&text, 200));
206        }
207
208        let data: serde_json::Value = resp.json().await?;
209        Ok(data["choices"][0]["message"]["content"]
210            .as_str()
211            .unwrap_or("No results found")
212            .to_string())
213    }
214}
215
216struct SearchHit {
217    title: String,
218    url: String,
219    snippet: String,
220}
221
222/// Rotates the user agent so a long-running agent looks less uniform.
223fn pick_user_agent() -> &'static str {
224    let nanos = std::time::SystemTime::now()
225        .duration_since(std::time::UNIX_EPOCH)
226        .map(|d| d.subsec_nanos() as usize)
227        .unwrap_or(0);
228    DDG_USER_AGENTS[nanos % DDG_USER_AGENTS.len()]
229}
230
231fn looks_blocked(body: &str) -> bool {
232    DDG_BLOCK_MARKERS.iter().any(|m| body.contains(m))
233}
234
235/// Read one attribute out of an opening tag, tolerating either quote style.
236fn attr(tag: &str, name: &str) -> Option<String> {
237    let at = tag.find(&format!("{name}="))?;
238    let rest = &tag[at + name.len() + 1..];
239    let quote = rest.chars().next()?;
240    if quote != '"' && quote != '\'' {
241        return None;
242    }
243    let rest = &rest[quote.len_utf8()..];
244    let end = rest.find(quote)?;
245    Some(rest[..end].to_string())
246}
247
248/// Strip tags, decode the entities DuckDuckGo emits, and collapse whitespace.
249fn clean_text(fragment: &str) -> String {
250    let mut out = String::with_capacity(fragment.len());
251    let mut in_tag = false;
252    for ch in fragment.chars() {
253        match ch {
254            '<' => in_tag = true,
255            '>' => in_tag = false,
256            c if !in_tag => out.push(c),
257            _ => {}
258        }
259    }
260    // `&amp;` decodes last so an escaped entity is not decoded twice.
261    let decoded = out
262        .replace("&nbsp;", " ")
263        .replace("&lt;", "<")
264        .replace("&gt;", ">")
265        .replace("&quot;", "\"")
266        .replace("&#x27;", "'")
267        .replace("&#39;", "'")
268        .replace("&amp;", "&");
269    decoded.split_whitespace().collect::<Vec<_>>().join(" ")
270}
271
272/// Pull results out of the lite page, which lists each hit as an anchor of
273/// class `result-link` followed by an optional `result-snippet` cell.
274fn parse_lite_results(html: &str) -> Vec<SearchHit> {
275    let mut hits = Vec::new();
276    let mut rest = html;
277
278    while let Some(marker) = rest.find("result-link") {
279        let (before, after) = rest.split_at(marker);
280        let Some(gt) = after.find('>') else { break };
281        let body = &after[gt + 1..];
282        let Some(end) = body.find("</a>") else { break };
283        let tail = &body[end..];
284
285        let url = before
286            .rfind("<a ")
287            .and_then(|start| attr(&before[start..], "href"))
288            .unwrap_or_default();
289        let title = clean_text(&body[..end]);
290
291        // A snippet belongs to this hit only if it precedes the next one.
292        let next = tail.find("result-link").unwrap_or(tail.len());
293        let snippet = tail[..next]
294            .find("result-snippet")
295            .and_then(|at| {
296                let segment = &tail[at..next];
297                let open = segment.find('>')?;
298                let close = segment[open..].find("</td>")?;
299                Some(clean_text(&segment[open + 1..open + close]))
300            })
301            .unwrap_or_default();
302
303        if !url.is_empty() && !title.is_empty() {
304            hits.push(SearchHit {
305                title,
306                url,
307                snippet,
308            });
309        }
310        rest = tail;
311    }
312
313    hits
314}
315
316fn field(value: &serde_json::Value, key: &str) -> String {
317    value
318        .get(key)
319        .and_then(|v| v.as_str())
320        .unwrap_or_default()
321        .to_string()
322}
323
324fn format_results(hits: impl Iterator<Item = SearchHit>) -> String {
325    let mut out = String::new();
326    for (i, hit) in hits.enumerate() {
327        if !out.is_empty() {
328            out.push_str("\n\n");
329        }
330        out.push_str(&format!("{}. {}\n   {}", i + 1, hit.title, hit.url));
331        if !hit.snippet.trim().is_empty() {
332            out.push_str(&format!("\n   {}", hit.snippet.trim()));
333        }
334    }
335    if out.is_empty() {
336        "No results found".to_string()
337    } else {
338        truncate_chars(&out, MAX_RESULT_CHARS)
339    }
340}
341
342/// Render an Instant Answer payload: the abstract when there is one, then the
343/// related topics, which is all this endpoint offers in place of a result list.
344fn format_duckduckgo(body: &serde_json::Value) -> String {
345    let mut out = String::new();
346
347    let abstract_text = field(body, "AbstractText");
348    if !abstract_text.trim().is_empty() {
349        out.push_str(abstract_text.trim());
350        let source = field(body, "AbstractURL");
351        if !source.is_empty() {
352            out.push_str(&format!("\n\nSource: {source}"));
353        }
354    }
355
356    let answer = field(body, "Answer");
357    if !answer.trim().is_empty() {
358        if !out.is_empty() {
359            out.push_str("\n\n");
360        }
361        out.push_str(answer.trim());
362    }
363
364    // RelatedTopics nests one level for grouped results, so flatten it.
365    let mut topics: Vec<SearchHit> = Vec::new();
366    if let Some(list) = body.get("RelatedTopics").and_then(|t| t.as_array()) {
367        for entry in list {
368            match entry.get("Topics").and_then(|t| t.as_array()) {
369                Some(nested) => topics.extend(nested.iter().filter_map(related_topic)),
370                None => topics.extend(related_topic(entry)),
371            }
372        }
373    }
374
375    if !topics.is_empty() {
376        if !out.is_empty() {
377            out.push_str("\n\nRelated:\n");
378        }
379        out.push_str(&format_results(topics.into_iter().take(8)));
380    }
381
382    if out.trim().is_empty() {
383        "No results found".to_string()
384    } else {
385        truncate_chars(&out, MAX_RESULT_CHARS)
386    }
387}
388
389fn related_topic(entry: &serde_json::Value) -> Option<SearchHit> {
390    let text = field(entry, "Text");
391    if text.trim().is_empty() {
392        return None;
393    }
394    // Instant Answer topics have no separate title; the first clause reads as one.
395    let title = text.split(" - ").next().unwrap_or(&text).to_string();
396    Some(SearchHit {
397        title,
398        url: field(entry, "FirstURL"),
399        snippet: String::new(),
400    })
401}
402
403impl Default for WebSearchTool {
404    fn default() -> Self {
405        Self::new()
406    }
407}
408
409#[derive(Deserialize)]
410struct SearchArgs {
411    query: String,
412}
413
414#[async_trait]
415impl Tool for WebSearchTool {
416    fn name(&self) -> &str {
417        "web_search"
418    }
419
420    fn spec(&self) -> ToolSpec {
421        ToolSpec {
422            name: "web_search".to_string(),
423            description: "Search the web for information. Returns relevant results with snippets."
424                .to_string(),
425            parameters: serde_json::json!({
426                "type": "object",
427                "properties": {
428                    "query": {
429                        "type": "string",
430                        "description": "Search query"
431                    }
432                },
433                "required": ["query"]
434            }),
435        }
436    }
437
438    async fn execute(&self, arguments: &str) -> anyhow::Result<ToolResult> {
439        let args: SearchArgs = serde_json::from_str(arguments)?;
440        if args.query.trim().is_empty() {
441            return Ok(ToolResult::error("query must not be empty"));
442        }
443
444        let outcome = match &self.backend {
445            Backend::Searxng(url) => Self::search_searxng(url, &args.query).await,
446            Backend::DuckDuckGo => Self::search_duckduckgo(&args.query).await,
447            Backend::Perplexity(key) => Self::search_perplexity(key, &args.query).await,
448        };
449
450        match outcome {
451            Ok(text) => Ok(ToolResult::success(text)),
452            // A self-hosted instance that is down should not take search with
453            // it — fall back to the keyless backend before giving up.
454            Err(e) if matches!(self.backend, Backend::Searxng(_)) => {
455                tracing::warn!("searxng search failed, falling back to duckduckgo: {e}");
456                match Self::search_duckduckgo(&args.query).await {
457                    Ok(text) => Ok(ToolResult::success(text)),
458                    Err(e) => Ok(ToolResult::error(format!("search failed: {e}"))),
459                }
460            }
461            Err(e) => Ok(ToolResult::error(format!("search failed: {e}"))),
462        }
463    }
464}
465
466#[cfg(test)]
467mod tests {
468    use super::*;
469
470    #[test]
471    fn duckduckgo_is_the_keyless_default() {
472        // Nothing configured should still leave search usable.
473        assert_eq!(
474            WebSearchTool::new().backend(),
475            &Backend::DuckDuckGo,
476            "expected the keyless backend when no env vars are set"
477        );
478    }
479
480    #[test]
481    fn formats_an_abstract_with_its_source() {
482        let body = serde_json::json!({
483            "AbstractText": "Rust is a general-purpose programming language.",
484            "AbstractURL": "https://en.wikipedia.org/wiki/Rust",
485            "RelatedTopics": [],
486        });
487        let out = format_duckduckgo(&body);
488        assert!(out.contains("general-purpose programming language"));
489        assert!(out.contains("Source: https://en.wikipedia.org/wiki/Rust"));
490    }
491
492    #[test]
493    fn flattens_grouped_related_topics() {
494        let body = serde_json::json!({
495            "AbstractText": "",
496            "RelatedTopics": [
497                {"Text": "Top level - a description", "FirstURL": "https://example.com/a"},
498                {"Topics": [
499                    {"Text": "Nested one - detail", "FirstURL": "https://example.com/b"},
500                    {"Text": "Nested two - detail", "FirstURL": "https://example.com/c"},
501                ]},
502            ],
503        });
504        let out = format_duckduckgo(&body);
505        assert!(out.contains("Top level"));
506        assert!(out.contains("https://example.com/b"));
507        assert!(out.contains("https://example.com/c"));
508    }
509
510    #[test]
511    fn empty_payload_reports_no_results() {
512        let body = serde_json::json!({"AbstractText": "", "RelatedTopics": []});
513        assert_eq!(format_duckduckgo(&body), "No results found");
514    }
515
516    #[test]
517    fn skips_related_topics_without_text() {
518        let body = serde_json::json!({
519            "AbstractText": "",
520            "RelatedTopics": [
521                {"FirstURL": "https://example.com/no-text"},
522                {"Text": "Has text", "FirstURL": "https://example.com/ok"},
523            ],
524        });
525        let out = format_duckduckgo(&body);
526        assert!(out.contains("Has text"));
527        assert!(!out.contains("no-text"));
528    }
529
530    const LITE_PAGE: &str = r#"
531      <table border="0">
532        <tr><td valign="top">1.&nbsp;</td>
533            <td><a rel="nofollow" href="https://tokio.rs/" class='result-link'>Tokio - An asynchronous <b>Rust</b> runtime</a></td></tr>
534        <tr><td>&nbsp;</td>
535            <td class='result-snippet'><b>Tokio</b> is a library for fast &amp; reliable apps.</td></tr>
536        <tr><td valign="top">2.&nbsp;</td>
537            <td><a rel="nofollow" href="https://docs.rs/tokio" class='result-link'>tokio - Rust</a></td></tr>
538      </table>"#;
539
540    #[test]
541    fn parses_lite_result_page() {
542        let hits = parse_lite_results(LITE_PAGE);
543        assert_eq!(hits.len(), 2);
544        assert_eq!(hits[0].url, "https://tokio.rs/");
545        assert_eq!(hits[0].title, "Tokio - An asynchronous Rust runtime");
546        assert_eq!(
547            hits[0].snippet,
548            "Tokio is a library for fast & reliable apps."
549        );
550        assert_eq!(hits[1].url, "https://docs.rs/tokio");
551    }
552
553    #[test]
554    fn does_not_borrow_a_later_hits_snippet() {
555        // The second hit has no snippet of its own and must not inherit one.
556        let hits = parse_lite_results(LITE_PAGE);
557        assert!(hits[1].snippet.is_empty());
558    }
559
560    #[test]
561    fn parsing_a_page_without_results_yields_nothing() {
562        assert!(parse_lite_results("<html><body>nothing here</body></html>").is_empty());
563    }
564
565    #[test]
566    fn detects_challenge_pages() {
567        assert!(looks_blocked("<div id=\"anomaly-modal\">"));
568        assert!(looks_blocked("window.DDG.anomalyDetection = {}"));
569        assert!(!looks_blocked(LITE_PAGE));
570    }
571
572    #[test]
573    fn reads_attributes_in_either_quote_style() {
574        assert_eq!(attr("<a href=\"x\">", "href").as_deref(), Some("x"));
575        assert_eq!(attr("<a href='y'>", "href").as_deref(), Some("y"));
576        assert_eq!(attr("<a rel=nofollow>", "href"), None);
577    }
578
579    #[test]
580    fn user_agent_comes_from_the_pool() {
581        assert!(DDG_USER_AGENTS.contains(&pick_user_agent()));
582    }
583
584    /// Canary for the scraped path: DuckDuckGo can change its markup or start
585    /// refusing this client at any time. Ignored by default — it needs network.
586    /// Run with: `cargo test --lib duckduckgo_still_serves -- --ignored`
587    #[tokio::test]
588    #[ignore]
589    async fn duckduckgo_still_serves_a_result_page() {
590        let out = WebSearchTool::scrape_duckduckgo("rust tokio runtime")
591            .await
592            .expect("lite endpoint refused or markup changed");
593        assert!(out.contains("http"), "no links in: {out}");
594    }
595
596    #[test]
597    fn formats_searxng_style_hits() {
598        let hits = vec![
599            SearchHit {
600                title: "First".into(),
601                url: "https://example.com/1".into(),
602                snippet: "  a snippet  ".into(),
603            },
604            SearchHit {
605                title: "Second".into(),
606                url: "https://example.com/2".into(),
607                snippet: String::new(),
608            },
609        ];
610        let out = format_results(hits.into_iter());
611        assert!(out.starts_with("1. First"));
612        assert!(out.contains("   a snippet"));
613        assert!(out.contains("2. Second"));
614    }
615}
apollo/tools/web_search.rs

apollo/tools/
web_search.rs
