websearch/providers/
mod.rs

1//! Search backends.
2//!
3//! DuckDuckGo Lite needs no key and stays the default, so `webtools` keeps
4//! working with no configuration at all. It is also scraped HTML, which means
5//! it is rate-limited aggressively and can change shape without notice — the
6//! keyed providers exist so a caller who cares about reliability can pay for a
7//! real API contract instead.
8//!
9//! No provider ever puts its key in a URL: keys travel in headers, so they
10//! cannot leak through `reqwest` error messages (which embed the request URL),
11//! proxy logs, or server access logs.
12
13pub mod brave;
14pub mod ddg;
15pub mod searxng;
16pub mod tavily;
17
18use std::time::Duration;
19
20use reqwest::Client;
21use serde::Deserialize;
22
23use crate::tls::TlsConfig;
24use crate::types::{SearchOptions, SearchResult, SearchStatus};
25
26/// Which backend answers a search, and the credential it needs.
27#[derive(Clone, Default, Deserialize)]
28#[serde(tag = "name", rename_all = "lowercase")]
29pub enum Provider {
30    /// Scraped DuckDuckGo Lite. No key, no contract. The default, so the tool
31    /// works with no configuration at all.
32    #[default]
33    #[serde(alias = "ddg")]
34    Duckduckgo,
35    /// Brave Search API. Key travels in `X-Subscription-Token`.
36    Brave { api_key: String },
37    /// Tavily, which returns cleaned page content alongside each hit.
38    Tavily { api_key: String },
39    /// A self-hosted SearXNG instance's JSON API.
40    Searxng {
41        base_url: String,
42        #[serde(default)]
43        api_key: Option<String>,
44    },
45}
46
47/// Redacting `Debug`: `SearchOptions` derives `Debug` and is routinely printed
48/// in error paths, so a derived impl here would put API keys on stderr.
49impl std::fmt::Debug for Provider {
50    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
51        match self {
52            Provider::Duckduckgo => write!(f, "Duckduckgo"),
53            Provider::Brave { .. } => write!(f, "Brave {{ api_key: <redacted> }}"),
54            Provider::Tavily { .. } => write!(f, "Tavily {{ api_key: <redacted> }}"),
55            Provider::Searxng { base_url, api_key } => write!(
56                f,
57                "Searxng {{ base_url: {base_url:?}, api_key: {} }}",
58                if api_key.is_some() {
59                    "<redacted>"
60                } else {
61                    "None"
62                }
63            ),
64        }
65    }
66}
67
68impl Provider {
69    /// Stable identifier recorded on the output so a fallback is visible.
70    pub fn label(&self) -> &'static str {
71        match self {
72            Provider::Duckduckgo => "duckduckgo",
73            Provider::Brave { .. } => "brave",
74            Provider::Tavily { .. } => "tavily",
75            Provider::Searxng { .. } => "searxng",
76        }
77    }
78
79    /// Parse a provider name with no credential attached. Used by the CLI's
80    /// `--search-provider` flag, which selects an entry the config file (or an
81    /// environment variable) supplies the key for.
82    pub fn parse_name(s: &str) -> Option<&'static str> {
83        match s.trim().to_ascii_lowercase().as_str() {
84            "duckduckgo" | "ddg" => Some("duckduckgo"),
85            "brave" => Some("brave"),
86            "tavily" => Some("tavily"),
87            "searxng" | "searx" => Some("searxng"),
88            _ => None,
89        }
90    }
91
92    /// Run a query against this backend.
93    pub async fn search(
94        &self,
95        options: &SearchOptions,
96    ) -> anyhow::Result<(Vec<SearchResult>, SearchStatus)> {
97        let max = options.max_results.unwrap_or(5);
98        match self {
99            Provider::Duckduckgo => ddg::search(options, max).await,
100            Provider::Brave { api_key } => brave::search(options, max, api_key).await,
101            Provider::Tavily { api_key } => tavily::search(options, max, api_key).await,
102            Provider::Searxng { base_url, api_key } => {
103                searxng::search(options, max, base_url, api_key.as_deref()).await
104            }
105        }
106    }
107}
108
109/// Build the HTTP client every provider uses: shared user agent, shared TLS
110/// trust configuration, and the caller's timeout.
111pub(crate) fn client(timeout_secs: u64, tls: &TlsConfig) -> anyhow::Result<Client> {
112    let builder = Client::builder()
113        .timeout(Duration::from_secs(timeout_secs))
114        .user_agent(webfetch_core::http::USER_AGENT)
115        .gzip(true);
116    Ok(tls.apply(builder)?.build()?)
117}
118
119/// Turn a provider's JSON hit list into results, numbering references as we go.
120///
121/// Field names are read defensively out of a `serde_json::Value` rather than a
122/// typed struct: a provider adding or renaming a peripheral field then degrades
123/// one hit instead of failing the whole search.
124pub(crate) fn results_from_json(
125    items: &[serde_json::Value],
126    title_key: &str,
127    url_key: &str,
128    snippet_keys: &[&str],
129    max: usize,
130) -> Vec<SearchResult> {
131    let mut out = Vec::new();
132    for item in items {
133        if out.len() >= max {
134            break;
135        }
136        let url = item
137            .get(url_key)
138            .and_then(|v| v.as_str())
139            .unwrap_or("")
140            .to_string();
141        if url.is_empty() {
142            continue;
143        }
144        let title = item
145            .get(title_key)
146            .and_then(|v| v.as_str())
147            .unwrap_or("")
148            .to_string();
149        let snippet = snippet_keys
150            .iter()
151            .find_map(|k| item.get(*k).and_then(|v| v.as_str()))
152            .map(crate::compress::compress_text)
153            .unwrap_or_default();
154        out.push(SearchResult {
155            title: crate::compress::compress_text(&title),
156            snippet,
157            url,
158            ref_index: out.len() + 1,
159        });
160    }
161    out
162}
163
164#[cfg(test)]
165mod tests {
166    use super::*;
167
168    #[test]
169    fn debug_never_prints_a_key() {
170        let p = Provider::Brave {
171            api_key: "SECRET-KEY-VALUE".into(),
172        };
173        let shown = format!("{p:?}");
174        assert!(!shown.contains("SECRET-KEY-VALUE"), "leaked: {shown}");
175        assert!(shown.contains("<redacted>"));
176
177        let p = Provider::Searxng {
178            base_url: "https://searx.test".into(),
179            api_key: Some("SECRET-KEY-VALUE".into()),
180        };
181        let shown = format!("{p:?}");
182        assert!(!shown.contains("SECRET-KEY-VALUE"), "leaked: {shown}");
183    }
184
185    #[test]
186    fn provider_names_parse() {
187        assert_eq!(Provider::parse_name("Brave"), Some("brave"));
188        assert_eq!(Provider::parse_name(" ddg "), Some("duckduckgo"));
189        assert_eq!(Provider::parse_name("searx"), Some("searxng"));
190        assert_eq!(Provider::parse_name("nope"), None);
191    }
192
193    #[test]
194    fn json_results_skip_entries_without_a_url() {
195        let items: Vec<serde_json::Value> = serde_json::from_str(
196            r#"[{"title":"A","url":"https://a.test","description":"about a"},
197                {"title":"No URL","description":"skipped"},
198                {"title":"B","url":"https://b.test"}]"#,
199        )
200        .unwrap();
201        let got = results_from_json(&items, "title", "url", &["description"], 10);
202        assert_eq!(got.len(), 2);
203        assert_eq!(got[0].ref_index, 1);
204        assert_eq!(got[0].snippet, "about a");
205        assert_eq!(got[1].ref_index, 2, "indices stay contiguous after a skip");
206        assert_eq!(got[1].snippet, "");
207    }
208
209    #[test]
210    fn json_results_respect_max() {
211        let items: Vec<serde_json::Value> =
212            serde_json::from_str(r#"[{"url":"https://a"},{"url":"https://b"}]"#).unwrap();
213        assert_eq!(results_from_json(&items, "t", "url", &[], 1).len(), 1);
214    }
215}
websearch/providers/mod.rs

websearch/providers/
mod.rs
