Skip to main content

harness_websearch/
engine.rs

1use async_trait::async_trait;
2use std::collections::HashMap;
3use std::future::Future;
4use std::pin::Pin;
5use std::sync::Arc;
6use std::time::{Duration, Instant};
7use url::Url;
8
9use crate::types::{SafeSearch, WebSearchResultItem, WebSearchTimeRange};
10
11#[derive(Clone)]
12pub struct WebSearchEngineInput {
13    pub backend_url: String,
14    pub query: String,
15    pub count: usize,
16    pub time_range: WebSearchTimeRange,
17    pub language: String,
18    pub safe_search: SafeSearch,
19    pub categories: Vec<String>,
20    pub timeout_ms: u64,
21    pub headers: HashMap<String, String>,
22    /// Called BEFORE the request fires with the resolved backend host.
23    /// Returning Err aborts the search with an SSRF-shaped SearchError.
24    pub check_host: HostCheckFn,
25}
26
27pub type HostCheckFn = Arc<
28    dyn Fn(String) -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>>
29        + Send
30        + Sync,
31>;
32
33pub struct WebSearchEngineResult {
34    pub results: Vec<WebSearchResultItem>,
35    pub backend_host: String,
36    pub elapsed_ms: u64,
37    /// Which engine served this result (provenance). None until set by an
38    /// engine or the fallback layer.
39    pub engine: Option<String>,
40    /// Coverage class of the serving engine (set by the fallback layer).
41    pub engine_class: Option<EngineClass>,
42    /// When the fallback chain MERGED results from more than one engine, the
43    /// contributing engine names in chain order. None/single otherwise.
44    pub engines: Option<Vec<String>>,
45    /// Whether the serving engine applied the requested time_range. None when
46    /// time_range=all (nothing to apply).
47    pub time_range_applied: Option<bool>,
48}
49
50#[async_trait]
51pub trait WebSearchEngine: Send + Sync {
52    async fn search(
53        &self,
54        input: WebSearchEngineInput,
55    ) -> Result<WebSearchEngineResult, SearchError>;
56
57    /// Engine name for provenance / fallback diagnostics. Defaults to
58    /// "searxng" for the legacy ReqwestEngine; new engines override it.
59    fn name(&self) -> &str {
60        "searxng"
61    }
62
63    /// Coverage class, used by the fallback chain to decide whether an `empty`
64    /// result is authoritative. Defaults to General (SearXNG is broad web).
65    fn engine_class(&self) -> EngineClass {
66        EngineClass::General
67    }
68}
69
70/// Engine coverage class. A "general" engine's empty is a trustworthy "the web
71/// had nothing" signal; a niche/vertical empty says far less, so the fallback
72/// chain treats a niche/vertical-only empty while a general engine ERRORED as
73/// a degraded failure rather than a clean empty. Mirrors TS `EngineClass`.
74#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
75#[serde(rename_all = "snake_case")]
76pub enum EngineClass {
77    General,
78    Niche,
79    Vertical,
80}
81
82impl EngineClass {
83    /// Human/model-readable coverage label used in the output header.
84    pub fn label(self) -> &'static str {
85        match self {
86            Self::General => "general web",
87            Self::Niche => "indie/small-web index",
88            Self::Vertical => "encyclopedic",
89        }
90    }
91}
92
93/// Engine-local error code, distinct from `harness_core::ToolErrorCode`. The
94/// orchestrator translates these into a `ToolError` reusing the existing
95/// core code set.
96#[derive(Debug, Clone, Copy, PartialEq, Eq)]
97pub enum SearchErrorCode {
98    InvalidParam,
99    ServerNotAvailable,
100    DnsError,
101    TlsError,
102    Timeout,
103    ConnectionReset,
104    SsrfBlocked,
105    IoError,
106}
107
108#[derive(Debug, Clone)]
109pub struct SearchError {
110    pub code: SearchErrorCode,
111    pub message: String,
112}
113
114impl SearchError {
115    pub fn new(code: SearchErrorCode, message: impl Into<String>) -> Self {
116        Self {
117            code,
118            message: message.into(),
119        }
120    }
121}
122
123pub struct ReqwestEngine {
124    client: reqwest::Client,
125}
126
127impl ReqwestEngine {
128    pub fn new() -> Self {
129        Self {
130            client: shared_client(),
131        }
132    }
133}
134
135/// A reqwest client configured the way every engine wants it (no auto-redirect
136/// so we control SSRF; gzip; rustls). Cheap to clone (Arc inside).
137pub(crate) fn shared_client() -> reqwest::Client {
138    reqwest::Client::builder()
139        .redirect(reqwest::redirect::Policy::none())
140        .build()
141        .expect("reqwest client build")
142}
143
144impl Default for ReqwestEngine {
145    fn default() -> Self {
146        Self::new()
147    }
148}
149
150pub fn default_engine() -> Arc<dyn WebSearchEngine> {
151    Arc::new(ReqwestEngine::new())
152}
153
154#[async_trait]
155impl WebSearchEngine for ReqwestEngine {
156    async fn search(
157        &self,
158        input: WebSearchEngineInput,
159    ) -> Result<WebSearchEngineResult, SearchError> {
160        let base = Url::parse(&input.backend_url).map_err(|_| {
161            SearchError::new(
162                SearchErrorCode::IoError,
163                format!("Invalid backend URL: {}", input.backend_url),
164            )
165        })?;
166        let host = base.host_str().unwrap_or("").to_string();
167
168        // Re-run the SSRF check on the resolved backend host before dialing.
169        (input.check_host)(host.clone())
170            .await
171            .map_err(|msg| SearchError::new(SearchErrorCode::SsrfBlocked, msg))?;
172
173        let url = build_search_url(&base, &input);
174        let started = Instant::now();
175
176        let mut req = self
177            .client
178            .request(reqwest::Method::GET, url)
179            .timeout(Duration::from_millis(input.timeout_ms));
180        for (k, v) in &input.headers {
181            req = req.header(k, v);
182        }
183
184        let res = req.send().await.map_err(classify_reqwest_error)?;
185        let status = res.status().as_u16();
186
187        if status >= 400 {
188            if status >= 500 {
189                return Err(SearchError::new(
190                    SearchErrorCode::ServerNotAvailable,
191                    format!("Search backend returned HTTP {}", status),
192                ));
193            }
194            return Err(SearchError::new(
195                SearchErrorCode::InvalidParam,
196                format!("Search backend rejected the query with HTTP {}", status),
197            ));
198        }
199
200        let bytes = res.bytes().await.map_err(classify_reqwest_error)?;
201        let parsed: serde_json::Value = serde_json::from_slice(&bytes).map_err(|e| {
202            SearchError::new(
203                SearchErrorCode::IoError,
204                format!(
205                    "Could not parse the search backend response as JSON: {}",
206                    e
207                ),
208            )
209        })?;
210
211        let results = map_results(&parsed);
212        Ok(WebSearchEngineResult {
213            results,
214            backend_host: host,
215            elapsed_ms: started.elapsed().as_millis() as u64,
216            engine: Some("searxng".to_string()),
217            engine_class: None,
218            engines: None,
219            // SearXNG applies the time_range param when one is requested.
220            time_range_applied: if input.time_range == WebSearchTimeRange::All {
221                None
222            } else {
223                Some(true)
224            },
225        })
226    }
227}
228
229fn build_search_url(base: &Url, input: &WebSearchEngineInput) -> Url {
230    // Append /search to the configured base, preserving any base path.
231    let mut url = base.clone();
232    let trimmed = url.path().trim_end_matches('/').to_string();
233    url.set_path(&format!("{}/search", trimmed));
234    {
235        let mut p = url.query_pairs_mut();
236        p.append_pair("q", &input.query);
237        p.append_pair("format", "json");
238        p.append_pair(
239            "safesearch",
240            &input.safe_search.to_numeric().to_string(),
241        );
242        // "all" omits the time_range param (SearXNG treats absent as all-time).
243        if input.time_range != WebSearchTimeRange::All {
244            p.append_pair("time_range", input.time_range.as_str());
245        }
246        p.append_pair("language", &input.language);
247        p.append_pair("categories", &input.categories.join(","));
248        p.append_pair("pageno", "1");
249    }
250    url
251}
252
253fn map_results(parsed: &serde_json::Value) -> Vec<WebSearchResultItem> {
254    let raw = match parsed.get("results").and_then(|v| v.as_array()) {
255        Some(arr) => arr,
256        None => return Vec::new(),
257    };
258    let mut out: Vec<WebSearchResultItem> = Vec::new();
259    for entry in raw {
260        if !entry.is_object() {
261            continue;
262        }
263        let title = entry.get("title").and_then(|v| v.as_str()).unwrap_or("");
264        let url = entry.get("url").and_then(|v| v.as_str()).unwrap_or("");
265        // Missing title/url → skip.
266        if title.is_empty() || url.is_empty() {
267            continue;
268        }
269        let snippet = entry.get("content").and_then(|v| v.as_str()).unwrap_or("");
270        out.push(WebSearchResultItem {
271            title: title.to_string(),
272            url: url.to_string(),
273            snippet: snippet.to_string(),
274            age: None,
275            score: None,
276            source: None,
277        });
278    }
279    out
280}
281
282pub(crate) fn classify_reqwest_error(e: reqwest::Error) -> SearchError {
283    let msg = e.to_string();
284    // reqwest's top-level Display rarely carries the OS-level reason
285    // ("Connection refused"); walk the source chain so the orchestrator can
286    // tell a down backend (refused) from a mid-flight drop (reset).
287    let chain = error_chain(&e);
288    if e.is_timeout() {
289        return SearchError::new(SearchErrorCode::Timeout, msg);
290    }
291    if e.is_connect() {
292        let lower = chain.to_lowercase();
293        if lower.contains("dns")
294            || lower.contains("resolve")
295            || lower.contains("lookup")
296            || lower.contains("not known")
297            || lower.contains("no such host")
298        {
299            return SearchError::new(SearchErrorCode::DnsError, msg);
300        }
301        // A refused connection to a configured-but-down SearXNG surfaces
302        // here; tag it "refused" so the orchestrator emits the "start it"
303        // hint under SERVER_NOT_AVAILABLE.
304        return SearchError::new(
305            SearchErrorCode::ServerNotAvailable,
306            format!("connection refused: {}", chain),
307        );
308    }
309    let lower = chain.to_lowercase();
310    if lower.contains("tls") || lower.contains("certificate") || lower.contains("ssl") {
311        return SearchError::new(SearchErrorCode::TlsError, msg);
312    }
313    if lower.contains("reset") {
314        return SearchError::new(SearchErrorCode::ConnectionReset, msg);
315    }
316    SearchError::new(SearchErrorCode::IoError, msg)
317}
318
319/// Flatten an error and its source chain into one string for keyword sniffing.
320pub(crate) fn error_chain(e: &reqwest::Error) -> String {
321    let mut parts = vec![e.to_string()];
322    let mut src = std::error::Error::source(e);
323    while let Some(s) = src {
324        parts.push(s.to_string());
325        src = s.source();
326    }
327    parts.join("; ")
328}