lc_tools/hosted_search/
mod.rs

1// lc-tools/src/hosted_search/mod.rs
2//! Hosted web-search backends behind one unified result shape (B6, v0.22.4).
3//!
4//! The framework previously shipped only the keyless DuckDuckGo instant-answer
5//! tool ([`crate::DuckDuckGoSearchTool`]). Agentic web research in 2026 mostly
6//! runs through paid hosted search APIs with cleaner result quality and
7//! optional synthesized answers:
8//!
9//! - [Tavily](https://tavily.com) — `tavily_search` (RAG-tuned snippets + answer)
10//! - [Serper](https://serper.dev) — `serper_search` (Google result pages)
11//! - [Exa](https://exa.ai) — `exa_search` (neural/embedding search)
12//!
13//! All three share a single [`HostedSearchTool`] and a single result schema;
14//! only the [`SearchBackend`] implementations differ. Provider relevance
15//! signals are normalized into `0..=1`, results are re-ranked and URL-deduped
16//! before being handed back to the agent.
17//!
18//! ```no_run
19//! # async fn demo() -> Result<(), lc_tools::ToolError> {
20//! use lc_tools::hosted_search::HostedSearchTool;
21//! use lc_tools::BaseTool;
22//! let tool = HostedSearchTool::tavily_from_env()?;
23//! let out = tool
24//!     .run(serde_json::json!({"query": "Rust 1.85 release notes"}).to_string())
25//!     .await?;
26//! # let _ = out;
27//! # Ok(()) }
28//! ```
29
30pub mod exa;
31pub mod serper;
32pub mod tavily;
33
34pub use exa::ExaBackend;
35pub use serper::SerperBackend;
36pub use tavily::TavilyBackend;
37
38use async_trait::async_trait;
39use schemars::JsonSchema;
40use serde::{Deserialize, Serialize};
41use std::sync::Arc;
42
43use lc_core::tools::{BaseTool, ToolError};
44
45/// Default number of hits returned when the caller omits `top_k`.
46pub const DEFAULT_TOP_K: usize = 5;
47/// Maximum number of hits callers may request in one tool call.
48pub const MAX_TOP_K: usize = 20;
49
50/// One ranked web hit, independent of the hosting provider.
51#[derive(Debug, Clone, Serialize, PartialEq)]
52pub struct SearchResult {
53    /// Result title.
54    pub title: String,
55    /// Canonicalized result URL (fragment stripped).
56    pub url: String,
57    /// Provider text snippet / page excerpt.
58    pub snippet: String,
59    /// Relevance normalized into `0..=1`; higher is better.
60    ///
61    /// Providers that ship a native score (Tavily, Exa) keep it; position-based
62    /// providers (Serper) get a rank-derived score. The tool re-ranks by this
63    /// field, so callers never need provider-specific comparisons.
64    pub score: f64,
65    /// Publisher-reported date, when the API returns one.
66    pub published_date: Option<String>,
67    /// Publisher-reported author, when the API returns one.
68    pub author: Option<String>,
69    /// Backend label (`"tavily"`, `"serper"`, `"exa"`, …).
70    pub provider: &'static str,
71}
72
73/// Unified output of a hosted search call.
74#[derive(Debug, Clone, Serialize)]
75pub struct SearchOutput {
76    /// The query that was executed.
77    pub query: String,
78    /// Ranked, URL-deduped hits.
79    pub results: Vec<SearchResult>,
80    /// Synthesized answer, for backends/requests that produce one (Tavily).
81    pub answer: Option<String>,
82    /// Backend label.
83    pub provider: &'static str,
84}
85
86/// Parsed backend response before cross-provider ranking.
87#[derive(Debug, Clone, Default)]
88pub struct BackendResponse {
89    /// Raw hits in provider order; `score` already normalized into `0..=1`.
90    pub results: Vec<SearchResult>,
91    /// Synthesized answer if the backend produced one.
92    pub answer: Option<String>,
93}
94
95/// A hosted search provider: maps the common query shape to the provider API.
96#[async_trait]
97pub trait SearchBackend: Send + Sync {
98    /// Stable backend/tool label (`"tavily"`, …); the tool name is `{label}_search`.
99    fn label(&self) -> &'static str;
100
101    /// Runs the provider-specific HTTP call and parses the response.
102    async fn search(
103        &self,
104        query: &str,
105        top_k: usize,
106        include_answer: bool,
107    ) -> Result<BackendResponse, ToolError>;
108}
109
110/// Tool input for every hosted backend.
111#[derive(Debug, Deserialize, JsonSchema)]
112pub struct HostedSearchInput {
113    /// The search query.
114    pub query: String,
115    /// Number of results to return (default 5, capped at 20).
116    pub top_k: Option<usize>,
117    /// Whether to request a synthesized answer when the backend supports one (default: true).
118    pub include_answer: Option<bool>,
119}
120
121/// Hosted web-search tool parameterized by a [`SearchBackend`].
122///
123/// Construct with the provider constructors:
124/// [`HostedSearchTool::tavily`], [`HostedSearchTool::serper`],
125/// [`HostedSearchTool::exa`] (or their `*_from_env` variants).
126pub struct HostedSearchTool {
127    backend: Arc<dyn SearchBackend>,
128    tool_name: String,
129}
130
131impl HostedSearchTool {
132    /// Wraps an arbitrary backend (custom gateway, mock, …).
133    pub fn new(backend: impl SearchBackend + 'static) -> Self {
134        Self::from_arc(Arc::new(backend))
135    }
136
137    /// Wraps an already-shared backend.
138    pub fn from_arc(backend: Arc<dyn SearchBackend>) -> Self {
139        let tool_name = format!("{}_search", backend.label());
140        Self { backend, tool_name }
141    }
142
143    /// Backend label exposed by this tool instance.
144    pub fn provider(&self) -> &'static str {
145        self.backend.label()
146    }
147
148    /// Tavily with an explicit API key.
149    pub fn tavily(api_key: impl Into<String>) -> Self {
150        Self::new(TavilyBackend::new(api_key))
151    }
152
153    /// Tavily from `TAVILY_API_KEY`.
154    pub fn tavily_from_env() -> Result<Self, ToolError> {
155        Ok(Self::new(TavilyBackend::from_env()?))
156    }
157
158    /// Serper with an explicit API key.
159    pub fn serper(api_key: impl Into<String>) -> Self {
160        Self::new(SerperBackend::new(api_key))
161    }
162
163    /// Serper from `SERPER_API_KEY`.
164    pub fn serper_from_env() -> Result<Self, ToolError> {
165        Ok(Self::new(SerperBackend::from_env()?))
166    }
167
168    /// Exa with an explicit API key.
169    pub fn exa(api_key: impl Into<String>) -> Self {
170        Self::new(ExaBackend::new(api_key))
171    }
172
173    /// Exa from `EXA_API_KEY`.
174    pub fn exa_from_env() -> Result<Self, ToolError> {
175        Ok(Self::new(ExaBackend::from_env()?))
176    }
177
178    /// Typed entry point used by the [`lc_core::tools::Tool`] impls of backends.
179    pub async fn search(
180        &self,
181        query: &str,
182        top_k: Option<usize>,
183        include_answer: Option<bool>,
184    ) -> Result<SearchOutput, ToolError> {
185        let query = query.trim();
186        if query.is_empty() {
187            return Err(ToolError::InvalidInput(
188                "search query must not be empty".to_string(),
189            ));
190        }
191        let top_k = top_k.unwrap_or(DEFAULT_TOP_K).clamp(1, MAX_TOP_K);
192        let include_answer = include_answer.unwrap_or(true);
193
194        let mut response = self.backend.search(query, top_k, include_answer).await?;
195        rank_results(&mut response.results);
196        response.results.truncate(top_k);
197
198        Ok(SearchOutput {
199            query: query.to_string(),
200            results: response.results,
201            answer: response.answer,
202            provider: self.backend.label(),
203        })
204    }
205}
206
207/// Canonicalizes a URL for cross-provider dedup: parse, drop fragment,
208/// lowercase host, strip a trailing `/` from the path. Unparseable inputs are
209/// returned trimmed and lowercased so dedup still degrades gracefully.
210pub(crate) fn canonical_url(raw: &str) -> String {
211    let raw = raw.trim();
212    match url::Url::parse(raw) {
213        Ok(mut parsed) => {
214            parsed.set_fragment(None);
215            if let Some(host) = parsed.host_str().map(str::to_lowercase) {
216                let _ = parsed.set_host(Some(&host));
217            }
218            let path = parsed.path().to_string();
219            if path.len() > 1 && path.ends_with('/') {
220                parsed.set_path(path.trim_end_matches('/'));
221            }
222            parsed.to_string()
223        }
224        Err(_) => raw.to_lowercase(),
225    }
226}
227
228/// Sorts by normalized score descending (stable) and removes duplicate URLs,
229/// keeping the highest-ranked occurrence.
230pub(crate) fn rank_results(results: &mut Vec<SearchResult>) {
231    // sort_by is stable; NaN guards (score is produced by our parsers, but never
232    // trust a float enough to let NaN poison ordering).
233    results.sort_by(|a, b| {
234        b.score
235            .partial_cmp(&a.score)
236            .unwrap_or(std::cmp::Ordering::Equal)
237    });
238    let mut seen = std::collections::HashSet::new();
239    results.retain(|r| seen.insert(canonical_url(&r.url)));
240}
241
242/// Reads an API key from the environment with a provider-named error.
243pub(crate) fn require_env(key: &str) -> Result<String, ToolError> {
244    let value = std::env::var(key).map_err(|_| {
245        ToolError::InvalidInput(format!("{key} environment variable not set or empty"))
246    })?;
247    let value = value.trim();
248    if value.is_empty() {
249        return Err(ToolError::InvalidInput(format!(
250            "{key} environment variable not set or empty"
251        )));
252    }
253    Ok(value.to_string())
254}
255
256/// Strips the trailing slashes from a configured base URL.
257pub(crate) fn trim_trailing_slash(mut base: String) -> String {
258    while base.len() > 1 && base.ends_with('/') {
259        base.pop();
260    }
261    base
262}
263
264fn render_text(output: &SearchOutput) -> String {
265    let mut text = format!("{} 搜索结果(查询: {})\n\n", output.provider, output.query);
266    if let Some(answer) = output.answer.as_ref().filter(|a| !a.is_empty()) {
267        text.push_str(&format!("综合答案: {answer}\n\n"));
268    }
269    for (i, result) in output.results.iter().enumerate() {
270        text.push_str(&format!("{}. {}\n", i + 1, result.title));
271        text.push_str(&format!("   {}\n", result.snippet));
272        text.push_str(&format!("   URL: {}\n", result.url));
273        text.push_str(&format!("   相关度: {:.2}\n", result.score));
274        match (result.published_date.as_ref(), result.author.as_ref()) {
275            (Some(date), Some(author)) => {
276                text.push_str(&format!("   发布: {date} · {author}\n"));
277            }
278            (Some(date), None) => text.push_str(&format!("   发布: {date}\n")),
279            (None, Some(author)) => text.push_str(&format!("   作者: {author}\n")),
280            (None, None) => {}
281        }
282        text.push('\n');
283    }
284    if output.results.is_empty() {
285        text.push_str("未找到相关结果");
286    } else {
287        text.push_str(&format!("共 {} 条结果", output.results.len()));
288    }
289    text
290}
291
292#[async_trait]
293impl BaseTool for HostedSearchTool {
294    fn name(&self) -> &str {
295        &self.tool_name
296    }
297
298    fn description(&self) -> &str {
299        match self.backend.label() {
300            "tavily" => "Tavily 托管网页搜索工具(为 RAG/agent 优化的正文片段,可选综合答案)。\n\n参数:\n- query: 搜索关键词\n- top_k: 返回结果数量(默认 5,上限 20)\n- include_answer: 是否请求综合答案(默认 true)\n\n需设置 TAVILY_API_KEY。\n示例: {\"query\": \"Rust 1.85 async closure\", \"top_k\": 5}",
301            "serper" => "Serper 托管网页搜索工具(Google 结果页)。\n\n参数:\n- query: 搜索关键词\n- top_k: 返回结果数量(默认 5,上限 20)\n- include_answer: 此后端不支持综合答案,字段被忽略\n\n需设置 SERPER_API_KEY。\n示例: {\"query\": \"tokio tungstenite connect_async\", \"top_k\": 5}",
302            "exa" => "Exa 神经托管网页搜索工具(语义检索,适合研究类查询)。\n\n参数:\n- query: 搜索查询(自然语言描述)\n- top_k: 返回结果数量(默认 5,上限 20)\n- include_answer: 此后端不支持综合答案,字段被忽略\n\n需设置 EXA_API_KEY。\n示例: {\"query\": \"papers about long-context transformer memory\", \"top_k\": 5}",
303            _ => "Hosted web search tool.\n\n参数:\n- query: 搜索关键词\n- top_k: 返回结果数量(默认 5,上限 20)",
304        }
305    }
306
307    async fn run(&self, input: String) -> Result<String, ToolError> {
308        let parsed: HostedSearchInput = serde_json::from_str(&input)
309            .map_err(|e| ToolError::InvalidInput(format!("JSON parse failed: {e}")))?;
310        let output = self
311            .search(&parsed.query, parsed.top_k, parsed.include_answer)
312            .await?;
313        Ok(render_text(&output))
314    }
315
316    fn args_schema(&self) -> Option<serde_json::Value> {
317        serde_json::to_value(schemars::schema_for!(HostedSearchInput)).ok()
318    }
319}
320
321#[cfg(test)]
322pub(crate) mod test_support {
323    //! Env-var serialization for the per-backend `*_from_env` tests and a tiny
324    //! loopback JSON HTTP server so request construction is exercised without
325    //! hitting the real APIs.
326    use std::sync::Mutex;
327
328    /// All hosted-search env tests mutate process environment; serialize them.
329    pub(crate) static ENV_LOCK: Mutex<()> = Mutex::new(());
330
331    /// Starts a one-shot HTTP server on an ephemeral port that responds to the
332    /// first request with `reply` and returns the raw request bytes through the
333    /// oneshot so tests can assert method, headers and body.
334    pub(crate) async fn spawn_one_shot_json(
335        reply: serde_json::Value,
336    ) -> (String, tokio::sync::oneshot::Receiver<Vec<u8>>) {
337        use tokio::io::{AsyncReadExt, AsyncWriteExt};
338        use tokio::net::TcpListener;
339
340        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
341        let addr = listener.local_addr().unwrap();
342        let (tx, rx) = tokio::sync::oneshot::channel();
343        tokio::spawn(async move {
344            let (mut socket, _) = listener.accept().await.unwrap();
345            let mut request = Vec::new();
346            let mut buf = [0u8; 4096];
347            // Read until end of headers.
348            loop {
349                let n = socket.read(&mut buf).await.unwrap();
350                assert!(n > 0, "client closed request early");
351                request.extend_from_slice(&buf[..n]);
352                if request.windows(4).any(|w| w == b"\r\n\r\n") {
353                    break;
354                }
355            }
356            // Read the rest of the declared body.
357            let header_end = request
358                .windows(4)
359                .position(|w| w == b"\r\n\r\n")
360                .map(|p| p + 4)
361                .unwrap();
362            let content_length = String::from_utf8_lossy(&request[..header_end])
363                .lines()
364                .find_map(|line| {
365                    let line = line.to_ascii_lowercase();
366                    line.strip_prefix("content-length:")
367                        .map(|v| v.trim().parse::<usize>().unwrap())
368                })
369                .unwrap_or(0);
370            while request.len() < header_end + content_length {
371                let n = socket.read(&mut buf).await.unwrap();
372                assert!(n > 0);
373                request.extend_from_slice(&buf[..n]);
374            }
375            let body = serde_json::to_vec(&reply).unwrap();
376            let response = format!(
377                "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
378                body.len()
379            );
380            socket.write_all(response.as_bytes()).await.unwrap();
381            socket.write_all(&body).await.unwrap();
382            socket.flush().await.unwrap();
383            let _ = tx.send(request);
384        });
385        (format!("http://{addr}"), rx)
386    }
387}
388
389#[cfg(test)]
390mod tests {
391    use super::*;
392
393    #[test]
394    fn backend_debug_never_exposes_api_keys() {
395        let secret = "supersecret-key-DEBUG-LEAK";
396        for rendered in [
397            format!("{:?}", TavilyBackend::new(secret)),
398            format!("{:?}", SerperBackend::new(secret)),
399            format!("{:?}", ExaBackend::new(secret)),
400        ] {
401            assert!(
402                !rendered.contains(secret),
403                "key leaked via Debug: {rendered}"
404            );
405            assert!(rendered.contains("<redacted>"), "{rendered}");
406        }
407    }
408
409    #[test]
410    fn canonical_url_strips_fragment_and_trailing_slash() {
411        assert_eq!(
412            canonical_url("HTTPS://Example.COM/path/#section"),
413            "https://example.com/path"
414        );
415        assert_eq!(
416            canonical_url("https://a.example/x?b=1#frag"),
417            "https://a.example/x?b=1"
418        );
419        // Root path keeps its slash (url crate normalizes "" back to "/").
420        assert_eq!(canonical_url("http://b.example/"), "http://b.example/");
421        // Garbage still lowercases, so dedup is case-insensitive.
422        assert_eq!(canonical_url("  NOT-A-URL  "), "not-a-url");
423    }
424
425    #[test]
426    fn rank_orders_by_score_and_dedupes_url() {
427        let provider = "test";
428        let mk = |url: &str, score: f64| SearchResult {
429            title: url.to_string(),
430            url: url.to_string(),
431            snippet: String::new(),
432            score,
433            published_date: None,
434            author: None,
435            provider,
436        };
437        let mut results = vec![
438            mk("https://a.example/p", 0.2),
439            mk("https://a.example/p#frag", 0.9), // same canonical URL, higher score
440            mk("https://b.example/", 0.5),
441        ];
442        rank_results(&mut results);
443        assert_eq!(results.len(), 2, "fragment dup must collapse");
444        assert_eq!(results[0].url, "https://a.example/p#frag");
445        assert!(results[0].score > results[1].score);
446    }
447
448    /// Records the `top_k` the tool dispatched, so validation/clamping is proven
449    /// without any HTTP.
450    struct RecordingBackend {
451        seen_k: tokio::sync::Mutex<Option<usize>>,
452    }
453    #[async_trait]
454    impl SearchBackend for RecordingBackend {
455        fn label(&self) -> &'static str {
456            "recording"
457        }
458        async fn search(
459            &self,
460            _query: &str,
461            top_k: usize,
462            _include_answer: bool,
463        ) -> Result<BackendResponse, ToolError> {
464            *self.seen_k.lock().await = Some(top_k);
465            Ok(BackendResponse::default())
466        }
467    }
468
469    /// Minimal current-thread block_on (the crate pulls tokio full anyway).
470    fn block_on<F: std::future::Future>(fut: F) -> F::Output {
471        let rt = tokio::runtime::Builder::new_current_thread()
472            .enable_all()
473            .build()
474            .unwrap();
475        rt.block_on(fut)
476    }
477
478    #[test]
479    fn empty_query_is_rejected_before_backend() {
480        let tool = HostedSearchTool::new(RecordingBackend {
481            seen_k: tokio::sync::Mutex::new(None),
482        });
483        let err = block_on(tool.search("   ", None, None)).unwrap_err();
484        assert!(err.to_string().contains("empty"));
485    }
486
487    #[test]
488    fn top_k_is_clamped_before_dispatch() {
489        let backend = Arc::new(RecordingBackend {
490            seen_k: tokio::sync::Mutex::new(None),
491        });
492        let tool = HostedSearchTool::from_arc(backend.clone() as Arc<dyn SearchBackend>);
493        block_on(tool.search("q", Some(999), None)).unwrap();
494        assert_eq!(block_on(backend.seen_k.lock()).as_ref(), Some(&MAX_TOP_K));
495
496        let backend2 = Arc::new(RecordingBackend {
497            seen_k: tokio::sync::Mutex::new(None),
498        });
499        let tool2 = HostedSearchTool::from_arc(backend2.clone() as Arc<dyn SearchBackend>);
500        block_on(tool2.search("q", Some(0), None)).unwrap();
501        assert_eq!(block_on(backend2.seen_k.lock()).as_ref(), Some(&1));
502    }
503
504    #[test]
505    fn tool_metadata_is_provider_named() {
506        let tool = HostedSearchTool::tavily("tvly-test");
507        assert_eq!(BaseTool::name(&tool), "tavily_search");
508        assert!(tool.description().contains("Tavily"));
509        assert!(BaseTool::args_schema(&tool).is_some());
510        assert_eq!(tool.provider(), "tavily");
511    }
512
513    #[test]
514    fn render_includes_answer_scores_and_empty_state() {
515        let output = SearchOutput {
516            query: "q".into(),
517            results: vec![SearchResult {
518                title: "T".into(),
519                url: "https://x.example".into(),
520                snippet: "S".into(),
521                score: 0.91,
522                published_date: Some("2026-01-02".into()),
523                author: Some("A".into()),
524                provider: "tavily",
525            }],
526            answer: Some("synth".into()),
527            provider: "tavily",
528        };
529        let text = render_text(&output);
530        assert!(text.contains("综合答案: synth"));
531        assert!(text.contains("相关度: 0.91"));
532        assert!(text.contains("2026-01-02 · A"));
533
534        let empty = SearchOutput {
535            query: "q".into(),
536            results: vec![],
537            answer: None,
538            provider: "serper",
539        };
540        assert!(render_text(&empty).contains("未找到相关结果"));
541    }
542}
lc_tools/hosted_search/mod.rs

lc_tools/hosted_search/
mod.rs
