websearch/
lib.rs

1//! Web search with reference-style URL preservation.
2//!
3//! Results reuse the same reference-style URL preservation as the fetch path:
4//! each hit's title carries an inline `[N]` marker and the full URLs are
5//! collected into a reference block, keeping the context window tight while
6//! staying citable.
7//!
8//! DuckDuckGo Lite is the default backend and needs no key, so the tool works
9//! with zero configuration. Because it is scraped HTML it is also the least
10//! reliable: [`types::SearchStatus`] reports a challenge or unparseable page as
11//! a failure instead of an empty result, and [`providers::Provider`] offers
12//! keyed alternatives for callers who need a real API contract.
13
14// Shared primitives from webfetch-core; re-exported so internal modules can
15// keep using `crate::compress` / `crate::refs`.
16pub use webfetch_core::{compress, http, refs, tls};
17
18pub mod extract;
19pub mod providers;
20pub mod types;
21
22use crate::compress::estimate_tokens;
23pub use providers::Provider;
24use types::{Reference, SearchOptions, SearchOutput, SearchResult, SearchStatus};
25
26/// Build the reference block (index → URL) from parsed results.
27pub fn build_refs(results: &[SearchResult]) -> Vec<Reference> {
28    results
29        .iter()
30        .map(|r| Reference {
31            index: r.ref_index,
32            url: r.url.clone(),
33        })
34        .collect()
35}
36
37/// Render the inline body: each result as `title [N]` followed by its snippet.
38/// URLs are intentionally absent here — they live in the reference block.
39pub fn format_results(results: &[SearchResult]) -> String {
40    results
41        .iter()
42        .map(|r| {
43            if r.snippet.is_empty() {
44                format!("{} [{}]", r.title, r.ref_index)
45            } else {
46                format!("{} [{}]\n{}", r.title, r.ref_index, r.snippet)
47            }
48        })
49        .collect::<Vec<_>>()
50        .join("\n\n")
51}
52
53/// Render the reference block appended to text output.
54/// Thin wrapper over [`crate::refs::render_block`].
55pub fn render_references(refs: &[Reference]) -> String {
56    crate::refs::render_block(refs)
57}
58
59/// Render the human-readable form of a search: results, then the reference
60/// block, then a one-line note when the search did not actually answer.
61pub fn render_output(output: &SearchOutput) -> String {
62    let mut s = format_results(&output.results);
63    let refs = render_references(&output.references);
64    if !refs.is_empty() {
65        s.push_str(&format!("\n\n{refs}"));
66    }
67    if let Some(note) = status_note(output) {
68        if !s.is_empty() {
69            s.push_str("\n\n");
70        }
71        s.push_str(&note);
72    }
73    s
74}
75
76/// A plain-language explanation of a non-`Ok` status. `None` when results came
77/// back normally.
78pub fn status_note(output: &SearchOutput) -> Option<String> {
79    match output.status {
80        SearchStatus::Ok => None,
81        SearchStatus::Empty => Some(format!(
82            "No results for `{}` (provider: {}).",
83            output.query, output.provider
84        )),
85        SearchStatus::Blocked => Some(format!(
86            "Search was blocked or returned an unrecognized page (provider: {}). \
87             This is not the same as having no results — the query was not answered.",
88            output.provider
89        )),
90    }
91}
92
93/// Assemble an output from parsed results.
94pub fn build_output(
95    query: &str,
96    results: Vec<SearchResult>,
97    status: SearchStatus,
98    provider: &str,
99) -> SearchOutput {
100    let references = build_refs(&results);
101    let body = format_results(&results);
102    let refs_block = render_references(&references);
103    let full = if refs_block.is_empty() {
104        body
105    } else {
106        format!("{body}\n\n{refs_block}")
107    };
108
109    SearchOutput {
110        query: query.to_string(),
111        token_estimate: estimate_tokens(&full),
112        result_count: results.len(),
113        status,
114        provider: provider.to_string(),
115        references,
116        results,
117    }
118}
119
120/// Parse an already-fetched DuckDuckGo Lite page into a [`SearchOutput`]
121/// (no network). Kept for tests and offline callers.
122pub fn build_output_from_ddg(query: &str, html: &str, max_results: usize) -> SearchOutput {
123    let results = extract::parse_ddg_lite(html, max_results);
124    let status = extract::classify_page(html, results.len());
125    build_output(query, results, status, "duckduckgo")
126}
127
128/// Run a query against the configured provider, falling back to
129/// `options.fallback` when the primary errors or is blocked.
130///
131/// A fallback is never silent: [`SearchOutput::provider`] records which backend
132/// actually answered, so a caller can tell a Brave answer from a scraped one.
133pub async fn run_search(options: SearchOptions) -> anyhow::Result<SearchOutput> {
134    let primary = attempt_provider(&options.provider, &options).await;
135
136    let primary_answered = matches!(&primary, Ok(out) if !out.status.is_failure());
137    if primary_answered {
138        return primary;
139    }
140
141    if let Some(fallback) = &options.fallback {
142        if let Ok(out) = attempt_provider(fallback, &options).await {
143            if !out.status.is_failure() {
144                return Ok(out);
145            }
146        }
147    }
148    // No fallback, or the fallback failed too: report the primary's outcome so
149    // the error the caller sees is the one they configured for.
150    primary
151}
152
153async fn attempt_provider(
154    provider: &Provider,
155    options: &SearchOptions,
156) -> anyhow::Result<SearchOutput> {
157    let (results, status) = provider.search(options).await?;
158    Ok(build_output(
159        &options.query,
160        results,
161        status,
162        provider.label(),
163    ))
164}
websearch/lib.rs

websearch/
lib.rs
