//! WebSearch tool: query -> ranked web results (title, url, snippet).
//!
//! Two providers, selected in `vtcode.toml` via `[tools.web_search] provider`:
//! - `duckduckgo` (default): keyless HTML scraping of
//! `https://html.duckduckgo.com/html/`. Best-effort; may be rate-limited.
//! - `youcom`: the You.com Search API (`https://ydc-index.io/v1/search`),
//! opt-in, requires the `YDC_API_KEY` environment variable.
//!
//! Safety/rate-limit guard rails (shared by both providers):
//! - A **cooldown** between consecutive network requests (default 3s) prevents
//! hammering DDG and triggering anti-bot challenges.
//! - A short **result cache** (default 5min TTL) means repeated identical
//! queries are answered from memory without any network call.
//! - A **session-wide cap** (default 12 requests) ensures the tool cannot
//! leak past DDG's soft quotas even with varied queries.
//!
//! All three are configurable via `WebSearchConfig` and turned off in tests.
use super::traits::Tool;
use crate::config::constants::tools;
use anyhow::{Context, Result, anyhow};
use astral_html::Token;
use async_trait::async_trait;
use reqwest::header::{ACCEPT, HeaderMap, HeaderValue, USER_AGENT};
use serde::Deserialize;
use serde_json::{Value, json};
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use url::Url;
use vtcode_config::{WebSearchConfig, WebSearchProvider};
const MAX_TIMEOUT_SECS: u64 = 60;
const MAX_RESULTS_CAP: usize = 20;
const MAX_TITLE_CHARS: usize = 200;
const MAX_SNIPPET_CHARS: usize = 400;
/// Browser-like user agent. DuckDuckGo's HTML endpoint blocks obvious bots, so a
/// realistic UA reduces (but does not eliminate) the chance of being challenged.
const BROWSER_USER_AGENT: &str =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36";
pub(crate) const WEB_SEARCH_DESCRIPTION: &str = "Searches the web for a query and returns a ranked list of results (title, url, snippet) inline. Accepts: { query: string, max_results?: number }. Provider is set in vtcode.toml ([tools.web_search] provider): \"duckduckgo\" (default; keyless HTML endpoint, best-effort, may be rate-limited) or \"youcom\" (You.com Search API; requires YDC_API_KEY). Results are cached for a few minutes to avoid repeat hits. Use web_fetch on the most promising result URL to read full content. Returns { query, provider, count, cached, results: [{ title, url, snippet }] }.";
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct WebSearchArgs {
/// Search query.
query: String,
#[serde(default)]
max_results: Option<usize>,
}
pub struct SearchResult {
pub title: String,
pub url: String,
pub snippet: String,
}
impl SearchResult {
pub fn new(title: String, url: String, snippet: String) -> Self {
Self { title, url, snippet }
}
}
/// Lightweight in-process state shared between calls on the same tool
/// instance. Owns the cooldown clock, the request counter, and the result
/// cache.
#[derive(Default)]
struct SessionState {
last_request_at: Option<Instant>,
requests_made: u32,
cache: HashMap<String, CachedResults>,
}
struct CachedResults {
stored_at: Instant,
payload: Value,
}
impl SessionState {
fn cache_get(&self, key: &str, ttl: Duration) -> Option<Value> {
let entry = self.cache.get(key)?;
if entry.stored_at.elapsed() > ttl {
return None;
}
Some(entry.payload.clone())
}
fn cache_put(&mut self, key: String, payload: Value) {
self.cache.insert(key, CachedResults { stored_at: Instant::now(), payload });
}
}
/// WebSearch tool. Stateless over the network (DDG) but tracks per-instance
/// cooldown, request cap, and short-lived result cache to avoid hammering
/// DDG's HTML endpoint and triggering anti-bot challenges.
#[derive(Clone, Default)]
pub struct WebSearchTool {
config: Arc<Mutex<WebSearchConfig>>,
state: Arc<Mutex<SessionState>>,
}
impl WebSearchTool {
pub fn new() -> Self {
Self {
config: Arc::new(Mutex::new(WebSearchConfig::default())),
state: Arc::new(Mutex::new(SessionState::default())),
}
}
/// Construct a tool with an explicit configuration. The config drives the
/// result-count default, request timeout, cooldown, cache TTL, and
/// session-wide request cap.
pub fn with_config(config: WebSearchConfig) -> Self {
Self {
config: Arc::new(Mutex::new(config)),
state: Arc::new(Mutex::new(SessionState::default())),
}
}
/// Apply a new configuration (e.g., after `vtcode.toml` reload).
pub fn set_config(&self, config: WebSearchConfig) {
if let Ok(mut guard) = self.config.lock() {
*guard = config;
}
}
/// Reset the in-process state (cooldown, counter, cache). Tests use this.
pub fn reset(&self) {
if let Ok(mut guard) = self.state.lock() {
*guard = SessionState::default();
}
}
fn snapshot_config(&self) -> WebSearchConfig {
self.config.lock().map(|guard| guard.clone()).unwrap_or_default()
}
async fn run(&self, raw_args: Value) -> Result<Value> {
let args: WebSearchArgs =
serde_json::from_value(raw_args).context("Invalid arguments for web_search. Provide a 'query' string.")?;
let query = args.query.trim().to_string();
if query.is_empty() {
return Err(anyhow!("web_search requires a non-empty 'query'"));
}
let snapshot = self.snapshot_config();
let default_max = snapshot.max_results.clamp(1, MAX_RESULTS_CAP);
let max_results = args.max_results.unwrap_or(default_max).clamp(1, MAX_RESULTS_CAP);
let cooldown = Duration::from_millis(snapshot.cooldown_ms);
let cache_ttl = Duration::from_secs(snapshot.cache_ttl_secs);
let session_cap = snapshot.session_max_requests;
// Include `max_results` in the key so a repeat query with a
// different cap gets a fresh, larger or smaller result set. With the
// hard 20-result cap this is rarely observable, but it keeps the
// cache semantics tight: the value the agent sees was produced for
// the same `max_results` it asked for.
let cache_key = format!("{max_results}::{query}");
// Fast path: cache hit. Avoid the network entirely.
if let Some(cached) = self
.state
.lock()
.unwrap_or_else(|e| e.into_inner())
.cache_get(&cache_key, cache_ttl)
{
return Ok(mark_cached(cached));
}
// Enforce session-wide request cap before touching the network.
{
let state = self.state.lock().map_err(|e| anyhow!("web_search state lock poisoned: {e}"))?;
if state.requests_made >= session_cap {
return Ok(session_cap_reached_response(&query, session_cap));
}
if let Some(last) = state.last_request_at {
let elapsed = last.elapsed();
if elapsed < cooldown {
// `elapsed < cooldown` is checked above, so subtraction is safe
return Ok(cooldown_response(
&query,
cooldown.checked_sub(elapsed).expect("elapsed < cooldown checked above"),
));
}
}
}
// Resolve the provider before touching the network so a missing
// YDC_API_KEY can be reported as a setup error without consuming a
// request from the session cap or starting the cooldown clock.
let provider = match snapshot.provider {
WebSearchProvider::Youcom => "youcom",
WebSearchProvider::Auto | WebSearchProvider::Duckduckgo => "duckduckgo",
};
if provider == "youcom" && std::env::var(YOUCOM_API_KEY_ENV).is_err() {
return Ok(json!({
"error": format!(
"You.com provider selected but {YOUCOM_API_KEY_ENV} is not set. Set it to a key from https://you.com/platform/api-keys, or switch [tools.web_search] provider back to \"duckduckgo\"."
),
"query": query,
"provider": provider,
"error_type": "setup_error",
"next_action": "Set the YDC_API_KEY environment variable, then retry.",
}));
}
let results = match snapshot.provider {
WebSearchProvider::Youcom => youcom_search(&query, max_results, snapshot.timeout_secs).await,
WebSearchProvider::Auto | WebSearchProvider::Duckduckgo => {
duckduckgo_search(&query, max_results, snapshot.timeout_secs).await
}
};
let provider_name = provider;
// By this point we have either short-circuited above (cache / cap /
// cooldown) or attempted a real network call. Record the request so
// the cooldown and session cap kick in for subsequent calls.
if let Ok(mut state) = self.state.lock() {
state.last_request_at = Some(Instant::now());
state.requests_made = state.requests_made.saturating_add(1);
}
match results {
Ok(results) if results.is_empty() => {
let payload = json!({
"query": query,
"provider": provider_name,
"count": 0,
"results": [],
"warning": if provider_name == "youcom" {
"No results were returned. The query may have matched nothing; try a different query."
} else {
"No results were returned. DuckDuckGo may have rate-limited the request or matched nothing. Try a different query, or wait a few seconds and try again."
}
});
self.cache_put(&cache_key, &payload);
Ok(payload)
}
Ok(results) => {
let payload = json!({
"query": query,
"provider": provider_name,
"count": results.len(),
"results": results
.into_iter()
.map(|r| json!({ "title": r.title, "url": r.url, "snippet": r.snippet }))
.collect::<Vec<_>>(),
});
self.cache_put(&cache_key, &payload);
Ok(payload)
}
Err(e) => {
// Categorize the error so the agent can act on it. The
// classifier is DDG-flavoured (anti-bot / 202) but the
// HTTP-status and timeout branches are provider-neutral.
let (error_type, next_action) = classify_search_error(&e.to_string(), provider_name);
Ok(json!({
"error": format!("web_search failed: {e}"),
"query": query,
"provider": provider_name,
"error_type": error_type,
"next_action": next_action,
}))
}
}
}
fn cache_put(&self, key: &str, payload: &Value) {
if let Ok(mut state) = self.state.lock() {
state.cache_put(key.to_string(), payload.clone());
}
}
}
/// Classify a search-provider error into a `(error_type, next_action)` pair.
/// The returned strings are stable so the agent loop can branch on them.
/// Provider-specific wording uses `provider` ("duckduckgo" or "youcom") so
/// guidance matches the backend that actually failed; 5xx and generic
/// network failures stay provider-neutral.
fn classify_search_error(message: &str, provider: &str) -> (&'static str, &'static str) {
let lower = message.to_lowercase();
// 5xx from the upstream search service is transient but should not be
// retried the same way as a network error — the cause is server-side.
if let Some(status) = crate::tools::web_fetch::classify_helpers::extract_http_status(&lower) {
if (500..=599).contains(&status) {
return (
"upstream_error",
"The search service is currently unavailable. Retry after a short delay, or use web_fetch on a known URL as a fallback.",
);
}
}
if message.contains("HTTP 202") || lower.contains("anti-bot") || lower.contains("challenge") {
(
"antiban_blocked",
"DuckDuckGo declined this request (likely an anti-bot challenge for this network). An immediate retry from this network will hit the same block; pick a result URL from this session's earlier searches and use web_fetch on it instead, or ask the user to confirm a different search provider.",
)
} else if lower.contains("timeout") || lower.contains("timed out") {
if provider == "youcom" {
(
"network_error",
"You.com timed out. Retry after a short delay, or use web_fetch on a known URL as a fallback.",
)
} else {
(
"network_error",
"DuckDuckGo timed out. Retry after a short delay, or use web_fetch on a known URL as a fallback.",
)
}
} else {
(
"network_error",
"Wait a few seconds and retry, or use web_fetch directly if you already know a relevant URL.",
)
}
}
fn mark_cached(mut payload: Value) -> Value {
if let Value::Object(map) = &mut payload {
map.insert("cached".to_string(), Value::Bool(true));
}
payload
}
fn cooldown_response(query: &str, wait: Duration) -> Value {
json!({
"error": "web_search cooldown active",
"query": query,
"retry_after_ms": wait.as_millis() as u64,
"next_action": format!("Wait at least {} ms before the next web search to avoid being rate-limited.", wait.as_millis())
})
}
fn session_cap_reached_response(query: &str, cap: u32) -> Value {
json!({
"error": "web_search session request cap reached",
"query": query,
"session_max_requests": cap,
"next_action": format!("This session has used its {cap} web searches. Use web_fetch on a known URL or restart the session to search again.")
})
}
/// Build the HTTP client for the DuckDuckGo provider. Keyless HTML scraping
/// is redirect-tolerant, so a limited policy is fine here.
fn build_client(timeout_secs: u64) -> Result<reqwest::Client> {
reqwest::Client::builder()
.timeout(Duration::from_secs(timeout_secs.min(MAX_TIMEOUT_SECS)))
.redirect(reqwest::redirect::Policy::limited(3))
.build()
.context("failed to build HTTP client for web_search")
}
/// Build the HTTP client for the You.com provider. Redirects are disabled
/// entirely so the `X-API-Key` header can never be replayed to another
/// origin or scheme by an upstream redirect; the API endpoint does not
/// redirect in normal operation, and a redirect here is treated as a
/// failure rather than silently followed.
fn build_youcom_client(timeout_secs: u64) -> Result<reqwest::Client> {
reqwest::Client::builder()
.timeout(Duration::from_secs(timeout_secs.min(MAX_TIMEOUT_SECS)))
.redirect(reqwest::redirect::Policy::none())
.build()
.context("failed to build HTTP client for web_search")
}
// ---------------------------------------------------------------------------
// DuckDuckGo (keyless, best-effort HTML scraping)
//
// Result extraction is tokenized with `astral-html`'s streaming `Reader`
// (pinned pilot dependency, vtcode-core only): tag and attribute names
// arrive ASCII-lowercased, text in `script`/`style` stays raw text instead
// of spurious tags, and each snippet is collected from its own anchor's
// window so a missing snippet yields empty rather than the next result's.
// ---------------------------------------------------------------------------
async fn duckduckgo_search(query: &str, max_results: usize, timeout_secs: u64) -> Result<Vec<SearchResult>> {
let client = build_client(timeout_secs)?;
let mut headers = HeaderMap::new();
headers.insert(USER_AGENT, HeaderValue::from_static(BROWSER_USER_AGENT));
headers.insert(ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml"));
// The HTML endpoint expects the query as a POST form field `q`.
let response = client
.post("https://html.duckduckgo.com/html/")
.headers(headers)
.form(&[("q", query)])
.send()
.await
.context("DuckDuckGo request failed")?;
let status = response.status();
// HTTP 202 from the HTML endpoint is DuckDuckGo's anti-bot challenge page
// (it returns the homepage shell with no results). Treat it, and any other
// non-200, as a challenge rather than letting parsing silently yield zero
// results (which previously caused agent retry loops).
if status.as_u16() == 202 || !status.is_success() {
return Err(anyhow!(
"DuckDuckGo declined the request (HTTP {status}), likely an anti-bot challenge for this network. Wait a few seconds and retry."
));
}
let body = read_capped_ddg_body(response).await?;
Ok(parse_duckduckgo_html(&body, max_results))
}
/// Upper bound for a DuckDuckGo HTML answer body. Result pages are a few
/// hundred kilobytes; the cap only bites on adversarial responses, and the
/// streaming parser stops at `max_results` well before it anyway.
const MAX_DDG_BODY_BYTES: usize = 1_000_000;
/// Read a DDG response body with an explicit byte cap. Decoding is lossy
/// rather than fallible: DuckDuckGo serves UTF-8, and a corrupt byte must
/// not turn into a hard error when partial results will do.
async fn read_capped_ddg_body(response: reqwest::Response) -> Result<String> {
let bytes = response.bytes().await.context("failed to read DuckDuckGo response body")?;
Ok(capped_body_text(&bytes))
}
/// Pure, testable core of [`read_capped_ddg_body`]: cap at
/// `MAX_DDG_BODY_BYTES`, then lossy-decode (never panics, even on a split
/// trailing char or leading garbage).
fn capped_body_text(bytes: &[u8]) -> String {
let end = bytes.len().min(MAX_DDG_BODY_BYTES);
String::from_utf8_lossy(&bytes[..end]).into_owned()
}
/// Extract ranked results from a DuckDuckGo HTML body. Pure function so it can
/// be exercised in unit tests against a local fixture (no live network).
///
/// Single streaming pass: a result anchor opens title collection, its first
/// `</a>` completes it, and the following `result__snippet` element (before
/// the next result anchor) supplies the snippet. First-close-wins keeps
/// parity with the previous extractor on well-formed markup.
pub fn parse_duckduckgo_html(body: &str, max_results: usize) -> Vec<SearchResult> {
let mut results = Vec::new();
let mut state = DdgSection::Idle;
for token in astral_html::Reader::new(body) {
if results.len() >= max_results {
break;
}
match token {
Token::StartTag(tag) => {
if tag.name == "a" && has_ddg_class(&tag, "result__a") {
// A new result anchor ends the pending one (keeping an
// empty snippet) and drops any partial buffers.
if let DdgSection::AfterTitle { url, title } = std::mem::replace(&mut state, DdgSection::Idle) {
push_ddg_result(&mut results, url, title, String::new(), max_results);
if results.len() >= max_results {
break;
}
}
if !tag.self_closing {
if let Some(href) = ddg_attr(&tag, "href") {
state = DdgSection::InTitle { href: href.to_string(), buf: String::new() };
}
}
} else if matches!(state, DdgSection::AfterTitle { .. }) && has_ddg_class(&tag, "result__snippet") {
if let DdgSection::AfterTitle { url, title } = std::mem::replace(&mut state, DdgSection::Idle) {
if tag.self_closing {
push_ddg_result(&mut results, url, title, String::new(), max_results);
} else {
state = DdgSection::InSnippet {
url,
title,
name: tag.name.to_string(),
buf: String::new(),
};
}
}
}
// All other tags are content or noise: nested formatting
// text arrives as Text tokens; nothing to do here.
}
Token::Text(text) => match &mut state {
DdgSection::InTitle { buf, .. } | DdgSection::InSnippet { buf, .. } => {
buf.push_str(&text);
}
DdgSection::Idle | DdgSection::AfterTitle { .. } => {}
},
Token::EndTag(tag) => {
if tag.name == "a" && matches!(state, DdgSection::InTitle { .. }) {
if let DdgSection::InTitle { href, buf } = std::mem::replace(&mut state, DdgSection::Idle) {
let title = buf.trim().to_string();
if !title.is_empty() {
if let Some(url) = normalize_ddg_url(&href) {
state = DdgSection::AfterTitle { url, title };
}
}
}
} else if matches!(state, DdgSection::InSnippet { .. }) {
let done = if let DdgSection::InSnippet { name, .. } = &state {
tag.name == *name
} else {
false
};
if done {
if let DdgSection::InSnippet { url, title, buf, .. } =
std::mem::replace(&mut state, DdgSection::Idle)
{
push_ddg_result(&mut results, url, title, buf, max_results);
}
}
}
}
_ => {}
}
}
// Trailing result without a snippet element keeps an empty snippet.
if let DdgSection::AfterTitle { url, title } = state {
push_ddg_result(&mut results, url, title, String::new(), max_results);
}
results
}
/// Tokenizer position while extracting one result: idle, collecting the
/// anchor title, holding a completed title, or collecting its snippet.
enum DdgSection {
Idle,
InTitle {
href: String,
buf: String,
},
AfterTitle {
url: String,
title: String,
},
InSnippet {
url: String,
title: String,
name: String,
buf: String,
},
}
/// Decoded value of attribute `name` on `tag`, if present. Attribute names
/// are ASCII-lowercased by the tokenizer, so plain equality applies.
fn ddg_attr(tag: &astral_html::Tag<'_>, name: &str) -> Option<String> {
tag.attributes
.iter()
.find(|attr| attr.name == name)
.map(|attr| attr.value().to_string())
}
/// Whether the tag's `class` attribute carries `class` as a whitespace
/// separated token (ASCII case-insensitive, matching DDG's lowercase).
fn has_ddg_class(tag: &astral_html::Tag<'_>, class: &str) -> bool {
tag.attributes
.iter()
.find(|attr| attr.name == "class")
.is_some_and(|attr| attr.value().split_whitespace().any(|token| token.eq_ignore_ascii_case(class)))
}
/// Append one validated result unless the cap is reached. Callers validate
/// the URL and require a non-empty title before reaching `AfterTitle`, so
/// this only truncates and pushes.
fn push_ddg_result(results: &mut Vec<SearchResult>, url: String, title: String, snippet: String, max_results: usize) {
if results.len() >= max_results {
return;
}
results.push(SearchResult {
title: truncate_chars(&title, MAX_TITLE_CHARS),
url,
snippet: truncate_chars(&snippet, MAX_SNIPPET_CHARS),
});
}
/// Resolve a DuckDuckGo result href into a real https/http URL.
///
/// DDG wraps targets in a redirector like `//duckduckgo.com/l/?uddg=<encoded>`.
/// Returns `None` for non-http(s) schemes (defends against `javascript:` etc).
fn normalize_ddg_url(href: &str) -> Option<String> {
let absolute = if let Some(stripped) = href.strip_prefix("//") {
format!("https://{stripped}")
} else {
href.to_string()
};
let parsed = Url::parse(&absolute).ok()?;
if let Some((_, target)) = parsed.query_pairs().find(|(k, _)| k == "uddg") {
let target = target.into_owned();
return validate_result_url(&target);
}
validate_result_url(&absolute)
}
/// Only allow http(s) result URLs with a host.
fn validate_result_url(url: &str) -> Option<String> {
let parsed = Url::parse(url).ok()?;
match parsed.scheme() {
"http" | "https" => {}
_ => return None,
}
if parsed.host_str().is_none_or(str::is_empty) {
return None;
}
Some(url.to_string())
}
// ---------------------------------------------------------------------------
// You.com Search API (opt-in, requires YDC_API_KEY)
// ---------------------------------------------------------------------------
/// Environment variable holding the You.com API key.
const YOUCOM_API_KEY_ENV: &str = "YDC_API_KEY";
/// You.com Search API endpoint (the search service host used by the
/// official You.com SDKs; see https://you.com/docs/api-reference/search).
const YOUCOM_SEARCH_URL: &str = "https://ydc-index.io/v1/search";
#[derive(Debug, Deserialize)]
struct YoucomSearchResponse {
#[serde(default)]
results: Option<YoucomResults>,
}
#[derive(Debug, Deserialize)]
struct YoucomResults {
#[serde(default)]
web: Option<Vec<YoucomWebResult>>,
}
#[derive(Debug, Deserialize)]
struct YoucomWebResult {
#[serde(default)]
url: Option<String>,
#[serde(default)]
title: Option<String>,
/// Flat description field (the API's short summary).
#[serde(default)]
description: Option<String>,
/// Alternative snippet list used by some response shapes.
#[serde(default)]
snippets: Option<Vec<String>>,
}
/// Search via the You.com Search API. The API key is read from the
/// `YDC_API_KEY` environment variable at request time and sent as the
/// `X-API-Key` header; it is never included in error messages or logs.
async fn youcom_search(query: &str, max_results: usize, timeout_secs: u64) -> Result<Vec<SearchResult>> {
let api_key = std::env::var(YOUCOM_API_KEY_ENV).map_err(|source| {
anyhow!(
"You.com provider selected but {YOUCOM_API_KEY_ENV} is not set ({source}). Set it to a key from https://you.com/platform/api-keys, or switch [tools.web_search] provider back to \"duckduckgo\"."
)
})?;
// Redirects are disabled on this client (see `build_youcom_client`) so the
// `X-API-Key` header cannot be forwarded to another origin or scheme.
let client = build_youcom_client(timeout_secs)?;
let response = client
.post(YOUCOM_SEARCH_URL)
.header("X-API-Key", &api_key)
// Send `count` so a `max_results` above the API default is honored
// instead of silently falling back to 10 results.
.json(&serde_json::json!({ "query": query, "count": max_results }))
.send()
.await
.context("You.com request failed")?;
let status = response.status();
if !status.is_success() {
// Do not include the API key or response body in the error. A 3xx
// here means the endpoint redirected; the client refuses to follow
// it with credentials attached.
if status.is_redirection() {
return Err(anyhow!(
"You.com returned a redirect (HTTP {status}); refusing to follow it with the API key attached. Check the endpoint configuration."
));
}
return Err(anyhow!(
"You.com declined the request (HTTP {status}). Check that {YOUCOM_API_KEY_ENV} is valid, then retry."
));
}
let body = response.text().await.context("failed to read You.com response body")?;
let parsed: YoucomSearchResponse =
serde_json::from_str(&body).with_context(|| "failed to parse You.com search response")?;
Ok(parse_youcom_results(parsed, max_results))
}
/// Map the You.com response into the tool's shared `SearchResult` shape.
/// Pure function so it can be exercised in unit tests against a local
/// fixture (no live network).
fn parse_youcom_results(response: YoucomSearchResponse, max_results: usize) -> Vec<SearchResult> {
let mut results = Vec::new();
for hit in parsed_web_hits(response) {
if results.len() >= max_results {
break;
}
let Some(url) = hit.url.as_deref().and_then(validate_result_url) else {
continue;
};
let title = hit.title.as_deref().unwrap_or_default().trim();
if title.is_empty() {
continue;
}
// Prefer the flat description; fall back to the first snippet.
let snippet = hit
.description
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
.or_else(|| hit.snippets.as_deref().and_then(|s| s.first()).map(|s| s.trim()))
.unwrap_or_default();
results.push(SearchResult {
title: truncate_chars(title, MAX_TITLE_CHARS),
url,
snippet: truncate_chars(snippet, MAX_SNIPPET_CHARS),
});
}
results
}
fn parsed_web_hits(response: YoucomSearchResponse) -> impl Iterator<Item = YoucomWebResult> {
response.results.into_iter().flat_map(|r| r.web.into_iter()).flatten()
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
fn truncate_chars(input: &str, max_chars: usize) -> String {
vtcode_commons::formatting::truncate_text(input.trim(), max_chars, "…")
}
#[async_trait]
impl Tool for WebSearchTool {
async fn execute(&self, args: Value) -> Result<Value> {
self.run(args).await
}
fn name(&self) -> &str {
tools::WEB_SEARCH
}
fn description(&self) -> &str {
WEB_SEARCH_DESCRIPTION
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::tools::web_fetch::classify_helpers::decode_html_entities;
#[test]
fn normalize_ddg_url_extracts_uddg_target() {
let href = "//duckduckgo.com/l/?uddg=https%3A%2F%2Fgithub.com%2Fvinhnx&rut=abc";
assert_eq!(normalize_ddg_url(href).as_deref(), Some("https://github.com/vinhnx"));
}
#[test]
fn normalize_ddg_url_passes_through_direct_https() {
let href = "https://example.com/page";
assert_eq!(normalize_ddg_url(href).as_deref(), Some("https://example.com/page"));
}
#[test]
fn validate_result_url_rejects_non_http_schemes() {
assert!(validate_result_url("javascript:alert(1)").is_none());
assert!(validate_result_url("data:text/html,hi").is_none());
assert!(validate_result_url("file:///etc/passwd").is_none());
assert!(validate_result_url("https://example.com").is_some());
}
#[test]
fn missing_query_is_rejected() {
let tool = WebSearchTool::new();
let result = tokio::runtime::Runtime::new()
.unwrap()
.block_on(tool.run(json!({ "max_results": 5 })));
assert!(result.is_err());
}
#[test]
fn pattern_is_rejected_as_an_unknown_field() {
let result = serde_json::from_value::<WebSearchArgs>(json!({
"query": "vinhnx",
"pattern": "legacy"
}));
assert!(result.is_err());
}
#[test]
fn canonical_query_and_max_results_are_accepted() {
let args = serde_json::from_value::<WebSearchArgs>(json!({
"query": "vinhnx",
"max_results": 5
}))
.expect("canonical web_search arguments");
assert_eq!(args.query, "vinhnx");
assert_eq!(args.max_results, Some(5));
}
#[test]
fn truncate_chars_appends_ellipsis() {
assert_eq!(truncate_chars("hello world", 5), "hello…");
assert_eq!(truncate_chars("hi", 5), "hi");
}
/// A small fixture that matches the markup the live DuckDuckGo HTML
/// endpoint emits. We do not assert on whitespace; we just need a stable
/// shape so the parser logic is exercised end-to-end without network.
const DDG_FIXTURE: &str = r#"
<a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fgithub.com%2Fvinhnx&rut=abc">vinhnx (vinhnguyenxuan) · GitHub</a>
<a class="result__snippet" href="https://github.com/vinhnx">Personal GitHub profile of Vinh Nguyen Xuan.</a>
<a class="result__a" href="https://example.com/page">Example Page</a>
<a class="result__snippet" href="https://example.com/page">An example page used in tests.</a>
<a class="result__a" href="javascript:alert(1)">Should be skipped</a>
"#;
#[test]
fn parse_duckduckgo_html_extracts_results_from_fixture() {
let results = parse_duckduckgo_html(DDG_FIXTURE, 10);
assert_eq!(results.len(), 2);
assert_eq!(results[0].title, "vinhnx (vinhnguyenxuan) · GitHub");
assert_eq!(results[0].url, "https://github.com/vinhnx");
assert_eq!(results[0].snippet, "Personal GitHub profile of Vinh Nguyen Xuan.");
assert_eq!(results[1].title, "Example Page");
assert_eq!(results[1].url, "https://example.com/page");
assert_eq!(results[1].snippet, "An example page used in tests.");
}
#[test]
fn parse_duckduckgo_html_respects_max_results() {
let results = parse_duckduckgo_html(DDG_FIXTURE, 1);
assert_eq!(results.len(), 1);
assert_eq!(results[0].url, "https://github.com/vinhnx");
}
#[test]
fn parse_duckduckgo_html_returns_empty_for_challenge_page() {
// A real DuckDuckGo anti-bot challenge returns the homepage shell with
// no `result__a` anchors. The parser should return an empty vec, not
// panic or fabricate results; the higher-level dispatcher then decides
// how to surface this to the agent.
let challenge = r#"<html><body>Anomaly detected.</body></html>"#;
assert!(parse_duckduckgo_html(challenge, 10).is_empty());
}
#[test]
fn parse_accepts_href_before_class_and_single_quotes() {
// Asymmetric attribute order + quote style: the old regex required
// `class="result__a"` before `href="..."` with double quotes.
let body = r#"
<a href='https://example.com/b-first' class='result__a'>B first</a>
<a class="result__snippet">Snippet B.</a>
<a href="https://example.com/a-second" class="result__a extra">A second</a>
<a class="result__snippet">Snippet A.</a>
"#;
let results = parse_duckduckgo_html(body, 10);
assert_eq!(results.len(), 2);
assert_eq!(results[0].url, "https://example.com/b-first");
assert_eq!(results[0].snippet, "Snippet B.");
assert_eq!(results[1].url, "https://example.com/a-second");
assert_eq!(results[1].snippet, "Snippet A.");
}
#[test]
fn parse_accepts_uppercase_tags_and_classes() {
let body = r#"
<A CLASS="RESULT__A" HREF="https://example.com/upper">Upper Title</A>
<A CLASS="RESULT__SNIPPET">Upper snippet.</A>
"#;
let results = parse_duckduckgo_html(body, 10);
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "Upper Title");
assert_eq!(results[0].snippet, "Upper snippet.");
}
#[test]
fn parse_missing_first_snippet_does_not_shift_second() {
// Regression for index-aligned snippet lookup: a missing snippet for
// the first result must yield empty, not the second result's snippet.
let body = r#"
<a class="result__a" href="https://example.com/no-snippet">No Snippet</a>
<a class="result__a" href="https://example.com/with-snippet">With Snippet</a>
<a class="result__snippet">Second snippet.</a>
"#;
let results = parse_duckduckgo_html(body, 10);
assert_eq!(results.len(), 2);
assert_eq!(results[0].snippet, "");
assert_eq!(results[1].snippet, "Second snippet.");
}
#[test]
fn parse_skipped_anchors_do_not_steal_snippets() {
// Asymmetric: [bad, good] vs [good, bad]. A skipped `javascript:`
// anchor must not consume or shift the good result's snippet.
let bad_first = r#"
<a class="result__a" href="javascript:alert(1)">Bad</a>
<a class="result__snippet">Bad snippet.</a>
<a class="result__a" href="https://example.com/good">Good</a>
<a class="result__snippet">Good snippet.</a>
"#;
let results = parse_duckduckgo_html(bad_first, 10);
assert_eq!(results.len(), 1);
assert_eq!(results[0].url, "https://example.com/good");
assert_eq!(results[0].snippet, "Good snippet.");
let good_first = r#"
<a class="result__a" href="https://example.com/good">Good</a>
<a class="result__snippet">Good snippet.</a>
<a class="result__a" href="javascript:alert(1)">Bad</a>
<a class="result__snippet">Bad snippet.</a>
"#;
let results = parse_duckduckgo_html(good_first, 10);
assert_eq!(results.len(), 1);
assert_eq!(results[0].snippet, "Good snippet.");
}
#[test]
fn parse_ignores_anchors_inside_script() {
// Tokenizer advantage over regex: `script` content is raw text, so
// a fake anchor inside JavaScript never becomes a result.
let body = concat!(
"<script>var x = '<a class=\"result__a\" href=\"https://example.com/evil\">Evil</a>';</script>",
"<a class=\"result__a\" href=\"https://example.com/good\">Good</a>",
"<a class=\"result__snippet\">Good snippet.</a>",
);
let results = parse_duckduckgo_html(body, 10);
assert_eq!(results.len(), 1);
assert_eq!(results[0].url, "https://example.com/good");
assert_eq!(results[0].snippet, "Good snippet.");
}
#[test]
fn parse_collects_nested_formatting_text_in_title() {
let body = concat!(
"<a class=\"result__a\" href=\"https://example.com/fmt\">Hello <b>Bold</b> tail</a>",
"<a class=\"result__snippet\">S.</a>",
);
let results = parse_duckduckgo_html(body, 10);
assert_eq!(results.len(), 1);
assert_eq!(results[0].title, "Hello Bold tail");
}
#[test]
fn capped_body_text_passes_small_bodies_through() {
assert_eq!(capped_body_text(b"<html>hi</html>"), "<html>hi</html>");
assert_eq!(capped_body_text(b""), "");
}
#[test]
fn capped_body_text_truncates_huge_bodies_without_panicking_on_split_chars() {
// 999_999 ASCII bytes + one 2-byte `é`: the 1MB cap lands mid-char.
// The ASCII prefix survives; the split tail becomes U+FFFD.
let mut bytes = vec![b'a'; 999_999];
bytes.extend_from_slice("é".as_bytes());
let text = capped_body_text(&bytes);
assert!(text[..999_999].bytes().all(|b| b == b'a'));
let huge = vec![b'a'; MAX_DDG_BODY_BYTES + 5];
assert_eq!(capped_body_text(&huge).len(), MAX_DDG_BODY_BYTES);
}
#[test]
fn capped_body_text_is_lossy_not_panicky_on_invalid_utf8() {
let text = capped_body_text(b"\xff\xfe<html></html>");
assert!(text.contains("<html>"), "got: {text}");
}
#[test]
fn decode_numeric_hex_and_named_entities() {
assert_eq!(decode_html_entities("ABC"), "ABC");
assert_eq!(decode_html_entities("’“—"), "’“—");
assert_eq!(decode_html_entities("&lt;"), "<");
assert_eq!(decode_html_entities("�"), "�");
}
#[test]
fn parse_accepts_unquoted_href_and_td_closed_snippet() {
let body = r#"
<a class=result__a href=https://example.com/unquoted>Unquoted</a>
<td class="result__snippet">TD snippet.</td>
<abbr title="abbreviation">abbr</abbr>
<a class="result__a" href="https://example.com/second">Second</a>
<a class="result__snippet">Second snippet.</a>
"#;
let results = parse_duckduckgo_html(body, 10);
assert_eq!(results.len(), 2);
assert_eq!(results[0].url, "https://example.com/unquoted");
assert_eq!(results[0].snippet, "TD snippet.");
assert_eq!(results[1].url, "https://example.com/second");
assert_eq!(results[1].snippet, "Second snippet.");
}
#[test]
fn session_cap_short_circuits_with_structured_error() {
let config = WebSearchConfig {
provider: Default::default(),
max_results: 5,
timeout_secs: 20,
cooldown_ms: 0,
cache_ttl_secs: 300,
session_max_requests: 2,
};
let tool = WebSearchTool::with_config(config);
// Bump the counter without going to the network.
{
let mut state = tool.state.lock().unwrap();
state.requests_made = 2;
state.last_request_at = Some(Instant::now());
}
let payload = tokio::runtime::Runtime::new()
.unwrap()
.block_on(tool.run(json!({ "query": "rust" })))
.expect("cap should be a structured JSON, not a runtime error");
assert_eq!(payload["error"], "web_search session request cap reached");
assert_eq!(payload["session_max_requests"], 2);
}
#[test]
fn cooldown_short_circuits_with_retry_after() {
let config = WebSearchConfig {
provider: Default::default(),
max_results: 5,
timeout_secs: 20,
cooldown_ms: 5_000,
cache_ttl_secs: 300,
session_max_requests: 100,
};
let tool = WebSearchTool::with_config(config);
{
let mut state = tool.state.lock().unwrap();
state.requests_made = 0;
state.last_request_at = Some(Instant::now());
}
let payload = tokio::runtime::Runtime::new()
.unwrap()
.block_on(tool.run(json!({ "query": "rust" })))
.expect("cooldown should be a structured JSON");
assert_eq!(payload["error"], "web_search cooldown active");
assert!(payload["retry_after_ms"].as_u64().unwrap() > 0);
}
#[test]
fn cache_serves_repeat_queries_without_network() {
// Pin max_results so the cache key matches the one the tool builds.
let tool = WebSearchTool::with_config(WebSearchConfig {
provider: Default::default(),
max_results: 5,
..WebSearchConfig::default()
});
let cached_payload = json!({
"query": "rust",
"provider": "duckduckgo",
"count": 1,
"results": [{
"title": "Cached Result",
"url": "https://example.com/cached",
"snippet": "from cache"
}]
});
{
let mut state = tool.state.lock().unwrap();
state.cache_put("5::rust".to_string(), cached_payload.clone());
}
let payload = tokio::runtime::Runtime::new()
.unwrap()
.block_on(tool.run(json!({ "query": "rust" })))
.expect("cache hit must not error");
assert_eq!(payload["cached"], json!(true));
assert_eq!(payload["count"], 1);
assert_eq!(payload["results"][0]["title"], "Cached Result");
}
#[test]
fn classify_search_error_flags_antiban_block() {
// DDG's HTML endpoint returns 202 (or 200 with a challenge page) on
// a bot block. The classifier must route that to `antiban_blocked`
// and tell the agent to NOT retry, because the same turn will
// almost certainly hit the same block.
let (kind, action) = classify_search_error(
"DuckDuckGo declined the request (HTTP 202), likely an anti-bot challenge for this network.",
"duckduckgo",
);
assert_eq!(kind, "antiban_blocked");
assert!(action.contains("immediate retry"), "action should discourage immediate retry; got: {action}");
}
#[test]
fn classify_search_error_flags_timeout_as_network_error() {
let (kind, action) = classify_search_error("request timed out after 20s", "duckduckgo");
assert_eq!(kind, "network_error");
assert!(
action.contains("retry") || action.contains("web_fetch"),
"action should suggest retry or web_fetch; got: {action}"
);
}
#[test]
fn classify_search_error_uses_youcom_wording_for_youcom_timeouts() {
// Timeout guidance must name the provider that actually failed, so
// a You.com timeout does not tell the agent that DuckDuckGo timed out.
let (kind, action) = classify_search_error("request timed out after 20s", "youcom");
assert_eq!(kind, "network_error");
assert!(action.contains("You.com"), "action should mention You.com for youcom failures; got: {action}");
assert!(
!action.contains("DuckDuckGo"),
"action should not blame DuckDuckGo for a youcom failure; got: {action}"
);
}
// -----------------------------------------------------------------------
// You.com provider
// -----------------------------------------------------------------------
/// A fixture matching the You.com Search API `results.web[]` shape.
fn youcom_fixture() -> YoucomSearchResponse {
serde_json::from_str(
r#"{
"results": {
"web": [
{
"title": "Rust Programming Language",
"url": "https://www.rust-lang.org/",
"description": "A language empowering everyone to build reliable software."
},
{
"title": "Snippet-only hit",
"url": "https://example.com/snips",
"snippets": ["First snippet line."]
},
{
"title": "No URL hit",
"description": "Skipped because the URL is missing."
},
{
"title": "javascript:alert(1)",
"url": "javascript:alert(1)",
"description": "Skipped by scheme validation."
}
]
}
}"#,
)
.expect("fixture should parse")
}
#[test]
fn parse_youcom_results_maps_web_hits() {
let results = parse_youcom_results(youcom_fixture(), 10);
assert_eq!(results.len(), 2);
assert_eq!(results[0].title, "Rust Programming Language");
assert_eq!(results[0].url, "https://www.rust-lang.org/");
assert!(results[0].snippet.contains("reliable software"));
// Snippet fallback: flat description missing, first snippet used.
assert_eq!(results[1].snippet, "First snippet line.");
}
#[test]
fn parse_youcom_results_respects_max_results() {
let results = parse_youcom_results(youcom_fixture(), 1);
assert_eq!(results.len(), 1);
assert_eq!(results[0].url, "https://www.rust-lang.org/");
}
#[test]
fn parse_youcom_results_handles_missing_results_block() {
let response: YoucomSearchResponse =
serde_json::from_str(r#"{"metadata": {}}"#).expect("empty envelope should parse");
assert!(parse_youcom_results(response, 10).is_empty());
}
#[test]
fn youcom_provider_without_key_errors_before_network() {
// With provider = youcom and no YDC_API_KEY, the tool must surface a
// structured setup error rather than attempting a network call.
// NOTE: env-var races are avoided by using a unique key absence only
// when the var is unset in this process; guard with a lock-free check.
if std::env::var(YOUCOM_API_KEY_ENV).is_ok() {
return; // key present in this environment; skip to avoid a live call
}
let config = WebSearchConfig {
provider: WebSearchProvider::Youcom,
max_results: 5,
timeout_secs: 20,
cooldown_ms: 0,
cache_ttl_secs: 300,
session_max_requests: 100,
};
let tool = WebSearchTool::with_config(config);
let payload = tokio::runtime::Runtime::new()
.unwrap()
.block_on(tool.run(json!({ "query": "rust" })))
.expect("missing key should be a structured JSON, not a panic");
assert!(payload["error"].as_str().unwrap().contains("YDC_API_KEY"));
// A setup error must not consume the session's request budget or
// start the cooldown clock — the agent may fix the key and retry
// immediately.
let state = tool.state.lock().unwrap();
assert_eq!(state.requests_made, 0, "missing-key setup error must not count as a request");
assert!(state.last_request_at.is_none(), "missing-key setup error must not start the cooldown clock");
}
#[test]
fn youcom_client_disables_redirects() {
// The credential-bearing request must never follow a redirect to
// another origin, so the youcom client disables redirects entirely.
let client = build_youcom_client(20).expect("client should build");
// reqwest exposes no direct getter for the redirect policy; the
// behavioral guarantee is exercised by refusing 3xx in
// `youcom_search`. Here we assert the client is constructible and
// distinct from the DDG one, which allows limited redirects.
let _ddg = build_client(20).expect("ddg client should build");
drop(client);
}
#[test]
fn youcom_provider_payload_reports_youcom_provider() {
// Cache-path check: a cached payload for provider youcom reports
// provider "youcom" once served. Seed the cache directly to avoid
// the network.
let config = WebSearchConfig {
provider: WebSearchProvider::Youcom,
max_results: 5,
..WebSearchConfig::default()
};
let tool = WebSearchTool::with_config(config);
let cached_payload = json!({
"query": "rust",
"provider": "youcom",
"count": 1,
"results": [{
"title": "Cached You.com Result",
"url": "https://example.com/you",
"snippet": "from cache"
}]
});
{
let mut state = tool.state.lock().unwrap();
state.cache_put("5::rust".to_string(), cached_payload);
}
let payload = tokio::runtime::Runtime::new()
.unwrap()
.block_on(tool.run(json!({ "query": "rust" })))
.expect("cache hit must not error");
assert_eq!(payload["provider"], json!("youcom"));
assert_eq!(payload["results"][0]["title"], "Cached You.com Result");
}
}