pub struct SearchResult {
pub title: String,
pub url: String,
pub snippet: String,
}
pub fn percent_decode(s: &str) -> String {
let b = s.as_bytes();
let mut out: Vec<u8> = Vec::with_capacity(b.len());
let mut i = 0;
while i < b.len() {
if b[i] == b'%' && i + 3 <= b.len() {
if let Ok(h) = u8::from_str_radix(std::str::from_utf8(&b[i + 1..i + 3]).unwrap_or("zz"), 16) {
out.push(h);
i += 3;
continue;
}
out.push(b'%');
i += 1;
} else if b[i] == b'+' {
out.push(b' ');
i += 1;
} else {
out.push(b[i]);
i += 1;
}
}
String::from_utf8_lossy(&out).into_owned()
}
pub fn urlencode(s: &str) -> String {
let mut out = String::new();
for &b in s.as_bytes() {
if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'~') {
out.push(b as char);
} else {
out.push_str(&format!("%{b:02X}"));
}
}
out
}
pub fn parse_searxng_results(json: &str, max: usize) -> Vec<SearchResult> {
let v: serde_json::Value = match serde_json::from_str(json) { Ok(v) => v, Err(_) => return Vec::new() };
let mut out = Vec::new();
if let Some(arr) = v.get("results").and_then(|r| r.as_array()) {
for r in arr.iter().take(max) {
let get = |k: &str| r.get(k).and_then(|x| x.as_str()).unwrap_or("").to_string();
let url = get("url");
if !url.is_empty() {
out.push(SearchResult { title: get("title"), url, snippet: get("content") });
}
}
}
out
}
fn ddg_target_url(href: &str) -> String {
if let Some(idx) = href.find("uddg=") {
let rest = &href[idx + 5..];
percent_decode(rest.split('&').next().unwrap_or(rest))
} else {
href.to_string()
}
}
pub fn parse_ddg_results(html: &str, max: usize) -> Vec<SearchResult> {
use scraper::{Html, Selector};
let doc = Html::parse_document(html);
let link_sel = Selector::parse("a.result__a, .result__title a").unwrap();
let snip_sel = Selector::parse(".result__snippet").unwrap();
let snips: Vec<String> = doc.select(&snip_sel).map(|e| e.text().collect::<String>().trim().to_string()).collect();
let mut out = Vec::new();
for (i, a) in doc.select(&link_sel).enumerate() {
if out.len() >= max { break; }
let title = a.text().collect::<String>().trim().to_string();
let url = ddg_target_url(a.value().attr("href").unwrap_or(""));
if !title.is_empty() && !url.is_empty() {
out.push(SearchResult { title, url, snippet: snips.get(i).cloned().unwrap_or_default() });
}
}
out
}
pub fn format_results(results: &[SearchResult]) -> String {
if results.is_empty() {
return "No results.".to_string();
}
results.iter().enumerate()
.map(|(i, r)| format!("{}. {} — {}\n{}", i + 1, r.title, r.url, r.snippet))
.collect::<Vec<_>>().join("\n\n")
}
pub async fn web_search(client: &reqwest::Client, base_url: Option<&str>, max_results: usize, query: &str) -> Vec<SearchResult> {
if let Some(base) = base_url.filter(|s| !s.is_empty()) {
let url = format!("{}/search?q={}&format=json", base.trim_end_matches('/'), urlencode(query));
match crate::web::fetch_url(client, &url).await {
Ok(b) => parse_searxng_results(&b, max_results),
Err(_) => Vec::new(),
}
} else {
let url = format!("https://html.duckduckgo.com/html/?q={}", urlencode(query));
match crate::web::fetch_url(client, &url).await {
Ok(h) => parse_ddg_results(&h, max_results),
Err(_) => Vec::new(),
}
}
}
pub async fn fetch_page(client: &reqwest::Client, url: &str, allow_hosts: &[String], max_bytes: u64, max_chars: usize) -> String {
if let Err(e) = crate::net::url_is_allowed(url, allow_hosts) {
return format!("Blocked: {e}");
}
match crate::web::fetch_url_capped(client, url, max_bytes).await {
Ok(html) => crate::web::extract_main_text(&html).chars().take(max_chars).collect(),
Err(e) => format!("Fetch failed: {e}"),
}
}
pub fn web_tools() -> serde_json::Value {
serde_json::json!([
{"type":"function","function":{"name":"web_search","description":"Search the web; returns top results (title, url, snippet).","parameters":{"type":"object","properties":{"query":{"type":"string"}},"required":["query"]}}},
{"type":"function","function":{"name":"fetch_page","description":"Fetch and read a web page's main text.","parameters":{"type":"object","properties":{"url":{"type":"string"}},"required":["url"]}}}
])
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_searxng_extracts_results() {
let json = r#"{"results":[{"title":"T1","url":"https://a","content":"snip1"},{"title":"T2","url":"https://b","content":"snip2"}]}"#;
let r = parse_searxng_results(json, 5);
assert_eq!(r.len(), 2);
assert_eq!(r[0].url, "https://a");
assert_eq!(r[0].snippet, "snip1");
assert!(parse_searxng_results("not json", 5).is_empty());
}
#[test]
fn ddg_decodes_uddg_redirect() {
let html = r#"<a class="result__a" href="/l/?uddg=https%3A%2F%2Fexample.com%2Fp&rut=x">Example</a><div class="result__snippet">snip</div>"#;
let r = parse_ddg_results(html, 5);
assert_eq!(r.len(), 1);
assert_eq!(r[0].url, "https://example.com/p");
assert_eq!(r[0].title, "Example");
}
#[test]
fn format_and_tools() {
assert_eq!(format_results(&[]), "No results.");
let f = format_results(&[SearchResult { title: "T".into(), url: "U".into(), snippet: "S".into() }]);
assert!(f.contains("T") && f.contains("U") && f.contains("S"));
let t = web_tools();
let names: Vec<&str> = t.as_array().unwrap().iter().map(|x| x["function"]["name"].as_str().unwrap()).collect();
assert!(names.contains(&"web_search") && names.contains(&"fetch_page"));
}
}