#![cfg_attr(docsrs, feature(doc_cfg))]
#![doc = include_str!("../WASM.md")]
#![doc = include_str!("../RUST.md")]
use crate::colors::AnsiColor;
use crate::colors::AnsiStyle;
use crate::params::SearchParams;
use crate::response::*;
use crate::topic::Topic;
use anyhow::{Context, Result};
use chrono::TimeZone;
use regex::Regex;
use scraper::{Html, Selector};
use serde_json::Value;
const BASE_URL: &str = "https://api.duckduckgo.com/";
pub struct Browser {
client: reqwest::Client,
}
#[derive(Debug, Default)]
pub struct BrowserBuilder {
user_agent: Option<String>,
#[cfg(not(target_arch = "wasm32"))]
#[allow(dead_code)]
cookie_store: bool,
#[cfg(not(target_arch = "wasm32"))]
#[allow(dead_code)]
proxy: Option<String>,
}
impl BrowserBuilder {
pub fn user_agent(mut self, agent: impl Into<String>) -> Self {
self.user_agent = Some(agent.into());
self
}
#[cfg(not(target_arch = "wasm32"))]
pub fn cookie_store(mut self, enable: bool) -> Self {
self.cookie_store = enable;
self
}
#[cfg(not(target_arch = "wasm32"))]
pub fn proxy(mut self, url: impl Into<String>) -> Self {
self.proxy = Some(url.into());
self
}
pub fn build(self) -> Result<Browser> {
let mut builder = reqwest::Client::builder();
if let Some(agent) = self.user_agent {
builder = builder.user_agent(agent);
}
#[cfg(not(target_arch = "wasm32"))]
if self.cookie_store {
builder = builder.cookie_store(true);
}
#[cfg(not(target_arch = "wasm32"))]
if let Some(proxy_url) = self.proxy {
let proxy = reqwest::Proxy::all(&proxy_url)
.with_context(|| format!("Invalid proxy URL: {proxy_url}"))?;
builder = builder.proxy(proxy);
}
let client = builder
.build()
.context("Failed to build reqwest HTTP client")?;
Ok(Browser { client })
}
}
impl Browser {
pub fn new() -> Self {
Browser {
client: reqwest::Client::new(),
}
}
pub fn builder() -> BrowserBuilder {
BrowserBuilder::default()
}
pub async fn request(
&self,
method: reqwest::Method,
url: &str,
user_agent: &str,
params: &[(&str, &str)],
) -> Result<reqwest::Response> {
let req = self
.client
.request(method, url)
.query(params)
.header("User-Agent", user_agent)
.header("Accept", "application/json")
.header("Referer", "https://duckduckgo.com/")
.header("Accept-Language", "en-US,en;q=0.9");
let resp = req.send().await?.error_for_status()?;
Ok(resp)
}
pub async fn get_vqd(&self, query: &str, user_agent: &str) -> Result<String> {
let resp = self
.request(
reqwest::Method::GET,
"https://duckduckgo.com/",
user_agent,
&[("q", query)],
)
.await?;
let text = resp.text().await?;
let re = Regex::new(r#"vqd=.?['\"]?([\d-]+)['\"]?"#)?;
let vqd = re
.captures(&text)
.and_then(|c| c.get(1).map(|m| m.as_str().to_string()))
.context("Missing vqd in response")?;
Ok(vqd)
}
pub async fn lite_search(
&self,
query: &str,
region: &str,
limit: Option<usize>,
user_agent: &str,
) -> anyhow::Result<Vec<LiteSearchResult>> {
let resp = self
.request(
reqwest::Method::POST,
"https://lite.duckduckgo.com/lite/",
user_agent,
&[("q", query), ("kl", region)],
)
.await
.context("Failed to send request to DuckDuckGo Lite")?;
let body = resp.text().await.context("Failed to read response body")?;
let doc = Html::parse_document(&body);
let sel = Selector::parse("table tr").map_err(|e| anyhow::anyhow!("{e}"))?;
let mut results = Vec::new();
let a_sel = Selector::parse("a").map_err(|e| anyhow::anyhow!("{e}"))?;
let snippet_sel =
Selector::parse("td.result-snippet").map_err(|e| anyhow::anyhow!("{e}"))?;
for tr in doc.select(&sel) {
if let Some(a) = tr.select(&a_sel).next() {
let title = a.text().collect::<String>();
if let Some(href) = a.value().attr("href") {
let snippet = tr
.select(&snippet_sel)
.next()
.map(|n| n.text().collect())
.unwrap_or_default();
results.push(LiteSearchResult {
title,
url: href.to_string(),
snippet,
});
if limit.is_some_and(|l| results.len() >= l) {
break;
}
}
}
}
Ok(results)
}
pub async fn images(
&self,
query: &str,
region: &str,
safesearch: bool,
limit: Option<usize>,
user_agent: &str,
) -> Result<Vec<ImageResult>> {
let vqd = self.get_vqd(query, user_agent).await?;
let mut page_params = vec![
("q", query.to_string()),
("l", region.to_string()),
("vqd", vqd),
("o", "json".into()),
("p", if safesearch { "1" } else { "-1" }.into()),
];
let mut results = Vec::new();
loop {
let params_ref: Vec<(&str, &str)> =
page_params.iter().map(|(k, v)| (*k, v.as_ref())).collect();
let resp = self
.request(
reqwest::Method::GET,
"https://duckduckgo.com/i.js",
user_agent,
¶ms_ref,
)
.await?;
let j: Value = resp.json().await?;
if let Some(array) = j.get("results").and_then(|r| r.as_array()) {
for item in array.iter() {
results.push(ImageResult {
title: item["title"].as_str().unwrap_or("").to_string(),
image: item["image"].as_str().unwrap_or("").to_string(),
thumbnail: item["thumbnail"].as_str().unwrap_or("").to_string(),
url: item["url"].as_str().unwrap_or("").to_string(),
height: item["height"].as_u64().unwrap_or(0) as u32,
width: item["width"].as_u64().unwrap_or(0) as u32,
source: item["source"].as_str().unwrap_or("").to_string(),
});
if limit.is_some_and(|l| results.len() >= l) {
return Ok(results);
}
}
}
if let Some(next) = j.get("next").and_then(|n| n.as_str()) {
let s = next.split("s=").nth(1).unwrap_or("").to_string();
page_params.push(("s", s));
} else {
break;
}
}
Ok(results)
}
pub async fn news(
&self,
query: &str,
region: &str,
safesearch: bool,
limit: Option<usize>,
user_agent: &str,
) -> Result<Vec<NewsResult>> {
let vqd = self.get_vqd(query, user_agent).await?;
let mut page_params = vec![
("q", query.to_string()),
("l", region.to_string()),
("vqd", vqd),
("o", "json".into()),
("p", if safesearch { "1" } else { "-1" }.into()),
("noamp", "1".into()),
];
let mut results = Vec::new();
loop {
let params_ref: Vec<(&str, &str)> =
page_params.iter().map(|(k, v)| (*k, v.as_ref())).collect();
let resp = self
.request(
reqwest::Method::GET,
"https://duckduckgo.com/news.js",
user_agent,
¶ms_ref,
)
.await?;
let j: Value = resp.json().await?;
if let Some(array) = j.get("results").and_then(|r| r.as_array()) {
for item in array.iter() {
let date = item["date"]
.as_i64()
.map(|ts| {
chrono::Utc
.timestamp_opt(ts, 0)
.single()
.unwrap_or_else(chrono::Utc::now)
})
.unwrap_or_else(chrono::Utc::now);
results.push(NewsResult {
date: date.to_rfc3339(),
title: item["title"].as_str().unwrap_or("").to_string(),
body: item["excerpt"].as_str().unwrap_or("").to_string(),
url: item["url"].as_str().unwrap_or("").to_string(),
image: item
.get("image")
.and_then(|v| v.as_str())
.map(str::to_string),
source: item["source"].as_str().unwrap_or("").to_string(),
});
if limit.is_some_and(|l| results.len() >= l) {
return Ok(results);
}
}
}
if let Some(next) = j.get("next").and_then(|n| n.as_str()) {
let s = next.split("s=").nth(1).unwrap_or("").to_string();
page_params.push(("s", s));
} else {
break;
}
}
Ok(results)
}
pub async fn browse(
&self,
path: &str,
result_format: ResultFormat,
limit: Option<usize>,
search_params: Option<&SearchParams>,
) -> Result<()> {
let api_response = self.get_api_response(path, search_params).await?;
match result_format {
ResultFormat::List => self.print_results_list(api_response, limit),
ResultFormat::Detailed => self.print_results_detailed(api_response, limit),
}
Ok(())
}
pub async fn get_api_response(
&self,
path: &str,
search_params: Option<&SearchParams>,
) -> Result<Response> {
let separator = if path.contains('?') { '&' } else { '?' };
let mut url = format!("{}{}{}format=json", BASE_URL, path, separator);
if let Some(params) = search_params {
for (key, value) in params.to_query_pairs() {
url.push('&');
url.push_str(key);
url.push('=');
url.push_str(&value);
}
}
let response = self
.client
.get(&url)
.send()
.await
.with_context(|| format!("Failed to send request to {}", url))?;
let status = response.status();
let text = response
.text()
.await
.with_context(|| "Failed to read response body")?;
if !status.is_success() {
anyhow::bail!("Request failed with status {}: {}", status, text);
}
let api_response: Response = serde_json::from_str(&text)
.with_context(|| format!("Failed to parse JSON response: {}", text))?;
Ok(api_response)
}
pub fn print_results_list(&self, api_response: Response, limit: Option<usize>) {
if let Some(heading) = api_response.heading {
let style = AnsiStyle {
bold: true,
color: Some(AnsiColor::Gold),
};
println!(
"{}{}{}",
style.escape_code(),
heading,
AnsiStyle::reset_code()
);
}
let topics = &api_response.related_topics;
for (index, topic) in topics
.iter()
.enumerate()
.take(limit.unwrap_or(topics.len()))
{
self.print_related_topic(index + 1, topic);
}
}
pub fn print_related_topic(&self, index: usize, topic: &Topic) {
let style = AnsiStyle {
bold: false,
color: Some(AnsiColor::BrightGreen),
};
let text = match &topic.text {
Some(t) => t,
None => {
return;
}
};
let first_url = match &topic.first_url {
Some(url) => url,
None => {
return;
}
};
println!("{}. {} {}", index, text, style.escape_code());
println!("URL: {}{}", first_url, style.escape_code());
if let Some(icon) = &topic.icon {
let style = AnsiStyle {
bold: false,
color: Some(AnsiColor::BrightBlue),
};
if !icon.url.is_empty() {
let full_url = format!("https://duckduckgo.com{}", icon.url);
println!("Image URL: {}{}", full_url, style.escape_code());
}
}
println!("--------------------------------------------");
}
pub fn print_results_detailed(&self, api_response: Response, limit: Option<usize>) {
if let Some(heading) = api_response.heading {
let style = AnsiStyle {
bold: true,
color: None,
};
println!(
"{}{}{}",
style.escape_code(),
heading,
AnsiStyle::reset_code()
);
}
if let Some(abstract_text) = api_response.abstract_text {
let style = AnsiStyle {
bold: false,
color: Some(AnsiColor::LightGray),
};
println!("Abstract: {}{}", abstract_text, style.escape_code());
}
if let Some(abstract_source) = api_response.abstract_source {
let style = AnsiStyle {
bold: false,
color: Some(AnsiColor::Purple),
};
println!(
"Abstract Source: {}{}",
abstract_source,
style.escape_code()
);
}
if let Some(abstract_url) = api_response.abstract_url {
let style = AnsiStyle {
bold: false,
color: Some(AnsiColor::Silver),
};
println!("Abstract URL: {}{}", abstract_url, style.escape_code());
}
if let Some(image) = api_response.image {
let style = AnsiStyle {
bold: false,
color: Some(AnsiColor::SkyBlue),
};
if !image.is_empty() {
let full_url = format!("https://duckduckgo.com{}", image);
println!("Image URL: {}{}", full_url, style.escape_code());
}
}
let topics = &api_response.related_topics;
for (index, topic) in topics
.iter()
.enumerate()
.take(limit.unwrap_or(topics.len()))
{
self.print_related_topic(index + 1, topic);
}
}
pub async fn search(
&self,
query: &str,
safe_search: bool,
result_format: ResultFormat,
limit: Option<usize>,
search_params: Option<&SearchParams>,
) -> Result<()> {
let safe_param = if safe_search { "&kp=1" } else { "&kp=-2" };
let path = format!("?q={}{}", query, safe_param);
self.browse(&path, result_format, limit, search_params)
.await
.with_context(|| format!("Failed to perform search for query '{}'", query))
}
pub async fn advanced_search(
&self,
query: &str,
params: &str,
safe_search: bool,
result_format: ResultFormat,
limit: Option<usize>,
search_params: Option<&SearchParams>,
) -> Result<()> {
let safe_param = if safe_search { "&kp=1" } else { "&kp=-2" };
let path = format!("?q={}&kl={}{}", query, params, safe_param);
self.browse(&path, result_format, limit, search_params)
.await
.with_context(|| format!("Failed to perform advanced search for query '{}'", query))
}
pub async fn search_operators(
&self,
query: &str,
operators: &str,
safe_search: bool,
result_format: ResultFormat,
limit: Option<usize>,
search_params: Option<&SearchParams>,
) -> Result<()> {
let safe_param = if safe_search { "&kp=1" } else { "&kp=-2" };
let path = format!("?q={}&{}{}", query, operators, safe_param);
self.browse(&path, result_format, limit, search_params)
.await
.with_context(|| format!("Failed to perform operator search for query '{}'", query))
}
}
impl Default for Browser {
fn default() -> Self {
Self::new()
}
}