duckduckgo/browser.rs
1use crate::colors::AnsiColor;
2use crate::colors::AnsiStyle;
3use crate::response::*;
4use crate::topic::Topic;
5use anyhow::{Context, Result};
6use chrono::TimeZone;
7use regex::Regex;
8use reqwest;
9use scraper::{Html, Selector};
10use serde_json::Value;
11
12const BASE_URL: &str = "https://api.duckduckgo.com/";
13
14/// A struct representing a browser for interacting with the DuckDuckGo API.
15pub struct Browser {
16 /// The underlying HTTP client used for making requests.
17 pub client: reqwest::Client,
18}
19
20impl Browser {
21 /// Creates a new instance of `Browser` with the specified HTTP client.
22 ///
23 /// # Arguments
24 /// * `client` - The reqwest HTTP client to be used for making requests.
25 ///
26 /// # Examples
27 /// ```
28 /// use duckduckgo::browser::Browser;
29 /// use reqwest::Client;
30 ///
31 /// let client = Client::new();
32 /// let browser = Browser::new(client);
33 /// ```
34 pub fn new(client: reqwest::Client) -> Self {
35 Browser { client }
36 }
37
38 /// Sends an HTTP request to the given URL using the specified method and query parameters.
39 ///
40 /// # Arguments
41 /// * `method` - The HTTP method to use (GET, POST, etc.).
42 /// * `url` - The target URL.
43 /// * `params` - A slice of key-value string pairs to be included as query parameters.
44 ///
45 /// # Returns
46 /// A `Result` containing the HTTP response or an error.
47 ///
48 /// # Example
49 /// ```rust
50 /// use reqwest::Method;
51 /// use duckduckgo::browser::Browser;
52 /// use duckduckgo::user_agents::get;
53 ///
54 /// #[tokio::main]
55 /// async fn main() -> anyhow::Result<()> {
56 /// let browser = Browser::new(reqwest::Client::new());
57 /// let user_agent = get("firefox").unwrap();
58 /// let response = browser.request(Method::GET, "https://api.duckduckgo.com", user_agent, &[("test", "123")]).await?;
59 /// assert!(response.status().is_success());
60 /// Ok(())
61 /// }
62 /// ```
63 pub async fn request(
64 &self,
65 method: reqwest::Method,
66 url: &str,
67 user_agent: &str,
68 params: &[(&str, &str)],
69 ) -> Result<reqwest::Response> {
70 let req = self
71 .client
72 .request(method, url)
73 .query(params)
74 .header("User-Agent", user_agent)
75 .header("Accept", "application/json")
76 .header("Referer", "https://duckduckgo.com/")
77 .header("Accept-Language", "en-US,en;q=0.9");
78
79 let resp = req.send().await?.error_for_status()?;
80 Ok(resp)
81 }
82
83 /// Retrieves the `vqd` token required for JavaScript-based DuckDuckGo API endpoints.
84 ///
85 /// # Arguments
86 /// * `query` - The search query string.
87 ///
88 /// # Returns
89 /// A `Result` containing the extracted `vqd` string or an error if not found.
90 ///
91 /// # Example
92 /// ```rust
93 /// use duckduckgo::browser::Browser;
94 /// use duckduckgo::user_agents::get;
95 ///
96 ///
97 /// #[tokio::main]
98 /// async fn main() -> anyhow::Result<()> {
99 /// let browser = Browser::new(reqwest::Client::new());
100 /// let user_agent = get("firefox").unwrap();
101 /// let vqd = browser.get_vqd("rust programming", user_agent).await?;
102 /// assert!(!vqd.is_empty());
103 /// Ok(())
104 /// }
105 /// ```
106 pub async fn get_vqd(&self, query: &str, user_agent: &str) -> Result<String> {
107 let resp = self
108 .request(
109 reqwest::Method::GET,
110 "https://duckduckgo.com/",
111 user_agent,
112 &[("q", query)],
113 )
114 .await?;
115
116 let text = resp.text().await?;
117
118 let re = Regex::new(r#"vqd=.?['"]?([\d-]+)['"]?"#)?;
119
120 let vqd = re
121 .captures(&text)
122 .and_then(|c| c.get(1).map(|m| m.as_str().to_string()))
123 .context("Missing vqd in response")?;
124
125 Ok(vqd)
126 }
127
128 /// Performs a search using DuckDuckGo Lite, a text-only HTML interface.
129 ///
130 /// # Arguments
131 /// * `query` - The search query.
132 /// * `region` - The region code (e.g., `"wt-wt"` for worldwide).
133 /// * `limit` - Optional maximum number of results to return.
134 ///
135 /// # Returns
136 /// A list of `LiteSearchResult` items.
137 ///
138 /// # Example
139 /// ```rust
140 /// use duckduckgo::browser::Browser;
141 /// use duckduckgo::user_agents::get;
142 ///
143 ///
144 /// #[tokio::main]
145 /// async fn main() -> anyhow::Result<()> {
146 /// let browser = Browser::new(reqwest::Client::new());
147 /// let user_agent = get("firefox").unwrap();
148 /// let results = browser.lite_search("rust language", "wt-wt", Some(3), user_agent).await?;
149 /// assert!(results.len() <= 3);
150 /// Ok(())
151 /// }
152 /// ```
153 pub async fn lite_search(
154 &self,
155 query: &str,
156 region: &str,
157 limit: Option<usize>,
158 user_agent: &str,
159 ) -> anyhow::Result<Vec<LiteSearchResult>> {
160 let resp = self
161 .request(
162 reqwest::Method::POST,
163 "https://lite.duckduckgo.com/lite/",
164 user_agent,
165 &[("q", query), ("kl", region)],
166 )
167 .await
168 .context("Failed to send request to DuckDuckGo Lite")?;
169
170 let body = resp.text().await.context("Failed to read response body")?;
171 let doc = Html::parse_document(&body);
172 let sel = Selector::parse("table tr").map_err(|e| anyhow::anyhow!("{e}"))?;
173
174 let mut results = Vec::new();
175 let a_sel = Selector::parse("a").map_err(|e| anyhow::anyhow!("{e}"))?;
176 let snippet_sel =
177 Selector::parse("td.result-snippet").map_err(|e| anyhow::anyhow!("{e}"))?;
178
179 for tr in doc.select(&sel) {
180 if let Some(a) = tr.select(&a_sel).next() {
181 let title = a.text().collect::<String>();
182 if let Some(href) = a.value().attr("href") {
183 let snippet = tr
184 .select(&snippet_sel)
185 .next()
186 .map(|n| n.text().collect())
187 .unwrap_or_default();
188
189 results.push(LiteSearchResult {
190 title,
191 url: href.to_string(),
192 snippet,
193 });
194
195 if limit.is_some_and(|l| results.len() >= l) {
196 break;
197 }
198 }
199 }
200 }
201
202 Ok(results)
203 }
204
205 /// Performs an image search on DuckDuckGo.
206 ///
207 /// # Arguments
208 /// * `query` - The search query.
209 /// * `region` - The region code (e.g., `"wt-wt"`).
210 /// * `safesearch` - Whether to enable safe search.
211 /// * `limit` - Optional maximum number of image results.
212 ///
213 /// # Returns
214 /// A list of `ImageResult` items.
215 ///
216 /// # Example
217 /// ```rust
218 /// use duckduckgo::browser::Browser;
219 /// use duckduckgo::user_agents::get;
220 ///
221 ///
222 /// #[tokio::main]
223 /// async fn main() -> anyhow::Result<()> {
224 /// let browser = Browser::new(reqwest::Client::new());
225 /// let user_agent = get("firefox").unwrap();
226 /// let images = browser.images("rustacean", "wt-wt", true, Some(5), user_agent).await?;
227 /// assert!(!images.is_empty());
228 /// Ok(())
229 /// }
230 /// ```
231 pub async fn images(
232 &self,
233 query: &str,
234 region: &str,
235 safesearch: bool,
236 limit: Option<usize>,
237 user_agent: &str,
238 ) -> Result<Vec<ImageResult>> {
239 let vqd = self.get_vqd(query, user_agent).await?;
240 let mut page_params = vec![
241 ("q", query.to_string()),
242 ("l", region.to_string()),
243 ("vqd", vqd),
244 ("o", "json".into()),
245 ("p", if safesearch { "1" } else { "-1" }.into()),
246 ];
247
248 let mut results = Vec::new();
249
250 loop {
251 let params_ref: Vec<(&str, &str)> =
252 page_params.iter().map(|(k, v)| (*k, v.as_ref())).collect();
253
254 let resp = self
255 .request(
256 reqwest::Method::GET,
257 "https://duckduckgo.com/i.js",
258 user_agent,
259 ¶ms_ref,
260 )
261 .await?;
262
263 let j: Value = resp.json().await?;
264 if let Some(array) = j.get("results").and_then(|r| r.as_array()) {
265 for item in array.iter() {
266 results.push(ImageResult {
267 title: item["title"].as_str().unwrap_or("").to_string(),
268 image: item["image"].as_str().unwrap_or("").to_string(),
269 thumbnail: item["thumbnail"].as_str().unwrap_or("").to_string(),
270 url: item["url"].as_str().unwrap_or("").to_string(),
271 height: item["height"].as_u64().unwrap_or(0) as u32,
272 width: item["width"].as_u64().unwrap_or(0) as u32,
273 source: item["source"].as_str().unwrap_or("").to_string(),
274 });
275
276 if limit.is_some_and(|l| results.len() >= l) {
277 return Ok(results);
278 }
279 }
280 }
281
282 if let Some(next) = j.get("next").and_then(|n| n.as_str()) {
283 let s = next.split("s=").nth(1).unwrap_or("").to_string();
284 page_params.push(("s", s));
285 } else {
286 break;
287 }
288 }
289
290 Ok(results)
291 }
292
293 /// Performs a news search using DuckDuckGo's `news.js` API.
294 ///
295 /// # Arguments
296 /// * `query` - The search query.
297 /// * `region` - Region/language code (e.g., `"wt-wt"`).
298 /// * `safesearch` - Enables/disables safe search.
299 /// * `limit` - Optional limit for number of news results.
300 ///
301 /// # Returns
302 /// A list of `NewsResult` entries, including title, source, URL, and date.
303 ///
304 /// # Example
305 /// ```rust
306 /// use duckduckgo::browser::Browser;
307 /// use duckduckgo::user_agents::get;
308 ///
309 ///
310 /// #[tokio::main]
311 /// async fn main() -> anyhow::Result<()> {
312 /// let user_agent = get("firefox").unwrap();
313 /// let browser = Browser::new(reqwest::Client::new());
314 /// let news = browser.news("AI", "wt-wt", true, Some(5), user_agent).await?;
315 /// assert!(news.iter().any(|n| n.title.contains("AI")));
316 /// Ok(())
317 /// }
318 /// ```
319 pub async fn news(
320 &self,
321 query: &str,
322 region: &str,
323 safesearch: bool,
324 limit: Option<usize>,
325 user_agent: &str,
326 ) -> Result<Vec<NewsResult>> {
327 let vqd = self.get_vqd(query, user_agent).await?;
328 let mut page_params = vec![
329 ("q", query.to_string()),
330 ("l", region.to_string()),
331 ("vqd", vqd),
332 ("o", "json".into()),
333 ("p", if safesearch { "1" } else { "-1" }.into()),
334 ("noamp", "1".into()),
335 ];
336
337 let mut results = Vec::new();
338
339 loop {
340 let params_ref: Vec<(&str, &str)> =
341 page_params.iter().map(|(k, v)| (*k, v.as_ref())).collect();
342
343 let resp = self
344 .request(
345 reqwest::Method::GET,
346 "https://duckduckgo.com/news.js",
347 user_agent,
348 ¶ms_ref,
349 )
350 .await?;
351
352 let j: Value = resp.json().await?;
353 if let Some(array) = j.get("results").and_then(|r| r.as_array()) {
354 for item in array.iter() {
355 let date = item["date"]
356 .as_i64()
357 .map(|ts| {
358 chrono::Utc
359 .timestamp_opt(ts, 0)
360 .single()
361 .unwrap_or_else(chrono::Utc::now)
362 })
363 .unwrap_or_else(chrono::Utc::now);
364
365 results.push(NewsResult {
366 date: date.to_rfc3339(),
367 title: item["title"].as_str().unwrap_or("").to_string(),
368 body: item["excerpt"].as_str().unwrap_or("").to_string(),
369 url: item["url"].as_str().unwrap_or("").to_string(),
370 image: item
371 .get("image")
372 .and_then(|v| v.as_str())
373 .map(str::to_string),
374 source: item["source"].as_str().unwrap_or("").to_string(),
375 });
376
377 if limit.is_some_and(|l| results.len() >= l) {
378 return Ok(results);
379 }
380 }
381 }
382
383 if let Some(next) = j.get("next").and_then(|n| n.as_str()) {
384 let s = next.split("s=").nth(1).unwrap_or("").to_string();
385 page_params.push(("s", s));
386 } else {
387 break;
388 }
389 }
390
391 Ok(results)
392 }
393
394 /// Performs a DuckDuckGo search based on the provided path, result format, and optional result limit.
395 ///
396 /// # Arguments
397 /// * `path` - The path to be appended to the DuckDuckGo API base URL.
398 /// * `result_format` - The format in which the search results should be displayed (List or Detailed).
399 /// * `limit` - Optional limit for the number of search results to be displayed.
400 ///
401 /// # Returns
402 /// `Result<(), reqwest::Error>` - Result indicating success or failure of the search operation.
403 ///
404 /// # Examples
405 /// ```
406 /// use duckduckgo::browser::Browser;
407 /// use duckduckgo::response::ResultFormat;
408 /// use reqwest::Client;
409 ///
410 /// #[tokio::main]
411 /// async fn main() {
412 /// let client = Client::new();
413 /// let browser = Browser::new(client);
414 /// browser.browse("?q=Rust", ResultFormat::List, Some(5)).await.unwrap();
415 /// }
416 /// ```
417 pub async fn browse(
418 &self,
419 path: &str,
420 result_format: ResultFormat,
421 limit: Option<usize>,
422 ) -> Result<()> {
423 let separator = if path.contains('?') { '&' } else { '?' };
424 let url = format!("{}{}{}format=json", BASE_URL, path, separator);
425
426 let response = self
427 .client
428 .get(&url)
429 .send()
430 .await
431 .with_context(|| format!("Failed to send request to {}", url))?;
432
433 let status = response.status();
434 let text = response
435 .text()
436 .await
437 .with_context(|| "Failed to read response body")?;
438
439 if !status.is_success() {
440 anyhow::bail!("Request failed with status {}: {}", status, text);
441 }
442
443 let api_response: Response = serde_json::from_str(&text)
444 .with_context(|| format!("Failed to parse JSON response: {}", text))?;
445
446 match result_format {
447 ResultFormat::List => self.print_results_list(api_response, limit),
448 ResultFormat::Detailed => self.print_results_detailed(api_response, limit),
449 }
450
451 Ok(())
452 }
453
454 /// Prints search results in list format.
455 ///
456 /// # Arguments
457 /// * `api_response` - The response from the DuckDuckGo API.
458 /// * `limit` - Optional limit for the number of search results to be displayed.
459 pub fn print_results_list(&self, api_response: Response, limit: Option<usize>) {
460 if let Some(heading) = api_response.heading {
461 let style = AnsiStyle {
462 bold: true,
463 color: Some(AnsiColor::Gold),
464 };
465 println!(
466 "{}{}{}",
467 style.escape_code(),
468 heading,
469 AnsiStyle::reset_code()
470 );
471 }
472
473 let topics = &api_response.related_topics;
474
475 for (index, topic) in topics
476 .iter()
477 .enumerate()
478 .take(limit.unwrap_or(topics.len()))
479 {
480 self.print_related_topic(index + 1, topic);
481 }
482 }
483
484 /// Prints a related topic in a detailed format.
485 ///
486 /// # Arguments
487 /// * `index` - The index of the related topic.
488 /// * `topic` - The related topic to be printed.
489 pub fn print_related_topic(&self, index: usize, topic: &Topic) {
490 let style = AnsiStyle {
491 bold: false,
492 color: Some(AnsiColor::BrightGreen),
493 };
494
495 let text = match &topic.text {
496 Some(t) => t,
497 None => {
498 return;
499 }
500 };
501
502 let first_url = match &topic.first_url {
503 Some(url) => url,
504 None => {
505 return;
506 }
507 };
508
509 println!("{}. {} {}", index, text, style.escape_code());
510 println!("URL: {}{}", first_url, style.escape_code());
511 if let Some(icon) = &topic.icon {
512 let style = AnsiStyle {
513 bold: false,
514 color: Some(AnsiColor::BrightBlue),
515 };
516 if !icon.url.is_empty() {
517 let full_url = format!("https://duckduckgo.com{}", icon.url);
518 println!("Image URL: {}{}", full_url, style.escape_code());
519 }
520 }
521 println!("--------------------------------------------");
522 }
523
524 /// Prints search results in detailed format.
525 ///
526 /// # Arguments
527 /// * `api_response` - The response from the DuckDuckGo API.
528 /// * `limit` - Optional limit for the number of search results to be displayed.
529 pub fn print_results_detailed(&self, api_response: Response, limit: Option<usize>) {
530 if let Some(heading) = api_response.heading {
531 let style = AnsiStyle {
532 bold: true,
533 color: None,
534 };
535 println!(
536 "{}{}{}",
537 style.escape_code(),
538 heading,
539 AnsiStyle::reset_code()
540 );
541 }
542
543 if let Some(abstract_text) = api_response.abstract_text {
544 let style = AnsiStyle {
545 bold: false,
546 color: Some(AnsiColor::LightGray),
547 };
548 println!("Abstract: {}{}", abstract_text, style.escape_code());
549 }
550
551 if let Some(abstract_source) = api_response.abstract_source {
552 let style = AnsiStyle {
553 bold: false,
554 color: Some(AnsiColor::Purple),
555 };
556 println!(
557 "Abstract Source: {}{}",
558 abstract_source,
559 style.escape_code()
560 );
561 }
562
563 if let Some(abstract_url) = api_response.abstract_url {
564 let style = AnsiStyle {
565 bold: false,
566 color: Some(AnsiColor::Silver),
567 };
568 println!("Abstract URL: {}{}", abstract_url, style.escape_code());
569 }
570
571 if let Some(image) = api_response.image {
572 let style = AnsiStyle {
573 bold: false,
574 color: Some(AnsiColor::SkyBlue),
575 };
576 if !image.is_empty() {
577 let full_url = format!("https://duckduckgo.com{}", image);
578 println!("Image URL: {}{}", full_url, style.escape_code());
579 }
580 }
581
582 let topics = &api_response.related_topics;
583
584 for (index, topic) in topics
585 .iter()
586 .enumerate()
587 .take(limit.unwrap_or(topics.len()))
588 {
589 self.print_related_topic(index + 1, topic);
590 }
591 }
592
593 /// Performs a basic DuckDuckGo search with the provided parameters.
594 ///
595 /// # Arguments
596 /// * `query` - The search query.
597 /// * `safe_search` - A boolean indicating whether safe search is enabled.
598 /// * `result_format` - The format in which the search results should be displayed (List or Detailed).
599 /// * `limit` - Optional limit for the number of search results to be displayed.
600 ///
601 /// # Returns
602 /// `Result<(), reqwest::Error>` - Result indicating success or failure of the search operation.
603 ///
604 /// # Examples
605 /// ```
606 /// use duckduckgo::browser::Browser;
607 /// use duckduckgo::response::ResultFormat;
608 /// use reqwest::Client;
609 ///
610 /// #[tokio::main]
611 /// async fn main() {
612 /// let client = Client::new();
613 /// let browser = Browser::new(client);
614 /// browser.search("Rust", true, ResultFormat::Detailed, Some(5)).await.unwrap();
615 /// }
616 /// ```
617 pub async fn search(
618 &self,
619 query: &str,
620 safe_search: bool,
621 result_format: ResultFormat,
622 limit: Option<usize>,
623 ) -> Result<()> {
624 let safe_param = if safe_search { "&kp=1" } else { "&kp=-2" };
625 let path = format!("?q={}{}", query, safe_param);
626
627 self.browse(&path, result_format, limit)
628 .await
629 .with_context(|| format!("Failed to perform search for query '{}'", query))
630 }
631
632 /// Performs an advanced DuckDuckGo search with additional parameters.
633 ///
634 /// # Arguments
635 /// * `query` - The search query.
636 /// * `params` - Additional search parameters.
637 /// * `safe_search` - A boolean indicating whether safe search is enabled.
638 /// * `result_format` - The format in which the search results should be displayed (List or Detailed).
639 /// * `limit` - Optional limit for the number of search results to be displayed.
640 ///
641 /// # Returns
642 /// `Result<(), reqwest::Error>` - Result indicating success or failure of the search operation.
643 ///
644 /// # Examples
645 /// ```
646 /// use duckduckgo::browser::Browser;
647 /// use duckduckgo::response::ResultFormat;
648 /// use reqwest::Client;
649 ///
650 /// #[tokio::main]
651 /// async fn main() {
652 /// let client = Client::new();
653 /// let browser = Browser::new(client);
654 /// browser.advanced_search("Rust", "lang:en", true, ResultFormat::Detailed, Some(5)).await.unwrap();
655 /// }
656 /// ```
657 pub async fn advanced_search(
658 &self,
659 query: &str,
660 params: &str,
661 safe_search: bool,
662 result_format: ResultFormat,
663 limit: Option<usize>,
664 ) -> Result<()> {
665 let safe_param = if safe_search { "&kp=1" } else { "&kp=-2" };
666 let path = format!("?q={}&kl={}{}", query, params, safe_param);
667
668 self.browse(&path, result_format, limit)
669 .await
670 .with_context(|| format!("Failed to perform advanced search for query '{}'", query))
671 }
672
673 /// Performs a DuckDuckGo search with custom search operators.
674 ///
675 /// # Arguments
676 /// * `query` - The search query.
677 /// * `operators` - Custom search operators.
678 /// * `safe_search` - A boolean indicating whether safe search is enabled.
679 /// * `result_format` - The format in which the search results should be displayed (List or Detailed).
680 /// * `limit` - Optional limit for the number of search results to be displayed.
681 ///
682 /// # Returns
683 /// `Result<(), reqwest::Error>` - Result indicating success or failure of the search operation.
684 ///
685 /// # Examples
686 /// ```
687 /// use duckduckgo::browser::Browser;
688 /// use duckduckgo::response::ResultFormat;
689 /// use reqwest::Client;
690 ///
691 /// #[tokio::main]
692 /// async fn main() {
693 /// let client = Client::new();
694 /// let browser = Browser::new(client);
695 /// browser.search_operators("Rust", "site:github.com", true, ResultFormat::List, Some(5)).await.unwrap();
696 /// }
697 /// ```
698 pub async fn search_operators(
699 &self,
700 query: &str,
701 operators: &str,
702 safe_search: bool,
703 result_format: ResultFormat,
704 limit: Option<usize>,
705 ) -> Result<()> {
706 let safe_param = if safe_search { "&kp=1" } else { "&kp=-2" };
707 let path = format!("?q={}&{}{}", query, operators, safe_param);
708
709 self.browse(&path, result_format, limit)
710 .await
711 .with_context(|| format!("Failed to perform operator search for query '{}'", query))
712 }
713}