deepseek_rust_cli/tools/
web_ops.rs

1use anyhow::Result;
2use futures::StreamExt;
3use once_cell::sync::Lazy;
4
5static CLIENT: Lazy<reqwest::Client> = Lazy::new(|| {
6    reqwest::Client::builder()
7        .timeout(std::time::Duration::from_secs(30))
8        .build()
9        .expect("Failed to create reqwest client")
10});
11
12pub async fn fetch_url(url: &str) -> Result<String> {
13    let response = CLIENT.get(url).send().await?;
14
15    // Limit to 1MB
16    let max_size = 1024 * 1024;
17    let mut body = Vec::new();
18    let mut stream = response.bytes_stream();
19
20    while let Some(chunk_res) = stream.next().await {
21        let chunk = chunk_res?;
22        if body.len() + chunk.len() > max_size {
23            anyhow::bail!("Fetched content exceeds 1MB limit.");
24        }
25        body.extend_from_slice(&chunk);
26    }
27
28    let clean = html2text::from_read(&body[..], 80)?;
29    Ok(clean)
30}
31
32pub fn get_env_var(name: &str) -> String {
33    std::env::var(name).unwrap_or_else(|_| "Not set".to_string())
34}
35
36pub async fn web_search_duckduckgo(query: &str) -> Result<String> {
37    let response = CLIENT
38        .get("https://html.duckduckgo.com/html/")
39        .query(&[("q", query)])
40        .header(
41            "User-Agent",
42            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
43             Chrome/120.0.0.0 Safari/537.36",
44        )
45        .send()
46        .await?
47        .error_for_status()?;
48
49    let body = response.bytes().await?;
50    let clean = html2text::from_read(&body[..], 80)?;
51
52    let lines: Vec<&str> = clean
53        .lines()
54        .filter(|line| {
55            let l = line.trim();
56            !l.is_empty()
57                && !l.contains("duckduckgo.com")
58                && !l.starts_with("Ad")
59                && !l.contains("JavaScript")
60        })
61        .collect();
62
63    let count = lines.len().min(100);
64    let result_text = lines[..count].join("\n");
65    Ok(result_text)
66}
deepseek_rust_cli/tools/web_ops.rs

deepseek_rust_cli/tools/
web_ops.rs
