aegis_tools/
web_fetch_pro.rs

1//! # web_fetch_pro (opt-in, external)
2//!
3//! Layer-1 anti-bot web fetching via the external
4//! [`Scrapling`](https://github.com/D4Vinci/Scrapling) CLI. Adds what the
5//! built-in `web_extract` cannot do: TLS-fingerprint-impersonating HTTP and a
6//! stealth browser that can pass Cloudflare Turnstile.
7//!
8//! Off by default. Enable in config:
9//! ```toml
10//! [web_fetch_pro]
11//! enabled = true
12//! path    = "scrapling"     # binary on PATH or absolute path
13//! mode    = "http"          # "http" (light) | "stealth" (spins up a browser)
14//! timeout_secs = 120
15//! ```
16//!
17//! `stealth` mode launches a headless browser (hundreds of MB RAM), so keep it
18//! off on 1c1g hosts and use `http` there.
19
20use crate::registry::{Tool, ToolContext};
21use aegis_security::is_safe_url;
22use anyhow::Result;
23use async_trait::async_trait;
24use serde_json::{json, Value};
25use std::process::Stdio;
26
27/// Opt-in wrapper around the external `scrapling extract` CLI.
28pub struct WebFetchProTool {
29    /// Path/name of the `scrapling` executable.
30    pub binary: String,
31    /// "http" (light) or "stealth" (browser, Cloudflare bypass).
32    pub mode: String,
33    /// Per-invocation timeout in seconds.
34    pub timeout_secs: u64,
35}
36
37#[async_trait]
38impl Tool for WebFetchProTool {
39    fn name(&self) -> &str {
40        "web_fetch_pro"
41    }
42
43    fn description(&self) -> &str {
44        "Fetch a URL through anti-bot defenses (TLS-fingerprint HTTP, or a stealth browser that bypasses Cloudflare) via the external Scrapling tool. Returns Markdown. Use when web_extract is blocked."
45    }
46
47    fn parameters(&self) -> Value {
48        json!({
49            "type": "object",
50            "properties": {
51                "url": { "type": "string", "description": "URL to fetch" },
52                "css_selector": { "type": "string", "description": "Optional CSS selector to extract only matching elements" },
53                "solve_cloudflare": { "type": "boolean", "description": "stealth mode only: attempt to solve Cloudflare Turnstile (default false)" }
54            },
55            "required": ["url"]
56        })
57    }
58
59    async fn execute(&self, args: Value, _ctx: &ToolContext<'_>) -> Result<String> {
60        let url = args["url"].as_str().unwrap_or("").trim();
61        if url.is_empty() {
62            return Ok("Error: url is required".to_string());
63        }
64        is_safe_url(url).map_err(|e| anyhow::anyhow!("SSRF check failed: {e}"))?;
65
66        let css = args["css_selector"].as_str();
67        let solve_cf = args["solve_cloudflare"].as_bool().unwrap_or(false);
68
69        // Scrapling `extract` writes the result to a file whose extension picks
70        // the format (.md → Markdown). Use a unique temp file, read it back.
71        let tmp = std::env::temp_dir().join(format!("aegis_wfp_{}.md", std::process::id()));
72
73        // Sub-command: `get` (HTTP) or `stealthy-fetch` (browser).
74        let subcmd = if self.mode == "stealth" {
75            "stealthy-fetch"
76        } else {
77            "get"
78        };
79
80        let mut cmd = tokio::process::Command::new(&self.binary);
81        cmd.arg("extract")
82            .arg(subcmd)
83            .arg(url)
84            .arg(&tmp);
85        if let Some(sel) = css {
86            if !sel.trim().is_empty() {
87                cmd.arg("--css-selector").arg(sel);
88            }
89        }
90        if self.mode == "stealth" && solve_cf {
91            cmd.arg("--solve-cloudflare");
92        }
93        cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
94
95        let output = tokio::time::timeout(
96            std::time::Duration::from_secs(self.timeout_secs),
97            cmd.output(),
98        )
99        .await
100        .map_err(|_| anyhow::anyhow!("web_fetch_pro timed out after {}s", self.timeout_secs))?
101        .map_err(|e| {
102            anyhow::anyhow!(
103                "failed to run '{}': {e}. Is Scrapling installed (pip install \"scrapling[shell]\") and on PATH?",
104                self.binary
105            )
106        })?;
107
108        let result = std::fs::read_to_string(&tmp).ok();
109        let _ = std::fs::remove_file(&tmp); // best-effort cleanup
110
111        if let Some(md) = result {
112            if !md.trim().is_empty() {
113                return Ok(md);
114            }
115        }
116
117        let stderr = String::from_utf8_lossy(&output.stderr);
118        if !output.status.success() {
119            Ok(format!(
120                "[web_fetch_pro error] exit {}\n{stderr}",
121                output.status.code().unwrap_or(-1)
122            ))
123        } else {
124            Ok(format!("web_fetch_pro returned no content. stderr: {stderr}"))
125        }
126    }
127}
aegis_tools/web_fetch_pro.rs

aegis_tools/
web_fetch_pro.rs
