browser_automation_cli/
robots.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2//! robots.txt honor for one-shot invocations (no cross-process cache).
3//!
4//! Default policy is honor. Override requires BOTH `--ignore-robots` and
5//! `--i-accept-robots-risk`. Uses `robotstxt::DefaultMatcher` (Google port).
6//!
7//! # Workload
8//!
9//! **I/O-bound** (network GET of `/robots.txt`). Reuses process-wide
10//! [`shared_http_client`] so keep-alive / TLS session cache survive batch scrapes.
11//! Not CPU-bound → no rayon. Not a long-lived daemon → no connection pool beyond
12//! reqwest's internal pool for this one-shot process.
13
14use std::sync::OnceLock;
15use std::time::Duration;
16
17use robotstxt::DefaultMatcher;
18use url::Url;
19
20use crate::error::{CliError, ErrorKind};
21
22/// Product User-Agent for shared HTTP (aligned with scrape engine).
23const DEFAULT_HTTP_UA: &str = concat!(
24    "browser-automation-cli/",
25    env!("CARGO_PKG_VERSION"),
26    " (+https://github.com/danilo-aguiar-br/browser-automation-cli; local-scrape)"
27);
28
29/// Process-wide async HTTP client (rules: create `reqwest::Client` once, reuse).
30///
31/// Default timeout 30s; callers may override per-request (e.g. robots 5s).
32///
33/// Uses `get_or_init` (stable). `get_or_try_init` is still unstable (`once_cell_try`);
34/// on first failure we surface the error without poisoning the lock.
35pub fn shared_http_client() -> Result<&'static reqwest::Client, CliError> {
36    static CLIENT: OnceLock<reqwest::Client> = OnceLock::new();
37    if let Some(c) = CLIENT.get() {
38        return Ok(c);
39    }
40    let built = reqwest::Client::builder()
41        .user_agent(DEFAULT_HTTP_UA)
42        .timeout(Duration::from_secs(30))
43        .redirect(reqwest::redirect::Policy::limited(10))
44        .pool_max_idle_per_host(4)
45        // Explicit TCP_NODELAY: reduce small-response latency on robots/scrape
46        // (rules_rust_latencia_reduzir — socket options on owned HTTP path).
47        // CDP WebSocket is owned by chromiumoxide; not configurable here.
48        .tcp_nodelay(true)
49        .build()
50        .map_err(|e| CliError::new(ErrorKind::Software, format!("http client: {e}")))?;
51    Ok(CLIENT.get_or_init(|| built))
52}
53
54/// Effective robots policy for one invocation.
55#[derive(Debug, Clone, Copy, PartialEq, Eq)]
56pub enum RobotsPolicy {
57    /// Honor robots.txt rules (default).
58    Honor,
59    /// Skip robots.txt only when dual risk flags are set.
60    Ignore,
61}
62
63impl RobotsPolicy {
64    /// Stable string for JSON and logs (`honor` | `ignore`).
65    pub fn as_str(self) -> &'static str {
66        match self {
67            Self::Honor => "honor",
68            Self::Ignore => "ignore",
69        }
70    }
71
72    /// Build policy from CLI flags. Ignore only when both flags are set.
73    pub fn from_flags(ignore_robots: bool, accept_risk: bool) -> Result<Self, CliError> {
74        match (ignore_robots, accept_risk) {
75            (false, false) => Ok(Self::Honor),
76            (true, true) => Ok(Self::Ignore),
77            (true, false) => Err(CliError::with_suggestion(
78                ErrorKind::Usage,
79                // message stays EN (stable); suggestion localized at emit via catalog map
80                "--ignore-robots requires --i-accept-robots-risk",
81                "Pass both flags together when you intentionally skip robots.txt",
82            )),
83            (false, true) => Err(CliError::with_suggestion(
84                ErrorKind::Usage,
85                "--i-accept-robots-risk requires --ignore-robots",
86                "Pass both flags together when you intentionally skip robots.txt",
87            )),
88        }
89    }
90}
91
92/// Returns true when path is allowed by a simple robots Disallow set.
93/// Empty disallow list means allow all. Exact prefix match on path.
94pub fn path_allowed(path: &str, disallows: &[&str]) -> bool {
95    if disallows.is_empty() {
96        return true;
97    }
98    let path = if path.is_empty() { "/" } else { path };
99    !disallows.iter().any(|d| {
100        if d.is_empty() {
101            return false;
102        }
103        path.starts_with(d)
104    })
105}
106
107/// Schemes that skip robots.txt (local / non-network).
108pub fn scheme_skips_robots(url: &str) -> bool {
109    let lower = url.trim().to_ascii_lowercase();
110    lower.starts_with("about:")
111        || lower.starts_with("file:")
112        || lower.starts_with("data:")
113        || lower.starts_with("blob:")
114        || lower == "about:blank"
115}
116
117/// Check URL against robots.txt body for a user-agent using DefaultMatcher.
118pub fn url_allowed_by_robots_body(robots_body: &str, user_agent: &str, url: &str) -> bool {
119    let mut matcher = DefaultMatcher::default();
120    matcher.one_agent_allowed_by_robots(robots_body, user_agent, url)
121}
122
123/// Fetch origin robots.txt and enforce honor policy (async, one-shot).
124pub async fn enforce_robots(
125    url: &str,
126    policy: RobotsPolicy,
127    user_agent: &str,
128) -> Result<(), CliError> {
129    if matches!(policy, RobotsPolicy::Ignore) || scheme_skips_robots(url) {
130        return Ok(());
131    }
132
133    let parsed = Url::parse(url).map_err(|e| {
134        CliError::with_suggestion(
135            ErrorKind::Usage,
136            format!("invalid URL for robots check: {e}"),
137            "Pass an absolute http(s) URL or about:blank/file://",
138        )
139    })?;
140
141    if parsed.scheme() != "http" && parsed.scheme() != "https" {
142        return Ok(());
143    }
144
145    let origin = parsed.origin().ascii_serialization();
146    let robots_url = format!("{origin}/robots.txt");
147
148    // Reuse process-wide client; short per-request timeout for robots only.
149    let client = shared_http_client()?;
150    let resp = match client
151        .get(&robots_url)
152        .header(reqwest::header::USER_AGENT, user_agent)
153        .timeout(Duration::from_secs(5))
154        .send()
155        .await
156    {
157        Ok(r) => r,
158        Err(e) => {
159            // PRD: fetch failure → allow with warning (tracing → stderr / optional file).
160            tracing::warn!(
161                target: "browser_automation_cli::robots",
162                robots_url = %robots_url,
163                error = %e,
164                "robots fetch failed; treating as allow"
165            );
166            return Ok(());
167        }
168    };
169
170    if !resp.status().is_success() {
171        // Missing robots → allow
172        return Ok(());
173    }
174
175    let body = resp
176        .text()
177        .await
178        .map_err(|e| CliError::new(ErrorKind::Io, format!("robots body: {e}")))?;
179
180    if url_allowed_by_robots_body(&body, user_agent, url) {
181        return Ok(());
182    }
183
184    Err(CliError::with_suggestion(
185        ErrorKind::Data,
186        format!("robots.txt disallows URL: {url}"),
187        "Use --ignore-robots --i-accept-robots-risk only when you accept the risk",
188    ))
189}
190
191#[cfg(test)]
192mod tests {
193    use super::*;
194
195    #[test]
196    fn honor_disallow_prefix() {
197        assert!(!path_allowed("/private/x", &["/private"]));
198        assert!(path_allowed("/public/x", &["/private"]));
199        assert!(path_allowed("/any", &[]));
200    }
201
202    #[test]
203    fn scheme_skips_local() {
204        assert!(scheme_skips_robots("about:blank"));
205        assert!(scheme_skips_robots("file:///tmp/x.html"));
206        assert!(!scheme_skips_robots("https://example.com/"));
207    }
208
209    #[test]
210    fn policy_requires_both_flags() {
211        assert!(matches!(
212            RobotsPolicy::from_flags(false, false).unwrap(),
213            RobotsPolicy::Honor
214        ));
215        assert!(matches!(
216            RobotsPolicy::from_flags(true, true).unwrap(),
217            RobotsPolicy::Ignore
218        ));
219        assert!(RobotsPolicy::from_flags(true, false).is_err());
220        assert!(RobotsPolicy::from_flags(false, true).is_err());
221    }
222
223    #[test]
224    fn default_matcher_blocks_disallow_all() {
225        let body = "user-agent: *\ndisallow: /\n";
226        assert!(!url_allowed_by_robots_body(
227            body,
228            "browser-automation-cli",
229            "https://example.com/secret"
230        ));
231    }
232
233    #[test]
234    fn default_matcher_allows_empty() {
235        assert!(url_allowed_by_robots_body(
236            "",
237            "browser-automation-cli",
238            "https://example.com/"
239        ));
240    }
241}
browser_automation_cli/robots.rs

browser_automation_cli/
robots.rs
