Skip to main content

rustai_core/http/
mod.rs

1//! The fetch layer: impersonated HTTP, politeness, and a headless fallback.
2//!
3//! Three things separate this from a bare `reqwest` loop:
4//!
5//! * **Fingerprint.** Cloudflare and friends read the TLS ClientHello (JA3/JA4)
6//!   and the HTTP/2 SETTINGS frame, not just the User-Agent. `wreq` replays a
7//!   real Chrome's, so a plain `GET` gets a plain `200` where a stock client
8//!   gets a challenge page.
9//! * **Politeness.** `robots.txt`, per-host spacing and a concurrency ceiling
10//!   are on by default, because a fast crawler with none of those is a way to
11//!   get a user's IP banned.
12//! * **Escalation.** JS-gated pages are detected from the response and, when
13//!   the `browser` feature is on, retried through headless Chrome — which is
14//!   two orders of magnitude more expensive and therefore never the first move.
15
16pub mod robots;
17
18#[cfg(feature = "browser")]
19pub mod browser;
20
21use std::collections::HashMap;
22use std::net::{IpAddr, SocketAddr};
23use std::sync::Arc;
24use std::time::{Duration, Instant};
25
26use futures_util::StreamExt;
27use tokio::sync::{Mutex, Semaphore};
28use url::Url;
29use wreq::dns::{Addrs, GaiResolver, Name, Resolve, Resolving};
30
31use crate::error::{Error, Result};
32use robots::Robots;
33
34/// Default identity when impersonation is off, or for the `robots.txt` lookup.
35pub const DEFAULT_USER_AGENT: &str =
36    concat!("rustai/", env!("CARGO_PKG_VERSION"), " (+https://github.com/imhyensuk/rustai)");
37
38/// The token we match `robots.txt` groups against.
39pub const ROBOTS_AGENT: &str = "rustai";
40
41/// Hard cap on a `robots.txt`.
42///
43/// Every other body is capped while streaming; this one is fetched before a
44/// crawl can start, so an endless `robots.txt` would be a free way to exhaust
45/// a crawler's memory. Google stops reading at 500 KB and nothing legitimate
46/// comes close.
47const MAX_ROBOTS_BYTES: usize = 512 * 1024;
48
49/// HTML-level redirects followed before giving up.
50///
51/// Two is enough for the real pattern (a canonicalising stub pointing at the
52/// article) and low enough that a redirect loop costs almost nothing.
53const MAX_HTML_REDIRECTS: usize = 2;
54
55/// Above this much extracted text, a page is not an empty shell.
56const MAX_STATIC_TEXT: usize = 2_000;
57
58/// One character of text per this many bytes of markup is the floor below which
59/// a page is presumed to be rendered client-side.
60const TEXT_TO_MARKUP_FLOOR: usize = 200;
61
62/// Content types we are willing to parse as a document.
63const TEXTUAL: &[&str] = &[
64    "text/html",
65    "application/xhtml",
66    "text/plain",
67    "application/xml",
68    "text/xml",
69    "application/rss",
70    "application/atom",
71    "application/json",
72];
73
74/// How the client should present itself on the wire.
75#[derive(Debug, Clone, Default, PartialEq, Eq)]
76pub enum Impersonate {
77    /// A recent Chrome build. The default, and what almost everyone wants.
78    #[default]
79    Chrome,
80    /// A recent Firefox build.
81    Firefox,
82    /// A recent Safari build.
83    Safari,
84    /// A named `wreq-util` profile, e.g. `"chrome_143"` or `"safari_ios_26"`.
85    Named(String),
86    /// Market-share weighted random profile, re-rolled per client.
87    Random,
88    /// No emulation: plain TLS and a `rustai/x.y.z` User-Agent.
89    None,
90}
91
92impl Impersonate {
93    /// Parse the string form used by the Python API.
94    pub fn parse(s: &str) -> Result<Impersonate> {
95        Ok(match s.trim().to_ascii_lowercase().as_str() {
96            "" | "none" | "off" | "false" => Impersonate::None,
97            "chrome" | "true" | "default" => Impersonate::Chrome,
98            "firefox" => Impersonate::Firefox,
99            "safari" => Impersonate::Safari,
100            "random" => Impersonate::Random,
101            other => Impersonate::Named(other.to_string()),
102        })
103    }
104}
105
106/// Everything tunable about fetching.
107#[derive(Clone)]
108pub struct FetchConfig {
109    /// Maximum requests in flight across all hosts.
110    pub concurrency: usize,
111    /// Total per-request deadline.
112    pub timeout: Duration,
113    /// Connection establishment deadline.
114    pub connect_timeout: Duration,
115    /// Redirects followed before giving up.
116    pub max_redirects: usize,
117    /// Hard cap on a response body. Enforced while streaming, so an oversized
118    /// body costs one chunk, not the whole download.
119    pub max_body_bytes: usize,
120    /// Retries for retryable failures.
121    pub retries: u32,
122    /// Base delay for exponential backoff between retries.
123    pub retry_backoff: Duration,
124    /// Minimum spacing between two requests to the same host.
125    pub per_host_delay: Duration,
126    /// Honour `robots.txt`. Leave this on unless you own the target.
127    pub respect_robots: bool,
128    /// Honour a `Crawl-delay` larger than [`FetchConfig::per_host_delay`].
129    pub respect_crawl_delay: bool,
130    /// Browser fingerprint to present.
131    pub impersonate: Impersonate,
132    /// `Accept-Language` header.
133    pub accept_language: String,
134    /// Escalate JS-gated pages to headless Chrome. Needs the `browser` feature.
135    pub browser_fallback: bool,
136    /// Proxies to route requests through, rotated round-robin.
137    ///
138    /// The only real answer to IP-reputation blocking: a TLS fingerprint says
139    /// what your client is, and an address says who it is. Accepts `http://`,
140    /// `https://` and `socks5://` URLs, with optional `user:pass@`.
141    pub proxies: Vec<String>,
142    /// Honour a `Retry-After` header, up to this long. Set to zero to ignore it.
143    pub max_retry_after: Duration,
144    /// File to persist cookies to, so a session survives the process.
145    pub cookie_file: Option<std::path::PathBuf>,
146    /// Allow loopback, private and link-local addresses as fetch targets.
147    ///
148    /// Off by default. A search result or a redirect can point at
149    /// `169.254.169.254` or at a service bound to the machine running this,
150    /// and whatever comes back is extracted into a context window destined for
151    /// a model -- which turns an ordinary crawl into a way to read the cloud
152    /// metadata endpoint out loud. Turn it on to crawl your own development
153    /// server.
154    pub allow_private_hosts: bool,
155}
156
157/// Hand-written rather than derived: [`FetchConfig::proxies`] can carry
158/// `user:pass@`, and a derived `Debug` would put those credentials into any log
159/// that dumps a `Fetcher` or a `Pipeline`. Shows the fields worth seeing in a
160/// dump; `..` covers the rest, so a new field cannot reintroduce a leak.
161impl std::fmt::Debug for FetchConfig {
162    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
163        f.debug_struct("FetchConfig")
164            .field("concurrency", &self.concurrency)
165            .field("timeout", &self.timeout)
166            .field("max_body_bytes", &self.max_body_bytes)
167            .field("retries", &self.retries)
168            .field("respect_robots", &self.respect_robots)
169            .field("impersonate", &self.impersonate)
170            .field("browser_fallback", &self.browser_fallback)
171            .field("proxies", &self.proxies.iter().map(|p| mask_proxy(p)).collect::<Vec<_>>())
172            .field("cookie_file", &self.cookie_file)
173            .field("allow_private_hosts", &self.allow_private_hosts)
174            .finish_non_exhaustive()
175    }
176}
177
178/// Replace any `user:pass@` in a proxy URL with `***`, keeping scheme, host and
179/// port — which are what a reader actually needs to identify the proxy.
180fn mask_proxy(proxy: &str) -> String {
181    let Ok(mut parsed) = Url::parse(proxy) else { return proxy.to_string() };
182    if parsed.username().is_empty() && parsed.password().is_none() {
183        return proxy.to_string();
184    }
185    let _ = parsed.set_username("***");
186    let _ = parsed.set_password(Some("***"));
187    parsed.to_string()
188}
189
190impl Default for FetchConfig {
191    fn default() -> Self {
192        FetchConfig {
193            concurrency: 16,
194            timeout: Duration::from_secs(20),
195            connect_timeout: Duration::from_secs(8),
196            max_redirects: 5,
197            max_body_bytes: 8 * 1024 * 1024,
198            retries: 2,
199            retry_backoff: Duration::from_millis(400),
200            per_host_delay: Duration::from_millis(250),
201            respect_robots: true,
202            respect_crawl_delay: true,
203            impersonate: Impersonate::Chrome,
204            accept_language: "en-US,en;q=0.9".to_string(),
205            browser_fallback: false,
206            proxies: Vec::new(),
207            max_retry_after: Duration::from_secs(60),
208            cookie_file: None,
209            allow_private_hosts: false,
210        }
211    }
212}
213
214/// A fetched document.
215#[derive(Debug, Clone, serde::Serialize)]
216pub struct Page {
217    /// URL as requested.
218    pub url: String,
219    /// URL after redirects.
220    pub final_url: String,
221    /// HTTP status.
222    pub status: u16,
223    /// `Content-Type` header, if any.
224    pub content_type: Option<String>,
225    /// Decoded body.
226    pub body: String,
227    /// Wall time for the fetch, in milliseconds.
228    pub elapsed_ms: u64,
229    /// Whether the headless fallback produced this body.
230    pub rendered: bool,
231}
232
233/// A pooled, polite, fingerprint-aware HTTP client.
234#[derive(Clone)]
235pub struct Fetcher {
236    /// One client per proxy, or a single direct client. Rotated per request.
237    clients: Arc<Vec<wreq::Client>>,
238    next_client: Arc<std::sync::atomic::AtomicUsize>,
239    jar: Option<Arc<wreq::cookie::Jar>>,
240    /// Hosts this fetcher has touched. The cookie jar indexes by domain but
241    /// does not expose that key, so a host-only cookie — which is most session
242    /// cookies — cannot be attributed on the way back out without this.
243    visited: Arc<std::sync::Mutex<std::collections::BTreeSet<String>>>,
244    cfg: Arc<FetchConfig>,
245    gate: Arc<Semaphore>,
246    robots: Arc<Mutex<HashMap<String, Arc<Robots>>>>,
247    last_hit: Arc<Mutex<HashMap<String, Instant>>>,
248}
249
250impl std::fmt::Debug for Fetcher {
251    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
252        f.debug_struct("Fetcher").field("config", &self.cfg).finish_non_exhaustive()
253    }
254}
255
256impl Fetcher {
257    /// Build a fetcher with default settings.
258    pub fn new() -> Result<Self> {
259        Self::with_config(FetchConfig::default())
260    }
261
262    /// Build a fetcher from an explicit config.
263    pub fn with_config(cfg: FetchConfig) -> Result<Self> {
264        if cfg.concurrency == 0 {
265            return Err(Error::Config("concurrency must be at least 1".into()));
266        }
267        let jar = cfg.cookie_file.as_ref().map(|path| Arc::new(load_jar(path)));
268
269        let build_one = |proxy: Option<&str>| -> Result<wreq::Client> {
270            let mut builder = wreq::Client::builder()
271                .timeout(cfg.timeout)
272                .connect_timeout(cfg.connect_timeout)
273                // Followed in `fetch_following` instead: see there for why.
274                .redirect(wreq::redirect::Policy::none())
275                .pool_max_idle_per_host(cfg.concurrency.min(32))
276                .referer(true)
277                .gzip(true)
278                .brotli(true)
279                .deflate(true)
280                .zstd(true);
281
282            // Literal addresses are already refused by `Fetcher::guard`; this
283            // is the same judgement applied to what a hostname resolves to.
284            if !cfg.allow_private_hosts {
285                builder = builder.dns_resolver(Arc::new(PublicOnlyDns(GaiResolver::new())));
286            }
287
288            builder = match &jar {
289                Some(jar) => builder.cookie_provider(jar.clone()),
290                None => builder.cookie_store(true),
291            };
292            builder = apply_impersonation(builder, &cfg.impersonate)?;
293
294            if let Some(proxy) = proxy {
295                let proxy = wreq::Proxy::all(proxy)
296                    .map_err(|e| Error::Config(format!("invalid proxy `{proxy}`: {e}")))?;
297                builder = builder.proxy(proxy);
298            }
299
300            let mut headers = wreq::header::HeaderMap::new();
301            if let Ok(v) = wreq::header::HeaderValue::from_str(&cfg.accept_language) {
302                headers.insert(wreq::header::ACCEPT_LANGUAGE, v);
303            }
304            if matches!(cfg.impersonate, Impersonate::None) {
305                headers.insert(
306                    wreq::header::USER_AGENT,
307                    wreq::header::HeaderValue::from_static(DEFAULT_USER_AGENT),
308                );
309            }
310            builder = builder.default_headers(headers);
311            builder.build().map_err(|e| Error::Config(e.to_string()))
312        };
313
314        let clients = if cfg.proxies.is_empty() {
315            vec![build_one(None)?]
316        } else {
317            cfg.proxies.iter().map(|p| build_one(Some(p))).collect::<Result<Vec<_>>>()?
318        };
319
320        Ok(Fetcher {
321            clients: Arc::new(clients),
322            next_client: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
323            jar,
324            visited: Arc::new(std::sync::Mutex::new(std::collections::BTreeSet::new())),
325            gate: Arc::new(Semaphore::new(cfg.concurrency)),
326            cfg: Arc::new(cfg),
327            robots: Arc::new(Mutex::new(HashMap::new())),
328            last_hit: Arc::new(Mutex::new(HashMap::new())),
329        })
330    }
331
332    /// The next client in the rotation.
333    ///
334    /// Round-robin rather than random: with a small proxy pool, randomness
335    /// clusters, and clustering on one address is exactly what gets it banned.
336    fn client(&self) -> &wreq::Client {
337        if self.clients.len() == 1 {
338            return &self.clients[0];
339        }
340        let i = self.next_client.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
341        &self.clients[i % self.clients.len()]
342    }
343
344    /// Write the cookie jar to [`FetchConfig::cookie_file`].
345    ///
346    /// Clearance cookies are the expensive part of getting through a bot wall;
347    /// throwing them away when the process exits means paying for them again.
348    pub fn save_cookies(&self) -> Result<usize> {
349        let (Some(jar), Some(path)) = (&self.jar, &self.cfg.cookie_file) else { return Ok(0) };
350        let hosts: Vec<String> = match self.visited.lock() {
351            Ok(v) => v.iter().cloned().collect(),
352            Err(poisoned) => poisoned.into_inner().iter().cloned().collect(),
353        };
354        save_jar(jar, &hosts, path)
355    }
356
357    /// The configuration this fetcher was built with.
358    pub fn config(&self) -> &FetchConfig {
359        &self.cfg
360    }
361
362    /// Fetch one URL, honouring `robots.txt` when configured to.
363    pub async fn fetch(&self, url: &str) -> Result<Page> {
364        self.fetch_inner(url, self.cfg.respect_robots).await
365    }
366
367    /// Fetch a provider endpoint, skipping the `robots.txt` check.
368    ///
369    /// `robots.txt` governs crawling — the discovery of documents a site did
370    /// not hand you. A search or feed endpoint the caller named explicitly is
371    /// a query, not a crawl, and several of them (DuckDuckGo's HTML endpoint
372    /// among them) disallow the very path they exist to serve. Pages found
373    /// *through* those endpoints go through [`Fetcher::fetch`] and are checked
374    /// normally, which is where the rule actually protects anyone.
375    pub async fn fetch_api(&self, url: &str) -> Result<Page> {
376        self.fetch_inner(url, false).await
377    }
378
379    /// Refuse an address only this machine or this network can reach.
380    ///
381    /// Every path that turns a `Url` into a request calls this, rather than
382    /// the entry point alone: `wreq` follows HTTP 3xx behind our back (handled
383    /// by [`redirect_policy`]), and a `<meta refresh>` is followed by the loop
384    /// in [`Fetcher::fetch_inner`], which reaches the network through
385    /// [`Fetcher::get_with_retry`] without passing [`normalize`] again.
386    fn guard(&self, url: &Url) -> Result<()> {
387        if !self.cfg.allow_private_hosts && is_internal(url) {
388            return Err(Error::InvalidUrl(format!("{url} is an internal address")));
389        }
390        Ok(())
391    }
392
393    /// Follow HTTP redirects to a page, checking every hop.
394    ///
395    /// `wreq` is set to [`wreq::redirect::Policy::none`] so the chain is walked
396    /// here instead. All three things owed to a redirect target need `await`
397    /// and so cannot live in a redirect policy, which is a synchronous
398    /// closure: `robots.txt` has to be read for the host the redirect actually
399    /// lands on, the internal-address guard has to see each target, and pacing
400    /// is per-host. Before this, a 3xx to a disallowed path on another host
401    /// was fetched without ever consulting that host's `robots.txt`.
402    async fn fetch_following(&self, start: &Url, check_robots: bool) -> Result<Page> {
403        let mut current = start.clone();
404        for _ in 0..=self.cfg.max_redirects {
405            self.guard(&current)?;
406            let crawl_delay = if check_robots {
407                let robots = self.robots_for(&current).await;
408                if !robots.allows(current.path()) {
409                    return Err(Error::RobotsDenied(current.to_string()));
410                }
411                robots.crawl_delay
412            } else {
413                None
414            };
415            self.pace(&current, crawl_delay).await;
416            match self.get_with_retry(&current).await? {
417                Hop::Page(page) => return Ok(page),
418                Hop::Redirect(next) => current = next,
419            }
420        }
421        Err(Error::network(start, format!("more than {} redirects", self.cfg.max_redirects)))
422    }
423
424    async fn fetch_inner(&self, url: &str, check_robots: bool) -> Result<Page> {
425        let parsed = normalize(url)?;
426        // Early, so an internal target costs neither a permit nor a robots fetch.
427        self.guard(&parsed)?;
428        let _permit = self.gate.acquire().await.map_err(|e| Error::Config(e.to_string()))?;
429
430        let started = Instant::now();
431        let mut page = self.fetch_following(&parsed, check_robots).await?;
432
433        // HTML-level redirects. A stub that redirects via `<meta refresh>` or
434        // `location.replace` returns a perfectly good 200 containing no
435        // content -- which is what sites that canonicalise in the browser do.
436        let mut current = parsed.clone();
437        for _ in 0..MAX_HTML_REDIRECTS {
438            let Some(next) = detect_html_redirect(&page.body, &current) else { break };
439            // The stub is still a better answer than an error, so a refused or
440            // disallowed target ends the chain rather than failing the fetch.
441            let Ok(followed) = self.fetch_following(&next, check_robots).await else { break };
442            page = followed;
443            current = next;
444        }
445        page.url = parsed.to_string();
446        page.elapsed_ms = started.elapsed().as_millis() as u64;
447
448        if self.cfg.browser_fallback && needs_javascript(&page) {
449            // A failed escalation is not a failed fetch: the static body is
450            // still the best answer we have.
451            if let Ok(rendered) = self.render(&parsed).await {
452                page.body = rendered;
453                page.rendered = true;
454                page.elapsed_ms = started.elapsed().as_millis() as u64;
455            }
456        }
457        Ok(page)
458    }
459
460    /// Fetch many URLs concurrently, preserving input order in the output.
461    ///
462    /// The concurrency ceiling and per-host spacing still apply, so this is
463    /// safe to call with a few hundred URLs at once.
464    pub async fn fetch_many(&self, urls: &[String]) -> Vec<Result<Page>> {
465        let futures = urls.iter().map(|u| {
466            let this = self.clone();
467            let u = u.clone();
468            async move { this.fetch(&u).await }
469        });
470        futures_util::stream::iter(futures).buffered(self.cfg.concurrency).collect::<Vec<_>>().await
471    }
472
473    /// One URL, with exponential backoff on retryable failures.
474    async fn get_with_retry(&self, url: &Url) -> Result<Hop> {
475        self.guard(url)?;
476        let mut attempt = 0u32;
477        loop {
478            match self.get_once(url).await {
479                Ok(page) => return Ok(page),
480                Err(e) if e.is_retryable() && attempt < self.cfg.retries => {
481                    // Retrying a 429 immediately is how a client turns a soft
482                    // limit into a hard block. When the server says how long to
483                    // wait, that beats any backoff curve we could invent — but
484                    // an hour-long hint is a refusal, not a delay, so it is
485                    // capped and otherwise treated as a failure.
486                    let wait = match e.retry_after() {
487                        Some(hint) if hint <= self.cfg.max_retry_after => hint,
488                        Some(_) => return Err(e),
489                        None => self.cfg.retry_backoff * 2u32.pow(attempt),
490                    };
491                    tokio::time::sleep(wait).await;
492                    attempt += 1;
493                }
494                Err(e) => return Err(e),
495            }
496        }
497    }
498
499    async fn get_once(&self, url: &Url) -> Result<Hop> {
500        let resp =
501            self.client().get(url.as_str()).send().await.map_err(|e| Error::network(url, e))?;
502
503        let status = resp.status().as_u16();
504        let final_url = resp.uri().to_string();
505        let content_type = resp
506            .headers()
507            .get(wreq::header::CONTENT_TYPE)
508            .and_then(|v| v.to_str().ok())
509            .map(str::to_string);
510
511        // `Policy::none()` hands the 3xx back rather than following it. A
512        // redirect with no usable target is just a failed request.
513        if (300..400).contains(&status)
514            && let Some(next) = resp
515                .headers()
516                .get(wreq::header::LOCATION)
517                .and_then(|v| v.to_str().ok())
518                .and_then(|l| url.join(l).ok())
519                .filter(|next| matches!(next.scheme(), "http" | "https"))
520        {
521            return Ok(Hop::Redirect(next));
522        }
523        if !(200..300).contains(&status) {
524            let retry_after = resp
525                .headers()
526                .get(wreq::header::RETRY_AFTER)
527                .and_then(|v| v.to_str().ok())
528                .and_then(parse_retry_after);
529            return Err(Error::Status { url: url.to_string(), status, retry_after });
530        }
531        if let Some(ct) = &content_type {
532            let base = ct.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
533            if !TEXTUAL.iter().any(|t| base.starts_with(t)) {
534                return Err(Error::Parse(format!("{url}: unsupported content type {base}")));
535            }
536        }
537        if resp.content_length().is_some_and(|n| n as usize > self.cfg.max_body_bytes) {
538            return Err(Error::BodyTooLarge {
539                url: url.to_string(),
540                limit: self.cfg.max_body_bytes,
541            });
542        }
543
544        let charset = content_type.as_deref().and_then(charset_of);
545        let bytes = self.read_capped(url, resp, self.cfg.max_body_bytes).await?;
546        let body = decode(&bytes, charset.as_deref(), tld_of(&final_url).as_deref());
547
548        Ok(Hop::Page(Page {
549            url: url.to_string(),
550            final_url,
551            status,
552            content_type,
553            body,
554            elapsed_ms: 0,
555            rendered: false,
556        }))
557    }
558
559    /// Read a body, aborting as soon as it crosses `limit`.
560    async fn read_capped(&self, url: &Url, resp: wreq::Response, limit: usize) -> Result<Vec<u8>> {
561        let mut buf: Vec<u8> = Vec::with_capacity(64 * 1024);
562        let mut stream = resp.bytes_stream();
563        while let Some(chunk) = stream.next().await {
564            let chunk = chunk.map_err(|e| Error::network(url, e))?;
565            if buf.len() + chunk.len() > limit {
566                return Err(Error::BodyTooLarge { url: url.to_string(), limit });
567            }
568            buf.extend_from_slice(&chunk);
569        }
570        Ok(buf)
571    }
572
573    /// Fetch and cache `robots.txt` for a host.
574    ///
575    /// A missing or unreadable file is treated as "allow all", which is what
576    /// the RFC specifies for a 404.
577    async fn robots_for(&self, url: &Url) -> Arc<Robots> {
578        let key = robots_origin(url);
579        if let Some(cached) = self.robots.lock().await.get(&key) {
580            return cached.clone();
581        }
582        let robots_url = format!("{key}/robots.txt");
583        let parsed = match self.client().get(&robots_url).send().await {
584            Ok(resp) if resp.status().is_success() => {
585                match self.read_capped(url, resp, MAX_ROBOTS_BYTES).await {
586                    // RFC 9309 says UTF-8; a host that disagrees gets lossy
587                    // decoding rather than an unparsed file.
588                    Ok(bytes) => Robots::parse(&String::from_utf8_lossy(&bytes), ROBOTS_AGENT),
589                    Err(_) => Robots::allow_all(),
590                }
591            }
592            _ => Robots::allow_all(),
593        };
594        let parsed = Arc::new(parsed);
595        self.robots.lock().await.insert(key, parsed.clone());
596        parsed
597    }
598
599    /// Space requests to one host apart, honouring `Crawl-delay` when larger.
600    async fn pace(&self, url: &Url, crawl_delay: Option<f64>) {
601        let host = url.host_str().unwrap_or_default().to_string();
602        if let Ok(mut visited) = self.visited.lock() {
603            visited.insert(host.clone());
604        }
605        let mut delay = self.cfg.per_host_delay;
606        if self.cfg.respect_crawl_delay
607            && let Some(d) = crawl_delay
608        {
609            delay = delay.max(Duration::from_secs_f64(d.min(30.0)));
610        }
611        if delay.is_zero() {
612            return;
613        }
614        let wait = {
615            let mut map = self.last_hit.lock().await;
616            let now = Instant::now();
617            let wait = match map.get(&host) {
618                Some(prev) => delay.checked_sub(now.duration_since(*prev)),
619                None => None,
620            };
621            // Reserve our slot before releasing the lock, so concurrent callers
622            // to the same host queue up instead of all sleeping the same amount.
623            map.insert(host, now + wait.unwrap_or_default());
624            wait
625        };
626        if let Some(wait) = wait {
627            tokio::time::sleep(wait).await;
628        }
629    }
630
631    #[cfg(feature = "browser")]
632    async fn render(&self, url: &Url) -> Result<String> {
633        self.guard(url)?;
634        browser::render(url.as_str(), self.cfg.timeout).await
635    }
636
637    #[cfg(not(feature = "browser"))]
638    async fn render(&self, _url: &Url) -> Result<String> {
639        Err(Error::Browser("build rustai with the `browser` feature".into()))
640    }
641}
642
643fn apply_impersonation(
644    builder: wreq::ClientBuilder,
645    mode: &Impersonate,
646) -> Result<wreq::ClientBuilder> {
647    #[cfg(not(feature = "impersonate"))]
648    {
649        if !matches!(mode, Impersonate::None) {
650            return Err(Error::Config(
651                "browser impersonation needs the `impersonate` feature".into(),
652            ));
653        }
654        Ok(builder)
655    }
656    #[cfg(feature = "impersonate")]
657    {
658        use wreq_util::{Emulation, Profile};
659        let profile = match mode {
660            Impersonate::None => return Ok(builder),
661            Impersonate::Random => return Ok(builder.emulation(Emulation::weighted_random())),
662            Impersonate::Chrome => Profile::Chrome149,
663            Impersonate::Firefox => Profile::Firefox147,
664            Impersonate::Safari => Profile::Safari26_4,
665            Impersonate::Named(name) => named_profile(name)?,
666        };
667        Ok(builder.emulation(profile))
668    }
669}
670
671/// Resolve a `wreq-util` profile from its wire name, e.g. `"chrome_143"`.
672///
673/// The enum has no `FromStr`, but it does derive `Deserialize` with exactly
674/// these renames under the `emulation-serde` feature, so serde is the
675/// supported way in — and it stays correct as upstream adds profiles.
676#[cfg(feature = "impersonate")]
677fn named_profile(name: &str) -> Result<wreq_util::Profile> {
678    serde_json::from_value::<wreq_util::Profile>(serde_json::Value::String(name.to_string()))
679        .map_err(|_| {
680            Error::Config(format!(
681                "unknown impersonation profile `{name}`; try `chrome`, `firefox`, `safari`, \
682                 `random`, or a versioned name such as `chrome_143`"
683            ))
684        })
685}
686
687/// Parse a `Retry-After` value into seconds.
688///
689/// Only the delta-seconds form. The HTTP-date form is rare in practice and
690/// parsing it would mean carrying a date library for one header; an unparsed
691/// value simply falls back to exponential backoff.
692fn parse_retry_after(value: &str) -> Option<f64> {
693    let secs: f64 = value.trim().parse().ok()?;
694    secs.is_finite().then_some(secs.max(0.0))
695}
696
697/// One persisted cookie, with the host it belongs to.
698#[derive(serde::Serialize, serde::Deserialize)]
699struct StoredCookie {
700    name: String,
701    value: String,
702    /// Host the cookie was collected from.
703    host: String,
704    path: String,
705    /// `Domain` attribute, when the cookie carried one. Absent means host-only,
706    /// and reloading must keep it that way.
707    #[serde(default, skip_serializing_if = "Option::is_none")]
708    domain: Option<String>,
709}
710
711/// Read a cookie jar from disk. A missing or unreadable file yields an empty
712/// jar: a lost session is a slow start, never an error.
713fn load_jar(path: &std::path::Path) -> wreq::cookie::Jar {
714    let jar = wreq::cookie::Jar::default();
715    let Ok(raw) = std::fs::read_to_string(path) else { return jar };
716    let Ok(stored) = serde_json::from_str::<Vec<StoredCookie>>(&raw) else { return jar };
717    for c in stored {
718        let uri = format!("https://{}{}", c.host, c.path);
719        let mut cookie = format!("{}={}; Path={}", c.name, c.value, c.path);
720        if let Some(domain) = &c.domain {
721            cookie.push_str(&format!("; Domain={domain}"));
722        }
723        jar.add(cookie.as_str(), uri.as_str());
724    }
725    jar
726}
727
728/// Write a cookie jar to disk, returning how many cookies were saved.
729fn save_jar(jar: &wreq::cookie::Jar, hosts: &[String], path: &std::path::Path) -> Result<usize> {
730    // Asking the jar what it would send to each host we visited is the only way
731    // to recover a host-only cookie's owner: `get_all` returns the cookies but
732    // not the domain key they were filed under.
733    let mut seen = std::collections::HashSet::new();
734    let mut stored: Vec<StoredCookie> = Vec::new();
735    for host in hosts {
736        for c in jar.matches(format!("https://{host}/").as_str()) {
737            let cookie = StoredCookie {
738                name: c.name().to_string(),
739                value: c.value().to_string(),
740                host: host.clone(),
741                path: c.path().unwrap_or("/").to_string(),
742                domain: c.domain().map(str::to_string),
743            };
744            if seen.insert((cookie.host.clone(), cookie.name.clone(), cookie.path.clone())) {
745                stored.push(cookie);
746            }
747        }
748    }
749    let json = serde_json::to_string_pretty(&stored)
750        .map_err(|e| Error::Config(format!("could not serialise cookies: {e}")))?;
751    if let Some(parent) = path.parent()
752        && !parent.as_os_str().is_empty()
753    {
754        let _ = std::fs::create_dir_all(parent);
755    }
756    // Clearance and session cookies are credentials: on Unix the file is born
757    // 0600 rather than chmod'd afterwards, so it is never briefly world-readable.
758    // `mode` only applies to a file this call creates, hence the unlink first.
759    #[cfg(unix)]
760    let written = {
761        use std::io::Write;
762        use std::os::unix::fs::OpenOptionsExt;
763        let _ = std::fs::remove_file(path);
764        std::fs::OpenOptions::new()
765            .write(true)
766            .create_new(true)
767            .mode(0o600)
768            .open(path)
769            .and_then(|mut f| f.write_all(json.as_bytes()))
770    };
771    #[cfg(not(unix))]
772    let written = std::fs::write(path, &json);
773    written.map_err(|e| Error::Config(format!("could not write {}: {e}", path.display())))?;
774    Ok(stored.len())
775}
776
777/// What one request produced: the page, or a redirect still to follow.
778enum Hop {
779    Page(Page),
780    Redirect(Url),
781}
782
783/// Whether an address is one only this machine or this network can reach.
784fn is_internal_ip(ip: &IpAddr) -> bool {
785    match ip {
786        IpAddr::V4(ip) => {
787            ip.is_loopback()
788                || ip.is_private()
789                || ip.is_link_local()
790                || ip.is_unspecified()
791                || ip.is_broadcast()
792                || ip.is_documentation()
793        }
794        IpAddr::V6(ip) => {
795            ip.is_loopback()
796                || ip.is_unspecified()
797                || (ip.segments()[0] & 0xfe00) == 0xfc00 // unique local, fc00::/7
798                || (ip.segments()[0] & 0xffc0) == 0xfe80 // link-local, fe80::/10
799                || ip.to_ipv4_mapped().is_some_and(|v4| is_internal_ip(&IpAddr::V4(v4)))
800        }
801    }
802}
803
804/// Whether a URL names somewhere only this machine or this network can reach.
805///
806/// What the URL *says*, which is all that can be judged before resolving it.
807/// A hostname pointing at an internal address is caught by [`PublicOnlyDns`]
808/// instead, at the moment it is resolved.
809fn is_internal(url: &Url) -> bool {
810    match url.host() {
811        Some(url::Host::Ipv4(ip)) => is_internal_ip(&IpAddr::V4(ip)),
812        Some(url::Host::Ipv6(ip)) => is_internal_ip(&IpAddr::V6(ip)),
813        // `.internal` is reserved for private use and is where GCP parks its
814        // metadata service; a trailing dot names the same host.
815        Some(url::Host::Domain(name)) => {
816            let name = name.trim_end_matches('.').to_ascii_lowercase();
817            name == "localhost" || name.ends_with(".localhost") || name.ends_with(".internal")
818        }
819        None => false,
820    }
821}
822
823/// A resolver that drops internal addresses before anything is dialled.
824///
825/// [`is_internal`] can only read the URL. A public hostname whose record
826/// points at `127.0.0.1` passes it, and so does one whose record flips to an
827/// internal address *after* the check and before the connection -- DNS
828/// rebinding. Filtering here closes both, because `wreq` dials the addresses a
829/// resolver hands back: the address that was vetted is the address used, with
830/// no second lookup in between to disagree with the first.
831///
832/// A name that resolves to a mix keeps its public addresses. One that resolves
833/// only to internal ones fails to resolve at all, which is the truthful answer
834/// -- there is nowhere this is willing to connect.
835///
836/// Generic over the inner resolver so a test can supply the answers rather
837/// than depending on what the network says today.
838struct PublicOnlyDns<R>(R);
839
840impl<R: Resolve + 'static> Resolve for PublicOnlyDns<R> {
841    fn resolve(&self, name: Name) -> Resolving {
842        let host = name.as_str().to_string();
843        let inner = self.0.resolve(name);
844        Box::pin(async move {
845            let kept: Vec<SocketAddr> =
846                inner.await?.filter(|addr| !is_internal_ip(&addr.ip())).collect();
847            if kept.is_empty() {
848                return Err(format!("{host} resolves only to internal addresses").into());
849            }
850            Ok(Box::new(kept.into_iter()) as Addrs)
851        })
852    }
853}
854
855/// The origin whose `robots.txt` governs `url`, as a cache key and a base.
856///
857/// RFC 9309 scopes the file to an origin, so the port is part of it:
858/// `example.com:8080` has its own `robots.txt` and deserves its own cache
859/// entry. Keying on scheme and host alone fetched port 80's file for every
860/// non-default port, and when nothing answered there the miss read as "allow
861/// all" -- which silently disabled `robots.txt` for those hosts entirely.
862fn robots_origin(url: &Url) -> String {
863    url.origin().ascii_serialization()
864}
865
866/// Reject anything that is not an absolute http(s) URL.
867pub fn normalize(url: &str) -> Result<Url> {
868    let parsed = Url::parse(url.trim()).map_err(|_| Error::InvalidUrl(url.to_string()))?;
869    if !matches!(parsed.scheme(), "http" | "https") {
870        return Err(Error::InvalidUrl(url.to_string()));
871    }
872    if parsed.host_str().is_none() {
873        return Err(Error::InvalidUrl(url.to_string()));
874    }
875    Ok(parsed)
876}
877
878fn charset_of(content_type: &str) -> Option<String> {
879    content_type
880        .split(';')
881        .skip(1)
882        .filter_map(|p| p.trim().strip_prefix("charset="))
883        .map(|c| c.trim_matches(['"', '\'']).to_ascii_lowercase())
884        .next()
885}
886
887/// Decode a body, trusting the header, then a `<meta charset>`, then UTF-8,
888/// and only when the bytes are not UTF-8 either, a guess.
889///
890/// Legacy encodings are still common outside the anglosphere — EUC-KR and
891/// Shift_JIS in particular — and mojibake destroys every downstream heuristic,
892/// so this is worth getting right rather than assuming UTF-8. Old pages often
893/// declare nothing at all (math.snu.ac.kr serves EUC-KR that way), so the last
894/// step is the detector Firefox uses, hinted with the host's top-level domain
895/// the way a browser is.
896fn decode(bytes: &[u8], header_charset: Option<&str>, tld: Option<&str>) -> String {
897    let label = header_charset.map(str::to_string).or_else(|| sniff_meta_charset(bytes));
898    let encoding = label
899        .as_deref()
900        .and_then(|l| encoding_rs::Encoding::for_label(l.as_bytes()))
901        .unwrap_or_else(|| {
902            // A capped body can end inside a multi-byte character; up to three
903            // trailing bytes of invalid UTF-8 are a cut, not another encoding.
904            if bytes.len() - encoding_rs::Encoding::utf8_valid_up_to(bytes) <= 3 {
905                encoding_rs::UTF_8
906            } else {
907                let mut detector = chardetng::EncodingDetector::new();
908                detector.feed(bytes, true);
909                detector.guess(tld.map(str::as_bytes), true)
910            }
911        });
912    let (text, _, _) = encoding.decode(bytes);
913    text.into_owned()
914}
915
916/// The last label of a URL's host — the hint the encoding detector takes.
917fn tld_of(url: &str) -> Option<String> {
918    let host = Url::parse(url).ok()?.host_str()?.to_ascii_lowercase();
919    host.rsplit('.').next().map(str::to_string)
920}
921
922/// Look for `<meta charset=...>` in the first 4 KiB, as browsers do.
923fn sniff_meta_charset(bytes: &[u8]) -> Option<String> {
924    let head = &bytes[..bytes.len().min(4096)];
925    let text = String::from_utf8_lossy(head).to_ascii_lowercase();
926    if let Some(at) = text.find("charset=") {
927        let rest = &text[at + "charset=".len()..];
928        let value: String = rest
929            .trim_start_matches(['"', '\'', ' '])
930            .chars()
931            .take_while(|c| c.is_ascii_alphanumeric() || *c == '-' || *c == '_')
932            .collect();
933        if !value.is_empty() {
934            return Some(value);
935        }
936    }
937    None
938}
939
940/// Find an HTML-level redirect target in a page body.
941///
942/// Two forms matter in practice: `<meta http-equiv="refresh" content="0; url=…">`
943/// and a script that assigns `location`. Both are gated on the page being small
944/// and on a zero delay, because a redirect stub is always tiny — a long article
945/// that happens to contain `location.href` somewhere must not be mistaken for
946/// one.
947fn detect_html_redirect(body: &str, base: &Url) -> Option<Url> {
948    const MAX_STUB_BYTES: usize = 4096;
949    if body.len() > MAX_STUB_BYTES {
950        return None;
951    }
952    let target = meta_refresh_target(body).or_else(|| script_location_target(body))?;
953    let next = base.join(target.trim().trim_matches(['"', '\''])).ok()?;
954    if !matches!(next.scheme(), "http" | "https") {
955        return None;
956    }
957    // A stub pointing at itself is a loop, not a redirect.
958    let same = next.as_str().trim_end_matches('/') == base.as_str().trim_end_matches('/');
959    (!same).then_some(next)
960}
961
962/// `<meta http-equiv="refresh" content="0; url=TARGET">`, delay 0 only.
963fn meta_refresh_target(body: &str) -> Option<&str> {
964    let lower = body.to_ascii_lowercase();
965    let mut from = 0usize;
966    while let Some(at) = lower[from..].find("http-equiv") {
967        let tag_start = lower[..from + at].rfind('<')?;
968        let tag_end = lower[tag_start..].find('>')? + tag_start;
969        let tag = &lower[tag_start..tag_end];
970        if tag.contains("refresh")
971            && let Some(content_at) = tag.find("content")
972            && let Some(url_at) = tag[content_at..].find("url=")
973        {
974            // Only an immediate redirect; a delayed one is a page in its own right.
975            let delay: String = tag[content_at..]
976                .chars()
977                .skip_while(|c| !c.is_ascii_digit())
978                .take_while(|c| c.is_ascii_digit())
979                .collect();
980            if delay.parse::<u32>().unwrap_or(0) == 0 {
981                let start = tag_start + content_at + url_at + 4;
982                let rest = &body[start..];
983                let end = rest.find(['"', '\'', '>']).unwrap_or(rest.len());
984                let target = rest[..end].trim();
985                if !target.is_empty() {
986                    return Some(target);
987                }
988            }
989        }
990        from += at + "http-equiv".len();
991    }
992    None
993}
994
995/// `location.replace("TARGET")`, `location.href = "TARGET"`, and friends.
996fn script_location_target(body: &str) -> Option<&str> {
997    for marker in ["location.replace(", "location.href=", "location.href =", "location.assign("] {
998        let Some(at) = body.find(marker) else { continue };
999        let rest = &body[at + marker.len()..];
1000        let rest = rest.trim_start();
1001        let quote = rest.chars().next()?;
1002        if quote != '"' && quote != '\'' {
1003            // An indirection through a variable; resolving it needs a JS engine.
1004            continue;
1005        }
1006        let inner = &rest[quote.len_utf8()..];
1007        let end = inner.find(quote)?;
1008        let target = &inner[..end];
1009        if target.starts_with("http") || target.starts_with('/') {
1010            return Some(target);
1011        }
1012    }
1013    None
1014}
1015
1016/// Does this response look like an empty shell waiting for JavaScript?
1017///
1018/// Deliberately conservative. Rendering a page costs ~100x a fetch, so the
1019/// bar is "there is markup but essentially no text", not "the page mentions
1020/// JavaScript".
1021pub fn needs_javascript(page: &Page) -> bool {
1022    let html = &page.body;
1023    if html.len() < 512 {
1024        return false;
1025    }
1026    let Ok(article) = crate::parse::extract(html, Some(&page.final_url)) else {
1027        return true;
1028    };
1029    let text_len = crate::text::visible_len(&article.text);
1030
1031    // Plenty of text: whatever else the page is doing, we already have it.
1032    if text_len > MAX_STATIC_TEXT {
1033        return false;
1034    }
1035
1036    // Markup carrying essentially no text. An absolute floor misses the common
1037    // case — a 350 KB front page yielding 510 characters clears any fixed
1038    // threshold while plainly being an app shell — so the test is the ratio.
1039    // Real pages run percent-level text-to-markup; a shell runs a tenth of that.
1040    let starved = text_len.saturating_mul(TEXT_TO_MARKUP_FLOOR) < html.len();
1041    if text_len < 120 || starved {
1042        return true;
1043    }
1044
1045    let lower = html.to_lowercase();
1046    let spa_markers = ["__next_data__", "id=\"root\"", "id=\"app\"", "ng-app", "data-reactroot"];
1047    spa_markers.iter().any(|m| lower.contains(m))
1048}
1049
1050#[cfg(test)]
1051mod tests {
1052    use super::*;
1053
1054    #[test]
1055    fn rejects_non_http_urls() {
1056        assert!(normalize("file:///etc/passwd").is_err());
1057        assert!(normalize("javascript:alert(1)").is_err());
1058        assert!(normalize("not a url").is_err());
1059        assert!(normalize("https://example.com/a").is_ok());
1060    }
1061
1062    #[test]
1063    fn parses_charset_from_content_type() {
1064        assert_eq!(charset_of("text/html; charset=EUC-KR").as_deref(), Some("euc-kr"));
1065        assert_eq!(charset_of("text/html").as_deref(), None);
1066    }
1067
1068    #[test]
1069    fn decodes_legacy_korean_encoding() {
1070        let (bytes, _, _) = encoding_rs::EUC_KR.encode("한글 인코딩");
1071        assert_eq!(decode(&bytes, Some("euc-kr"), None), "한글 인코딩");
1072        // Without the header, the meta tag is the fallback.
1073        let mut doc = b"<html><head><meta charset=\"euc-kr\"></head><body>".to_vec();
1074        doc.extend_from_slice(&bytes);
1075        assert!(decode(&doc, None, None).contains("한글 인코딩"));
1076    }
1077
1078    #[test]
1079    fn guesses_a_legacy_encoding_that_nothing_declares() {
1080        // math.snu.ac.kr serves EUC-KR with neither a header charset nor a meta tag.
1081        let (bytes, _, _) = encoding_rs::EUC_KR.encode(concat!(
1082            "<html><head><title>합성함수 미분법 = 연쇄법칙(Chain Rule)</title></head><body>",
1083            "<p>함수 y = f(x) 와 함수 z = g(y)를 합성한 함수의 미분법은 다음과 같다.</p></body></html>",
1084        ));
1085        assert!(decode(&bytes, None, Some("kr")).contains("합성함수 미분법 = 연쇄법칙"));
1086    }
1087
1088    #[test]
1089    fn undeclared_utf8_stays_utf8() {
1090        assert_eq!(
1091            decode("<p>한글 UTF-8 본문</p>".as_bytes(), None, Some("kr")),
1092            "<p>한글 UTF-8 본문</p>"
1093        );
1094    }
1095
1096    #[test]
1097    fn a_body_cut_mid_character_is_still_read_as_utf8() {
1098        // The body cap can stop a fetch inside a multi-byte character.
1099        let full = "<p>가나다라마바사</p>".repeat(20);
1100        let cut = &full.as_bytes()[..full.len() - 7];
1101        assert!(decode(cut, None, Some("kr")).starts_with("<p>가나다라마바사</p>"));
1102    }
1103
1104    #[test]
1105    fn tld_is_the_last_host_label() {
1106        assert_eq!(tld_of("https://www.math.snu.ac.kr/a.htm").as_deref(), Some("kr"));
1107        assert_eq!(tld_of("not a url"), None);
1108    }
1109
1110    #[test]
1111    fn impersonate_strings_parse() {
1112        assert_eq!(Impersonate::parse("chrome").unwrap(), Impersonate::Chrome);
1113        assert_eq!(Impersonate::parse("none").unwrap(), Impersonate::None);
1114        assert_eq!(
1115            Impersonate::parse("chrome_143").unwrap(),
1116            Impersonate::Named("chrome_143".into())
1117        );
1118    }
1119
1120    #[cfg(feature = "impersonate")]
1121    #[test]
1122    fn named_profiles_resolve_and_reject() {
1123        assert!(named_profile("chrome_143").is_ok());
1124        assert!(named_profile("netscape_2").is_err());
1125    }
1126
1127    #[test]
1128    fn client_builds_with_every_preset() {
1129        for mode in ["chrome", "firefox", "safari", "random", "none"] {
1130            let cfg = FetchConfig {
1131                impersonate: Impersonate::parse(mode).unwrap(),
1132                ..Default::default()
1133            };
1134            assert!(Fetcher::with_config(cfg).is_ok(), "{mode} failed to build");
1135        }
1136    }
1137
1138    #[test]
1139    fn retry_after_is_parsed_and_capped() {
1140        assert_eq!(parse_retry_after("120"), Some(120.0));
1141        assert_eq!(parse_retry_after("  0 "), Some(0.0));
1142        assert_eq!(parse_retry_after("-5"), Some(0.0));
1143        // The HTTP-date form is not parsed; backoff takes over.
1144        assert_eq!(parse_retry_after("Wed, 21 Oct 2026 07:28:00 GMT"), None);
1145        assert_eq!(parse_retry_after(""), None);
1146
1147        let e =
1148            Error::Status { url: "https://x.dev/".into(), status: 429, retry_after: Some(30.0) };
1149        assert_eq!(e.retry_after(), Some(Duration::from_secs(30)));
1150        assert!(e.is_retryable());
1151        let none = Error::Status { url: "https://x.dev/".into(), status: 429, retry_after: None };
1152        assert_eq!(none.retry_after(), None);
1153    }
1154
1155    #[test]
1156    fn proxies_build_one_client_each() {
1157        let cfg = FetchConfig {
1158            proxies: vec!["http://127.0.0.1:8080".into(), "socks5://127.0.0.1:1080".into()],
1159            ..Default::default()
1160        };
1161        let fetcher = Fetcher::with_config(cfg).expect("proxied client");
1162        assert_eq!(fetcher.clients.len(), 2);
1163        // Round-robin, so a small pool cannot cluster on one address.
1164        let first = fetcher.client() as *const _;
1165        let second = fetcher.client() as *const _;
1166        assert_ne!(first, second);
1167        assert_eq!(fetcher.client() as *const _, first);
1168    }
1169
1170    #[test]
1171    fn an_invalid_proxy_is_rejected_at_build_time() {
1172        let cfg = FetchConfig { proxies: vec!["not a proxy".into()], ..Default::default() };
1173        assert!(Fetcher::with_config(cfg).is_err());
1174    }
1175
1176    #[test]
1177    fn cookies_survive_a_round_trip_through_disk() {
1178        let dir = std::env::temp_dir().join(format!("rustai-cookies-{}", std::process::id()));
1179        let path = dir.join("jar.json");
1180        let _ = std::fs::remove_file(&path);
1181
1182        let jar = wreq::cookie::Jar::default();
1183        // A domain cookie and a host-only one: the latter is what most session
1184        // cookies actually are, and it has no Domain attribute to save.
1185        jar.add("cf_clearance=abc123; Domain=example.com; Path=/", "https://example.com/");
1186        jar.add("session=xyz789; Path=/", "https://example.com/");
1187        let hosts = vec!["example.com".to_string()];
1188        assert_eq!(save_jar(&jar, &hosts, &path).unwrap(), 2);
1189
1190        let reloaded = load_jar(&path);
1191        assert!(reloaded.contains("session", "https://example.com/"), "host-only cookie lost");
1192        assert!(reloaded.contains("cf_clearance", "https://example.com/"), "cookie was lost");
1193        assert_eq!(
1194            reloaded.get("cf_clearance", "https://example.com/").map(|c| c.value().to_string()),
1195            Some("abc123".to_string())
1196        );
1197
1198        // A missing file is a slow start, not an error.
1199        assert_eq!(load_jar(&dir.join("nope.json")).len(), 0);
1200        let _ = std::fs::remove_dir_all(&dir);
1201    }
1202
1203    #[cfg(unix)]
1204    #[test]
1205    fn a_saved_jar_is_readable_only_by_its_owner() {
1206        use std::os::unix::fs::PermissionsExt;
1207        let dir = std::env::temp_dir().join(format!("rustai-cookie-perms-{}", std::process::id()));
1208        std::fs::create_dir_all(&dir).unwrap();
1209        let path = dir.join("jar.json");
1210
1211        // Start from a world-readable file: saving has to replace the mode, not
1212        // inherit it, or a jar written once stays exposed for every later save.
1213        std::fs::write(&path, "[]").unwrap();
1214        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap();
1215
1216        let jar = wreq::cookie::Jar::default();
1217        jar.add("cf_clearance=secret; Path=/", "https://example.com/");
1218        save_jar(&jar, &["example.com".to_string()], &path).unwrap();
1219
1220        let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777;
1221        assert_eq!(mode, 0o600, "cookie jar left at {mode:o}");
1222        let _ = std::fs::remove_dir_all(&dir);
1223    }
1224
1225    #[test]
1226    fn robots_is_scoped_to_an_origin_including_its_port() {
1227        let key = |u: &str| robots_origin(&Url::parse(u).unwrap());
1228        // The port belongs in the key. Without it a non-default port fetched
1229        // port 80's file, found nothing, and read the miss as "allow all".
1230        assert_eq!(key("http://example.com:8080/a/b"), "http://example.com:8080");
1231        assert_ne!(key("http://example.com:8080/a"), key("http://example.com:9090/a"));
1232        // A default port is not spelled out, so it shares one entry with a
1233        // URL that omits it -- which is the same origin.
1234        assert_eq!(key("https://example.com/a"), "https://example.com");
1235        assert_eq!(key("https://example.com:443/a"), key("https://example.com/b"));
1236        assert_eq!(key("http://example.com:80/a"), key("http://example.com/b"));
1237        // Scheme still separates them.
1238        assert_ne!(key("http://example.com/a"), key("https://example.com/a"));
1239    }
1240
1241    /// A resolver that answers with whatever it was built from.
1242    struct Fixed(Vec<SocketAddr>);
1243
1244    impl Resolve for Fixed {
1245        fn resolve(&self, _: Name) -> Resolving {
1246            let addrs = self.0.clone();
1247            Box::pin(async move { Ok(Box::new(addrs.into_iter()) as Addrs) })
1248        }
1249    }
1250
1251    fn addr(s: &str) -> SocketAddr {
1252        s.parse().expect("socket addr")
1253    }
1254
1255    #[tokio::test]
1256    async fn resolving_drops_internal_addresses() {
1257        // The rebinding case: a name that looks public and answers otherwise.
1258        // `is_internal` cannot see this, because the URL says nothing wrong.
1259        let only_internal = PublicOnlyDns(Fixed(vec![addr("127.0.0.1:80")]));
1260        assert!(
1261            only_internal.resolve(Name::from("rebind.test")).await.is_err(),
1262            "a name with nowhere public to go must not resolve"
1263        );
1264
1265        // A mixed answer keeps the public addresses and loses the rest, so one
1266        // poisoned record cannot redirect a connection inward.
1267        let mixed = PublicOnlyDns(Fixed(vec![
1268            addr("127.0.0.1:80"),
1269            addr("93.184.216.34:80"),
1270            addr("10.0.0.7:80"),
1271            addr("[::1]:80"),
1272            addr("[2606:4700::1]:80"),
1273        ]));
1274        let kept: Vec<SocketAddr> =
1275            mixed.resolve(Name::from("mixed.test")).await.expect("resolves").collect();
1276        assert_eq!(kept, vec![addr("93.184.216.34:80"), addr("[2606:4700::1]:80")]);
1277    }
1278
1279    #[test]
1280    fn internal_ip_ranges_are_recognised() {
1281        let internal = [
1282            "127.0.0.1",
1283            "127.9.9.9",
1284            "10.0.0.5",
1285            "192.168.1.1",
1286            "172.16.0.1",
1287            "169.254.169.254",
1288            "0.0.0.0",
1289            "255.255.255.255",
1290            "::1",
1291            "::",
1292            "fe80::1",
1293            "fd00::1",
1294            "::ffff:127.0.0.1",
1295            "::ffff:10.1.2.3",
1296        ];
1297        for ip in internal {
1298            let ip: IpAddr = ip.parse().expect(ip);
1299            assert!(is_internal_ip(&ip), "{ip} should be internal");
1300        }
1301
1302        let public = ["8.8.8.8", "93.184.216.34", "172.32.0.1", "2606:4700::1", "::ffff:8.8.8.8"];
1303        for ip in public {
1304            let ip: IpAddr = ip.parse().expect(ip);
1305            assert!(!is_internal_ip(&ip), "{ip} should be reachable");
1306        }
1307    }
1308
1309    #[test]
1310    fn internal_addresses_are_recognised() {
1311        let internal = [
1312            "http://127.0.0.1/",
1313            "http://localhost:3000/",
1314            "http://sub.localhost/",
1315            "http://LOCALHOST./",
1316            "http://169.254.169.254/latest/meta-data/", // AWS metadata
1317            "http://metadata.google.internal/computeMetadata/v1/",
1318            "http://10.0.0.5/",
1319            "http://192.168.1.1/",
1320            "http://172.16.0.1/",
1321            "http://0.0.0.0/",
1322            "http://[::1]/",
1323            "http://[fe80::1]/",
1324            "http://[fd00::1]/",
1325            "http://[::ffff:127.0.0.1]/",
1326        ];
1327        for u in internal {
1328            assert!(is_internal(&Url::parse(u).unwrap()), "{u} should be internal");
1329        }
1330
1331        let public = [
1332            "https://example.com/",
1333            "https://en.wikipedia.org/wiki/BM25",
1334            "http://8.8.8.8/",
1335            "http://172.32.0.1/", // just outside 172.16/12
1336            "http://[2606:4700::1]/",
1337            "https://notlocalhost.com/", // suffix match must not be a substring match
1338            "https://internal.example.com/",
1339        ];
1340        for u in public {
1341            assert!(!is_internal(&Url::parse(u).unwrap()), "{u} should be reachable");
1342        }
1343    }
1344
1345    #[tokio::test]
1346    async fn fetching_an_internal_address_is_refused_before_the_network() {
1347        let f = Fetcher::new().unwrap();
1348        let err = f.fetch("http://169.254.169.254/latest/meta-data/").await.unwrap_err();
1349        assert!(matches!(err, Error::InvalidUrl(_)), "got {err:?}");
1350        // `fetch_api` skips robots, not this.
1351        assert!(f.fetch_api("http://127.0.0.1:8080/").await.is_err());
1352
1353        // Opt in and the gate is gone -- the fetch then fails on connection,
1354        // which is a different error entirely.
1355        let cfg = FetchConfig { allow_private_hosts: true, ..Default::default() };
1356        let f = Fetcher::with_config(cfg).unwrap();
1357        let err = f.fetch("http://127.0.0.1:9/").await.unwrap_err();
1358        assert!(!matches!(err, Error::InvalidUrl(_)), "gate still firing: {err:?}");
1359    }
1360
1361    #[tokio::test]
1362    async fn every_request_path_shares_one_internal_address_guard() {
1363        let f = Fetcher::new().unwrap();
1364        let internal = Url::parse("http://169.254.169.254/latest/meta-data/").unwrap();
1365        // `fetch_inner` gates the entry point, but a `<meta refresh>` hop and a
1366        // headless render reach the network through their own calls -- so the
1367        // guard lives where they all pass, not only at the door.
1368        assert!(f.guard(&internal).is_err());
1369        assert!(f.get_with_retry(&internal).await.is_err());
1370        assert!(f.guard(&Url::parse("https://example.com/").unwrap()).is_ok());
1371
1372        let open =
1373            Fetcher::with_config(FetchConfig { allow_private_hosts: true, ..Default::default() })
1374                .unwrap();
1375        assert!(open.guard(&internal).is_ok());
1376    }
1377
1378    #[test]
1379    fn a_meta_refresh_to_an_internal_address_is_still_an_internal_address() {
1380        // `detect_html_redirect` only vets the scheme; the guard is what stops
1381        // the hop, so the two have to agree on what this URL is.
1382        let base = Url::parse("https://evil.test/post").unwrap();
1383        let body = r#"<!doctype html><meta http-equiv="refresh"
1384            content="0;url=http://169.254.169.254/latest/meta-data/">"#;
1385        let next = detect_html_redirect(body, &base).expect("redirect should parse");
1386        assert!(is_internal(&next), "{next} must be caught by the guard");
1387    }
1388
1389    #[test]
1390    fn proxy_credentials_do_not_reach_debug_output() {
1391        // `Url` normalises an empty path to `/` on the way out; this string is
1392        // only ever displayed, never handed back to the proxy builder.
1393        assert_eq!(
1394            mask_proxy("http://user:hunter2@proxy.dev:8080"),
1395            "http://***:***@proxy.dev:8080/"
1396        );
1397        // Nothing to hide, nothing rewritten.
1398        assert_eq!(mask_proxy("socks5://127.0.0.1:1080"), "socks5://127.0.0.1:1080");
1399        assert_eq!(mask_proxy("not a url"), "not a url");
1400
1401        let cfg = FetchConfig {
1402            proxies: vec!["http://bob:s3cr3t@proxy.dev:8080".into()],
1403            ..Default::default()
1404        };
1405        let dumped = format!("{cfg:?}");
1406        assert!(!dumped.contains("s3cr3t"), "password leaked: {dumped}");
1407        assert!(dumped.contains("proxy.dev"), "host should survive: {dumped}");
1408    }
1409
1410    #[test]
1411    fn a_fetcher_without_a_cookie_file_saves_nothing() {
1412        let fetcher = Fetcher::new().unwrap();
1413        assert_eq!(fetcher.save_cookies().unwrap(), 0);
1414    }
1415
1416    #[test]
1417    fn zero_concurrency_is_rejected() {
1418        let cfg = FetchConfig { concurrency: 0, ..Default::default() };
1419        assert!(Fetcher::with_config(cfg).is_err());
1420    }
1421
1422    fn page(body: &str) -> Page {
1423        Page {
1424            url: "https://x.dev/".into(),
1425            final_url: "https://x.dev/".into(),
1426            status: 200,
1427            content_type: Some("text/html".into()),
1428            body: body.into(),
1429            elapsed_ms: 0,
1430            rendered: false,
1431        }
1432    }
1433
1434    #[test]
1435    fn follows_a_meta_refresh_stub() {
1436        let base = Url::parse("https://blog.dev/2024/02/08/Post.html").unwrap();
1437        let body = r#"<!doctype html><title>Redirect</title>
1438            <noscript><meta http-equiv="refresh" content="0; url=https://blog.dev/2024/02/08/Post/"></noscript>"#;
1439        assert_eq!(
1440            detect_html_redirect(body, &base).unwrap().as_str(),
1441            "https://blog.dev/2024/02/08/Post/"
1442        );
1443    }
1444
1445    #[test]
1446    fn follows_a_script_location_stub() {
1447        let base = Url::parse("https://blog.dev/a.html").unwrap();
1448        let body = r#"<!doctype html><title>Redirect</title><script>
1449            const target = "https://blog.dev/a/";
1450            window.location.replace("https://blog.dev/a/");
1451        </script>"#;
1452        assert_eq!(detect_html_redirect(body, &base).unwrap().as_str(), "https://blog.dev/a/");
1453    }
1454
1455    #[test]
1456    fn resolves_a_relative_redirect_target() {
1457        let base = Url::parse("https://blog.dev/x/a.html").unwrap();
1458        let body = r#"<meta http-equiv="refresh" content="0;url=/y/b.html">"#;
1459        assert_eq!(
1460            detect_html_redirect(body, &base).unwrap().as_str(),
1461            "https://blog.dev/y/b.html"
1462        );
1463    }
1464
1465    #[test]
1466    fn ignores_delayed_refreshes_and_self_loops() {
1467        let base = Url::parse("https://blog.dev/a").unwrap();
1468        // A 5-second refresh is a real page that reloads, not a redirect.
1469        assert!(
1470            detect_html_redirect(
1471                r#"<meta http-equiv="refresh" content="5; url=https://blog.dev/b">"#,
1472                &base
1473            )
1474            .is_none()
1475        );
1476        // Pointing at itself is a loop.
1477        assert!(
1478            detect_html_redirect(
1479                r#"<meta http-equiv="refresh" content="0; url=https://blog.dev/a">"#,
1480                &base
1481            )
1482            .is_none()
1483        );
1484    }
1485
1486    #[test]
1487    fn ignores_a_long_page_that_merely_mentions_location() {
1488        let base = Url::parse("https://blog.dev/a").unwrap();
1489        let article = format!(
1490            "<article><p>{}</p><p>You can call location.replace(\"https://evil.dev/\") in JS.</p></article>",
1491            "Real prose. ".repeat(500)
1492        );
1493        assert!(
1494            detect_html_redirect(&article, &base).is_none(),
1495            "a full article was treated as a stub"
1496        );
1497    }
1498
1499    #[test]
1500    fn ignores_non_http_and_unresolvable_targets() {
1501        let base = Url::parse("https://blog.dev/a").unwrap();
1502        assert!(
1503            detect_html_redirect(
1504                r#"<meta http-equiv="refresh" content="0;url=javascript:x()">"#,
1505                &base
1506            )
1507            .is_none()
1508        );
1509        assert!(detect_html_redirect("<p>nothing here</p>", &base).is_none());
1510        // A variable indirection needs a JS engine; we must not guess.
1511        assert!(
1512            detect_html_redirect(r#"<script>window.location.replace(target);</script>"#, &base)
1513                .is_none()
1514        );
1515    }
1516
1517    #[test]
1518    fn detects_an_spa_shell() {
1519        let shell = format!(
1520            "<html><body><div id=\"root\"></div><script>{}</script></body></html>",
1521            "x".repeat(2000)
1522        );
1523        assert!(needs_javascript(&page(&shell)));
1524    }
1525
1526    #[test]
1527    fn does_not_escalate_a_real_article() {
1528        let real = format!(
1529            "<html><body><article><h1>T</h1><p>{}</p></article></body></html>",
1530            "Real prose that a reader can actually read. ".repeat(20)
1531        );
1532        assert!(!needs_javascript(&page(&real)));
1533    }
1534}
1535
1536#[cfg(test)]
1537mod js_gate_tests {
1538    use super::*;
1539
1540    fn page(body: String) -> Page {
1541        Page {
1542            url: "https://x.dev/".into(),
1543            final_url: "https://x.dev/".into(),
1544            status: 200,
1545            content_type: Some("text/html".into()),
1546            body,
1547            elapsed_ms: 0,
1548            rendered: false,
1549        }
1550    }
1551
1552    /// Modelled on a real front page whose stories render client-side: ~350 KB
1553    /// of markup carrying ~500 characters of text. An absolute text threshold
1554    /// waves that through — 500 clears any sane floor — while the ratio does
1555    /// not. Index detection cannot rescue such a page either: there is nothing
1556    /// in the HTML to index.
1557    #[test]
1558    fn a_client_rendered_front_page_is_flagged() {
1559        let shell = format!(
1560            "<html><body><nav>{}</nav><main></main><script>{}</script></body></html>",
1561            "<a href=\"/section\">Section link</a>".repeat(20),
1562            "x".repeat(340_000)
1563        );
1564        assert!(needs_javascript(&page(shell)));
1565    }
1566
1567    /// The same size of page, but the markup actually carries the article.
1568    #[test]
1569    fn a_large_real_article_is_not_flagged() {
1570        let real = format!(
1571            "<html><body><article><h1>T</h1><p>{}</p></article></body></html>",
1572            "Real prose that a reader can actually read and act on. ".repeat(400)
1573        );
1574        assert!(!needs_javascript(&page(real)));
1575    }
1576}