Skip to main content

rustai_core/http/
mod.rs

1//! The fetch layer: impersonated HTTP, politeness, and a headless fallback.
2//!
3//! Three things separate this from a bare `reqwest` loop:
4//!
5//! * **Fingerprint.** Cloudflare and friends read the TLS ClientHello (JA3/JA4)
6//!   and the HTTP/2 SETTINGS frame, not just the User-Agent. `wreq` replays a
7//!   real Chrome's, so a plain `GET` gets a plain `200` where a stock client
8//!   gets a challenge page.
9//! * **Politeness.** `robots.txt`, per-host spacing and a concurrency ceiling
10//!   are on by default, because a fast crawler with none of those is a way to
11//!   get a user's IP banned.
12//! * **Escalation.** JS-gated pages are detected from the response and, when
13//!   the `browser` feature is on, retried through headless Chrome — which is
14//!   two orders of magnitude more expensive and therefore never the first move.
15
16pub mod robots;
17
18#[cfg(feature = "browser")]
19pub mod browser;
20
21use std::collections::HashMap;
22use std::sync::Arc;
23use std::time::{Duration, Instant};
24
25use futures_util::StreamExt;
26use tokio::sync::{Mutex, Semaphore};
27use url::Url;
28
29use crate::error::{Error, Result};
30use robots::Robots;
31
32/// Default identity when impersonation is off, or for the `robots.txt` lookup.
33pub const DEFAULT_USER_AGENT: &str =
34    concat!("rustai/", env!("CARGO_PKG_VERSION"), " (+https://github.com/imhyensuk/rustai)");
35
36/// The token we match `robots.txt` groups against.
37pub const ROBOTS_AGENT: &str = "rustai";
38
39/// HTML-level redirects followed before giving up.
40///
41/// Two is enough for the real pattern (a canonicalising stub pointing at the
42/// article) and low enough that a redirect loop costs almost nothing.
43const MAX_HTML_REDIRECTS: usize = 2;
44
45/// Above this much extracted text, a page is not an empty shell.
46const MAX_STATIC_TEXT: usize = 2_000;
47
48/// One character of text per this many bytes of markup is the floor below which
49/// a page is presumed to be rendered client-side.
50const TEXT_TO_MARKUP_FLOOR: usize = 200;
51
52/// Content types we are willing to parse as a document.
53const TEXTUAL: &[&str] = &[
54    "text/html",
55    "application/xhtml",
56    "text/plain",
57    "application/xml",
58    "text/xml",
59    "application/rss",
60    "application/atom",
61    "application/json",
62];
63
64/// How the client should present itself on the wire.
65#[derive(Debug, Clone, Default, PartialEq, Eq)]
66pub enum Impersonate {
67    /// A recent Chrome build. The default, and what almost everyone wants.
68    #[default]
69    Chrome,
70    /// A recent Firefox build.
71    Firefox,
72    /// A recent Safari build.
73    Safari,
74    /// A named `wreq-util` profile, e.g. `"chrome_143"` or `"safari_ios_26"`.
75    Named(String),
76    /// Market-share weighted random profile, re-rolled per client.
77    Random,
78    /// No emulation: plain TLS and a `rustai/x.y.z` User-Agent.
79    None,
80}
81
82impl Impersonate {
83    /// Parse the string form used by the Python API.
84    pub fn parse(s: &str) -> Result<Impersonate> {
85        Ok(match s.trim().to_ascii_lowercase().as_str() {
86            "" | "none" | "off" | "false" => Impersonate::None,
87            "chrome" | "true" | "default" => Impersonate::Chrome,
88            "firefox" => Impersonate::Firefox,
89            "safari" => Impersonate::Safari,
90            "random" => Impersonate::Random,
91            other => Impersonate::Named(other.to_string()),
92        })
93    }
94}
95
96/// Everything tunable about fetching.
97#[derive(Debug, Clone)]
98pub struct FetchConfig {
99    /// Maximum requests in flight across all hosts.
100    pub concurrency: usize,
101    /// Total per-request deadline.
102    pub timeout: Duration,
103    /// Connection establishment deadline.
104    pub connect_timeout: Duration,
105    /// Redirects followed before giving up.
106    pub max_redirects: usize,
107    /// Hard cap on a response body. Enforced while streaming, so an oversized
108    /// body costs one chunk, not the whole download.
109    pub max_body_bytes: usize,
110    /// Retries for retryable failures.
111    pub retries: u32,
112    /// Base delay for exponential backoff between retries.
113    pub retry_backoff: Duration,
114    /// Minimum spacing between two requests to the same host.
115    pub per_host_delay: Duration,
116    /// Honour `robots.txt`. Leave this on unless you own the target.
117    pub respect_robots: bool,
118    /// Honour a `Crawl-delay` larger than [`FetchConfig::per_host_delay`].
119    pub respect_crawl_delay: bool,
120    /// Browser fingerprint to present.
121    pub impersonate: Impersonate,
122    /// `Accept-Language` header.
123    pub accept_language: String,
124    /// Escalate JS-gated pages to headless Chrome. Needs the `browser` feature.
125    pub browser_fallback: bool,
126    /// Proxies to route requests through, rotated round-robin.
127    ///
128    /// The only real answer to IP-reputation blocking: a TLS fingerprint says
129    /// what your client is, and an address says who it is. Accepts `http://`,
130    /// `https://` and `socks5://` URLs, with optional `user:pass@`.
131    pub proxies: Vec<String>,
132    /// Honour a `Retry-After` header, up to this long. Set to zero to ignore it.
133    pub max_retry_after: Duration,
134    /// File to persist cookies to, so a session survives the process.
135    pub cookie_file: Option<std::path::PathBuf>,
136}
137
138impl Default for FetchConfig {
139    fn default() -> Self {
140        FetchConfig {
141            concurrency: 16,
142            timeout: Duration::from_secs(20),
143            connect_timeout: Duration::from_secs(8),
144            max_redirects: 5,
145            max_body_bytes: 8 * 1024 * 1024,
146            retries: 2,
147            retry_backoff: Duration::from_millis(400),
148            per_host_delay: Duration::from_millis(250),
149            respect_robots: true,
150            respect_crawl_delay: true,
151            impersonate: Impersonate::Chrome,
152            accept_language: "en-US,en;q=0.9".to_string(),
153            browser_fallback: false,
154            proxies: Vec::new(),
155            max_retry_after: Duration::from_secs(60),
156            cookie_file: None,
157        }
158    }
159}
160
161/// A fetched document.
162#[derive(Debug, Clone, serde::Serialize)]
163pub struct Page {
164    /// URL as requested.
165    pub url: String,
166    /// URL after redirects.
167    pub final_url: String,
168    /// HTTP status.
169    pub status: u16,
170    /// `Content-Type` header, if any.
171    pub content_type: Option<String>,
172    /// Decoded body.
173    pub body: String,
174    /// Wall time for the fetch, in milliseconds.
175    pub elapsed_ms: u64,
176    /// Whether the headless fallback produced this body.
177    pub rendered: bool,
178}
179
180/// A pooled, polite, fingerprint-aware HTTP client.
181#[derive(Clone)]
182pub struct Fetcher {
183    /// One client per proxy, or a single direct client. Rotated per request.
184    clients: Arc<Vec<wreq::Client>>,
185    next_client: Arc<std::sync::atomic::AtomicUsize>,
186    jar: Option<Arc<wreq::cookie::Jar>>,
187    /// Hosts this fetcher has touched. The cookie jar indexes by domain but
188    /// does not expose that key, so a host-only cookie — which is most session
189    /// cookies — cannot be attributed on the way back out without this.
190    visited: Arc<std::sync::Mutex<std::collections::BTreeSet<String>>>,
191    cfg: Arc<FetchConfig>,
192    gate: Arc<Semaphore>,
193    robots: Arc<Mutex<HashMap<String, Arc<Robots>>>>,
194    last_hit: Arc<Mutex<HashMap<String, Instant>>>,
195}
196
197impl std::fmt::Debug for Fetcher {
198    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
199        f.debug_struct("Fetcher").field("config", &self.cfg).finish_non_exhaustive()
200    }
201}
202
203impl Fetcher {
204    /// Build a fetcher with default settings.
205    pub fn new() -> Result<Self> {
206        Self::with_config(FetchConfig::default())
207    }
208
209    /// Build a fetcher from an explicit config.
210    pub fn with_config(cfg: FetchConfig) -> Result<Self> {
211        if cfg.concurrency == 0 {
212            return Err(Error::Config("concurrency must be at least 1".into()));
213        }
214        let jar = cfg.cookie_file.as_ref().map(|path| Arc::new(load_jar(path)));
215
216        let build_one = |proxy: Option<&str>| -> Result<wreq::Client> {
217            let mut builder = wreq::Client::builder()
218                .timeout(cfg.timeout)
219                .connect_timeout(cfg.connect_timeout)
220                .redirect(wreq::redirect::Policy::limited(cfg.max_redirects))
221                .pool_max_idle_per_host(cfg.concurrency.min(32))
222                .referer(true)
223                .gzip(true)
224                .brotli(true)
225                .deflate(true)
226                .zstd(true);
227
228            builder = match &jar {
229                Some(jar) => builder.cookie_provider(jar.clone()),
230                None => builder.cookie_store(true),
231            };
232            builder = apply_impersonation(builder, &cfg.impersonate)?;
233
234            if let Some(proxy) = proxy {
235                let proxy = wreq::Proxy::all(proxy)
236                    .map_err(|e| Error::Config(format!("invalid proxy `{proxy}`: {e}")))?;
237                builder = builder.proxy(proxy);
238            }
239
240            let mut headers = wreq::header::HeaderMap::new();
241            if let Ok(v) = wreq::header::HeaderValue::from_str(&cfg.accept_language) {
242                headers.insert(wreq::header::ACCEPT_LANGUAGE, v);
243            }
244            if matches!(cfg.impersonate, Impersonate::None) {
245                headers.insert(
246                    wreq::header::USER_AGENT,
247                    wreq::header::HeaderValue::from_static(DEFAULT_USER_AGENT),
248                );
249            }
250            builder = builder.default_headers(headers);
251            builder.build().map_err(|e| Error::Config(e.to_string()))
252        };
253
254        let clients = if cfg.proxies.is_empty() {
255            vec![build_one(None)?]
256        } else {
257            cfg.proxies.iter().map(|p| build_one(Some(p))).collect::<Result<Vec<_>>>()?
258        };
259
260        Ok(Fetcher {
261            clients: Arc::new(clients),
262            next_client: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
263            jar,
264            visited: Arc::new(std::sync::Mutex::new(std::collections::BTreeSet::new())),
265            gate: Arc::new(Semaphore::new(cfg.concurrency)),
266            cfg: Arc::new(cfg),
267            robots: Arc::new(Mutex::new(HashMap::new())),
268            last_hit: Arc::new(Mutex::new(HashMap::new())),
269        })
270    }
271
272    /// The next client in the rotation.
273    ///
274    /// Round-robin rather than random: with a small proxy pool, randomness
275    /// clusters, and clustering on one address is exactly what gets it banned.
276    fn client(&self) -> &wreq::Client {
277        if self.clients.len() == 1 {
278            return &self.clients[0];
279        }
280        let i = self.next_client.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
281        &self.clients[i % self.clients.len()]
282    }
283
284    /// Write the cookie jar to [`FetchConfig::cookie_file`].
285    ///
286    /// Clearance cookies are the expensive part of getting through a bot wall;
287    /// throwing them away when the process exits means paying for them again.
288    pub fn save_cookies(&self) -> Result<usize> {
289        let (Some(jar), Some(path)) = (&self.jar, &self.cfg.cookie_file) else { return Ok(0) };
290        let hosts: Vec<String> = match self.visited.lock() {
291            Ok(v) => v.iter().cloned().collect(),
292            Err(poisoned) => poisoned.into_inner().iter().cloned().collect(),
293        };
294        save_jar(jar, &hosts, path)
295    }
296
297    /// The configuration this fetcher was built with.
298    pub fn config(&self) -> &FetchConfig {
299        &self.cfg
300    }
301
302    /// Fetch one URL, honouring `robots.txt` when configured to.
303    pub async fn fetch(&self, url: &str) -> Result<Page> {
304        self.fetch_inner(url, self.cfg.respect_robots).await
305    }
306
307    /// Fetch a provider endpoint, skipping the `robots.txt` check.
308    ///
309    /// `robots.txt` governs crawling — the discovery of documents a site did
310    /// not hand you. A search or feed endpoint the caller named explicitly is
311    /// a query, not a crawl, and several of them (DuckDuckGo's HTML endpoint
312    /// among them) disallow the very path they exist to serve. Pages found
313    /// *through* those endpoints go through [`Fetcher::fetch`] and are checked
314    /// normally, which is where the rule actually protects anyone.
315    pub async fn fetch_api(&self, url: &str) -> Result<Page> {
316        self.fetch_inner(url, false).await
317    }
318
319    async fn fetch_inner(&self, url: &str, check_robots: bool) -> Result<Page> {
320        let parsed = normalize(url)?;
321        let _permit = self.gate.acquire().await.map_err(|e| Error::Config(e.to_string()))?;
322
323        let crawl_delay = if check_robots {
324            let robots = self.robots_for(&parsed).await;
325            if !robots.allows(parsed.path()) {
326                return Err(Error::RobotsDenied(parsed.to_string()));
327            }
328            robots.crawl_delay
329        } else {
330            None
331        };
332        self.pace(&parsed, crawl_delay).await;
333
334        let started = Instant::now();
335        let mut page = self.get_with_retry(&parsed).await?;
336
337        // Follow HTML-level redirects. `wreq` follows HTTP 3xx, but a stub page
338        // that redirects via `<meta refresh>` or `location.replace` returns a
339        // perfectly good 200 containing no content — which is exactly what
340        // happens on sites that canonicalise URLs in the browser.
341        let mut current = parsed.clone();
342        for _ in 0..MAX_HTML_REDIRECTS {
343            let Some(next) = detect_html_redirect(&page.body, &current) else { break };
344            if check_robots && !self.robots_for(&next).await.allows(next.path()) {
345                break;
346            }
347            self.pace(&next, crawl_delay).await;
348            match self.get_with_retry(&next).await {
349                Ok(followed) => {
350                    page = followed;
351                    current = next;
352                }
353                // The stub is still a better answer than an error.
354                Err(_) => break,
355            }
356        }
357        page.url = parsed.to_string();
358        page.elapsed_ms = started.elapsed().as_millis() as u64;
359
360        if self.cfg.browser_fallback && needs_javascript(&page) {
361            // A failed escalation is not a failed fetch: the static body is
362            // still the best answer we have.
363            if let Ok(rendered) = self.render(&parsed).await {
364                page.body = rendered;
365                page.rendered = true;
366                page.elapsed_ms = started.elapsed().as_millis() as u64;
367            }
368        }
369        Ok(page)
370    }
371
372    /// Fetch many URLs concurrently, preserving input order in the output.
373    ///
374    /// The concurrency ceiling and per-host spacing still apply, so this is
375    /// safe to call with a few hundred URLs at once.
376    pub async fn fetch_many(&self, urls: &[String]) -> Vec<Result<Page>> {
377        let futures = urls.iter().map(|u| {
378            let this = self.clone();
379            let u = u.clone();
380            async move { this.fetch(&u).await }
381        });
382        futures_util::stream::iter(futures).buffered(self.cfg.concurrency).collect::<Vec<_>>().await
383    }
384
385    /// One URL, with exponential backoff on retryable failures.
386    async fn get_with_retry(&self, url: &Url) -> Result<Page> {
387        let mut attempt = 0u32;
388        loop {
389            match self.get_once(url).await {
390                Ok(page) => return Ok(page),
391                Err(e) if e.is_retryable() && attempt < self.cfg.retries => {
392                    // Retrying a 429 immediately is how a client turns a soft
393                    // limit into a hard block. When the server says how long to
394                    // wait, that beats any backoff curve we could invent — but
395                    // an hour-long hint is a refusal, not a delay, so it is
396                    // capped and otherwise treated as a failure.
397                    let wait = match e.retry_after() {
398                        Some(hint) if hint <= self.cfg.max_retry_after => hint,
399                        Some(_) => return Err(e),
400                        None => self.cfg.retry_backoff * 2u32.pow(attempt),
401                    };
402                    tokio::time::sleep(wait).await;
403                    attempt += 1;
404                }
405                Err(e) => return Err(e),
406            }
407        }
408    }
409
410    async fn get_once(&self, url: &Url) -> Result<Page> {
411        let resp =
412            self.client().get(url.as_str()).send().await.map_err(|e| Error::network(url, e))?;
413
414        let status = resp.status().as_u16();
415        let final_url = resp.uri().to_string();
416        let content_type = resp
417            .headers()
418            .get(wreq::header::CONTENT_TYPE)
419            .and_then(|v| v.to_str().ok())
420            .map(str::to_string);
421
422        if !(200..300).contains(&status) {
423            let retry_after = resp
424                .headers()
425                .get(wreq::header::RETRY_AFTER)
426                .and_then(|v| v.to_str().ok())
427                .and_then(parse_retry_after);
428            return Err(Error::Status { url: url.to_string(), status, retry_after });
429        }
430        if let Some(ct) = &content_type {
431            let base = ct.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
432            if !TEXTUAL.iter().any(|t| base.starts_with(t)) {
433                return Err(Error::Parse(format!("{url}: unsupported content type {base}")));
434            }
435        }
436        if resp.content_length().is_some_and(|n| n as usize > self.cfg.max_body_bytes) {
437            return Err(Error::BodyTooLarge {
438                url: url.to_string(),
439                limit: self.cfg.max_body_bytes,
440            });
441        }
442
443        let charset = content_type.as_deref().and_then(charset_of);
444        let bytes = self.read_capped(url, resp).await?;
445        let body = decode(&bytes, charset.as_deref(), tld_of(&final_url).as_deref());
446
447        Ok(Page {
448            url: url.to_string(),
449            final_url,
450            status,
451            content_type,
452            body,
453            elapsed_ms: 0,
454            rendered: false,
455        })
456    }
457
458    /// Read a body, aborting as soon as it crosses the cap.
459    async fn read_capped(&self, url: &Url, resp: wreq::Response) -> Result<Vec<u8>> {
460        let mut buf: Vec<u8> = Vec::with_capacity(64 * 1024);
461        let mut stream = resp.bytes_stream();
462        while let Some(chunk) = stream.next().await {
463            let chunk = chunk.map_err(|e| Error::network(url, e))?;
464            if buf.len() + chunk.len() > self.cfg.max_body_bytes {
465                return Err(Error::BodyTooLarge {
466                    url: url.to_string(),
467                    limit: self.cfg.max_body_bytes,
468                });
469            }
470            buf.extend_from_slice(&chunk);
471        }
472        Ok(buf)
473    }
474
475    /// Fetch and cache `robots.txt` for a host.
476    ///
477    /// A missing or unreadable file is treated as "allow all", which is what
478    /// the RFC specifies for a 404.
479    async fn robots_for(&self, url: &Url) -> Arc<Robots> {
480        let host = url.host_str().unwrap_or_default().to_string();
481        let key = format!("{}://{host}", url.scheme());
482        if let Some(cached) = self.robots.lock().await.get(&key) {
483            return cached.clone();
484        }
485        let robots_url = format!("{key}/robots.txt");
486        let parsed = match self.client().get(&robots_url).send().await {
487            Ok(resp) if resp.status().is_success() => match resp.text().await {
488                Ok(body) => Robots::parse(&body, ROBOTS_AGENT),
489                Err(_) => Robots::allow_all(),
490            },
491            _ => Robots::allow_all(),
492        };
493        let parsed = Arc::new(parsed);
494        self.robots.lock().await.insert(key, parsed.clone());
495        parsed
496    }
497
498    /// Space requests to one host apart, honouring `Crawl-delay` when larger.
499    async fn pace(&self, url: &Url, crawl_delay: Option<f64>) {
500        let host = url.host_str().unwrap_or_default().to_string();
501        if let Ok(mut visited) = self.visited.lock() {
502            visited.insert(host.clone());
503        }
504        let mut delay = self.cfg.per_host_delay;
505        if self.cfg.respect_crawl_delay
506            && let Some(d) = crawl_delay
507        {
508            delay = delay.max(Duration::from_secs_f64(d.min(30.0)));
509        }
510        if delay.is_zero() {
511            return;
512        }
513        let wait = {
514            let mut map = self.last_hit.lock().await;
515            let now = Instant::now();
516            let wait = match map.get(&host) {
517                Some(prev) => delay.checked_sub(now.duration_since(*prev)),
518                None => None,
519            };
520            // Reserve our slot before releasing the lock, so concurrent callers
521            // to the same host queue up instead of all sleeping the same amount.
522            map.insert(host, now + wait.unwrap_or_default());
523            wait
524        };
525        if let Some(wait) = wait {
526            tokio::time::sleep(wait).await;
527        }
528    }
529
530    #[cfg(feature = "browser")]
531    async fn render(&self, url: &Url) -> Result<String> {
532        browser::render(url.as_str(), self.cfg.timeout).await
533    }
534
535    #[cfg(not(feature = "browser"))]
536    async fn render(&self, _url: &Url) -> Result<String> {
537        Err(Error::Browser("build rustai with the `browser` feature".into()))
538    }
539}
540
541fn apply_impersonation(
542    builder: wreq::ClientBuilder,
543    mode: &Impersonate,
544) -> Result<wreq::ClientBuilder> {
545    #[cfg(not(feature = "impersonate"))]
546    {
547        if !matches!(mode, Impersonate::None) {
548            return Err(Error::Config(
549                "browser impersonation needs the `impersonate` feature".into(),
550            ));
551        }
552        Ok(builder)
553    }
554    #[cfg(feature = "impersonate")]
555    {
556        use wreq_util::{Emulation, Profile};
557        let profile = match mode {
558            Impersonate::None => return Ok(builder),
559            Impersonate::Random => return Ok(builder.emulation(Emulation::weighted_random())),
560            Impersonate::Chrome => Profile::Chrome149,
561            Impersonate::Firefox => Profile::Firefox147,
562            Impersonate::Safari => Profile::Safari26_4,
563            Impersonate::Named(name) => named_profile(name)?,
564        };
565        Ok(builder.emulation(profile))
566    }
567}
568
569/// Resolve a `wreq-util` profile from its wire name, e.g. `"chrome_143"`.
570///
571/// The enum has no `FromStr`, but it does derive `Deserialize` with exactly
572/// these renames under the `emulation-serde` feature, so serde is the
573/// supported way in — and it stays correct as upstream adds profiles.
574#[cfg(feature = "impersonate")]
575fn named_profile(name: &str) -> Result<wreq_util::Profile> {
576    serde_json::from_value::<wreq_util::Profile>(serde_json::Value::String(name.to_string()))
577        .map_err(|_| {
578            Error::Config(format!(
579                "unknown impersonation profile `{name}`; try `chrome`, `firefox`, `safari`, \
580                 `random`, or a versioned name such as `chrome_143`"
581            ))
582        })
583}
584
585/// Parse a `Retry-After` value into seconds.
586///
587/// Only the delta-seconds form. The HTTP-date form is rare in practice and
588/// parsing it would mean carrying a date library for one header; an unparsed
589/// value simply falls back to exponential backoff.
590fn parse_retry_after(value: &str) -> Option<f64> {
591    let secs: f64 = value.trim().parse().ok()?;
592    secs.is_finite().then_some(secs.max(0.0))
593}
594
595/// One persisted cookie, with the host it belongs to.
596#[derive(serde::Serialize, serde::Deserialize)]
597struct StoredCookie {
598    name: String,
599    value: String,
600    /// Host the cookie was collected from.
601    host: String,
602    path: String,
603    /// `Domain` attribute, when the cookie carried one. Absent means host-only,
604    /// and reloading must keep it that way.
605    #[serde(default, skip_serializing_if = "Option::is_none")]
606    domain: Option<String>,
607}
608
609/// Read a cookie jar from disk. A missing or unreadable file yields an empty
610/// jar: a lost session is a slow start, never an error.
611fn load_jar(path: &std::path::Path) -> wreq::cookie::Jar {
612    let jar = wreq::cookie::Jar::default();
613    let Ok(raw) = std::fs::read_to_string(path) else { return jar };
614    let Ok(stored) = serde_json::from_str::<Vec<StoredCookie>>(&raw) else { return jar };
615    for c in stored {
616        let uri = format!("https://{}{}", c.host, c.path);
617        let mut cookie = format!("{}={}; Path={}", c.name, c.value, c.path);
618        if let Some(domain) = &c.domain {
619            cookie.push_str(&format!("; Domain={domain}"));
620        }
621        jar.add(cookie.as_str(), uri.as_str());
622    }
623    jar
624}
625
626/// Write a cookie jar to disk, returning how many cookies were saved.
627fn save_jar(jar: &wreq::cookie::Jar, hosts: &[String], path: &std::path::Path) -> Result<usize> {
628    // Asking the jar what it would send to each host we visited is the only way
629    // to recover a host-only cookie's owner: `get_all` returns the cookies but
630    // not the domain key they were filed under.
631    let mut seen = std::collections::HashSet::new();
632    let mut stored: Vec<StoredCookie> = Vec::new();
633    for host in hosts {
634        for c in jar.matches(format!("https://{host}/").as_str()) {
635            let cookie = StoredCookie {
636                name: c.name().to_string(),
637                value: c.value().to_string(),
638                host: host.clone(),
639                path: c.path().unwrap_or("/").to_string(),
640                domain: c.domain().map(str::to_string),
641            };
642            if seen.insert((cookie.host.clone(), cookie.name.clone(), cookie.path.clone())) {
643                stored.push(cookie);
644            }
645        }
646    }
647    let json = serde_json::to_string_pretty(&stored)
648        .map_err(|e| Error::Config(format!("could not serialise cookies: {e}")))?;
649    if let Some(parent) = path.parent()
650        && !parent.as_os_str().is_empty()
651    {
652        let _ = std::fs::create_dir_all(parent);
653    }
654    std::fs::write(path, json)
655        .map_err(|e| Error::Config(format!("could not write {}: {e}", path.display())))?;
656    Ok(stored.len())
657}
658
659/// Reject anything that is not an absolute http(s) URL.
660pub fn normalize(url: &str) -> Result<Url> {
661    let parsed = Url::parse(url.trim()).map_err(|_| Error::InvalidUrl(url.to_string()))?;
662    if !matches!(parsed.scheme(), "http" | "https") {
663        return Err(Error::InvalidUrl(url.to_string()));
664    }
665    if parsed.host_str().is_none() {
666        return Err(Error::InvalidUrl(url.to_string()));
667    }
668    Ok(parsed)
669}
670
671fn charset_of(content_type: &str) -> Option<String> {
672    content_type
673        .split(';')
674        .skip(1)
675        .filter_map(|p| p.trim().strip_prefix("charset="))
676        .map(|c| c.trim_matches(['"', '\'']).to_ascii_lowercase())
677        .next()
678}
679
680/// Decode a body, trusting the header, then a `<meta charset>`, then UTF-8,
681/// and only when the bytes are not UTF-8 either, a guess.
682///
683/// Legacy encodings are still common outside the anglosphere — EUC-KR and
684/// Shift_JIS in particular — and mojibake destroys every downstream heuristic,
685/// so this is worth getting right rather than assuming UTF-8. Old pages often
686/// declare nothing at all (math.snu.ac.kr serves EUC-KR that way), so the last
687/// step is the detector Firefox uses, hinted with the host's top-level domain
688/// the way a browser is.
689fn decode(bytes: &[u8], header_charset: Option<&str>, tld: Option<&str>) -> String {
690    let label = header_charset.map(str::to_string).or_else(|| sniff_meta_charset(bytes));
691    let encoding = label
692        .as_deref()
693        .and_then(|l| encoding_rs::Encoding::for_label(l.as_bytes()))
694        .unwrap_or_else(|| {
695            // A capped body can end inside a multi-byte character; up to three
696            // trailing bytes of invalid UTF-8 are a cut, not another encoding.
697            if bytes.len() - encoding_rs::Encoding::utf8_valid_up_to(bytes) <= 3 {
698                encoding_rs::UTF_8
699            } else {
700                let mut detector = chardetng::EncodingDetector::new();
701                detector.feed(bytes, true);
702                detector.guess(tld.map(str::as_bytes), true)
703            }
704        });
705    let (text, _, _) = encoding.decode(bytes);
706    text.into_owned()
707}
708
709/// The last label of a URL's host — the hint the encoding detector takes.
710fn tld_of(url: &str) -> Option<String> {
711    let host = Url::parse(url).ok()?.host_str()?.to_ascii_lowercase();
712    host.rsplit('.').next().map(str::to_string)
713}
714
715/// Look for `<meta charset=...>` in the first 4 KiB, as browsers do.
716fn sniff_meta_charset(bytes: &[u8]) -> Option<String> {
717    let head = &bytes[..bytes.len().min(4096)];
718    let text = String::from_utf8_lossy(head).to_ascii_lowercase();
719    if let Some(at) = text.find("charset=") {
720        let rest = &text[at + "charset=".len()..];
721        let value: String = rest
722            .trim_start_matches(['"', '\'', ' '])
723            .chars()
724            .take_while(|c| c.is_ascii_alphanumeric() || *c == '-' || *c == '_')
725            .collect();
726        if !value.is_empty() {
727            return Some(value);
728        }
729    }
730    None
731}
732
733/// Find an HTML-level redirect target in a page body.
734///
735/// Two forms matter in practice: `<meta http-equiv="refresh" content="0; url=…">`
736/// and a script that assigns `location`. Both are gated on the page being small
737/// and on a zero delay, because a redirect stub is always tiny — a long article
738/// that happens to contain `location.href` somewhere must not be mistaken for
739/// one.
740fn detect_html_redirect(body: &str, base: &Url) -> Option<Url> {
741    const MAX_STUB_BYTES: usize = 4096;
742    if body.len() > MAX_STUB_BYTES {
743        return None;
744    }
745    let target = meta_refresh_target(body).or_else(|| script_location_target(body))?;
746    let next = base.join(target.trim().trim_matches(['"', '\''])).ok()?;
747    if !matches!(next.scheme(), "http" | "https") {
748        return None;
749    }
750    // A stub pointing at itself is a loop, not a redirect.
751    let same = next.as_str().trim_end_matches('/') == base.as_str().trim_end_matches('/');
752    (!same).then_some(next)
753}
754
755/// `<meta http-equiv="refresh" content="0; url=TARGET">`, delay 0 only.
756fn meta_refresh_target(body: &str) -> Option<&str> {
757    let lower = body.to_ascii_lowercase();
758    let mut from = 0usize;
759    while let Some(at) = lower[from..].find("http-equiv") {
760        let tag_start = lower[..from + at].rfind('<')?;
761        let tag_end = lower[tag_start..].find('>')? + tag_start;
762        let tag = &lower[tag_start..tag_end];
763        if tag.contains("refresh")
764            && let Some(content_at) = tag.find("content")
765            && let Some(url_at) = tag[content_at..].find("url=")
766        {
767            // Only an immediate redirect; a delayed one is a page in its own right.
768            let delay: String = tag[content_at..]
769                .chars()
770                .skip_while(|c| !c.is_ascii_digit())
771                .take_while(|c| c.is_ascii_digit())
772                .collect();
773            if delay.parse::<u32>().unwrap_or(0) == 0 {
774                let start = tag_start + content_at + url_at + 4;
775                let rest = &body[start..];
776                let end = rest.find(['"', '\'', '>']).unwrap_or(rest.len());
777                let target = rest[..end].trim();
778                if !target.is_empty() {
779                    return Some(target);
780                }
781            }
782        }
783        from += at + "http-equiv".len();
784    }
785    None
786}
787
788/// `location.replace("TARGET")`, `location.href = "TARGET"`, and friends.
789fn script_location_target(body: &str) -> Option<&str> {
790    for marker in ["location.replace(", "location.href=", "location.href =", "location.assign("] {
791        let Some(at) = body.find(marker) else { continue };
792        let rest = &body[at + marker.len()..];
793        let rest = rest.trim_start();
794        let quote = rest.chars().next()?;
795        if quote != '"' && quote != '\'' {
796            // An indirection through a variable; resolving it needs a JS engine.
797            continue;
798        }
799        let inner = &rest[quote.len_utf8()..];
800        let end = inner.find(quote)?;
801        let target = &inner[..end];
802        if target.starts_with("http") || target.starts_with('/') {
803            return Some(target);
804        }
805    }
806    None
807}
808
809/// Does this response look like an empty shell waiting for JavaScript?
810///
811/// Deliberately conservative. Rendering a page costs ~100x a fetch, so the
812/// bar is "there is markup but essentially no text", not "the page mentions
813/// JavaScript".
814pub fn needs_javascript(page: &Page) -> bool {
815    let html = &page.body;
816    if html.len() < 512 {
817        return false;
818    }
819    let Ok(article) = crate::parse::extract(html, Some(&page.final_url)) else {
820        return true;
821    };
822    let text_len = crate::text::visible_len(&article.text);
823
824    // Plenty of text: whatever else the page is doing, we already have it.
825    if text_len > MAX_STATIC_TEXT {
826        return false;
827    }
828
829    // Markup carrying essentially no text. An absolute floor misses the common
830    // case — a 350 KB front page yielding 510 characters clears any fixed
831    // threshold while plainly being an app shell — so the test is the ratio.
832    // Real pages run percent-level text-to-markup; a shell runs a tenth of that.
833    let starved = text_len.saturating_mul(TEXT_TO_MARKUP_FLOOR) < html.len();
834    if text_len < 120 || starved {
835        return true;
836    }
837
838    let lower = html.to_lowercase();
839    let spa_markers = ["__next_data__", "id=\"root\"", "id=\"app\"", "ng-app", "data-reactroot"];
840    spa_markers.iter().any(|m| lower.contains(m))
841}
842
843#[cfg(test)]
844mod tests {
845    use super::*;
846
847    #[test]
848    fn rejects_non_http_urls() {
849        assert!(normalize("file:///etc/passwd").is_err());
850        assert!(normalize("javascript:alert(1)").is_err());
851        assert!(normalize("not a url").is_err());
852        assert!(normalize("https://example.com/a").is_ok());
853    }
854
855    #[test]
856    fn parses_charset_from_content_type() {
857        assert_eq!(charset_of("text/html; charset=EUC-KR").as_deref(), Some("euc-kr"));
858        assert_eq!(charset_of("text/html").as_deref(), None);
859    }
860
861    #[test]
862    fn decodes_legacy_korean_encoding() {
863        let (bytes, _, _) = encoding_rs::EUC_KR.encode("한글 인코딩");
864        assert_eq!(decode(&bytes, Some("euc-kr"), None), "한글 인코딩");
865        // Without the header, the meta tag is the fallback.
866        let mut doc = b"<html><head><meta charset=\"euc-kr\"></head><body>".to_vec();
867        doc.extend_from_slice(&bytes);
868        assert!(decode(&doc, None, None).contains("한글 인코딩"));
869    }
870
871    #[test]
872    fn guesses_a_legacy_encoding_that_nothing_declares() {
873        // math.snu.ac.kr serves EUC-KR with neither a header charset nor a meta tag.
874        let (bytes, _, _) = encoding_rs::EUC_KR.encode(concat!(
875            "<html><head><title>합성함수 미분법 = 연쇄법칙(Chain Rule)</title></head><body>",
876            "<p>함수 y = f(x) 와 함수 z = g(y)를 합성한 함수의 미분법은 다음과 같다.</p></body></html>",
877        ));
878        assert!(decode(&bytes, None, Some("kr")).contains("합성함수 미분법 = 연쇄법칙"));
879    }
880
881    #[test]
882    fn undeclared_utf8_stays_utf8() {
883        assert_eq!(
884            decode("<p>한글 UTF-8 본문</p>".as_bytes(), None, Some("kr")),
885            "<p>한글 UTF-8 본문</p>"
886        );
887    }
888
889    #[test]
890    fn a_body_cut_mid_character_is_still_read_as_utf8() {
891        // The body cap can stop a fetch inside a multi-byte character.
892        let full = "<p>가나다라마바사</p>".repeat(20);
893        let cut = &full.as_bytes()[..full.len() - 7];
894        assert!(decode(cut, None, Some("kr")).starts_with("<p>가나다라마바사</p>"));
895    }
896
897    #[test]
898    fn tld_is_the_last_host_label() {
899        assert_eq!(tld_of("https://www.math.snu.ac.kr/a.htm").as_deref(), Some("kr"));
900        assert_eq!(tld_of("not a url"), None);
901    }
902
903    #[test]
904    fn impersonate_strings_parse() {
905        assert_eq!(Impersonate::parse("chrome").unwrap(), Impersonate::Chrome);
906        assert_eq!(Impersonate::parse("none").unwrap(), Impersonate::None);
907        assert_eq!(
908            Impersonate::parse("chrome_143").unwrap(),
909            Impersonate::Named("chrome_143".into())
910        );
911    }
912
913    #[cfg(feature = "impersonate")]
914    #[test]
915    fn named_profiles_resolve_and_reject() {
916        assert!(named_profile("chrome_143").is_ok());
917        assert!(named_profile("netscape_2").is_err());
918    }
919
920    #[test]
921    fn client_builds_with_every_preset() {
922        for mode in ["chrome", "firefox", "safari", "random", "none"] {
923            let cfg = FetchConfig {
924                impersonate: Impersonate::parse(mode).unwrap(),
925                ..Default::default()
926            };
927            assert!(Fetcher::with_config(cfg).is_ok(), "{mode} failed to build");
928        }
929    }
930
931    #[test]
932    fn retry_after_is_parsed_and_capped() {
933        assert_eq!(parse_retry_after("120"), Some(120.0));
934        assert_eq!(parse_retry_after("  0 "), Some(0.0));
935        assert_eq!(parse_retry_after("-5"), Some(0.0));
936        // The HTTP-date form is not parsed; backoff takes over.
937        assert_eq!(parse_retry_after("Wed, 21 Oct 2026 07:28:00 GMT"), None);
938        assert_eq!(parse_retry_after(""), None);
939
940        let e =
941            Error::Status { url: "https://x.dev/".into(), status: 429, retry_after: Some(30.0) };
942        assert_eq!(e.retry_after(), Some(Duration::from_secs(30)));
943        assert!(e.is_retryable());
944        let none = Error::Status { url: "https://x.dev/".into(), status: 429, retry_after: None };
945        assert_eq!(none.retry_after(), None);
946    }
947
948    #[test]
949    fn proxies_build_one_client_each() {
950        let cfg = FetchConfig {
951            proxies: vec!["http://127.0.0.1:8080".into(), "socks5://127.0.0.1:1080".into()],
952            ..Default::default()
953        };
954        let fetcher = Fetcher::with_config(cfg).expect("proxied client");
955        assert_eq!(fetcher.clients.len(), 2);
956        // Round-robin, so a small pool cannot cluster on one address.
957        let first = fetcher.client() as *const _;
958        let second = fetcher.client() as *const _;
959        assert_ne!(first, second);
960        assert_eq!(fetcher.client() as *const _, first);
961    }
962
963    #[test]
964    fn an_invalid_proxy_is_rejected_at_build_time() {
965        let cfg = FetchConfig { proxies: vec!["not a proxy".into()], ..Default::default() };
966        assert!(Fetcher::with_config(cfg).is_err());
967    }
968
969    #[test]
970    fn cookies_survive_a_round_trip_through_disk() {
971        let dir = std::env::temp_dir().join(format!("rustai-cookies-{}", std::process::id()));
972        let path = dir.join("jar.json");
973        let _ = std::fs::remove_file(&path);
974
975        let jar = wreq::cookie::Jar::default();
976        // A domain cookie and a host-only one: the latter is what most session
977        // cookies actually are, and it has no Domain attribute to save.
978        jar.add("cf_clearance=abc123; Domain=example.com; Path=/", "https://example.com/");
979        jar.add("session=xyz789; Path=/", "https://example.com/");
980        let hosts = vec!["example.com".to_string()];
981        assert_eq!(save_jar(&jar, &hosts, &path).unwrap(), 2);
982
983        let reloaded = load_jar(&path);
984        assert!(reloaded.contains("session", "https://example.com/"), "host-only cookie lost");
985        assert!(reloaded.contains("cf_clearance", "https://example.com/"), "cookie was lost");
986        assert_eq!(
987            reloaded.get("cf_clearance", "https://example.com/").map(|c| c.value().to_string()),
988            Some("abc123".to_string())
989        );
990
991        // A missing file is a slow start, not an error.
992        assert_eq!(load_jar(&dir.join("nope.json")).len(), 0);
993        let _ = std::fs::remove_dir_all(&dir);
994    }
995
996    #[test]
997    fn a_fetcher_without_a_cookie_file_saves_nothing() {
998        let fetcher = Fetcher::new().unwrap();
999        assert_eq!(fetcher.save_cookies().unwrap(), 0);
1000    }
1001
1002    #[test]
1003    fn zero_concurrency_is_rejected() {
1004        let cfg = FetchConfig { concurrency: 0, ..Default::default() };
1005        assert!(Fetcher::with_config(cfg).is_err());
1006    }
1007
1008    fn page(body: &str) -> Page {
1009        Page {
1010            url: "https://x.dev/".into(),
1011            final_url: "https://x.dev/".into(),
1012            status: 200,
1013            content_type: Some("text/html".into()),
1014            body: body.into(),
1015            elapsed_ms: 0,
1016            rendered: false,
1017        }
1018    }
1019
1020    #[test]
1021    fn follows_a_meta_refresh_stub() {
1022        let base = Url::parse("https://blog.dev/2024/02/08/Post.html").unwrap();
1023        let body = r#"<!doctype html><title>Redirect</title>
1024            <noscript><meta http-equiv="refresh" content="0; url=https://blog.dev/2024/02/08/Post/"></noscript>"#;
1025        assert_eq!(
1026            detect_html_redirect(body, &base).unwrap().as_str(),
1027            "https://blog.dev/2024/02/08/Post/"
1028        );
1029    }
1030
1031    #[test]
1032    fn follows_a_script_location_stub() {
1033        let base = Url::parse("https://blog.dev/a.html").unwrap();
1034        let body = r#"<!doctype html><title>Redirect</title><script>
1035            const target = "https://blog.dev/a/";
1036            window.location.replace("https://blog.dev/a/");
1037        </script>"#;
1038        assert_eq!(detect_html_redirect(body, &base).unwrap().as_str(), "https://blog.dev/a/");
1039    }
1040
1041    #[test]
1042    fn resolves_a_relative_redirect_target() {
1043        let base = Url::parse("https://blog.dev/x/a.html").unwrap();
1044        let body = r#"<meta http-equiv="refresh" content="0;url=/y/b.html">"#;
1045        assert_eq!(
1046            detect_html_redirect(body, &base).unwrap().as_str(),
1047            "https://blog.dev/y/b.html"
1048        );
1049    }
1050
1051    #[test]
1052    fn ignores_delayed_refreshes_and_self_loops() {
1053        let base = Url::parse("https://blog.dev/a").unwrap();
1054        // A 5-second refresh is a real page that reloads, not a redirect.
1055        assert!(
1056            detect_html_redirect(
1057                r#"<meta http-equiv="refresh" content="5; url=https://blog.dev/b">"#,
1058                &base
1059            )
1060            .is_none()
1061        );
1062        // Pointing at itself is a loop.
1063        assert!(
1064            detect_html_redirect(
1065                r#"<meta http-equiv="refresh" content="0; url=https://blog.dev/a">"#,
1066                &base
1067            )
1068            .is_none()
1069        );
1070    }
1071
1072    #[test]
1073    fn ignores_a_long_page_that_merely_mentions_location() {
1074        let base = Url::parse("https://blog.dev/a").unwrap();
1075        let article = format!(
1076            "<article><p>{}</p><p>You can call location.replace(\"https://evil.dev/\") in JS.</p></article>",
1077            "Real prose. ".repeat(500)
1078        );
1079        assert!(
1080            detect_html_redirect(&article, &base).is_none(),
1081            "a full article was treated as a stub"
1082        );
1083    }
1084
1085    #[test]
1086    fn ignores_non_http_and_unresolvable_targets() {
1087        let base = Url::parse("https://blog.dev/a").unwrap();
1088        assert!(
1089            detect_html_redirect(
1090                r#"<meta http-equiv="refresh" content="0;url=javascript:x()">"#,
1091                &base
1092            )
1093            .is_none()
1094        );
1095        assert!(detect_html_redirect("<p>nothing here</p>", &base).is_none());
1096        // A variable indirection needs a JS engine; we must not guess.
1097        assert!(
1098            detect_html_redirect(r#"<script>window.location.replace(target);</script>"#, &base)
1099                .is_none()
1100        );
1101    }
1102
1103    #[test]
1104    fn detects_an_spa_shell() {
1105        let shell = format!(
1106            "<html><body><div id=\"root\"></div><script>{}</script></body></html>",
1107            "x".repeat(2000)
1108        );
1109        assert!(needs_javascript(&page(&shell)));
1110    }
1111
1112    #[test]
1113    fn does_not_escalate_a_real_article() {
1114        let real = format!(
1115            "<html><body><article><h1>T</h1><p>{}</p></article></body></html>",
1116            "Real prose that a reader can actually read. ".repeat(20)
1117        );
1118        assert!(!needs_javascript(&page(&real)));
1119    }
1120}
1121
1122#[cfg(test)]
1123mod js_gate_tests {
1124    use super::*;
1125
1126    fn page(body: String) -> Page {
1127        Page {
1128            url: "https://x.dev/".into(),
1129            final_url: "https://x.dev/".into(),
1130            status: 200,
1131            content_type: Some("text/html".into()),
1132            body,
1133            elapsed_ms: 0,
1134            rendered: false,
1135        }
1136    }
1137
1138    /// Modelled on a real front page whose stories render client-side: ~350 KB
1139    /// of markup carrying ~500 characters of text. An absolute text threshold
1140    /// waves that through — 500 clears any sane floor — while the ratio does
1141    /// not. Index detection cannot rescue such a page either: there is nothing
1142    /// in the HTML to index.
1143    #[test]
1144    fn a_client_rendered_front_page_is_flagged() {
1145        let shell = format!(
1146            "<html><body><nav>{}</nav><main></main><script>{}</script></body></html>",
1147            "<a href=\"/section\">Section link</a>".repeat(20),
1148            "x".repeat(340_000)
1149        );
1150        assert!(needs_javascript(&page(shell)));
1151    }
1152
1153    /// The same size of page, but the markup actually carries the article.
1154    #[test]
1155    fn a_large_real_article_is_not_flagged() {
1156        let real = format!(
1157            "<html><body><article><h1>T</h1><p>{}</p></article></body></html>",
1158            "Real prose that a reader can actually read and act on. ".repeat(400)
1159        );
1160        assert!(!needs_javascript(&page(real)));
1161    }
1162}