Skip to main content

rustai_core/http/
mod.rs

1//! The fetch layer: impersonated HTTP, politeness, and a headless fallback.
2//!
3//! Three things separate this from a bare `reqwest` loop:
4//!
5//! * **Fingerprint.** Cloudflare and friends read the TLS ClientHello (JA3/JA4)
6//!   and the HTTP/2 SETTINGS frame, not just the User-Agent. `wreq` replays a
7//!   real Chrome's, so a plain `GET` gets a plain `200` where a stock client
8//!   gets a challenge page.
9//! * **Politeness.** `robots.txt`, per-host spacing and a concurrency ceiling
10//!   are on by default, because a fast crawler with none of those is a way to
11//!   get a user's IP banned.
12//! * **Escalation.** JS-gated pages are detected from the response and, when
13//!   the `browser` feature is on, retried through headless Chrome — which is
14//!   two orders of magnitude more expensive and therefore never the first move.
15
16pub mod robots;
17
18#[cfg(feature = "browser")]
19pub mod browser;
20
21use std::collections::HashMap;
22use std::sync::Arc;
23use std::time::{Duration, Instant};
24
25use futures_util::StreamExt;
26use tokio::sync::{Mutex, Semaphore};
27use url::Url;
28
29use crate::error::{Error, Result};
30use robots::Robots;
31
32/// Default identity when impersonation is off, or for the `robots.txt` lookup.
33pub const DEFAULT_USER_AGENT: &str =
34    concat!("rustai/", env!("CARGO_PKG_VERSION"), " (+https://github.com/imhyensuk/rustai)");
35
36/// The token we match `robots.txt` groups against.
37pub const ROBOTS_AGENT: &str = "rustai";
38
39/// HTML-level redirects followed before giving up.
40///
41/// Two is enough for the real pattern (a canonicalising stub pointing at the
42/// article) and low enough that a redirect loop costs almost nothing.
43const MAX_HTML_REDIRECTS: usize = 2;
44
45/// Above this much extracted text, a page is not an empty shell.
46const MAX_STATIC_TEXT: usize = 2_000;
47
48/// One character of text per this many bytes of markup is the floor below which
49/// a page is presumed to be rendered client-side.
50const TEXT_TO_MARKUP_FLOOR: usize = 200;
51
52/// Content types we are willing to parse as a document.
53const TEXTUAL: &[&str] = &[
54    "text/html",
55    "application/xhtml",
56    "text/plain",
57    "application/xml",
58    "text/xml",
59    "application/rss",
60    "application/atom",
61    "application/json",
62];
63
64/// How the client should present itself on the wire.
65#[derive(Debug, Clone, Default, PartialEq, Eq)]
66pub enum Impersonate {
67    /// A recent Chrome build. The default, and what almost everyone wants.
68    #[default]
69    Chrome,
70    /// A recent Firefox build.
71    Firefox,
72    /// A recent Safari build.
73    Safari,
74    /// A named `wreq-util` profile, e.g. `"chrome_143"` or `"safari_ios_26"`.
75    Named(String),
76    /// Market-share weighted random profile, re-rolled per client.
77    Random,
78    /// No emulation: plain TLS and a `rustai/x.y.z` User-Agent.
79    None,
80}
81
82impl Impersonate {
83    /// Parse the string form used by the Python API.
84    pub fn parse(s: &str) -> Result<Impersonate> {
85        Ok(match s.trim().to_ascii_lowercase().as_str() {
86            "" | "none" | "off" | "false" => Impersonate::None,
87            "chrome" | "true" | "default" => Impersonate::Chrome,
88            "firefox" => Impersonate::Firefox,
89            "safari" => Impersonate::Safari,
90            "random" => Impersonate::Random,
91            other => Impersonate::Named(other.to_string()),
92        })
93    }
94}
95
96/// Everything tunable about fetching.
97#[derive(Debug, Clone)]
98pub struct FetchConfig {
99    /// Maximum requests in flight across all hosts.
100    pub concurrency: usize,
101    /// Total per-request deadline.
102    pub timeout: Duration,
103    /// Connection establishment deadline.
104    pub connect_timeout: Duration,
105    /// Redirects followed before giving up.
106    pub max_redirects: usize,
107    /// Hard cap on a response body. Enforced while streaming, so an oversized
108    /// body costs one chunk, not the whole download.
109    pub max_body_bytes: usize,
110    /// Retries for retryable failures.
111    pub retries: u32,
112    /// Base delay for exponential backoff between retries.
113    pub retry_backoff: Duration,
114    /// Minimum spacing between two requests to the same host.
115    pub per_host_delay: Duration,
116    /// Honour `robots.txt`. Leave this on unless you own the target.
117    pub respect_robots: bool,
118    /// Honour a `Crawl-delay` larger than [`FetchConfig::per_host_delay`].
119    pub respect_crawl_delay: bool,
120    /// Browser fingerprint to present.
121    pub impersonate: Impersonate,
122    /// `Accept-Language` header.
123    pub accept_language: String,
124    /// Escalate JS-gated pages to headless Chrome. Needs the `browser` feature.
125    pub browser_fallback: bool,
126    /// Proxies to route requests through, rotated round-robin.
127    ///
128    /// The only real answer to IP-reputation blocking: a TLS fingerprint says
129    /// what your client is, and an address says who it is. Accepts `http://`,
130    /// `https://` and `socks5://` URLs, with optional `user:pass@`.
131    pub proxies: Vec<String>,
132    /// Honour a `Retry-After` header, up to this long. Set to zero to ignore it.
133    pub max_retry_after: Duration,
134    /// File to persist cookies to, so a session survives the process.
135    pub cookie_file: Option<std::path::PathBuf>,
136}
137
138impl Default for FetchConfig {
139    fn default() -> Self {
140        FetchConfig {
141            concurrency: 16,
142            timeout: Duration::from_secs(20),
143            connect_timeout: Duration::from_secs(8),
144            max_redirects: 5,
145            max_body_bytes: 8 * 1024 * 1024,
146            retries: 2,
147            retry_backoff: Duration::from_millis(400),
148            per_host_delay: Duration::from_millis(250),
149            respect_robots: true,
150            respect_crawl_delay: true,
151            impersonate: Impersonate::Chrome,
152            accept_language: "en-US,en;q=0.9".to_string(),
153            browser_fallback: false,
154            proxies: Vec::new(),
155            max_retry_after: Duration::from_secs(60),
156            cookie_file: None,
157        }
158    }
159}
160
161/// A fetched document.
162#[derive(Debug, Clone, serde::Serialize)]
163pub struct Page {
164    /// URL as requested.
165    pub url: String,
166    /// URL after redirects.
167    pub final_url: String,
168    /// HTTP status.
169    pub status: u16,
170    /// `Content-Type` header, if any.
171    pub content_type: Option<String>,
172    /// Decoded body.
173    pub body: String,
174    /// Wall time for the fetch, in milliseconds.
175    pub elapsed_ms: u64,
176    /// Whether the headless fallback produced this body.
177    pub rendered: bool,
178}
179
180/// A pooled, polite, fingerprint-aware HTTP client.
181#[derive(Clone)]
182pub struct Fetcher {
183    /// One client per proxy, or a single direct client. Rotated per request.
184    clients: Arc<Vec<wreq::Client>>,
185    next_client: Arc<std::sync::atomic::AtomicUsize>,
186    jar: Option<Arc<wreq::cookie::Jar>>,
187    /// Hosts this fetcher has touched. The cookie jar indexes by domain but
188    /// does not expose that key, so a host-only cookie — which is most session
189    /// cookies — cannot be attributed on the way back out without this.
190    visited: Arc<std::sync::Mutex<std::collections::BTreeSet<String>>>,
191    cfg: Arc<FetchConfig>,
192    gate: Arc<Semaphore>,
193    robots: Arc<Mutex<HashMap<String, Arc<Robots>>>>,
194    last_hit: Arc<Mutex<HashMap<String, Instant>>>,
195}
196
197impl std::fmt::Debug for Fetcher {
198    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
199        f.debug_struct("Fetcher").field("config", &self.cfg).finish_non_exhaustive()
200    }
201}
202
203impl Fetcher {
204    /// Build a fetcher with default settings.
205    pub fn new() -> Result<Self> {
206        Self::with_config(FetchConfig::default())
207    }
208
209    /// Build a fetcher from an explicit config.
210    pub fn with_config(cfg: FetchConfig) -> Result<Self> {
211        if cfg.concurrency == 0 {
212            return Err(Error::Config("concurrency must be at least 1".into()));
213        }
214        let jar = cfg.cookie_file.as_ref().map(|path| Arc::new(load_jar(path)));
215
216        let build_one = |proxy: Option<&str>| -> Result<wreq::Client> {
217            let mut builder = wreq::Client::builder()
218                .timeout(cfg.timeout)
219                .connect_timeout(cfg.connect_timeout)
220                .redirect(wreq::redirect::Policy::limited(cfg.max_redirects))
221                .pool_max_idle_per_host(cfg.concurrency.min(32))
222                .referer(true)
223                .gzip(true)
224                .brotli(true)
225                .deflate(true)
226                .zstd(true);
227
228            builder = match &jar {
229                Some(jar) => builder.cookie_provider(jar.clone()),
230                None => builder.cookie_store(true),
231            };
232            builder = apply_impersonation(builder, &cfg.impersonate)?;
233
234            if let Some(proxy) = proxy {
235                let proxy = wreq::Proxy::all(proxy)
236                    .map_err(|e| Error::Config(format!("invalid proxy `{proxy}`: {e}")))?;
237                builder = builder.proxy(proxy);
238            }
239
240            let mut headers = wreq::header::HeaderMap::new();
241            if let Ok(v) = wreq::header::HeaderValue::from_str(&cfg.accept_language) {
242                headers.insert(wreq::header::ACCEPT_LANGUAGE, v);
243            }
244            if matches!(cfg.impersonate, Impersonate::None) {
245                headers.insert(
246                    wreq::header::USER_AGENT,
247                    wreq::header::HeaderValue::from_static(DEFAULT_USER_AGENT),
248                );
249            }
250            builder = builder.default_headers(headers);
251            builder.build().map_err(|e| Error::Config(e.to_string()))
252        };
253
254        let clients = if cfg.proxies.is_empty() {
255            vec![build_one(None)?]
256        } else {
257            cfg.proxies.iter().map(|p| build_one(Some(p))).collect::<Result<Vec<_>>>()?
258        };
259
260        Ok(Fetcher {
261            clients: Arc::new(clients),
262            next_client: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
263            jar,
264            visited: Arc::new(std::sync::Mutex::new(std::collections::BTreeSet::new())),
265            gate: Arc::new(Semaphore::new(cfg.concurrency)),
266            cfg: Arc::new(cfg),
267            robots: Arc::new(Mutex::new(HashMap::new())),
268            last_hit: Arc::new(Mutex::new(HashMap::new())),
269        })
270    }
271
272    /// The next client in the rotation.
273    ///
274    /// Round-robin rather than random: with a small proxy pool, randomness
275    /// clusters, and clustering on one address is exactly what gets it banned.
276    fn client(&self) -> &wreq::Client {
277        if self.clients.len() == 1 {
278            return &self.clients[0];
279        }
280        let i = self.next_client.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
281        &self.clients[i % self.clients.len()]
282    }
283
284    /// Write the cookie jar to [`FetchConfig::cookie_file`].
285    ///
286    /// Clearance cookies are the expensive part of getting through a bot wall;
287    /// throwing them away when the process exits means paying for them again.
288    pub fn save_cookies(&self) -> Result<usize> {
289        let (Some(jar), Some(path)) = (&self.jar, &self.cfg.cookie_file) else { return Ok(0) };
290        let hosts: Vec<String> = match self.visited.lock() {
291            Ok(v) => v.iter().cloned().collect(),
292            Err(poisoned) => poisoned.into_inner().iter().cloned().collect(),
293        };
294        save_jar(jar, &hosts, path)
295    }
296
297    /// The configuration this fetcher was built with.
298    pub fn config(&self) -> &FetchConfig {
299        &self.cfg
300    }
301
302    /// Fetch one URL, honouring `robots.txt` when configured to.
303    pub async fn fetch(&self, url: &str) -> Result<Page> {
304        self.fetch_inner(url, self.cfg.respect_robots).await
305    }
306
307    /// Fetch a provider endpoint, skipping the `robots.txt` check.
308    ///
309    /// `robots.txt` governs crawling — the discovery of documents a site did
310    /// not hand you. A search or feed endpoint the caller named explicitly is
311    /// a query, not a crawl, and several of them (DuckDuckGo's HTML endpoint
312    /// among them) disallow the very path they exist to serve. Pages found
313    /// *through* those endpoints go through [`Fetcher::fetch`] and are checked
314    /// normally, which is where the rule actually protects anyone.
315    pub async fn fetch_api(&self, url: &str) -> Result<Page> {
316        self.fetch_inner(url, false).await
317    }
318
319    async fn fetch_inner(&self, url: &str, check_robots: bool) -> Result<Page> {
320        let parsed = normalize(url)?;
321        let _permit = self.gate.acquire().await.map_err(|e| Error::Config(e.to_string()))?;
322
323        let crawl_delay = if check_robots {
324            let robots = self.robots_for(&parsed).await;
325            if !robots.allows(parsed.path()) {
326                return Err(Error::RobotsDenied(parsed.to_string()));
327            }
328            robots.crawl_delay
329        } else {
330            None
331        };
332        self.pace(&parsed, crawl_delay).await;
333
334        let started = Instant::now();
335        let mut page = self.get_with_retry(&parsed).await?;
336
337        // Follow HTML-level redirects. `wreq` follows HTTP 3xx, but a stub page
338        // that redirects via `<meta refresh>` or `location.replace` returns a
339        // perfectly good 200 containing no content — which is exactly what
340        // happens on sites that canonicalise URLs in the browser.
341        let mut current = parsed.clone();
342        for _ in 0..MAX_HTML_REDIRECTS {
343            let Some(next) = detect_html_redirect(&page.body, &current) else { break };
344            if check_robots && !self.robots_for(&next).await.allows(next.path()) {
345                break;
346            }
347            self.pace(&next, crawl_delay).await;
348            match self.get_with_retry(&next).await {
349                Ok(followed) => {
350                    page = followed;
351                    current = next;
352                }
353                // The stub is still a better answer than an error.
354                Err(_) => break,
355            }
356        }
357        page.url = parsed.to_string();
358        page.elapsed_ms = started.elapsed().as_millis() as u64;
359
360        if self.cfg.browser_fallback && needs_javascript(&page) {
361            // A failed escalation is not a failed fetch: the static body is
362            // still the best answer we have.
363            if let Ok(rendered) = self.render(&parsed).await {
364                page.body = rendered;
365                page.rendered = true;
366                page.elapsed_ms = started.elapsed().as_millis() as u64;
367            }
368        }
369        Ok(page)
370    }
371
372    /// Fetch many URLs concurrently, preserving input order in the output.
373    ///
374    /// The concurrency ceiling and per-host spacing still apply, so this is
375    /// safe to call with a few hundred URLs at once.
376    pub async fn fetch_many(&self, urls: &[String]) -> Vec<Result<Page>> {
377        let futures = urls.iter().map(|u| {
378            let this = self.clone();
379            let u = u.clone();
380            async move { this.fetch(&u).await }
381        });
382        futures_util::stream::iter(futures).buffered(self.cfg.concurrency).collect::<Vec<_>>().await
383    }
384
385    /// One URL, with exponential backoff on retryable failures.
386    async fn get_with_retry(&self, url: &Url) -> Result<Page> {
387        let mut attempt = 0u32;
388        loop {
389            match self.get_once(url).await {
390                Ok(page) => return Ok(page),
391                Err(e) if e.is_retryable() && attempt < self.cfg.retries => {
392                    // Retrying a 429 immediately is how a client turns a soft
393                    // limit into a hard block. When the server says how long to
394                    // wait, that beats any backoff curve we could invent — but
395                    // an hour-long hint is a refusal, not a delay, so it is
396                    // capped and otherwise treated as a failure.
397                    let wait = match e.retry_after() {
398                        Some(hint) if hint <= self.cfg.max_retry_after => hint,
399                        Some(_) => return Err(e),
400                        None => self.cfg.retry_backoff * 2u32.pow(attempt),
401                    };
402                    tokio::time::sleep(wait).await;
403                    attempt += 1;
404                }
405                Err(e) => return Err(e),
406            }
407        }
408    }
409
410    async fn get_once(&self, url: &Url) -> Result<Page> {
411        let resp =
412            self.client().get(url.as_str()).send().await.map_err(|e| Error::network(url, e))?;
413
414        let status = resp.status().as_u16();
415        let final_url = resp.uri().to_string();
416        let content_type = resp
417            .headers()
418            .get(wreq::header::CONTENT_TYPE)
419            .and_then(|v| v.to_str().ok())
420            .map(str::to_string);
421
422        if !(200..300).contains(&status) {
423            let retry_after = resp
424                .headers()
425                .get(wreq::header::RETRY_AFTER)
426                .and_then(|v| v.to_str().ok())
427                .and_then(parse_retry_after);
428            return Err(Error::Status { url: url.to_string(), status, retry_after });
429        }
430        if let Some(ct) = &content_type {
431            let base = ct.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
432            if !TEXTUAL.iter().any(|t| base.starts_with(t)) {
433                return Err(Error::Parse(format!("{url}: unsupported content type {base}")));
434            }
435        }
436        if resp.content_length().is_some_and(|n| n as usize > self.cfg.max_body_bytes) {
437            return Err(Error::BodyTooLarge {
438                url: url.to_string(),
439                limit: self.cfg.max_body_bytes,
440            });
441        }
442
443        let charset = content_type.as_deref().and_then(charset_of);
444        let bytes = self.read_capped(url, resp).await?;
445        let body = decode(&bytes, charset.as_deref());
446
447        Ok(Page {
448            url: url.to_string(),
449            final_url,
450            status,
451            content_type,
452            body,
453            elapsed_ms: 0,
454            rendered: false,
455        })
456    }
457
458    /// Read a body, aborting as soon as it crosses the cap.
459    async fn read_capped(&self, url: &Url, resp: wreq::Response) -> Result<Vec<u8>> {
460        let mut buf: Vec<u8> = Vec::with_capacity(64 * 1024);
461        let mut stream = resp.bytes_stream();
462        while let Some(chunk) = stream.next().await {
463            let chunk = chunk.map_err(|e| Error::network(url, e))?;
464            if buf.len() + chunk.len() > self.cfg.max_body_bytes {
465                return Err(Error::BodyTooLarge {
466                    url: url.to_string(),
467                    limit: self.cfg.max_body_bytes,
468                });
469            }
470            buf.extend_from_slice(&chunk);
471        }
472        Ok(buf)
473    }
474
475    /// Fetch and cache `robots.txt` for a host.
476    ///
477    /// A missing or unreadable file is treated as "allow all", which is what
478    /// the RFC specifies for a 404.
479    async fn robots_for(&self, url: &Url) -> Arc<Robots> {
480        let host = url.host_str().unwrap_or_default().to_string();
481        let key = format!("{}://{host}", url.scheme());
482        if let Some(cached) = self.robots.lock().await.get(&key) {
483            return cached.clone();
484        }
485        let robots_url = format!("{key}/robots.txt");
486        let parsed = match self.client().get(&robots_url).send().await {
487            Ok(resp) if resp.status().is_success() => match resp.text().await {
488                Ok(body) => Robots::parse(&body, ROBOTS_AGENT),
489                Err(_) => Robots::allow_all(),
490            },
491            _ => Robots::allow_all(),
492        };
493        let parsed = Arc::new(parsed);
494        self.robots.lock().await.insert(key, parsed.clone());
495        parsed
496    }
497
498    /// Space requests to one host apart, honouring `Crawl-delay` when larger.
499    async fn pace(&self, url: &Url, crawl_delay: Option<f64>) {
500        let host = url.host_str().unwrap_or_default().to_string();
501        if let Ok(mut visited) = self.visited.lock() {
502            visited.insert(host.clone());
503        }
504        let mut delay = self.cfg.per_host_delay;
505        if self.cfg.respect_crawl_delay
506            && let Some(d) = crawl_delay
507        {
508            delay = delay.max(Duration::from_secs_f64(d.min(30.0)));
509        }
510        if delay.is_zero() {
511            return;
512        }
513        let wait = {
514            let mut map = self.last_hit.lock().await;
515            let now = Instant::now();
516            let wait = match map.get(&host) {
517                Some(prev) => delay.checked_sub(now.duration_since(*prev)),
518                None => None,
519            };
520            // Reserve our slot before releasing the lock, so concurrent callers
521            // to the same host queue up instead of all sleeping the same amount.
522            map.insert(host, now + wait.unwrap_or_default());
523            wait
524        };
525        if let Some(wait) = wait {
526            tokio::time::sleep(wait).await;
527        }
528    }
529
530    #[cfg(feature = "browser")]
531    async fn render(&self, url: &Url) -> Result<String> {
532        browser::render(url.as_str(), self.cfg.timeout).await
533    }
534
535    #[cfg(not(feature = "browser"))]
536    async fn render(&self, _url: &Url) -> Result<String> {
537        Err(Error::Browser("build rustai with the `browser` feature".into()))
538    }
539}
540
541fn apply_impersonation(
542    builder: wreq::ClientBuilder,
543    mode: &Impersonate,
544) -> Result<wreq::ClientBuilder> {
545    #[cfg(not(feature = "impersonate"))]
546    {
547        if !matches!(mode, Impersonate::None) {
548            return Err(Error::Config(
549                "browser impersonation needs the `impersonate` feature".into(),
550            ));
551        }
552        Ok(builder)
553    }
554    #[cfg(feature = "impersonate")]
555    {
556        use wreq_util::{Emulation, Profile};
557        let profile = match mode {
558            Impersonate::None => return Ok(builder),
559            Impersonate::Random => return Ok(builder.emulation(Emulation::weighted_random())),
560            Impersonate::Chrome => Profile::Chrome149,
561            Impersonate::Firefox => Profile::Firefox147,
562            Impersonate::Safari => Profile::Safari26_4,
563            Impersonate::Named(name) => named_profile(name)?,
564        };
565        Ok(builder.emulation(profile))
566    }
567}
568
569/// Resolve a `wreq-util` profile from its wire name, e.g. `"chrome_143"`.
570///
571/// The enum has no `FromStr`, but it does derive `Deserialize` with exactly
572/// these renames under the `emulation-serde` feature, so serde is the
573/// supported way in — and it stays correct as upstream adds profiles.
574#[cfg(feature = "impersonate")]
575fn named_profile(name: &str) -> Result<wreq_util::Profile> {
576    serde_json::from_value::<wreq_util::Profile>(serde_json::Value::String(name.to_string()))
577        .map_err(|_| {
578            Error::Config(format!(
579                "unknown impersonation profile `{name}`; try `chrome`, `firefox`, `safari`, \
580                 `random`, or a versioned name such as `chrome_143`"
581            ))
582        })
583}
584
585/// Parse a `Retry-After` value into seconds.
586///
587/// Only the delta-seconds form. The HTTP-date form is rare in practice and
588/// parsing it would mean carrying a date library for one header; an unparsed
589/// value simply falls back to exponential backoff.
590fn parse_retry_after(value: &str) -> Option<f64> {
591    let secs: f64 = value.trim().parse().ok()?;
592    secs.is_finite().then_some(secs.max(0.0))
593}
594
595/// One persisted cookie, with the host it belongs to.
596#[derive(serde::Serialize, serde::Deserialize)]
597struct StoredCookie {
598    name: String,
599    value: String,
600    /// Host the cookie was collected from.
601    host: String,
602    path: String,
603    /// `Domain` attribute, when the cookie carried one. Absent means host-only,
604    /// and reloading must keep it that way.
605    #[serde(default, skip_serializing_if = "Option::is_none")]
606    domain: Option<String>,
607}
608
609/// Read a cookie jar from disk. A missing or unreadable file yields an empty
610/// jar: a lost session is a slow start, never an error.
611fn load_jar(path: &std::path::Path) -> wreq::cookie::Jar {
612    let jar = wreq::cookie::Jar::default();
613    let Ok(raw) = std::fs::read_to_string(path) else { return jar };
614    let Ok(stored) = serde_json::from_str::<Vec<StoredCookie>>(&raw) else { return jar };
615    for c in stored {
616        let uri = format!("https://{}{}", c.host, c.path);
617        let mut cookie = format!("{}={}; Path={}", c.name, c.value, c.path);
618        if let Some(domain) = &c.domain {
619            cookie.push_str(&format!("; Domain={domain}"));
620        }
621        jar.add(cookie.as_str(), uri.as_str());
622    }
623    jar
624}
625
626/// Write a cookie jar to disk, returning how many cookies were saved.
627fn save_jar(jar: &wreq::cookie::Jar, hosts: &[String], path: &std::path::Path) -> Result<usize> {
628    // Asking the jar what it would send to each host we visited is the only way
629    // to recover a host-only cookie's owner: `get_all` returns the cookies but
630    // not the domain key they were filed under.
631    let mut seen = std::collections::HashSet::new();
632    let mut stored: Vec<StoredCookie> = Vec::new();
633    for host in hosts {
634        for c in jar.matches(format!("https://{host}/").as_str()) {
635            let cookie = StoredCookie {
636                name: c.name().to_string(),
637                value: c.value().to_string(),
638                host: host.clone(),
639                path: c.path().unwrap_or("/").to_string(),
640                domain: c.domain().map(str::to_string),
641            };
642            if seen.insert((cookie.host.clone(), cookie.name.clone(), cookie.path.clone())) {
643                stored.push(cookie);
644            }
645        }
646    }
647    let json = serde_json::to_string_pretty(&stored)
648        .map_err(|e| Error::Config(format!("could not serialise cookies: {e}")))?;
649    if let Some(parent) = path.parent()
650        && !parent.as_os_str().is_empty()
651    {
652        let _ = std::fs::create_dir_all(parent);
653    }
654    std::fs::write(path, json)
655        .map_err(|e| Error::Config(format!("could not write {}: {e}", path.display())))?;
656    Ok(stored.len())
657}
658
659/// Reject anything that is not an absolute http(s) URL.
660pub fn normalize(url: &str) -> Result<Url> {
661    let parsed = Url::parse(url.trim()).map_err(|_| Error::InvalidUrl(url.to_string()))?;
662    if !matches!(parsed.scheme(), "http" | "https") {
663        return Err(Error::InvalidUrl(url.to_string()));
664    }
665    if parsed.host_str().is_none() {
666        return Err(Error::InvalidUrl(url.to_string()));
667    }
668    Ok(parsed)
669}
670
671fn charset_of(content_type: &str) -> Option<String> {
672    content_type
673        .split(';')
674        .skip(1)
675        .filter_map(|p| p.trim().strip_prefix("charset="))
676        .map(|c| c.trim_matches(['"', '\'']).to_ascii_lowercase())
677        .next()
678}
679
680/// Decode a body, trusting the header, then a `<meta charset>`, then UTF-8.
681///
682/// Legacy encodings are still common outside the anglosphere — EUC-KR and
683/// Shift_JIS in particular — and mojibake destroys every downstream heuristic,
684/// so this is worth getting right rather than assuming UTF-8.
685fn decode(bytes: &[u8], header_charset: Option<&str>) -> String {
686    let label = header_charset.map(str::to_string).or_else(|| sniff_meta_charset(bytes));
687    let encoding = label
688        .as_deref()
689        .and_then(|l| encoding_rs::Encoding::for_label(l.as_bytes()))
690        .unwrap_or(encoding_rs::UTF_8);
691    let (text, _, _) = encoding.decode(bytes);
692    text.into_owned()
693}
694
695/// Look for `<meta charset=...>` in the first 4 KiB, as browsers do.
696fn sniff_meta_charset(bytes: &[u8]) -> Option<String> {
697    let head = &bytes[..bytes.len().min(4096)];
698    let text = String::from_utf8_lossy(head).to_ascii_lowercase();
699    if let Some(at) = text.find("charset=") {
700        let rest = &text[at + "charset=".len()..];
701        let value: String = rest
702            .trim_start_matches(['"', '\'', ' '])
703            .chars()
704            .take_while(|c| c.is_ascii_alphanumeric() || *c == '-' || *c == '_')
705            .collect();
706        if !value.is_empty() {
707            return Some(value);
708        }
709    }
710    None
711}
712
713/// Find an HTML-level redirect target in a page body.
714///
715/// Two forms matter in practice: `<meta http-equiv="refresh" content="0; url=…">`
716/// and a script that assigns `location`. Both are gated on the page being small
717/// and on a zero delay, because a redirect stub is always tiny — a long article
718/// that happens to contain `location.href` somewhere must not be mistaken for
719/// one.
720fn detect_html_redirect(body: &str, base: &Url) -> Option<Url> {
721    const MAX_STUB_BYTES: usize = 4096;
722    if body.len() > MAX_STUB_BYTES {
723        return None;
724    }
725    let target = meta_refresh_target(body).or_else(|| script_location_target(body))?;
726    let next = base.join(target.trim().trim_matches(['"', '\''])).ok()?;
727    if !matches!(next.scheme(), "http" | "https") {
728        return None;
729    }
730    // A stub pointing at itself is a loop, not a redirect.
731    let same = next.as_str().trim_end_matches('/') == base.as_str().trim_end_matches('/');
732    (!same).then_some(next)
733}
734
735/// `<meta http-equiv="refresh" content="0; url=TARGET">`, delay 0 only.
736fn meta_refresh_target(body: &str) -> Option<&str> {
737    let lower = body.to_ascii_lowercase();
738    let mut from = 0usize;
739    while let Some(at) = lower[from..].find("http-equiv") {
740        let tag_start = lower[..from + at].rfind('<')?;
741        let tag_end = lower[tag_start..].find('>')? + tag_start;
742        let tag = &lower[tag_start..tag_end];
743        if tag.contains("refresh")
744            && let Some(content_at) = tag.find("content")
745            && let Some(url_at) = tag[content_at..].find("url=")
746        {
747            // Only an immediate redirect; a delayed one is a page in its own right.
748            let delay: String = tag[content_at..]
749                .chars()
750                .skip_while(|c| !c.is_ascii_digit())
751                .take_while(|c| c.is_ascii_digit())
752                .collect();
753            if delay.parse::<u32>().unwrap_or(0) == 0 {
754                let start = tag_start + content_at + url_at + 4;
755                let rest = &body[start..];
756                let end = rest.find(['"', '\'', '>']).unwrap_or(rest.len());
757                let target = rest[..end].trim();
758                if !target.is_empty() {
759                    return Some(target);
760                }
761            }
762        }
763        from += at + "http-equiv".len();
764    }
765    None
766}
767
768/// `location.replace("TARGET")`, `location.href = "TARGET"`, and friends.
769fn script_location_target(body: &str) -> Option<&str> {
770    for marker in ["location.replace(", "location.href=", "location.href =", "location.assign("] {
771        let Some(at) = body.find(marker) else { continue };
772        let rest = &body[at + marker.len()..];
773        let rest = rest.trim_start();
774        let quote = rest.chars().next()?;
775        if quote != '"' && quote != '\'' {
776            // An indirection through a variable; resolving it needs a JS engine.
777            continue;
778        }
779        let inner = &rest[quote.len_utf8()..];
780        let end = inner.find(quote)?;
781        let target = &inner[..end];
782        if target.starts_with("http") || target.starts_with('/') {
783            return Some(target);
784        }
785    }
786    None
787}
788
789/// Does this response look like an empty shell waiting for JavaScript?
790///
791/// Deliberately conservative. Rendering a page costs ~100x a fetch, so the
792/// bar is "there is markup but essentially no text", not "the page mentions
793/// JavaScript".
794pub fn needs_javascript(page: &Page) -> bool {
795    let html = &page.body;
796    if html.len() < 512 {
797        return false;
798    }
799    let Ok(article) = crate::parse::extract(html, Some(&page.final_url)) else {
800        return true;
801    };
802    let text_len = crate::text::visible_len(&article.text);
803
804    // Plenty of text: whatever else the page is doing, we already have it.
805    if text_len > MAX_STATIC_TEXT {
806        return false;
807    }
808
809    // Markup carrying essentially no text. An absolute floor misses the common
810    // case — a 350 KB front page yielding 510 characters clears any fixed
811    // threshold while plainly being an app shell — so the test is the ratio.
812    // Real pages run percent-level text-to-markup; a shell runs a tenth of that.
813    let starved = text_len.saturating_mul(TEXT_TO_MARKUP_FLOOR) < html.len();
814    if text_len < 120 || starved {
815        return true;
816    }
817
818    let lower = html.to_lowercase();
819    let spa_markers = ["__next_data__", "id=\"root\"", "id=\"app\"", "ng-app", "data-reactroot"];
820    spa_markers.iter().any(|m| lower.contains(m))
821}
822
823#[cfg(test)]
824mod tests {
825    use super::*;
826
827    #[test]
828    fn rejects_non_http_urls() {
829        assert!(normalize("file:///etc/passwd").is_err());
830        assert!(normalize("javascript:alert(1)").is_err());
831        assert!(normalize("not a url").is_err());
832        assert!(normalize("https://example.com/a").is_ok());
833    }
834
835    #[test]
836    fn parses_charset_from_content_type() {
837        assert_eq!(charset_of("text/html; charset=EUC-KR").as_deref(), Some("euc-kr"));
838        assert_eq!(charset_of("text/html").as_deref(), None);
839    }
840
841    #[test]
842    fn decodes_legacy_korean_encoding() {
843        let (bytes, _, _) = encoding_rs::EUC_KR.encode("한글 인코딩");
844        assert_eq!(decode(&bytes, Some("euc-kr")), "한글 인코딩");
845        // Without the header, the meta tag is the fallback.
846        let mut doc = b"<html><head><meta charset=\"euc-kr\"></head><body>".to_vec();
847        doc.extend_from_slice(&bytes);
848        assert!(decode(&doc, None).contains("한글 인코딩"));
849    }
850
851    #[test]
852    fn impersonate_strings_parse() {
853        assert_eq!(Impersonate::parse("chrome").unwrap(), Impersonate::Chrome);
854        assert_eq!(Impersonate::parse("none").unwrap(), Impersonate::None);
855        assert_eq!(
856            Impersonate::parse("chrome_143").unwrap(),
857            Impersonate::Named("chrome_143".into())
858        );
859    }
860
861    #[cfg(feature = "impersonate")]
862    #[test]
863    fn named_profiles_resolve_and_reject() {
864        assert!(named_profile("chrome_143").is_ok());
865        assert!(named_profile("netscape_2").is_err());
866    }
867
868    #[test]
869    fn client_builds_with_every_preset() {
870        for mode in ["chrome", "firefox", "safari", "random", "none"] {
871            let cfg = FetchConfig {
872                impersonate: Impersonate::parse(mode).unwrap(),
873                ..Default::default()
874            };
875            assert!(Fetcher::with_config(cfg).is_ok(), "{mode} failed to build");
876        }
877    }
878
879    #[test]
880    fn retry_after_is_parsed_and_capped() {
881        assert_eq!(parse_retry_after("120"), Some(120.0));
882        assert_eq!(parse_retry_after("  0 "), Some(0.0));
883        assert_eq!(parse_retry_after("-5"), Some(0.0));
884        // The HTTP-date form is not parsed; backoff takes over.
885        assert_eq!(parse_retry_after("Wed, 21 Oct 2026 07:28:00 GMT"), None);
886        assert_eq!(parse_retry_after(""), None);
887
888        let e =
889            Error::Status { url: "https://x.dev/".into(), status: 429, retry_after: Some(30.0) };
890        assert_eq!(e.retry_after(), Some(Duration::from_secs(30)));
891        assert!(e.is_retryable());
892        let none = Error::Status { url: "https://x.dev/".into(), status: 429, retry_after: None };
893        assert_eq!(none.retry_after(), None);
894    }
895
896    #[test]
897    fn proxies_build_one_client_each() {
898        let cfg = FetchConfig {
899            proxies: vec!["http://127.0.0.1:8080".into(), "socks5://127.0.0.1:1080".into()],
900            ..Default::default()
901        };
902        let fetcher = Fetcher::with_config(cfg).expect("proxied client");
903        assert_eq!(fetcher.clients.len(), 2);
904        // Round-robin, so a small pool cannot cluster on one address.
905        let first = fetcher.client() as *const _;
906        let second = fetcher.client() as *const _;
907        assert_ne!(first, second);
908        assert_eq!(fetcher.client() as *const _, first);
909    }
910
911    #[test]
912    fn an_invalid_proxy_is_rejected_at_build_time() {
913        let cfg = FetchConfig { proxies: vec!["not a proxy".into()], ..Default::default() };
914        assert!(Fetcher::with_config(cfg).is_err());
915    }
916
917    #[test]
918    fn cookies_survive_a_round_trip_through_disk() {
919        let dir = std::env::temp_dir().join(format!("rustai-cookies-{}", std::process::id()));
920        let path = dir.join("jar.json");
921        let _ = std::fs::remove_file(&path);
922
923        let jar = wreq::cookie::Jar::default();
924        // A domain cookie and a host-only one: the latter is what most session
925        // cookies actually are, and it has no Domain attribute to save.
926        jar.add("cf_clearance=abc123; Domain=example.com; Path=/", "https://example.com/");
927        jar.add("session=xyz789; Path=/", "https://example.com/");
928        let hosts = vec!["example.com".to_string()];
929        assert_eq!(save_jar(&jar, &hosts, &path).unwrap(), 2);
930
931        let reloaded = load_jar(&path);
932        assert!(reloaded.contains("session", "https://example.com/"), "host-only cookie lost");
933        assert!(reloaded.contains("cf_clearance", "https://example.com/"), "cookie was lost");
934        assert_eq!(
935            reloaded.get("cf_clearance", "https://example.com/").map(|c| c.value().to_string()),
936            Some("abc123".to_string())
937        );
938
939        // A missing file is a slow start, not an error.
940        assert_eq!(load_jar(&dir.join("nope.json")).len(), 0);
941        let _ = std::fs::remove_dir_all(&dir);
942    }
943
944    #[test]
945    fn a_fetcher_without_a_cookie_file_saves_nothing() {
946        let fetcher = Fetcher::new().unwrap();
947        assert_eq!(fetcher.save_cookies().unwrap(), 0);
948    }
949
950    #[test]
951    fn zero_concurrency_is_rejected() {
952        let cfg = FetchConfig { concurrency: 0, ..Default::default() };
953        assert!(Fetcher::with_config(cfg).is_err());
954    }
955
956    fn page(body: &str) -> Page {
957        Page {
958            url: "https://x.dev/".into(),
959            final_url: "https://x.dev/".into(),
960            status: 200,
961            content_type: Some("text/html".into()),
962            body: body.into(),
963            elapsed_ms: 0,
964            rendered: false,
965        }
966    }
967
968    #[test]
969    fn follows_a_meta_refresh_stub() {
970        let base = Url::parse("https://blog.dev/2024/02/08/Post.html").unwrap();
971        let body = r#"<!doctype html><title>Redirect</title>
972            <noscript><meta http-equiv="refresh" content="0; url=https://blog.dev/2024/02/08/Post/"></noscript>"#;
973        assert_eq!(
974            detect_html_redirect(body, &base).unwrap().as_str(),
975            "https://blog.dev/2024/02/08/Post/"
976        );
977    }
978
979    #[test]
980    fn follows_a_script_location_stub() {
981        let base = Url::parse("https://blog.dev/a.html").unwrap();
982        let body = r#"<!doctype html><title>Redirect</title><script>
983            const target = "https://blog.dev/a/";
984            window.location.replace("https://blog.dev/a/");
985        </script>"#;
986        assert_eq!(detect_html_redirect(body, &base).unwrap().as_str(), "https://blog.dev/a/");
987    }
988
989    #[test]
990    fn resolves_a_relative_redirect_target() {
991        let base = Url::parse("https://blog.dev/x/a.html").unwrap();
992        let body = r#"<meta http-equiv="refresh" content="0;url=/y/b.html">"#;
993        assert_eq!(
994            detect_html_redirect(body, &base).unwrap().as_str(),
995            "https://blog.dev/y/b.html"
996        );
997    }
998
999    #[test]
1000    fn ignores_delayed_refreshes_and_self_loops() {
1001        let base = Url::parse("https://blog.dev/a").unwrap();
1002        // A 5-second refresh is a real page that reloads, not a redirect.
1003        assert!(
1004            detect_html_redirect(
1005                r#"<meta http-equiv="refresh" content="5; url=https://blog.dev/b">"#,
1006                &base
1007            )
1008            .is_none()
1009        );
1010        // Pointing at itself is a loop.
1011        assert!(
1012            detect_html_redirect(
1013                r#"<meta http-equiv="refresh" content="0; url=https://blog.dev/a">"#,
1014                &base
1015            )
1016            .is_none()
1017        );
1018    }
1019
1020    #[test]
1021    fn ignores_a_long_page_that_merely_mentions_location() {
1022        let base = Url::parse("https://blog.dev/a").unwrap();
1023        let article = format!(
1024            "<article><p>{}</p><p>You can call location.replace(\"https://evil.dev/\") in JS.</p></article>",
1025            "Real prose. ".repeat(500)
1026        );
1027        assert!(
1028            detect_html_redirect(&article, &base).is_none(),
1029            "a full article was treated as a stub"
1030        );
1031    }
1032
1033    #[test]
1034    fn ignores_non_http_and_unresolvable_targets() {
1035        let base = Url::parse("https://blog.dev/a").unwrap();
1036        assert!(
1037            detect_html_redirect(
1038                r#"<meta http-equiv="refresh" content="0;url=javascript:x()">"#,
1039                &base
1040            )
1041            .is_none()
1042        );
1043        assert!(detect_html_redirect("<p>nothing here</p>", &base).is_none());
1044        // A variable indirection needs a JS engine; we must not guess.
1045        assert!(
1046            detect_html_redirect(r#"<script>window.location.replace(target);</script>"#, &base)
1047                .is_none()
1048        );
1049    }
1050
1051    #[test]
1052    fn detects_an_spa_shell() {
1053        let shell = format!(
1054            "<html><body><div id=\"root\"></div><script>{}</script></body></html>",
1055            "x".repeat(2000)
1056        );
1057        assert!(needs_javascript(&page(&shell)));
1058    }
1059
1060    #[test]
1061    fn does_not_escalate_a_real_article() {
1062        let real = format!(
1063            "<html><body><article><h1>T</h1><p>{}</p></article></body></html>",
1064            "Real prose that a reader can actually read. ".repeat(20)
1065        );
1066        assert!(!needs_javascript(&page(&real)));
1067    }
1068}
1069
1070#[cfg(test)]
1071mod js_gate_tests {
1072    use super::*;
1073
1074    fn page(body: String) -> Page {
1075        Page {
1076            url: "https://x.dev/".into(),
1077            final_url: "https://x.dev/".into(),
1078            status: 200,
1079            content_type: Some("text/html".into()),
1080            body,
1081            elapsed_ms: 0,
1082            rendered: false,
1083        }
1084    }
1085
1086    /// Modelled on a real front page whose stories render client-side: ~350 KB
1087    /// of markup carrying ~500 characters of text. An absolute text threshold
1088    /// waves that through — 500 clears any sane floor — while the ratio does
1089    /// not. Index detection cannot rescue such a page either: there is nothing
1090    /// in the HTML to index.
1091    #[test]
1092    fn a_client_rendered_front_page_is_flagged() {
1093        let shell = format!(
1094            "<html><body><nav>{}</nav><main></main><script>{}</script></body></html>",
1095            "<a href=\"/section\">Section link</a>".repeat(20),
1096            "x".repeat(340_000)
1097        );
1098        assert!(needs_javascript(&page(shell)));
1099    }
1100
1101    /// The same size of page, but the markup actually carries the article.
1102    #[test]
1103    fn a_large_real_article_is_not_flagged() {
1104        let real = format!(
1105            "<html><body><article><h1>T</h1><p>{}</p></article></body></html>",
1106            "Real prose that a reader can actually read and act on. ".repeat(400)
1107        );
1108        assert!(!needs_javascript(&page(real)));
1109    }
1110}