1pub mod robots;
17
18#[cfg(feature = "browser")]
19pub mod browser;
20
21use std::collections::HashMap;
22use std::sync::Arc;
23use std::time::{Duration, Instant};
24
25use futures_util::StreamExt;
26use tokio::sync::{Mutex, Semaphore};
27use url::Url;
28
29use crate::error::{Error, Result};
30use robots::Robots;
31
32pub const DEFAULT_USER_AGENT: &str =
34 concat!("rustai/", env!("CARGO_PKG_VERSION"), " (+https://github.com/imhyensuk/rustai)");
35
36pub const ROBOTS_AGENT: &str = "rustai";
38
39const MAX_HTML_REDIRECTS: usize = 2;
44
45const MAX_STATIC_TEXT: usize = 2_000;
47
48const TEXT_TO_MARKUP_FLOOR: usize = 200;
51
52const TEXTUAL: &[&str] = &[
54 "text/html",
55 "application/xhtml",
56 "text/plain",
57 "application/xml",
58 "text/xml",
59 "application/rss",
60 "application/atom",
61 "application/json",
62];
63
64#[derive(Debug, Clone, Default, PartialEq, Eq)]
66pub enum Impersonate {
67 #[default]
69 Chrome,
70 Firefox,
72 Safari,
74 Named(String),
76 Random,
78 None,
80}
81
82impl Impersonate {
83 pub fn parse(s: &str) -> Result<Impersonate> {
85 Ok(match s.trim().to_ascii_lowercase().as_str() {
86 "" | "none" | "off" | "false" => Impersonate::None,
87 "chrome" | "true" | "default" => Impersonate::Chrome,
88 "firefox" => Impersonate::Firefox,
89 "safari" => Impersonate::Safari,
90 "random" => Impersonate::Random,
91 other => Impersonate::Named(other.to_string()),
92 })
93 }
94}
95
96#[derive(Debug, Clone)]
98pub struct FetchConfig {
99 pub concurrency: usize,
101 pub timeout: Duration,
103 pub connect_timeout: Duration,
105 pub max_redirects: usize,
107 pub max_body_bytes: usize,
110 pub retries: u32,
112 pub retry_backoff: Duration,
114 pub per_host_delay: Duration,
116 pub respect_robots: bool,
118 pub respect_crawl_delay: bool,
120 pub impersonate: Impersonate,
122 pub accept_language: String,
124 pub browser_fallback: bool,
126 pub proxies: Vec<String>,
132 pub max_retry_after: Duration,
134 pub cookie_file: Option<std::path::PathBuf>,
136}
137
138impl Default for FetchConfig {
139 fn default() -> Self {
140 FetchConfig {
141 concurrency: 16,
142 timeout: Duration::from_secs(20),
143 connect_timeout: Duration::from_secs(8),
144 max_redirects: 5,
145 max_body_bytes: 8 * 1024 * 1024,
146 retries: 2,
147 retry_backoff: Duration::from_millis(400),
148 per_host_delay: Duration::from_millis(250),
149 respect_robots: true,
150 respect_crawl_delay: true,
151 impersonate: Impersonate::Chrome,
152 accept_language: "en-US,en;q=0.9".to_string(),
153 browser_fallback: false,
154 proxies: Vec::new(),
155 max_retry_after: Duration::from_secs(60),
156 cookie_file: None,
157 }
158 }
159}
160
161#[derive(Debug, Clone, serde::Serialize)]
163pub struct Page {
164 pub url: String,
166 pub final_url: String,
168 pub status: u16,
170 pub content_type: Option<String>,
172 pub body: String,
174 pub elapsed_ms: u64,
176 pub rendered: bool,
178}
179
180#[derive(Clone)]
182pub struct Fetcher {
183 clients: Arc<Vec<wreq::Client>>,
185 next_client: Arc<std::sync::atomic::AtomicUsize>,
186 jar: Option<Arc<wreq::cookie::Jar>>,
187 visited: Arc<std::sync::Mutex<std::collections::BTreeSet<String>>>,
191 cfg: Arc<FetchConfig>,
192 gate: Arc<Semaphore>,
193 robots: Arc<Mutex<HashMap<String, Arc<Robots>>>>,
194 last_hit: Arc<Mutex<HashMap<String, Instant>>>,
195}
196
197impl std::fmt::Debug for Fetcher {
198 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
199 f.debug_struct("Fetcher").field("config", &self.cfg).finish_non_exhaustive()
200 }
201}
202
203impl Fetcher {
204 pub fn new() -> Result<Self> {
206 Self::with_config(FetchConfig::default())
207 }
208
209 pub fn with_config(cfg: FetchConfig) -> Result<Self> {
211 if cfg.concurrency == 0 {
212 return Err(Error::Config("concurrency must be at least 1".into()));
213 }
214 let jar = cfg.cookie_file.as_ref().map(|path| Arc::new(load_jar(path)));
215
216 let build_one = |proxy: Option<&str>| -> Result<wreq::Client> {
217 let mut builder = wreq::Client::builder()
218 .timeout(cfg.timeout)
219 .connect_timeout(cfg.connect_timeout)
220 .redirect(wreq::redirect::Policy::limited(cfg.max_redirects))
221 .pool_max_idle_per_host(cfg.concurrency.min(32))
222 .referer(true)
223 .gzip(true)
224 .brotli(true)
225 .deflate(true)
226 .zstd(true);
227
228 builder = match &jar {
229 Some(jar) => builder.cookie_provider(jar.clone()),
230 None => builder.cookie_store(true),
231 };
232 builder = apply_impersonation(builder, &cfg.impersonate)?;
233
234 if let Some(proxy) = proxy {
235 let proxy = wreq::Proxy::all(proxy)
236 .map_err(|e| Error::Config(format!("invalid proxy `{proxy}`: {e}")))?;
237 builder = builder.proxy(proxy);
238 }
239
240 let mut headers = wreq::header::HeaderMap::new();
241 if let Ok(v) = wreq::header::HeaderValue::from_str(&cfg.accept_language) {
242 headers.insert(wreq::header::ACCEPT_LANGUAGE, v);
243 }
244 if matches!(cfg.impersonate, Impersonate::None) {
245 headers.insert(
246 wreq::header::USER_AGENT,
247 wreq::header::HeaderValue::from_static(DEFAULT_USER_AGENT),
248 );
249 }
250 builder = builder.default_headers(headers);
251 builder.build().map_err(|e| Error::Config(e.to_string()))
252 };
253
254 let clients = if cfg.proxies.is_empty() {
255 vec![build_one(None)?]
256 } else {
257 cfg.proxies.iter().map(|p| build_one(Some(p))).collect::<Result<Vec<_>>>()?
258 };
259
260 Ok(Fetcher {
261 clients: Arc::new(clients),
262 next_client: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
263 jar,
264 visited: Arc::new(std::sync::Mutex::new(std::collections::BTreeSet::new())),
265 gate: Arc::new(Semaphore::new(cfg.concurrency)),
266 cfg: Arc::new(cfg),
267 robots: Arc::new(Mutex::new(HashMap::new())),
268 last_hit: Arc::new(Mutex::new(HashMap::new())),
269 })
270 }
271
272 fn client(&self) -> &wreq::Client {
277 if self.clients.len() == 1 {
278 return &self.clients[0];
279 }
280 let i = self.next_client.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
281 &self.clients[i % self.clients.len()]
282 }
283
284 pub fn save_cookies(&self) -> Result<usize> {
289 let (Some(jar), Some(path)) = (&self.jar, &self.cfg.cookie_file) else { return Ok(0) };
290 let hosts: Vec<String> = match self.visited.lock() {
291 Ok(v) => v.iter().cloned().collect(),
292 Err(poisoned) => poisoned.into_inner().iter().cloned().collect(),
293 };
294 save_jar(jar, &hosts, path)
295 }
296
297 pub fn config(&self) -> &FetchConfig {
299 &self.cfg
300 }
301
302 pub async fn fetch(&self, url: &str) -> Result<Page> {
304 self.fetch_inner(url, self.cfg.respect_robots).await
305 }
306
307 pub async fn fetch_api(&self, url: &str) -> Result<Page> {
316 self.fetch_inner(url, false).await
317 }
318
319 async fn fetch_inner(&self, url: &str, check_robots: bool) -> Result<Page> {
320 let parsed = normalize(url)?;
321 let _permit = self.gate.acquire().await.map_err(|e| Error::Config(e.to_string()))?;
322
323 let crawl_delay = if check_robots {
324 let robots = self.robots_for(&parsed).await;
325 if !robots.allows(parsed.path()) {
326 return Err(Error::RobotsDenied(parsed.to_string()));
327 }
328 robots.crawl_delay
329 } else {
330 None
331 };
332 self.pace(&parsed, crawl_delay).await;
333
334 let started = Instant::now();
335 let mut page = self.get_with_retry(&parsed).await?;
336
337 let mut current = parsed.clone();
342 for _ in 0..MAX_HTML_REDIRECTS {
343 let Some(next) = detect_html_redirect(&page.body, ¤t) else { break };
344 if check_robots && !self.robots_for(&next).await.allows(next.path()) {
345 break;
346 }
347 self.pace(&next, crawl_delay).await;
348 match self.get_with_retry(&next).await {
349 Ok(followed) => {
350 page = followed;
351 current = next;
352 }
353 Err(_) => break,
355 }
356 }
357 page.url = parsed.to_string();
358 page.elapsed_ms = started.elapsed().as_millis() as u64;
359
360 if self.cfg.browser_fallback && needs_javascript(&page) {
361 if let Ok(rendered) = self.render(&parsed).await {
364 page.body = rendered;
365 page.rendered = true;
366 page.elapsed_ms = started.elapsed().as_millis() as u64;
367 }
368 }
369 Ok(page)
370 }
371
372 pub async fn fetch_many(&self, urls: &[String]) -> Vec<Result<Page>> {
377 let futures = urls.iter().map(|u| {
378 let this = self.clone();
379 let u = u.clone();
380 async move { this.fetch(&u).await }
381 });
382 futures_util::stream::iter(futures).buffered(self.cfg.concurrency).collect::<Vec<_>>().await
383 }
384
385 async fn get_with_retry(&self, url: &Url) -> Result<Page> {
387 let mut attempt = 0u32;
388 loop {
389 match self.get_once(url).await {
390 Ok(page) => return Ok(page),
391 Err(e) if e.is_retryable() && attempt < self.cfg.retries => {
392 let wait = match e.retry_after() {
398 Some(hint) if hint <= self.cfg.max_retry_after => hint,
399 Some(_) => return Err(e),
400 None => self.cfg.retry_backoff * 2u32.pow(attempt),
401 };
402 tokio::time::sleep(wait).await;
403 attempt += 1;
404 }
405 Err(e) => return Err(e),
406 }
407 }
408 }
409
410 async fn get_once(&self, url: &Url) -> Result<Page> {
411 let resp =
412 self.client().get(url.as_str()).send().await.map_err(|e| Error::network(url, e))?;
413
414 let status = resp.status().as_u16();
415 let final_url = resp.uri().to_string();
416 let content_type = resp
417 .headers()
418 .get(wreq::header::CONTENT_TYPE)
419 .and_then(|v| v.to_str().ok())
420 .map(str::to_string);
421
422 if !(200..300).contains(&status) {
423 let retry_after = resp
424 .headers()
425 .get(wreq::header::RETRY_AFTER)
426 .and_then(|v| v.to_str().ok())
427 .and_then(parse_retry_after);
428 return Err(Error::Status { url: url.to_string(), status, retry_after });
429 }
430 if let Some(ct) = &content_type {
431 let base = ct.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
432 if !TEXTUAL.iter().any(|t| base.starts_with(t)) {
433 return Err(Error::Parse(format!("{url}: unsupported content type {base}")));
434 }
435 }
436 if resp.content_length().is_some_and(|n| n as usize > self.cfg.max_body_bytes) {
437 return Err(Error::BodyTooLarge {
438 url: url.to_string(),
439 limit: self.cfg.max_body_bytes,
440 });
441 }
442
443 let charset = content_type.as_deref().and_then(charset_of);
444 let bytes = self.read_capped(url, resp).await?;
445 let body = decode(&bytes, charset.as_deref(), tld_of(&final_url).as_deref());
446
447 Ok(Page {
448 url: url.to_string(),
449 final_url,
450 status,
451 content_type,
452 body,
453 elapsed_ms: 0,
454 rendered: false,
455 })
456 }
457
458 async fn read_capped(&self, url: &Url, resp: wreq::Response) -> Result<Vec<u8>> {
460 let mut buf: Vec<u8> = Vec::with_capacity(64 * 1024);
461 let mut stream = resp.bytes_stream();
462 while let Some(chunk) = stream.next().await {
463 let chunk = chunk.map_err(|e| Error::network(url, e))?;
464 if buf.len() + chunk.len() > self.cfg.max_body_bytes {
465 return Err(Error::BodyTooLarge {
466 url: url.to_string(),
467 limit: self.cfg.max_body_bytes,
468 });
469 }
470 buf.extend_from_slice(&chunk);
471 }
472 Ok(buf)
473 }
474
475 async fn robots_for(&self, url: &Url) -> Arc<Robots> {
480 let host = url.host_str().unwrap_or_default().to_string();
481 let key = format!("{}://{host}", url.scheme());
482 if let Some(cached) = self.robots.lock().await.get(&key) {
483 return cached.clone();
484 }
485 let robots_url = format!("{key}/robots.txt");
486 let parsed = match self.client().get(&robots_url).send().await {
487 Ok(resp) if resp.status().is_success() => match resp.text().await {
488 Ok(body) => Robots::parse(&body, ROBOTS_AGENT),
489 Err(_) => Robots::allow_all(),
490 },
491 _ => Robots::allow_all(),
492 };
493 let parsed = Arc::new(parsed);
494 self.robots.lock().await.insert(key, parsed.clone());
495 parsed
496 }
497
498 async fn pace(&self, url: &Url, crawl_delay: Option<f64>) {
500 let host = url.host_str().unwrap_or_default().to_string();
501 if let Ok(mut visited) = self.visited.lock() {
502 visited.insert(host.clone());
503 }
504 let mut delay = self.cfg.per_host_delay;
505 if self.cfg.respect_crawl_delay
506 && let Some(d) = crawl_delay
507 {
508 delay = delay.max(Duration::from_secs_f64(d.min(30.0)));
509 }
510 if delay.is_zero() {
511 return;
512 }
513 let wait = {
514 let mut map = self.last_hit.lock().await;
515 let now = Instant::now();
516 let wait = match map.get(&host) {
517 Some(prev) => delay.checked_sub(now.duration_since(*prev)),
518 None => None,
519 };
520 map.insert(host, now + wait.unwrap_or_default());
523 wait
524 };
525 if let Some(wait) = wait {
526 tokio::time::sleep(wait).await;
527 }
528 }
529
530 #[cfg(feature = "browser")]
531 async fn render(&self, url: &Url) -> Result<String> {
532 browser::render(url.as_str(), self.cfg.timeout).await
533 }
534
535 #[cfg(not(feature = "browser"))]
536 async fn render(&self, _url: &Url) -> Result<String> {
537 Err(Error::Browser("build rustai with the `browser` feature".into()))
538 }
539}
540
541fn apply_impersonation(
542 builder: wreq::ClientBuilder,
543 mode: &Impersonate,
544) -> Result<wreq::ClientBuilder> {
545 #[cfg(not(feature = "impersonate"))]
546 {
547 if !matches!(mode, Impersonate::None) {
548 return Err(Error::Config(
549 "browser impersonation needs the `impersonate` feature".into(),
550 ));
551 }
552 Ok(builder)
553 }
554 #[cfg(feature = "impersonate")]
555 {
556 use wreq_util::{Emulation, Profile};
557 let profile = match mode {
558 Impersonate::None => return Ok(builder),
559 Impersonate::Random => return Ok(builder.emulation(Emulation::weighted_random())),
560 Impersonate::Chrome => Profile::Chrome149,
561 Impersonate::Firefox => Profile::Firefox147,
562 Impersonate::Safari => Profile::Safari26_4,
563 Impersonate::Named(name) => named_profile(name)?,
564 };
565 Ok(builder.emulation(profile))
566 }
567}
568
569#[cfg(feature = "impersonate")]
575fn named_profile(name: &str) -> Result<wreq_util::Profile> {
576 serde_json::from_value::<wreq_util::Profile>(serde_json::Value::String(name.to_string()))
577 .map_err(|_| {
578 Error::Config(format!(
579 "unknown impersonation profile `{name}`; try `chrome`, `firefox`, `safari`, \
580 `random`, or a versioned name such as `chrome_143`"
581 ))
582 })
583}
584
585fn parse_retry_after(value: &str) -> Option<f64> {
591 let secs: f64 = value.trim().parse().ok()?;
592 secs.is_finite().then_some(secs.max(0.0))
593}
594
595#[derive(serde::Serialize, serde::Deserialize)]
597struct StoredCookie {
598 name: String,
599 value: String,
600 host: String,
602 path: String,
603 #[serde(default, skip_serializing_if = "Option::is_none")]
606 domain: Option<String>,
607}
608
609fn load_jar(path: &std::path::Path) -> wreq::cookie::Jar {
612 let jar = wreq::cookie::Jar::default();
613 let Ok(raw) = std::fs::read_to_string(path) else { return jar };
614 let Ok(stored) = serde_json::from_str::<Vec<StoredCookie>>(&raw) else { return jar };
615 for c in stored {
616 let uri = format!("https://{}{}", c.host, c.path);
617 let mut cookie = format!("{}={}; Path={}", c.name, c.value, c.path);
618 if let Some(domain) = &c.domain {
619 cookie.push_str(&format!("; Domain={domain}"));
620 }
621 jar.add(cookie.as_str(), uri.as_str());
622 }
623 jar
624}
625
626fn save_jar(jar: &wreq::cookie::Jar, hosts: &[String], path: &std::path::Path) -> Result<usize> {
628 let mut seen = std::collections::HashSet::new();
632 let mut stored: Vec<StoredCookie> = Vec::new();
633 for host in hosts {
634 for c in jar.matches(format!("https://{host}/").as_str()) {
635 let cookie = StoredCookie {
636 name: c.name().to_string(),
637 value: c.value().to_string(),
638 host: host.clone(),
639 path: c.path().unwrap_or("/").to_string(),
640 domain: c.domain().map(str::to_string),
641 };
642 if seen.insert((cookie.host.clone(), cookie.name.clone(), cookie.path.clone())) {
643 stored.push(cookie);
644 }
645 }
646 }
647 let json = serde_json::to_string_pretty(&stored)
648 .map_err(|e| Error::Config(format!("could not serialise cookies: {e}")))?;
649 if let Some(parent) = path.parent()
650 && !parent.as_os_str().is_empty()
651 {
652 let _ = std::fs::create_dir_all(parent);
653 }
654 std::fs::write(path, json)
655 .map_err(|e| Error::Config(format!("could not write {}: {e}", path.display())))?;
656 Ok(stored.len())
657}
658
659pub fn normalize(url: &str) -> Result<Url> {
661 let parsed = Url::parse(url.trim()).map_err(|_| Error::InvalidUrl(url.to_string()))?;
662 if !matches!(parsed.scheme(), "http" | "https") {
663 return Err(Error::InvalidUrl(url.to_string()));
664 }
665 if parsed.host_str().is_none() {
666 return Err(Error::InvalidUrl(url.to_string()));
667 }
668 Ok(parsed)
669}
670
671fn charset_of(content_type: &str) -> Option<String> {
672 content_type
673 .split(';')
674 .skip(1)
675 .filter_map(|p| p.trim().strip_prefix("charset="))
676 .map(|c| c.trim_matches(['"', '\'']).to_ascii_lowercase())
677 .next()
678}
679
680fn decode(bytes: &[u8], header_charset: Option<&str>, tld: Option<&str>) -> String {
690 let label = header_charset.map(str::to_string).or_else(|| sniff_meta_charset(bytes));
691 let encoding = label
692 .as_deref()
693 .and_then(|l| encoding_rs::Encoding::for_label(l.as_bytes()))
694 .unwrap_or_else(|| {
695 if bytes.len() - encoding_rs::Encoding::utf8_valid_up_to(bytes) <= 3 {
698 encoding_rs::UTF_8
699 } else {
700 let mut detector = chardetng::EncodingDetector::new();
701 detector.feed(bytes, true);
702 detector.guess(tld.map(str::as_bytes), true)
703 }
704 });
705 let (text, _, _) = encoding.decode(bytes);
706 text.into_owned()
707}
708
709fn tld_of(url: &str) -> Option<String> {
711 let host = Url::parse(url).ok()?.host_str()?.to_ascii_lowercase();
712 host.rsplit('.').next().map(str::to_string)
713}
714
715fn sniff_meta_charset(bytes: &[u8]) -> Option<String> {
717 let head = &bytes[..bytes.len().min(4096)];
718 let text = String::from_utf8_lossy(head).to_ascii_lowercase();
719 if let Some(at) = text.find("charset=") {
720 let rest = &text[at + "charset=".len()..];
721 let value: String = rest
722 .trim_start_matches(['"', '\'', ' '])
723 .chars()
724 .take_while(|c| c.is_ascii_alphanumeric() || *c == '-' || *c == '_')
725 .collect();
726 if !value.is_empty() {
727 return Some(value);
728 }
729 }
730 None
731}
732
733fn detect_html_redirect(body: &str, base: &Url) -> Option<Url> {
741 const MAX_STUB_BYTES: usize = 4096;
742 if body.len() > MAX_STUB_BYTES {
743 return None;
744 }
745 let target = meta_refresh_target(body).or_else(|| script_location_target(body))?;
746 let next = base.join(target.trim().trim_matches(['"', '\''])).ok()?;
747 if !matches!(next.scheme(), "http" | "https") {
748 return None;
749 }
750 let same = next.as_str().trim_end_matches('/') == base.as_str().trim_end_matches('/');
752 (!same).then_some(next)
753}
754
755fn meta_refresh_target(body: &str) -> Option<&str> {
757 let lower = body.to_ascii_lowercase();
758 let mut from = 0usize;
759 while let Some(at) = lower[from..].find("http-equiv") {
760 let tag_start = lower[..from + at].rfind('<')?;
761 let tag_end = lower[tag_start..].find('>')? + tag_start;
762 let tag = &lower[tag_start..tag_end];
763 if tag.contains("refresh")
764 && let Some(content_at) = tag.find("content")
765 && let Some(url_at) = tag[content_at..].find("url=")
766 {
767 let delay: String = tag[content_at..]
769 .chars()
770 .skip_while(|c| !c.is_ascii_digit())
771 .take_while(|c| c.is_ascii_digit())
772 .collect();
773 if delay.parse::<u32>().unwrap_or(0) == 0 {
774 let start = tag_start + content_at + url_at + 4;
775 let rest = &body[start..];
776 let end = rest.find(['"', '\'', '>']).unwrap_or(rest.len());
777 let target = rest[..end].trim();
778 if !target.is_empty() {
779 return Some(target);
780 }
781 }
782 }
783 from += at + "http-equiv".len();
784 }
785 None
786}
787
788fn script_location_target(body: &str) -> Option<&str> {
790 for marker in ["location.replace(", "location.href=", "location.href =", "location.assign("] {
791 let Some(at) = body.find(marker) else { continue };
792 let rest = &body[at + marker.len()..];
793 let rest = rest.trim_start();
794 let quote = rest.chars().next()?;
795 if quote != '"' && quote != '\'' {
796 continue;
798 }
799 let inner = &rest[quote.len_utf8()..];
800 let end = inner.find(quote)?;
801 let target = &inner[..end];
802 if target.starts_with("http") || target.starts_with('/') {
803 return Some(target);
804 }
805 }
806 None
807}
808
809pub fn needs_javascript(page: &Page) -> bool {
815 let html = &page.body;
816 if html.len() < 512 {
817 return false;
818 }
819 let Ok(article) = crate::parse::extract(html, Some(&page.final_url)) else {
820 return true;
821 };
822 let text_len = crate::text::visible_len(&article.text);
823
824 if text_len > MAX_STATIC_TEXT {
826 return false;
827 }
828
829 let starved = text_len.saturating_mul(TEXT_TO_MARKUP_FLOOR) < html.len();
834 if text_len < 120 || starved {
835 return true;
836 }
837
838 let lower = html.to_lowercase();
839 let spa_markers = ["__next_data__", "id=\"root\"", "id=\"app\"", "ng-app", "data-reactroot"];
840 spa_markers.iter().any(|m| lower.contains(m))
841}
842
843#[cfg(test)]
844mod tests {
845 use super::*;
846
847 #[test]
848 fn rejects_non_http_urls() {
849 assert!(normalize("file:///etc/passwd").is_err());
850 assert!(normalize("javascript:alert(1)").is_err());
851 assert!(normalize("not a url").is_err());
852 assert!(normalize("https://example.com/a").is_ok());
853 }
854
855 #[test]
856 fn parses_charset_from_content_type() {
857 assert_eq!(charset_of("text/html; charset=EUC-KR").as_deref(), Some("euc-kr"));
858 assert_eq!(charset_of("text/html").as_deref(), None);
859 }
860
861 #[test]
862 fn decodes_legacy_korean_encoding() {
863 let (bytes, _, _) = encoding_rs::EUC_KR.encode("한글 인코딩");
864 assert_eq!(decode(&bytes, Some("euc-kr"), None), "한글 인코딩");
865 let mut doc = b"<html><head><meta charset=\"euc-kr\"></head><body>".to_vec();
867 doc.extend_from_slice(&bytes);
868 assert!(decode(&doc, None, None).contains("한글 인코딩"));
869 }
870
871 #[test]
872 fn guesses_a_legacy_encoding_that_nothing_declares() {
873 let (bytes, _, _) = encoding_rs::EUC_KR.encode(concat!(
875 "<html><head><title>합성함수 미분법 = 연쇄법칙(Chain Rule)</title></head><body>",
876 "<p>함수 y = f(x) 와 함수 z = g(y)를 합성한 함수의 미분법은 다음과 같다.</p></body></html>",
877 ));
878 assert!(decode(&bytes, None, Some("kr")).contains("합성함수 미분법 = 연쇄법칙"));
879 }
880
881 #[test]
882 fn undeclared_utf8_stays_utf8() {
883 assert_eq!(
884 decode("<p>한글 UTF-8 본문</p>".as_bytes(), None, Some("kr")),
885 "<p>한글 UTF-8 본문</p>"
886 );
887 }
888
889 #[test]
890 fn a_body_cut_mid_character_is_still_read_as_utf8() {
891 let full = "<p>가나다라마바사</p>".repeat(20);
893 let cut = &full.as_bytes()[..full.len() - 7];
894 assert!(decode(cut, None, Some("kr")).starts_with("<p>가나다라마바사</p>"));
895 }
896
897 #[test]
898 fn tld_is_the_last_host_label() {
899 assert_eq!(tld_of("https://www.math.snu.ac.kr/a.htm").as_deref(), Some("kr"));
900 assert_eq!(tld_of("not a url"), None);
901 }
902
903 #[test]
904 fn impersonate_strings_parse() {
905 assert_eq!(Impersonate::parse("chrome").unwrap(), Impersonate::Chrome);
906 assert_eq!(Impersonate::parse("none").unwrap(), Impersonate::None);
907 assert_eq!(
908 Impersonate::parse("chrome_143").unwrap(),
909 Impersonate::Named("chrome_143".into())
910 );
911 }
912
913 #[cfg(feature = "impersonate")]
914 #[test]
915 fn named_profiles_resolve_and_reject() {
916 assert!(named_profile("chrome_143").is_ok());
917 assert!(named_profile("netscape_2").is_err());
918 }
919
920 #[test]
921 fn client_builds_with_every_preset() {
922 for mode in ["chrome", "firefox", "safari", "random", "none"] {
923 let cfg = FetchConfig {
924 impersonate: Impersonate::parse(mode).unwrap(),
925 ..Default::default()
926 };
927 assert!(Fetcher::with_config(cfg).is_ok(), "{mode} failed to build");
928 }
929 }
930
931 #[test]
932 fn retry_after_is_parsed_and_capped() {
933 assert_eq!(parse_retry_after("120"), Some(120.0));
934 assert_eq!(parse_retry_after(" 0 "), Some(0.0));
935 assert_eq!(parse_retry_after("-5"), Some(0.0));
936 assert_eq!(parse_retry_after("Wed, 21 Oct 2026 07:28:00 GMT"), None);
938 assert_eq!(parse_retry_after(""), None);
939
940 let e =
941 Error::Status { url: "https://x.dev/".into(), status: 429, retry_after: Some(30.0) };
942 assert_eq!(e.retry_after(), Some(Duration::from_secs(30)));
943 assert!(e.is_retryable());
944 let none = Error::Status { url: "https://x.dev/".into(), status: 429, retry_after: None };
945 assert_eq!(none.retry_after(), None);
946 }
947
948 #[test]
949 fn proxies_build_one_client_each() {
950 let cfg = FetchConfig {
951 proxies: vec!["http://127.0.0.1:8080".into(), "socks5://127.0.0.1:1080".into()],
952 ..Default::default()
953 };
954 let fetcher = Fetcher::with_config(cfg).expect("proxied client");
955 assert_eq!(fetcher.clients.len(), 2);
956 let first = fetcher.client() as *const _;
958 let second = fetcher.client() as *const _;
959 assert_ne!(first, second);
960 assert_eq!(fetcher.client() as *const _, first);
961 }
962
963 #[test]
964 fn an_invalid_proxy_is_rejected_at_build_time() {
965 let cfg = FetchConfig { proxies: vec!["not a proxy".into()], ..Default::default() };
966 assert!(Fetcher::with_config(cfg).is_err());
967 }
968
969 #[test]
970 fn cookies_survive_a_round_trip_through_disk() {
971 let dir = std::env::temp_dir().join(format!("rustai-cookies-{}", std::process::id()));
972 let path = dir.join("jar.json");
973 let _ = std::fs::remove_file(&path);
974
975 let jar = wreq::cookie::Jar::default();
976 jar.add("cf_clearance=abc123; Domain=example.com; Path=/", "https://example.com/");
979 jar.add("session=xyz789; Path=/", "https://example.com/");
980 let hosts = vec!["example.com".to_string()];
981 assert_eq!(save_jar(&jar, &hosts, &path).unwrap(), 2);
982
983 let reloaded = load_jar(&path);
984 assert!(reloaded.contains("session", "https://example.com/"), "host-only cookie lost");
985 assert!(reloaded.contains("cf_clearance", "https://example.com/"), "cookie was lost");
986 assert_eq!(
987 reloaded.get("cf_clearance", "https://example.com/").map(|c| c.value().to_string()),
988 Some("abc123".to_string())
989 );
990
991 assert_eq!(load_jar(&dir.join("nope.json")).len(), 0);
993 let _ = std::fs::remove_dir_all(&dir);
994 }
995
996 #[test]
997 fn a_fetcher_without_a_cookie_file_saves_nothing() {
998 let fetcher = Fetcher::new().unwrap();
999 assert_eq!(fetcher.save_cookies().unwrap(), 0);
1000 }
1001
1002 #[test]
1003 fn zero_concurrency_is_rejected() {
1004 let cfg = FetchConfig { concurrency: 0, ..Default::default() };
1005 assert!(Fetcher::with_config(cfg).is_err());
1006 }
1007
1008 fn page(body: &str) -> Page {
1009 Page {
1010 url: "https://x.dev/".into(),
1011 final_url: "https://x.dev/".into(),
1012 status: 200,
1013 content_type: Some("text/html".into()),
1014 body: body.into(),
1015 elapsed_ms: 0,
1016 rendered: false,
1017 }
1018 }
1019
1020 #[test]
1021 fn follows_a_meta_refresh_stub() {
1022 let base = Url::parse("https://blog.dev/2024/02/08/Post.html").unwrap();
1023 let body = r#"<!doctype html><title>Redirect</title>
1024 <noscript><meta http-equiv="refresh" content="0; url=https://blog.dev/2024/02/08/Post/"></noscript>"#;
1025 assert_eq!(
1026 detect_html_redirect(body, &base).unwrap().as_str(),
1027 "https://blog.dev/2024/02/08/Post/"
1028 );
1029 }
1030
1031 #[test]
1032 fn follows_a_script_location_stub() {
1033 let base = Url::parse("https://blog.dev/a.html").unwrap();
1034 let body = r#"<!doctype html><title>Redirect</title><script>
1035 const target = "https://blog.dev/a/";
1036 window.location.replace("https://blog.dev/a/");
1037 </script>"#;
1038 assert_eq!(detect_html_redirect(body, &base).unwrap().as_str(), "https://blog.dev/a/");
1039 }
1040
1041 #[test]
1042 fn resolves_a_relative_redirect_target() {
1043 let base = Url::parse("https://blog.dev/x/a.html").unwrap();
1044 let body = r#"<meta http-equiv="refresh" content="0;url=/y/b.html">"#;
1045 assert_eq!(
1046 detect_html_redirect(body, &base).unwrap().as_str(),
1047 "https://blog.dev/y/b.html"
1048 );
1049 }
1050
1051 #[test]
1052 fn ignores_delayed_refreshes_and_self_loops() {
1053 let base = Url::parse("https://blog.dev/a").unwrap();
1054 assert!(
1056 detect_html_redirect(
1057 r#"<meta http-equiv="refresh" content="5; url=https://blog.dev/b">"#,
1058 &base
1059 )
1060 .is_none()
1061 );
1062 assert!(
1064 detect_html_redirect(
1065 r#"<meta http-equiv="refresh" content="0; url=https://blog.dev/a">"#,
1066 &base
1067 )
1068 .is_none()
1069 );
1070 }
1071
1072 #[test]
1073 fn ignores_a_long_page_that_merely_mentions_location() {
1074 let base = Url::parse("https://blog.dev/a").unwrap();
1075 let article = format!(
1076 "<article><p>{}</p><p>You can call location.replace(\"https://evil.dev/\") in JS.</p></article>",
1077 "Real prose. ".repeat(500)
1078 );
1079 assert!(
1080 detect_html_redirect(&article, &base).is_none(),
1081 "a full article was treated as a stub"
1082 );
1083 }
1084
1085 #[test]
1086 fn ignores_non_http_and_unresolvable_targets() {
1087 let base = Url::parse("https://blog.dev/a").unwrap();
1088 assert!(
1089 detect_html_redirect(
1090 r#"<meta http-equiv="refresh" content="0;url=javascript:x()">"#,
1091 &base
1092 )
1093 .is_none()
1094 );
1095 assert!(detect_html_redirect("<p>nothing here</p>", &base).is_none());
1096 assert!(
1098 detect_html_redirect(r#"<script>window.location.replace(target);</script>"#, &base)
1099 .is_none()
1100 );
1101 }
1102
1103 #[test]
1104 fn detects_an_spa_shell() {
1105 let shell = format!(
1106 "<html><body><div id=\"root\"></div><script>{}</script></body></html>",
1107 "x".repeat(2000)
1108 );
1109 assert!(needs_javascript(&page(&shell)));
1110 }
1111
1112 #[test]
1113 fn does_not_escalate_a_real_article() {
1114 let real = format!(
1115 "<html><body><article><h1>T</h1><p>{}</p></article></body></html>",
1116 "Real prose that a reader can actually read. ".repeat(20)
1117 );
1118 assert!(!needs_javascript(&page(&real)));
1119 }
1120}
1121
1122#[cfg(test)]
1123mod js_gate_tests {
1124 use super::*;
1125
1126 fn page(body: String) -> Page {
1127 Page {
1128 url: "https://x.dev/".into(),
1129 final_url: "https://x.dev/".into(),
1130 status: 200,
1131 content_type: Some("text/html".into()),
1132 body,
1133 elapsed_ms: 0,
1134 rendered: false,
1135 }
1136 }
1137
1138 #[test]
1144 fn a_client_rendered_front_page_is_flagged() {
1145 let shell = format!(
1146 "<html><body><nav>{}</nav><main></main><script>{}</script></body></html>",
1147 "<a href=\"/section\">Section link</a>".repeat(20),
1148 "x".repeat(340_000)
1149 );
1150 assert!(needs_javascript(&page(shell)));
1151 }
1152
1153 #[test]
1155 fn a_large_real_article_is_not_flagged() {
1156 let real = format!(
1157 "<html><body><article><h1>T</h1><p>{}</p></article></body></html>",
1158 "Real prose that a reader can actually read and act on. ".repeat(400)
1159 );
1160 assert!(!needs_javascript(&page(real)));
1161 }
1162}