1pub mod robots;
17
18#[cfg(feature = "browser")]
19pub mod browser;
20
21use std::collections::HashMap;
22use std::sync::Arc;
23use std::time::{Duration, Instant};
24
25use futures_util::StreamExt;
26use tokio::sync::{Mutex, Semaphore};
27use url::Url;
28
29use crate::error::{Error, Result};
30use robots::Robots;
31
32pub const DEFAULT_USER_AGENT: &str =
34 concat!("rustai/", env!("CARGO_PKG_VERSION"), " (+https://github.com/imhyensuk/rustai)");
35
36pub const ROBOTS_AGENT: &str = "rustai";
38
39const MAX_HTML_REDIRECTS: usize = 2;
44
45const MAX_STATIC_TEXT: usize = 2_000;
47
48const TEXT_TO_MARKUP_FLOOR: usize = 200;
51
52const TEXTUAL: &[&str] = &[
54 "text/html",
55 "application/xhtml",
56 "text/plain",
57 "application/xml",
58 "text/xml",
59 "application/rss",
60 "application/atom",
61 "application/json",
62];
63
64#[derive(Debug, Clone, Default, PartialEq, Eq)]
66pub enum Impersonate {
67 #[default]
69 Chrome,
70 Firefox,
72 Safari,
74 Named(String),
76 Random,
78 None,
80}
81
82impl Impersonate {
83 pub fn parse(s: &str) -> Result<Impersonate> {
85 Ok(match s.trim().to_ascii_lowercase().as_str() {
86 "" | "none" | "off" | "false" => Impersonate::None,
87 "chrome" | "true" | "default" => Impersonate::Chrome,
88 "firefox" => Impersonate::Firefox,
89 "safari" => Impersonate::Safari,
90 "random" => Impersonate::Random,
91 other => Impersonate::Named(other.to_string()),
92 })
93 }
94}
95
96#[derive(Debug, Clone)]
98pub struct FetchConfig {
99 pub concurrency: usize,
101 pub timeout: Duration,
103 pub connect_timeout: Duration,
105 pub max_redirects: usize,
107 pub max_body_bytes: usize,
110 pub retries: u32,
112 pub retry_backoff: Duration,
114 pub per_host_delay: Duration,
116 pub respect_robots: bool,
118 pub respect_crawl_delay: bool,
120 pub impersonate: Impersonate,
122 pub accept_language: String,
124 pub browser_fallback: bool,
126 pub proxies: Vec<String>,
132 pub max_retry_after: Duration,
134 pub cookie_file: Option<std::path::PathBuf>,
136}
137
138impl Default for FetchConfig {
139 fn default() -> Self {
140 FetchConfig {
141 concurrency: 16,
142 timeout: Duration::from_secs(20),
143 connect_timeout: Duration::from_secs(8),
144 max_redirects: 5,
145 max_body_bytes: 8 * 1024 * 1024,
146 retries: 2,
147 retry_backoff: Duration::from_millis(400),
148 per_host_delay: Duration::from_millis(250),
149 respect_robots: true,
150 respect_crawl_delay: true,
151 impersonate: Impersonate::Chrome,
152 accept_language: "en-US,en;q=0.9".to_string(),
153 browser_fallback: false,
154 proxies: Vec::new(),
155 max_retry_after: Duration::from_secs(60),
156 cookie_file: None,
157 }
158 }
159}
160
161#[derive(Debug, Clone, serde::Serialize)]
163pub struct Page {
164 pub url: String,
166 pub final_url: String,
168 pub status: u16,
170 pub content_type: Option<String>,
172 pub body: String,
174 pub elapsed_ms: u64,
176 pub rendered: bool,
178}
179
180#[derive(Clone)]
182pub struct Fetcher {
183 clients: Arc<Vec<wreq::Client>>,
185 next_client: Arc<std::sync::atomic::AtomicUsize>,
186 jar: Option<Arc<wreq::cookie::Jar>>,
187 visited: Arc<std::sync::Mutex<std::collections::BTreeSet<String>>>,
191 cfg: Arc<FetchConfig>,
192 gate: Arc<Semaphore>,
193 robots: Arc<Mutex<HashMap<String, Arc<Robots>>>>,
194 last_hit: Arc<Mutex<HashMap<String, Instant>>>,
195}
196
197impl std::fmt::Debug for Fetcher {
198 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
199 f.debug_struct("Fetcher").field("config", &self.cfg).finish_non_exhaustive()
200 }
201}
202
203impl Fetcher {
204 pub fn new() -> Result<Self> {
206 Self::with_config(FetchConfig::default())
207 }
208
209 pub fn with_config(cfg: FetchConfig) -> Result<Self> {
211 if cfg.concurrency == 0 {
212 return Err(Error::Config("concurrency must be at least 1".into()));
213 }
214 let jar = cfg.cookie_file.as_ref().map(|path| Arc::new(load_jar(path)));
215
216 let build_one = |proxy: Option<&str>| -> Result<wreq::Client> {
217 let mut builder = wreq::Client::builder()
218 .timeout(cfg.timeout)
219 .connect_timeout(cfg.connect_timeout)
220 .redirect(wreq::redirect::Policy::limited(cfg.max_redirects))
221 .pool_max_idle_per_host(cfg.concurrency.min(32))
222 .referer(true)
223 .gzip(true)
224 .brotli(true)
225 .deflate(true)
226 .zstd(true);
227
228 builder = match &jar {
229 Some(jar) => builder.cookie_provider(jar.clone()),
230 None => builder.cookie_store(true),
231 };
232 builder = apply_impersonation(builder, &cfg.impersonate)?;
233
234 if let Some(proxy) = proxy {
235 let proxy = wreq::Proxy::all(proxy)
236 .map_err(|e| Error::Config(format!("invalid proxy `{proxy}`: {e}")))?;
237 builder = builder.proxy(proxy);
238 }
239
240 let mut headers = wreq::header::HeaderMap::new();
241 if let Ok(v) = wreq::header::HeaderValue::from_str(&cfg.accept_language) {
242 headers.insert(wreq::header::ACCEPT_LANGUAGE, v);
243 }
244 if matches!(cfg.impersonate, Impersonate::None) {
245 headers.insert(
246 wreq::header::USER_AGENT,
247 wreq::header::HeaderValue::from_static(DEFAULT_USER_AGENT),
248 );
249 }
250 builder = builder.default_headers(headers);
251 builder.build().map_err(|e| Error::Config(e.to_string()))
252 };
253
254 let clients = if cfg.proxies.is_empty() {
255 vec![build_one(None)?]
256 } else {
257 cfg.proxies.iter().map(|p| build_one(Some(p))).collect::<Result<Vec<_>>>()?
258 };
259
260 Ok(Fetcher {
261 clients: Arc::new(clients),
262 next_client: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
263 jar,
264 visited: Arc::new(std::sync::Mutex::new(std::collections::BTreeSet::new())),
265 gate: Arc::new(Semaphore::new(cfg.concurrency)),
266 cfg: Arc::new(cfg),
267 robots: Arc::new(Mutex::new(HashMap::new())),
268 last_hit: Arc::new(Mutex::new(HashMap::new())),
269 })
270 }
271
272 fn client(&self) -> &wreq::Client {
277 if self.clients.len() == 1 {
278 return &self.clients[0];
279 }
280 let i = self.next_client.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
281 &self.clients[i % self.clients.len()]
282 }
283
284 pub fn save_cookies(&self) -> Result<usize> {
289 let (Some(jar), Some(path)) = (&self.jar, &self.cfg.cookie_file) else { return Ok(0) };
290 let hosts: Vec<String> = match self.visited.lock() {
291 Ok(v) => v.iter().cloned().collect(),
292 Err(poisoned) => poisoned.into_inner().iter().cloned().collect(),
293 };
294 save_jar(jar, &hosts, path)
295 }
296
297 pub fn config(&self) -> &FetchConfig {
299 &self.cfg
300 }
301
302 pub async fn fetch(&self, url: &str) -> Result<Page> {
304 self.fetch_inner(url, self.cfg.respect_robots).await
305 }
306
307 pub async fn fetch_api(&self, url: &str) -> Result<Page> {
316 self.fetch_inner(url, false).await
317 }
318
319 async fn fetch_inner(&self, url: &str, check_robots: bool) -> Result<Page> {
320 let parsed = normalize(url)?;
321 let _permit = self.gate.acquire().await.map_err(|e| Error::Config(e.to_string()))?;
322
323 let crawl_delay = if check_robots {
324 let robots = self.robots_for(&parsed).await;
325 if !robots.allows(parsed.path()) {
326 return Err(Error::RobotsDenied(parsed.to_string()));
327 }
328 robots.crawl_delay
329 } else {
330 None
331 };
332 self.pace(&parsed, crawl_delay).await;
333
334 let started = Instant::now();
335 let mut page = self.get_with_retry(&parsed).await?;
336
337 let mut current = parsed.clone();
342 for _ in 0..MAX_HTML_REDIRECTS {
343 let Some(next) = detect_html_redirect(&page.body, ¤t) else { break };
344 if check_robots && !self.robots_for(&next).await.allows(next.path()) {
345 break;
346 }
347 self.pace(&next, crawl_delay).await;
348 match self.get_with_retry(&next).await {
349 Ok(followed) => {
350 page = followed;
351 current = next;
352 }
353 Err(_) => break,
355 }
356 }
357 page.url = parsed.to_string();
358 page.elapsed_ms = started.elapsed().as_millis() as u64;
359
360 if self.cfg.browser_fallback && needs_javascript(&page) {
361 if let Ok(rendered) = self.render(&parsed).await {
364 page.body = rendered;
365 page.rendered = true;
366 page.elapsed_ms = started.elapsed().as_millis() as u64;
367 }
368 }
369 Ok(page)
370 }
371
372 pub async fn fetch_many(&self, urls: &[String]) -> Vec<Result<Page>> {
377 let futures = urls.iter().map(|u| {
378 let this = self.clone();
379 let u = u.clone();
380 async move { this.fetch(&u).await }
381 });
382 futures_util::stream::iter(futures).buffered(self.cfg.concurrency).collect::<Vec<_>>().await
383 }
384
385 async fn get_with_retry(&self, url: &Url) -> Result<Page> {
387 let mut attempt = 0u32;
388 loop {
389 match self.get_once(url).await {
390 Ok(page) => return Ok(page),
391 Err(e) if e.is_retryable() && attempt < self.cfg.retries => {
392 let wait = match e.retry_after() {
398 Some(hint) if hint <= self.cfg.max_retry_after => hint,
399 Some(_) => return Err(e),
400 None => self.cfg.retry_backoff * 2u32.pow(attempt),
401 };
402 tokio::time::sleep(wait).await;
403 attempt += 1;
404 }
405 Err(e) => return Err(e),
406 }
407 }
408 }
409
410 async fn get_once(&self, url: &Url) -> Result<Page> {
411 let resp =
412 self.client().get(url.as_str()).send().await.map_err(|e| Error::network(url, e))?;
413
414 let status = resp.status().as_u16();
415 let final_url = resp.uri().to_string();
416 let content_type = resp
417 .headers()
418 .get(wreq::header::CONTENT_TYPE)
419 .and_then(|v| v.to_str().ok())
420 .map(str::to_string);
421
422 if !(200..300).contains(&status) {
423 let retry_after = resp
424 .headers()
425 .get(wreq::header::RETRY_AFTER)
426 .and_then(|v| v.to_str().ok())
427 .and_then(parse_retry_after);
428 return Err(Error::Status { url: url.to_string(), status, retry_after });
429 }
430 if let Some(ct) = &content_type {
431 let base = ct.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
432 if !TEXTUAL.iter().any(|t| base.starts_with(t)) {
433 return Err(Error::Parse(format!("{url}: unsupported content type {base}")));
434 }
435 }
436 if resp.content_length().is_some_and(|n| n as usize > self.cfg.max_body_bytes) {
437 return Err(Error::BodyTooLarge {
438 url: url.to_string(),
439 limit: self.cfg.max_body_bytes,
440 });
441 }
442
443 let charset = content_type.as_deref().and_then(charset_of);
444 let bytes = self.read_capped(url, resp).await?;
445 let body = decode(&bytes, charset.as_deref());
446
447 Ok(Page {
448 url: url.to_string(),
449 final_url,
450 status,
451 content_type,
452 body,
453 elapsed_ms: 0,
454 rendered: false,
455 })
456 }
457
458 async fn read_capped(&self, url: &Url, resp: wreq::Response) -> Result<Vec<u8>> {
460 let mut buf: Vec<u8> = Vec::with_capacity(64 * 1024);
461 let mut stream = resp.bytes_stream();
462 while let Some(chunk) = stream.next().await {
463 let chunk = chunk.map_err(|e| Error::network(url, e))?;
464 if buf.len() + chunk.len() > self.cfg.max_body_bytes {
465 return Err(Error::BodyTooLarge {
466 url: url.to_string(),
467 limit: self.cfg.max_body_bytes,
468 });
469 }
470 buf.extend_from_slice(&chunk);
471 }
472 Ok(buf)
473 }
474
475 async fn robots_for(&self, url: &Url) -> Arc<Robots> {
480 let host = url.host_str().unwrap_or_default().to_string();
481 let key = format!("{}://{host}", url.scheme());
482 if let Some(cached) = self.robots.lock().await.get(&key) {
483 return cached.clone();
484 }
485 let robots_url = format!("{key}/robots.txt");
486 let parsed = match self.client().get(&robots_url).send().await {
487 Ok(resp) if resp.status().is_success() => match resp.text().await {
488 Ok(body) => Robots::parse(&body, ROBOTS_AGENT),
489 Err(_) => Robots::allow_all(),
490 },
491 _ => Robots::allow_all(),
492 };
493 let parsed = Arc::new(parsed);
494 self.robots.lock().await.insert(key, parsed.clone());
495 parsed
496 }
497
498 async fn pace(&self, url: &Url, crawl_delay: Option<f64>) {
500 let host = url.host_str().unwrap_or_default().to_string();
501 if let Ok(mut visited) = self.visited.lock() {
502 visited.insert(host.clone());
503 }
504 let mut delay = self.cfg.per_host_delay;
505 if self.cfg.respect_crawl_delay
506 && let Some(d) = crawl_delay
507 {
508 delay = delay.max(Duration::from_secs_f64(d.min(30.0)));
509 }
510 if delay.is_zero() {
511 return;
512 }
513 let wait = {
514 let mut map = self.last_hit.lock().await;
515 let now = Instant::now();
516 let wait = match map.get(&host) {
517 Some(prev) => delay.checked_sub(now.duration_since(*prev)),
518 None => None,
519 };
520 map.insert(host, now + wait.unwrap_or_default());
523 wait
524 };
525 if let Some(wait) = wait {
526 tokio::time::sleep(wait).await;
527 }
528 }
529
530 #[cfg(feature = "browser")]
531 async fn render(&self, url: &Url) -> Result<String> {
532 browser::render(url.as_str(), self.cfg.timeout).await
533 }
534
535 #[cfg(not(feature = "browser"))]
536 async fn render(&self, _url: &Url) -> Result<String> {
537 Err(Error::Browser("build rustai with the `browser` feature".into()))
538 }
539}
540
541fn apply_impersonation(
542 builder: wreq::ClientBuilder,
543 mode: &Impersonate,
544) -> Result<wreq::ClientBuilder> {
545 #[cfg(not(feature = "impersonate"))]
546 {
547 if !matches!(mode, Impersonate::None) {
548 return Err(Error::Config(
549 "browser impersonation needs the `impersonate` feature".into(),
550 ));
551 }
552 Ok(builder)
553 }
554 #[cfg(feature = "impersonate")]
555 {
556 use wreq_util::{Emulation, Profile};
557 let profile = match mode {
558 Impersonate::None => return Ok(builder),
559 Impersonate::Random => return Ok(builder.emulation(Emulation::weighted_random())),
560 Impersonate::Chrome => Profile::Chrome149,
561 Impersonate::Firefox => Profile::Firefox147,
562 Impersonate::Safari => Profile::Safari26_4,
563 Impersonate::Named(name) => named_profile(name)?,
564 };
565 Ok(builder.emulation(profile))
566 }
567}
568
569#[cfg(feature = "impersonate")]
575fn named_profile(name: &str) -> Result<wreq_util::Profile> {
576 serde_json::from_value::<wreq_util::Profile>(serde_json::Value::String(name.to_string()))
577 .map_err(|_| {
578 Error::Config(format!(
579 "unknown impersonation profile `{name}`; try `chrome`, `firefox`, `safari`, \
580 `random`, or a versioned name such as `chrome_143`"
581 ))
582 })
583}
584
585fn parse_retry_after(value: &str) -> Option<f64> {
591 let secs: f64 = value.trim().parse().ok()?;
592 secs.is_finite().then_some(secs.max(0.0))
593}
594
595#[derive(serde::Serialize, serde::Deserialize)]
597struct StoredCookie {
598 name: String,
599 value: String,
600 host: String,
602 path: String,
603 #[serde(default, skip_serializing_if = "Option::is_none")]
606 domain: Option<String>,
607}
608
609fn load_jar(path: &std::path::Path) -> wreq::cookie::Jar {
612 let jar = wreq::cookie::Jar::default();
613 let Ok(raw) = std::fs::read_to_string(path) else { return jar };
614 let Ok(stored) = serde_json::from_str::<Vec<StoredCookie>>(&raw) else { return jar };
615 for c in stored {
616 let uri = format!("https://{}{}", c.host, c.path);
617 let mut cookie = format!("{}={}; Path={}", c.name, c.value, c.path);
618 if let Some(domain) = &c.domain {
619 cookie.push_str(&format!("; Domain={domain}"));
620 }
621 jar.add(cookie.as_str(), uri.as_str());
622 }
623 jar
624}
625
626fn save_jar(jar: &wreq::cookie::Jar, hosts: &[String], path: &std::path::Path) -> Result<usize> {
628 let mut seen = std::collections::HashSet::new();
632 let mut stored: Vec<StoredCookie> = Vec::new();
633 for host in hosts {
634 for c in jar.matches(format!("https://{host}/").as_str()) {
635 let cookie = StoredCookie {
636 name: c.name().to_string(),
637 value: c.value().to_string(),
638 host: host.clone(),
639 path: c.path().unwrap_or("/").to_string(),
640 domain: c.domain().map(str::to_string),
641 };
642 if seen.insert((cookie.host.clone(), cookie.name.clone(), cookie.path.clone())) {
643 stored.push(cookie);
644 }
645 }
646 }
647 let json = serde_json::to_string_pretty(&stored)
648 .map_err(|e| Error::Config(format!("could not serialise cookies: {e}")))?;
649 if let Some(parent) = path.parent()
650 && !parent.as_os_str().is_empty()
651 {
652 let _ = std::fs::create_dir_all(parent);
653 }
654 std::fs::write(path, json)
655 .map_err(|e| Error::Config(format!("could not write {}: {e}", path.display())))?;
656 Ok(stored.len())
657}
658
659pub fn normalize(url: &str) -> Result<Url> {
661 let parsed = Url::parse(url.trim()).map_err(|_| Error::InvalidUrl(url.to_string()))?;
662 if !matches!(parsed.scheme(), "http" | "https") {
663 return Err(Error::InvalidUrl(url.to_string()));
664 }
665 if parsed.host_str().is_none() {
666 return Err(Error::InvalidUrl(url.to_string()));
667 }
668 Ok(parsed)
669}
670
671fn charset_of(content_type: &str) -> Option<String> {
672 content_type
673 .split(';')
674 .skip(1)
675 .filter_map(|p| p.trim().strip_prefix("charset="))
676 .map(|c| c.trim_matches(['"', '\'']).to_ascii_lowercase())
677 .next()
678}
679
680fn decode(bytes: &[u8], header_charset: Option<&str>) -> String {
686 let label = header_charset.map(str::to_string).or_else(|| sniff_meta_charset(bytes));
687 let encoding = label
688 .as_deref()
689 .and_then(|l| encoding_rs::Encoding::for_label(l.as_bytes()))
690 .unwrap_or(encoding_rs::UTF_8);
691 let (text, _, _) = encoding.decode(bytes);
692 text.into_owned()
693}
694
695fn sniff_meta_charset(bytes: &[u8]) -> Option<String> {
697 let head = &bytes[..bytes.len().min(4096)];
698 let text = String::from_utf8_lossy(head).to_ascii_lowercase();
699 if let Some(at) = text.find("charset=") {
700 let rest = &text[at + "charset=".len()..];
701 let value: String = rest
702 .trim_start_matches(['"', '\'', ' '])
703 .chars()
704 .take_while(|c| c.is_ascii_alphanumeric() || *c == '-' || *c == '_')
705 .collect();
706 if !value.is_empty() {
707 return Some(value);
708 }
709 }
710 None
711}
712
713fn detect_html_redirect(body: &str, base: &Url) -> Option<Url> {
721 const MAX_STUB_BYTES: usize = 4096;
722 if body.len() > MAX_STUB_BYTES {
723 return None;
724 }
725 let target = meta_refresh_target(body).or_else(|| script_location_target(body))?;
726 let next = base.join(target.trim().trim_matches(['"', '\''])).ok()?;
727 if !matches!(next.scheme(), "http" | "https") {
728 return None;
729 }
730 let same = next.as_str().trim_end_matches('/') == base.as_str().trim_end_matches('/');
732 (!same).then_some(next)
733}
734
735fn meta_refresh_target(body: &str) -> Option<&str> {
737 let lower = body.to_ascii_lowercase();
738 let mut from = 0usize;
739 while let Some(at) = lower[from..].find("http-equiv") {
740 let tag_start = lower[..from + at].rfind('<')?;
741 let tag_end = lower[tag_start..].find('>')? + tag_start;
742 let tag = &lower[tag_start..tag_end];
743 if tag.contains("refresh")
744 && let Some(content_at) = tag.find("content")
745 && let Some(url_at) = tag[content_at..].find("url=")
746 {
747 let delay: String = tag[content_at..]
749 .chars()
750 .skip_while(|c| !c.is_ascii_digit())
751 .take_while(|c| c.is_ascii_digit())
752 .collect();
753 if delay.parse::<u32>().unwrap_or(0) == 0 {
754 let start = tag_start + content_at + url_at + 4;
755 let rest = &body[start..];
756 let end = rest.find(['"', '\'', '>']).unwrap_or(rest.len());
757 let target = rest[..end].trim();
758 if !target.is_empty() {
759 return Some(target);
760 }
761 }
762 }
763 from += at + "http-equiv".len();
764 }
765 None
766}
767
768fn script_location_target(body: &str) -> Option<&str> {
770 for marker in ["location.replace(", "location.href=", "location.href =", "location.assign("] {
771 let Some(at) = body.find(marker) else { continue };
772 let rest = &body[at + marker.len()..];
773 let rest = rest.trim_start();
774 let quote = rest.chars().next()?;
775 if quote != '"' && quote != '\'' {
776 continue;
778 }
779 let inner = &rest[quote.len_utf8()..];
780 let end = inner.find(quote)?;
781 let target = &inner[..end];
782 if target.starts_with("http") || target.starts_with('/') {
783 return Some(target);
784 }
785 }
786 None
787}
788
789pub fn needs_javascript(page: &Page) -> bool {
795 let html = &page.body;
796 if html.len() < 512 {
797 return false;
798 }
799 let Ok(article) = crate::parse::extract(html, Some(&page.final_url)) else {
800 return true;
801 };
802 let text_len = crate::text::visible_len(&article.text);
803
804 if text_len > MAX_STATIC_TEXT {
806 return false;
807 }
808
809 let starved = text_len.saturating_mul(TEXT_TO_MARKUP_FLOOR) < html.len();
814 if text_len < 120 || starved {
815 return true;
816 }
817
818 let lower = html.to_lowercase();
819 let spa_markers = ["__next_data__", "id=\"root\"", "id=\"app\"", "ng-app", "data-reactroot"];
820 spa_markers.iter().any(|m| lower.contains(m))
821}
822
823#[cfg(test)]
824mod tests {
825 use super::*;
826
827 #[test]
828 fn rejects_non_http_urls() {
829 assert!(normalize("file:///etc/passwd").is_err());
830 assert!(normalize("javascript:alert(1)").is_err());
831 assert!(normalize("not a url").is_err());
832 assert!(normalize("https://example.com/a").is_ok());
833 }
834
835 #[test]
836 fn parses_charset_from_content_type() {
837 assert_eq!(charset_of("text/html; charset=EUC-KR").as_deref(), Some("euc-kr"));
838 assert_eq!(charset_of("text/html").as_deref(), None);
839 }
840
841 #[test]
842 fn decodes_legacy_korean_encoding() {
843 let (bytes, _, _) = encoding_rs::EUC_KR.encode("한글 인코딩");
844 assert_eq!(decode(&bytes, Some("euc-kr")), "한글 인코딩");
845 let mut doc = b"<html><head><meta charset=\"euc-kr\"></head><body>".to_vec();
847 doc.extend_from_slice(&bytes);
848 assert!(decode(&doc, None).contains("한글 인코딩"));
849 }
850
851 #[test]
852 fn impersonate_strings_parse() {
853 assert_eq!(Impersonate::parse("chrome").unwrap(), Impersonate::Chrome);
854 assert_eq!(Impersonate::parse("none").unwrap(), Impersonate::None);
855 assert_eq!(
856 Impersonate::parse("chrome_143").unwrap(),
857 Impersonate::Named("chrome_143".into())
858 );
859 }
860
861 #[cfg(feature = "impersonate")]
862 #[test]
863 fn named_profiles_resolve_and_reject() {
864 assert!(named_profile("chrome_143").is_ok());
865 assert!(named_profile("netscape_2").is_err());
866 }
867
868 #[test]
869 fn client_builds_with_every_preset() {
870 for mode in ["chrome", "firefox", "safari", "random", "none"] {
871 let cfg = FetchConfig {
872 impersonate: Impersonate::parse(mode).unwrap(),
873 ..Default::default()
874 };
875 assert!(Fetcher::with_config(cfg).is_ok(), "{mode} failed to build");
876 }
877 }
878
879 #[test]
880 fn retry_after_is_parsed_and_capped() {
881 assert_eq!(parse_retry_after("120"), Some(120.0));
882 assert_eq!(parse_retry_after(" 0 "), Some(0.0));
883 assert_eq!(parse_retry_after("-5"), Some(0.0));
884 assert_eq!(parse_retry_after("Wed, 21 Oct 2026 07:28:00 GMT"), None);
886 assert_eq!(parse_retry_after(""), None);
887
888 let e =
889 Error::Status { url: "https://x.dev/".into(), status: 429, retry_after: Some(30.0) };
890 assert_eq!(e.retry_after(), Some(Duration::from_secs(30)));
891 assert!(e.is_retryable());
892 let none = Error::Status { url: "https://x.dev/".into(), status: 429, retry_after: None };
893 assert_eq!(none.retry_after(), None);
894 }
895
896 #[test]
897 fn proxies_build_one_client_each() {
898 let cfg = FetchConfig {
899 proxies: vec!["http://127.0.0.1:8080".into(), "socks5://127.0.0.1:1080".into()],
900 ..Default::default()
901 };
902 let fetcher = Fetcher::with_config(cfg).expect("proxied client");
903 assert_eq!(fetcher.clients.len(), 2);
904 let first = fetcher.client() as *const _;
906 let second = fetcher.client() as *const _;
907 assert_ne!(first, second);
908 assert_eq!(fetcher.client() as *const _, first);
909 }
910
911 #[test]
912 fn an_invalid_proxy_is_rejected_at_build_time() {
913 let cfg = FetchConfig { proxies: vec!["not a proxy".into()], ..Default::default() };
914 assert!(Fetcher::with_config(cfg).is_err());
915 }
916
917 #[test]
918 fn cookies_survive_a_round_trip_through_disk() {
919 let dir = std::env::temp_dir().join(format!("rustai-cookies-{}", std::process::id()));
920 let path = dir.join("jar.json");
921 let _ = std::fs::remove_file(&path);
922
923 let jar = wreq::cookie::Jar::default();
924 jar.add("cf_clearance=abc123; Domain=example.com; Path=/", "https://example.com/");
927 jar.add("session=xyz789; Path=/", "https://example.com/");
928 let hosts = vec!["example.com".to_string()];
929 assert_eq!(save_jar(&jar, &hosts, &path).unwrap(), 2);
930
931 let reloaded = load_jar(&path);
932 assert!(reloaded.contains("session", "https://example.com/"), "host-only cookie lost");
933 assert!(reloaded.contains("cf_clearance", "https://example.com/"), "cookie was lost");
934 assert_eq!(
935 reloaded.get("cf_clearance", "https://example.com/").map(|c| c.value().to_string()),
936 Some("abc123".to_string())
937 );
938
939 assert_eq!(load_jar(&dir.join("nope.json")).len(), 0);
941 let _ = std::fs::remove_dir_all(&dir);
942 }
943
944 #[test]
945 fn a_fetcher_without_a_cookie_file_saves_nothing() {
946 let fetcher = Fetcher::new().unwrap();
947 assert_eq!(fetcher.save_cookies().unwrap(), 0);
948 }
949
950 #[test]
951 fn zero_concurrency_is_rejected() {
952 let cfg = FetchConfig { concurrency: 0, ..Default::default() };
953 assert!(Fetcher::with_config(cfg).is_err());
954 }
955
956 fn page(body: &str) -> Page {
957 Page {
958 url: "https://x.dev/".into(),
959 final_url: "https://x.dev/".into(),
960 status: 200,
961 content_type: Some("text/html".into()),
962 body: body.into(),
963 elapsed_ms: 0,
964 rendered: false,
965 }
966 }
967
968 #[test]
969 fn follows_a_meta_refresh_stub() {
970 let base = Url::parse("https://blog.dev/2024/02/08/Post.html").unwrap();
971 let body = r#"<!doctype html><title>Redirect</title>
972 <noscript><meta http-equiv="refresh" content="0; url=https://blog.dev/2024/02/08/Post/"></noscript>"#;
973 assert_eq!(
974 detect_html_redirect(body, &base).unwrap().as_str(),
975 "https://blog.dev/2024/02/08/Post/"
976 );
977 }
978
979 #[test]
980 fn follows_a_script_location_stub() {
981 let base = Url::parse("https://blog.dev/a.html").unwrap();
982 let body = r#"<!doctype html><title>Redirect</title><script>
983 const target = "https://blog.dev/a/";
984 window.location.replace("https://blog.dev/a/");
985 </script>"#;
986 assert_eq!(detect_html_redirect(body, &base).unwrap().as_str(), "https://blog.dev/a/");
987 }
988
989 #[test]
990 fn resolves_a_relative_redirect_target() {
991 let base = Url::parse("https://blog.dev/x/a.html").unwrap();
992 let body = r#"<meta http-equiv="refresh" content="0;url=/y/b.html">"#;
993 assert_eq!(
994 detect_html_redirect(body, &base).unwrap().as_str(),
995 "https://blog.dev/y/b.html"
996 );
997 }
998
999 #[test]
1000 fn ignores_delayed_refreshes_and_self_loops() {
1001 let base = Url::parse("https://blog.dev/a").unwrap();
1002 assert!(
1004 detect_html_redirect(
1005 r#"<meta http-equiv="refresh" content="5; url=https://blog.dev/b">"#,
1006 &base
1007 )
1008 .is_none()
1009 );
1010 assert!(
1012 detect_html_redirect(
1013 r#"<meta http-equiv="refresh" content="0; url=https://blog.dev/a">"#,
1014 &base
1015 )
1016 .is_none()
1017 );
1018 }
1019
1020 #[test]
1021 fn ignores_a_long_page_that_merely_mentions_location() {
1022 let base = Url::parse("https://blog.dev/a").unwrap();
1023 let article = format!(
1024 "<article><p>{}</p><p>You can call location.replace(\"https://evil.dev/\") in JS.</p></article>",
1025 "Real prose. ".repeat(500)
1026 );
1027 assert!(
1028 detect_html_redirect(&article, &base).is_none(),
1029 "a full article was treated as a stub"
1030 );
1031 }
1032
1033 #[test]
1034 fn ignores_non_http_and_unresolvable_targets() {
1035 let base = Url::parse("https://blog.dev/a").unwrap();
1036 assert!(
1037 detect_html_redirect(
1038 r#"<meta http-equiv="refresh" content="0;url=javascript:x()">"#,
1039 &base
1040 )
1041 .is_none()
1042 );
1043 assert!(detect_html_redirect("<p>nothing here</p>", &base).is_none());
1044 assert!(
1046 detect_html_redirect(r#"<script>window.location.replace(target);</script>"#, &base)
1047 .is_none()
1048 );
1049 }
1050
1051 #[test]
1052 fn detects_an_spa_shell() {
1053 let shell = format!(
1054 "<html><body><div id=\"root\"></div><script>{}</script></body></html>",
1055 "x".repeat(2000)
1056 );
1057 assert!(needs_javascript(&page(&shell)));
1058 }
1059
1060 #[test]
1061 fn does_not_escalate_a_real_article() {
1062 let real = format!(
1063 "<html><body><article><h1>T</h1><p>{}</p></article></body></html>",
1064 "Real prose that a reader can actually read. ".repeat(20)
1065 );
1066 assert!(!needs_javascript(&page(&real)));
1067 }
1068}
1069
1070#[cfg(test)]
1071mod js_gate_tests {
1072 use super::*;
1073
1074 fn page(body: String) -> Page {
1075 Page {
1076 url: "https://x.dev/".into(),
1077 final_url: "https://x.dev/".into(),
1078 status: 200,
1079 content_type: Some("text/html".into()),
1080 body,
1081 elapsed_ms: 0,
1082 rendered: false,
1083 }
1084 }
1085
1086 #[test]
1092 fn a_client_rendered_front_page_is_flagged() {
1093 let shell = format!(
1094 "<html><body><nav>{}</nav><main></main><script>{}</script></body></html>",
1095 "<a href=\"/section\">Section link</a>".repeat(20),
1096 "x".repeat(340_000)
1097 );
1098 assert!(needs_javascript(&page(shell)));
1099 }
1100
1101 #[test]
1103 fn a_large_real_article_is_not_flagged() {
1104 let real = format!(
1105 "<html><body><article><h1>T</h1><p>{}</p></article></body></html>",
1106 "Real prose that a reader can actually read and act on. ".repeat(400)
1107 );
1108 assert!(!needs_javascript(&page(real)));
1109 }
1110}