1pub mod robots;
17
18#[cfg(feature = "browser")]
19pub mod browser;
20
21use std::collections::HashMap;
22use std::net::{IpAddr, SocketAddr};
23use std::sync::Arc;
24use std::time::{Duration, Instant};
25
26use futures_util::StreamExt;
27use tokio::sync::{Mutex, Semaphore};
28use url::Url;
29use wreq::dns::{Addrs, GaiResolver, Name, Resolve, Resolving};
30
31use crate::error::{Error, Result};
32use robots::Robots;
33
34pub const DEFAULT_USER_AGENT: &str =
36 concat!("rustai/", env!("CARGO_PKG_VERSION"), " (+https://github.com/imhyensuk/rustai)");
37
38pub const ROBOTS_AGENT: &str = "rustai";
40
41const MAX_ROBOTS_BYTES: usize = 512 * 1024;
48
49const MAX_HTML_REDIRECTS: usize = 2;
54
55const MAX_STATIC_TEXT: usize = 2_000;
57
58const TEXT_TO_MARKUP_FLOOR: usize = 200;
61
62const TEXTUAL: &[&str] = &[
64 "text/html",
65 "application/xhtml",
66 "text/plain",
67 "application/xml",
68 "text/xml",
69 "application/rss",
70 "application/atom",
71 "application/json",
72];
73
74#[derive(Debug, Clone, Default, PartialEq, Eq)]
76pub enum Impersonate {
77 #[default]
79 Chrome,
80 Firefox,
82 Safari,
84 Named(String),
86 Random,
88 None,
90}
91
92impl Impersonate {
93 pub fn parse(s: &str) -> Result<Impersonate> {
95 Ok(match s.trim().to_ascii_lowercase().as_str() {
96 "" | "none" | "off" | "false" => Impersonate::None,
97 "chrome" | "true" | "default" => Impersonate::Chrome,
98 "firefox" => Impersonate::Firefox,
99 "safari" => Impersonate::Safari,
100 "random" => Impersonate::Random,
101 other => Impersonate::Named(other.to_string()),
102 })
103 }
104}
105
106#[derive(Clone)]
108pub struct FetchConfig {
109 pub concurrency: usize,
111 pub timeout: Duration,
113 pub connect_timeout: Duration,
115 pub max_redirects: usize,
117 pub max_body_bytes: usize,
120 pub retries: u32,
122 pub retry_backoff: Duration,
124 pub per_host_delay: Duration,
126 pub respect_robots: bool,
128 pub respect_crawl_delay: bool,
130 pub impersonate: Impersonate,
132 pub accept_language: String,
134 pub browser_fallback: bool,
136 pub proxies: Vec<String>,
142 pub max_retry_after: Duration,
144 pub cookie_file: Option<std::path::PathBuf>,
146 pub allow_private_hosts: bool,
155}
156
157impl std::fmt::Debug for FetchConfig {
162 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
163 f.debug_struct("FetchConfig")
164 .field("concurrency", &self.concurrency)
165 .field("timeout", &self.timeout)
166 .field("max_body_bytes", &self.max_body_bytes)
167 .field("retries", &self.retries)
168 .field("respect_robots", &self.respect_robots)
169 .field("impersonate", &self.impersonate)
170 .field("browser_fallback", &self.browser_fallback)
171 .field("proxies", &self.proxies.iter().map(|p| mask_proxy(p)).collect::<Vec<_>>())
172 .field("cookie_file", &self.cookie_file)
173 .field("allow_private_hosts", &self.allow_private_hosts)
174 .finish_non_exhaustive()
175 }
176}
177
178fn mask_proxy(proxy: &str) -> String {
181 let Ok(mut parsed) = Url::parse(proxy) else { return proxy.to_string() };
182 if parsed.username().is_empty() && parsed.password().is_none() {
183 return proxy.to_string();
184 }
185 let _ = parsed.set_username("***");
186 let _ = parsed.set_password(Some("***"));
187 parsed.to_string()
188}
189
190impl Default for FetchConfig {
191 fn default() -> Self {
192 FetchConfig {
193 concurrency: 16,
194 timeout: Duration::from_secs(20),
195 connect_timeout: Duration::from_secs(8),
196 max_redirects: 5,
197 max_body_bytes: 8 * 1024 * 1024,
198 retries: 2,
199 retry_backoff: Duration::from_millis(400),
200 per_host_delay: Duration::from_millis(250),
201 respect_robots: true,
202 respect_crawl_delay: true,
203 impersonate: Impersonate::Chrome,
204 accept_language: "en-US,en;q=0.9".to_string(),
205 browser_fallback: false,
206 proxies: Vec::new(),
207 max_retry_after: Duration::from_secs(60),
208 cookie_file: None,
209 allow_private_hosts: false,
210 }
211 }
212}
213
214#[derive(Debug, Clone, serde::Serialize)]
216pub struct Page {
217 pub url: String,
219 pub final_url: String,
221 pub status: u16,
223 pub content_type: Option<String>,
225 pub body: String,
227 pub elapsed_ms: u64,
229 pub rendered: bool,
231}
232
233#[derive(Clone)]
235pub struct Fetcher {
236 clients: Arc<Vec<wreq::Client>>,
238 next_client: Arc<std::sync::atomic::AtomicUsize>,
239 jar: Option<Arc<wreq::cookie::Jar>>,
240 visited: Arc<std::sync::Mutex<std::collections::BTreeSet<String>>>,
244 cfg: Arc<FetchConfig>,
245 gate: Arc<Semaphore>,
246 robots: Arc<Mutex<HashMap<String, Arc<Robots>>>>,
247 last_hit: Arc<Mutex<HashMap<String, Instant>>>,
248}
249
250impl std::fmt::Debug for Fetcher {
251 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
252 f.debug_struct("Fetcher").field("config", &self.cfg).finish_non_exhaustive()
253 }
254}
255
256impl Fetcher {
257 pub fn new() -> Result<Self> {
259 Self::with_config(FetchConfig::default())
260 }
261
262 pub fn with_config(cfg: FetchConfig) -> Result<Self> {
264 if cfg.concurrency == 0 {
265 return Err(Error::Config("concurrency must be at least 1".into()));
266 }
267 let jar = cfg.cookie_file.as_ref().map(|path| Arc::new(load_jar(path)));
268
269 let build_one = |proxy: Option<&str>| -> Result<wreq::Client> {
270 let mut builder = wreq::Client::builder()
271 .timeout(cfg.timeout)
272 .connect_timeout(cfg.connect_timeout)
273 .redirect(wreq::redirect::Policy::none())
275 .pool_max_idle_per_host(cfg.concurrency.min(32))
276 .referer(true)
277 .gzip(true)
278 .brotli(true)
279 .deflate(true)
280 .zstd(true);
281
282 if !cfg.allow_private_hosts {
285 builder = builder.dns_resolver(Arc::new(PublicOnlyDns(GaiResolver::new())));
286 }
287
288 builder = match &jar {
289 Some(jar) => builder.cookie_provider(jar.clone()),
290 None => builder.cookie_store(true),
291 };
292 builder = apply_impersonation(builder, &cfg.impersonate)?;
293
294 if let Some(proxy) = proxy {
295 let proxy = wreq::Proxy::all(proxy)
296 .map_err(|e| Error::Config(format!("invalid proxy `{proxy}`: {e}")))?;
297 builder = builder.proxy(proxy);
298 }
299
300 let mut headers = wreq::header::HeaderMap::new();
301 if let Ok(v) = wreq::header::HeaderValue::from_str(&cfg.accept_language) {
302 headers.insert(wreq::header::ACCEPT_LANGUAGE, v);
303 }
304 if matches!(cfg.impersonate, Impersonate::None) {
305 headers.insert(
306 wreq::header::USER_AGENT,
307 wreq::header::HeaderValue::from_static(DEFAULT_USER_AGENT),
308 );
309 }
310 builder = builder.default_headers(headers);
311 builder.build().map_err(|e| Error::Config(e.to_string()))
312 };
313
314 let clients = if cfg.proxies.is_empty() {
315 vec![build_one(None)?]
316 } else {
317 cfg.proxies.iter().map(|p| build_one(Some(p))).collect::<Result<Vec<_>>>()?
318 };
319
320 Ok(Fetcher {
321 clients: Arc::new(clients),
322 next_client: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
323 jar,
324 visited: Arc::new(std::sync::Mutex::new(std::collections::BTreeSet::new())),
325 gate: Arc::new(Semaphore::new(cfg.concurrency)),
326 cfg: Arc::new(cfg),
327 robots: Arc::new(Mutex::new(HashMap::new())),
328 last_hit: Arc::new(Mutex::new(HashMap::new())),
329 })
330 }
331
332 fn client(&self) -> &wreq::Client {
337 if self.clients.len() == 1 {
338 return &self.clients[0];
339 }
340 let i = self.next_client.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
341 &self.clients[i % self.clients.len()]
342 }
343
344 pub fn save_cookies(&self) -> Result<usize> {
349 let (Some(jar), Some(path)) = (&self.jar, &self.cfg.cookie_file) else { return Ok(0) };
350 let hosts: Vec<String> = match self.visited.lock() {
351 Ok(v) => v.iter().cloned().collect(),
352 Err(poisoned) => poisoned.into_inner().iter().cloned().collect(),
353 };
354 save_jar(jar, &hosts, path)
355 }
356
357 pub fn config(&self) -> &FetchConfig {
359 &self.cfg
360 }
361
362 pub async fn fetch(&self, url: &str) -> Result<Page> {
364 self.fetch_inner(url, self.cfg.respect_robots).await
365 }
366
367 pub async fn fetch_api(&self, url: &str) -> Result<Page> {
376 self.fetch_inner(url, false).await
377 }
378
379 fn guard(&self, url: &Url) -> Result<()> {
387 if !self.cfg.allow_private_hosts && is_internal(url) {
388 return Err(Error::InvalidUrl(format!("{url} is an internal address")));
389 }
390 Ok(())
391 }
392
393 async fn fetch_following(&self, start: &Url, check_robots: bool) -> Result<Page> {
403 let mut current = start.clone();
404 for _ in 0..=self.cfg.max_redirects {
405 self.guard(¤t)?;
406 let crawl_delay = if check_robots {
407 let robots = self.robots_for(¤t).await;
408 if !robots.allows(current.path()) {
409 return Err(Error::RobotsDenied(current.to_string()));
410 }
411 robots.crawl_delay
412 } else {
413 None
414 };
415 self.pace(¤t, crawl_delay).await;
416 match self.get_with_retry(¤t).await? {
417 Hop::Page(page) => return Ok(page),
418 Hop::Redirect(next) => current = next,
419 }
420 }
421 Err(Error::network(start, format!("more than {} redirects", self.cfg.max_redirects)))
422 }
423
424 async fn fetch_inner(&self, url: &str, check_robots: bool) -> Result<Page> {
425 let parsed = normalize(url)?;
426 self.guard(&parsed)?;
428 let _permit = self.gate.acquire().await.map_err(|e| Error::Config(e.to_string()))?;
429
430 let started = Instant::now();
431 let mut page = self.fetch_following(&parsed, check_robots).await?;
432
433 let mut current = parsed.clone();
437 for _ in 0..MAX_HTML_REDIRECTS {
438 let Some(next) = detect_html_redirect(&page.body, ¤t) else { break };
439 let Ok(followed) = self.fetch_following(&next, check_robots).await else { break };
442 page = followed;
443 current = next;
444 }
445 page.url = parsed.to_string();
446 page.elapsed_ms = started.elapsed().as_millis() as u64;
447
448 if self.cfg.browser_fallback && needs_javascript(&page) {
449 if let Ok(rendered) = self.render(&parsed).await {
452 page.body = rendered;
453 page.rendered = true;
454 page.elapsed_ms = started.elapsed().as_millis() as u64;
455 }
456 }
457 Ok(page)
458 }
459
460 pub async fn fetch_many(&self, urls: &[String]) -> Vec<Result<Page>> {
465 let futures = urls.iter().map(|u| {
466 let this = self.clone();
467 let u = u.clone();
468 async move { this.fetch(&u).await }
469 });
470 futures_util::stream::iter(futures).buffered(self.cfg.concurrency).collect::<Vec<_>>().await
471 }
472
473 async fn get_with_retry(&self, url: &Url) -> Result<Hop> {
475 self.guard(url)?;
476 let mut attempt = 0u32;
477 loop {
478 match self.get_once(url).await {
479 Ok(page) => return Ok(page),
480 Err(e) if e.is_retryable() && attempt < self.cfg.retries => {
481 let wait = match e.retry_after() {
487 Some(hint) if hint <= self.cfg.max_retry_after => hint,
488 Some(_) => return Err(e),
489 None => self.cfg.retry_backoff * 2u32.pow(attempt),
490 };
491 tokio::time::sleep(wait).await;
492 attempt += 1;
493 }
494 Err(e) => return Err(e),
495 }
496 }
497 }
498
499 async fn get_once(&self, url: &Url) -> Result<Hop> {
500 let resp =
501 self.client().get(url.as_str()).send().await.map_err(|e| Error::network(url, e))?;
502
503 let status = resp.status().as_u16();
504 let final_url = resp.uri().to_string();
505 let content_type = resp
506 .headers()
507 .get(wreq::header::CONTENT_TYPE)
508 .and_then(|v| v.to_str().ok())
509 .map(str::to_string);
510
511 if (300..400).contains(&status)
514 && let Some(next) = resp
515 .headers()
516 .get(wreq::header::LOCATION)
517 .and_then(|v| v.to_str().ok())
518 .and_then(|l| url.join(l).ok())
519 .filter(|next| matches!(next.scheme(), "http" | "https"))
520 {
521 return Ok(Hop::Redirect(next));
522 }
523 if !(200..300).contains(&status) {
524 let retry_after = resp
525 .headers()
526 .get(wreq::header::RETRY_AFTER)
527 .and_then(|v| v.to_str().ok())
528 .and_then(parse_retry_after);
529 return Err(Error::Status { url: url.to_string(), status, retry_after });
530 }
531 if let Some(ct) = &content_type {
532 let base = ct.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
533 if !TEXTUAL.iter().any(|t| base.starts_with(t)) {
534 return Err(Error::Parse(format!("{url}: unsupported content type {base}")));
535 }
536 }
537 if resp.content_length().is_some_and(|n| n as usize > self.cfg.max_body_bytes) {
538 return Err(Error::BodyTooLarge {
539 url: url.to_string(),
540 limit: self.cfg.max_body_bytes,
541 });
542 }
543
544 let charset = content_type.as_deref().and_then(charset_of);
545 let bytes = self.read_capped(url, resp, self.cfg.max_body_bytes).await?;
546 let body = decode(&bytes, charset.as_deref(), tld_of(&final_url).as_deref());
547
548 Ok(Hop::Page(Page {
549 url: url.to_string(),
550 final_url,
551 status,
552 content_type,
553 body,
554 elapsed_ms: 0,
555 rendered: false,
556 }))
557 }
558
559 async fn read_capped(&self, url: &Url, resp: wreq::Response, limit: usize) -> Result<Vec<u8>> {
561 let mut buf: Vec<u8> = Vec::with_capacity(64 * 1024);
562 let mut stream = resp.bytes_stream();
563 while let Some(chunk) = stream.next().await {
564 let chunk = chunk.map_err(|e| Error::network(url, e))?;
565 if buf.len() + chunk.len() > limit {
566 return Err(Error::BodyTooLarge { url: url.to_string(), limit });
567 }
568 buf.extend_from_slice(&chunk);
569 }
570 Ok(buf)
571 }
572
573 async fn robots_for(&self, url: &Url) -> Arc<Robots> {
578 let key = robots_origin(url);
579 if let Some(cached) = self.robots.lock().await.get(&key) {
580 return cached.clone();
581 }
582 let robots_url = format!("{key}/robots.txt");
583 let parsed = match self.client().get(&robots_url).send().await {
584 Ok(resp) if resp.status().is_success() => {
585 match self.read_capped(url, resp, MAX_ROBOTS_BYTES).await {
586 Ok(bytes) => Robots::parse(&String::from_utf8_lossy(&bytes), ROBOTS_AGENT),
589 Err(_) => Robots::allow_all(),
590 }
591 }
592 _ => Robots::allow_all(),
593 };
594 let parsed = Arc::new(parsed);
595 self.robots.lock().await.insert(key, parsed.clone());
596 parsed
597 }
598
599 async fn pace(&self, url: &Url, crawl_delay: Option<f64>) {
601 let host = url.host_str().unwrap_or_default().to_string();
602 if let Ok(mut visited) = self.visited.lock() {
603 visited.insert(host.clone());
604 }
605 let mut delay = self.cfg.per_host_delay;
606 if self.cfg.respect_crawl_delay
607 && let Some(d) = crawl_delay
608 {
609 delay = delay.max(Duration::from_secs_f64(d.min(30.0)));
610 }
611 if delay.is_zero() {
612 return;
613 }
614 let wait = {
615 let mut map = self.last_hit.lock().await;
616 let now = Instant::now();
617 let wait = match map.get(&host) {
618 Some(prev) => delay.checked_sub(now.duration_since(*prev)),
619 None => None,
620 };
621 map.insert(host, now + wait.unwrap_or_default());
624 wait
625 };
626 if let Some(wait) = wait {
627 tokio::time::sleep(wait).await;
628 }
629 }
630
631 #[cfg(feature = "browser")]
632 async fn render(&self, url: &Url) -> Result<String> {
633 self.guard(url)?;
634 browser::render(url.as_str(), self.cfg.timeout).await
635 }
636
637 #[cfg(not(feature = "browser"))]
638 async fn render(&self, _url: &Url) -> Result<String> {
639 Err(Error::Browser("build rustai with the `browser` feature".into()))
640 }
641}
642
643fn apply_impersonation(
644 builder: wreq::ClientBuilder,
645 mode: &Impersonate,
646) -> Result<wreq::ClientBuilder> {
647 #[cfg(not(feature = "impersonate"))]
648 {
649 if !matches!(mode, Impersonate::None) {
650 return Err(Error::Config(
651 "browser impersonation needs the `impersonate` feature".into(),
652 ));
653 }
654 Ok(builder)
655 }
656 #[cfg(feature = "impersonate")]
657 {
658 use wreq_util::{Emulation, Profile};
659 let profile = match mode {
660 Impersonate::None => return Ok(builder),
661 Impersonate::Random => return Ok(builder.emulation(Emulation::weighted_random())),
662 Impersonate::Chrome => Profile::Chrome149,
663 Impersonate::Firefox => Profile::Firefox147,
664 Impersonate::Safari => Profile::Safari26_4,
665 Impersonate::Named(name) => named_profile(name)?,
666 };
667 Ok(builder.emulation(profile))
668 }
669}
670
671#[cfg(feature = "impersonate")]
677fn named_profile(name: &str) -> Result<wreq_util::Profile> {
678 serde_json::from_value::<wreq_util::Profile>(serde_json::Value::String(name.to_string()))
679 .map_err(|_| {
680 Error::Config(format!(
681 "unknown impersonation profile `{name}`; try `chrome`, `firefox`, `safari`, \
682 `random`, or a versioned name such as `chrome_143`"
683 ))
684 })
685}
686
687fn parse_retry_after(value: &str) -> Option<f64> {
693 let secs: f64 = value.trim().parse().ok()?;
694 secs.is_finite().then_some(secs.max(0.0))
695}
696
697#[derive(serde::Serialize, serde::Deserialize)]
699struct StoredCookie {
700 name: String,
701 value: String,
702 host: String,
704 path: String,
705 #[serde(default, skip_serializing_if = "Option::is_none")]
708 domain: Option<String>,
709}
710
711fn load_jar(path: &std::path::Path) -> wreq::cookie::Jar {
714 let jar = wreq::cookie::Jar::default();
715 let Ok(raw) = std::fs::read_to_string(path) else { return jar };
716 let Ok(stored) = serde_json::from_str::<Vec<StoredCookie>>(&raw) else { return jar };
717 for c in stored {
718 let uri = format!("https://{}{}", c.host, c.path);
719 let mut cookie = format!("{}={}; Path={}", c.name, c.value, c.path);
720 if let Some(domain) = &c.domain {
721 cookie.push_str(&format!("; Domain={domain}"));
722 }
723 jar.add(cookie.as_str(), uri.as_str());
724 }
725 jar
726}
727
728fn save_jar(jar: &wreq::cookie::Jar, hosts: &[String], path: &std::path::Path) -> Result<usize> {
730 let mut seen = std::collections::HashSet::new();
734 let mut stored: Vec<StoredCookie> = Vec::new();
735 for host in hosts {
736 for c in jar.matches(format!("https://{host}/").as_str()) {
737 let cookie = StoredCookie {
738 name: c.name().to_string(),
739 value: c.value().to_string(),
740 host: host.clone(),
741 path: c.path().unwrap_or("/").to_string(),
742 domain: c.domain().map(str::to_string),
743 };
744 if seen.insert((cookie.host.clone(), cookie.name.clone(), cookie.path.clone())) {
745 stored.push(cookie);
746 }
747 }
748 }
749 let json = serde_json::to_string_pretty(&stored)
750 .map_err(|e| Error::Config(format!("could not serialise cookies: {e}")))?;
751 if let Some(parent) = path.parent()
752 && !parent.as_os_str().is_empty()
753 {
754 let _ = std::fs::create_dir_all(parent);
755 }
756 #[cfg(unix)]
760 let written = {
761 use std::io::Write;
762 use std::os::unix::fs::OpenOptionsExt;
763 let _ = std::fs::remove_file(path);
764 std::fs::OpenOptions::new()
765 .write(true)
766 .create_new(true)
767 .mode(0o600)
768 .open(path)
769 .and_then(|mut f| f.write_all(json.as_bytes()))
770 };
771 #[cfg(not(unix))]
772 let written = std::fs::write(path, &json);
773 written.map_err(|e| Error::Config(format!("could not write {}: {e}", path.display())))?;
774 Ok(stored.len())
775}
776
777enum Hop {
779 Page(Page),
780 Redirect(Url),
781}
782
783fn is_internal_ip(ip: &IpAddr) -> bool {
785 match ip {
786 IpAddr::V4(ip) => {
787 ip.is_loopback()
788 || ip.is_private()
789 || ip.is_link_local()
790 || ip.is_unspecified()
791 || ip.is_broadcast()
792 || ip.is_documentation()
793 }
794 IpAddr::V6(ip) => {
795 ip.is_loopback()
796 || ip.is_unspecified()
797 || (ip.segments()[0] & 0xfe00) == 0xfc00 || (ip.segments()[0] & 0xffc0) == 0xfe80 || ip.to_ipv4_mapped().is_some_and(|v4| is_internal_ip(&IpAddr::V4(v4)))
800 }
801 }
802}
803
804fn is_internal(url: &Url) -> bool {
810 match url.host() {
811 Some(url::Host::Ipv4(ip)) => is_internal_ip(&IpAddr::V4(ip)),
812 Some(url::Host::Ipv6(ip)) => is_internal_ip(&IpAddr::V6(ip)),
813 Some(url::Host::Domain(name)) => {
816 let name = name.trim_end_matches('.').to_ascii_lowercase();
817 name == "localhost" || name.ends_with(".localhost") || name.ends_with(".internal")
818 }
819 None => false,
820 }
821}
822
823struct PublicOnlyDns<R>(R);
839
840impl<R: Resolve + 'static> Resolve for PublicOnlyDns<R> {
841 fn resolve(&self, name: Name) -> Resolving {
842 let host = name.as_str().to_string();
843 let inner = self.0.resolve(name);
844 Box::pin(async move {
845 let kept: Vec<SocketAddr> =
846 inner.await?.filter(|addr| !is_internal_ip(&addr.ip())).collect();
847 if kept.is_empty() {
848 return Err(format!("{host} resolves only to internal addresses").into());
849 }
850 Ok(Box::new(kept.into_iter()) as Addrs)
851 })
852 }
853}
854
855fn robots_origin(url: &Url) -> String {
863 url.origin().ascii_serialization()
864}
865
866pub fn normalize(url: &str) -> Result<Url> {
868 let parsed = Url::parse(url.trim()).map_err(|_| Error::InvalidUrl(url.to_string()))?;
869 if !matches!(parsed.scheme(), "http" | "https") {
870 return Err(Error::InvalidUrl(url.to_string()));
871 }
872 if parsed.host_str().is_none() {
873 return Err(Error::InvalidUrl(url.to_string()));
874 }
875 Ok(parsed)
876}
877
878fn charset_of(content_type: &str) -> Option<String> {
879 content_type
880 .split(';')
881 .skip(1)
882 .filter_map(|p| p.trim().strip_prefix("charset="))
883 .map(|c| c.trim_matches(['"', '\'']).to_ascii_lowercase())
884 .next()
885}
886
887fn decode(bytes: &[u8], header_charset: Option<&str>, tld: Option<&str>) -> String {
897 let label = header_charset.map(str::to_string).or_else(|| sniff_meta_charset(bytes));
898 let encoding = label
899 .as_deref()
900 .and_then(|l| encoding_rs::Encoding::for_label(l.as_bytes()))
901 .unwrap_or_else(|| {
902 if bytes.len() - encoding_rs::Encoding::utf8_valid_up_to(bytes) <= 3 {
905 encoding_rs::UTF_8
906 } else {
907 let mut detector = chardetng::EncodingDetector::new();
908 detector.feed(bytes, true);
909 detector.guess(tld.map(str::as_bytes), true)
910 }
911 });
912 let (text, _, _) = encoding.decode(bytes);
913 text.into_owned()
914}
915
916fn tld_of(url: &str) -> Option<String> {
918 let host = Url::parse(url).ok()?.host_str()?.to_ascii_lowercase();
919 host.rsplit('.').next().map(str::to_string)
920}
921
922fn sniff_meta_charset(bytes: &[u8]) -> Option<String> {
924 let head = &bytes[..bytes.len().min(4096)];
925 let text = String::from_utf8_lossy(head).to_ascii_lowercase();
926 if let Some(at) = text.find("charset=") {
927 let rest = &text[at + "charset=".len()..];
928 let value: String = rest
929 .trim_start_matches(['"', '\'', ' '])
930 .chars()
931 .take_while(|c| c.is_ascii_alphanumeric() || *c == '-' || *c == '_')
932 .collect();
933 if !value.is_empty() {
934 return Some(value);
935 }
936 }
937 None
938}
939
940fn detect_html_redirect(body: &str, base: &Url) -> Option<Url> {
948 const MAX_STUB_BYTES: usize = 4096;
949 if body.len() > MAX_STUB_BYTES {
950 return None;
951 }
952 let target = meta_refresh_target(body).or_else(|| script_location_target(body))?;
953 let next = base.join(target.trim().trim_matches(['"', '\''])).ok()?;
954 if !matches!(next.scheme(), "http" | "https") {
955 return None;
956 }
957 let same = next.as_str().trim_end_matches('/') == base.as_str().trim_end_matches('/');
959 (!same).then_some(next)
960}
961
962fn meta_refresh_target(body: &str) -> Option<&str> {
964 let lower = body.to_ascii_lowercase();
965 let mut from = 0usize;
966 while let Some(at) = lower[from..].find("http-equiv") {
967 let tag_start = lower[..from + at].rfind('<')?;
968 let tag_end = lower[tag_start..].find('>')? + tag_start;
969 let tag = &lower[tag_start..tag_end];
970 if tag.contains("refresh")
971 && let Some(content_at) = tag.find("content")
972 && let Some(url_at) = tag[content_at..].find("url=")
973 {
974 let delay: String = tag[content_at..]
976 .chars()
977 .skip_while(|c| !c.is_ascii_digit())
978 .take_while(|c| c.is_ascii_digit())
979 .collect();
980 if delay.parse::<u32>().unwrap_or(0) == 0 {
981 let start = tag_start + content_at + url_at + 4;
982 let rest = &body[start..];
983 let end = rest.find(['"', '\'', '>']).unwrap_or(rest.len());
984 let target = rest[..end].trim();
985 if !target.is_empty() {
986 return Some(target);
987 }
988 }
989 }
990 from += at + "http-equiv".len();
991 }
992 None
993}
994
995fn script_location_target(body: &str) -> Option<&str> {
997 for marker in ["location.replace(", "location.href=", "location.href =", "location.assign("] {
998 let Some(at) = body.find(marker) else { continue };
999 let rest = &body[at + marker.len()..];
1000 let rest = rest.trim_start();
1001 let quote = rest.chars().next()?;
1002 if quote != '"' && quote != '\'' {
1003 continue;
1005 }
1006 let inner = &rest[quote.len_utf8()..];
1007 let end = inner.find(quote)?;
1008 let target = &inner[..end];
1009 if target.starts_with("http") || target.starts_with('/') {
1010 return Some(target);
1011 }
1012 }
1013 None
1014}
1015
1016pub fn needs_javascript(page: &Page) -> bool {
1022 let html = &page.body;
1023 if html.len() < 512 {
1024 return false;
1025 }
1026 let Ok(article) = crate::parse::extract(html, Some(&page.final_url)) else {
1027 return true;
1028 };
1029 let text_len = crate::text::visible_len(&article.text);
1030
1031 if text_len > MAX_STATIC_TEXT {
1033 return false;
1034 }
1035
1036 let starved = text_len.saturating_mul(TEXT_TO_MARKUP_FLOOR) < html.len();
1041 if text_len < 120 || starved {
1042 return true;
1043 }
1044
1045 let lower = html.to_lowercase();
1046 let spa_markers = ["__next_data__", "id=\"root\"", "id=\"app\"", "ng-app", "data-reactroot"];
1047 spa_markers.iter().any(|m| lower.contains(m))
1048}
1049
1050#[cfg(test)]
1051mod tests {
1052 use super::*;
1053
1054 #[test]
1055 fn rejects_non_http_urls() {
1056 assert!(normalize("file:///etc/passwd").is_err());
1057 assert!(normalize("javascript:alert(1)").is_err());
1058 assert!(normalize("not a url").is_err());
1059 assert!(normalize("https://example.com/a").is_ok());
1060 }
1061
1062 #[test]
1063 fn parses_charset_from_content_type() {
1064 assert_eq!(charset_of("text/html; charset=EUC-KR").as_deref(), Some("euc-kr"));
1065 assert_eq!(charset_of("text/html").as_deref(), None);
1066 }
1067
1068 #[test]
1069 fn decodes_legacy_korean_encoding() {
1070 let (bytes, _, _) = encoding_rs::EUC_KR.encode("한글 인코딩");
1071 assert_eq!(decode(&bytes, Some("euc-kr"), None), "한글 인코딩");
1072 let mut doc = b"<html><head><meta charset=\"euc-kr\"></head><body>".to_vec();
1074 doc.extend_from_slice(&bytes);
1075 assert!(decode(&doc, None, None).contains("한글 인코딩"));
1076 }
1077
1078 #[test]
1079 fn guesses_a_legacy_encoding_that_nothing_declares() {
1080 let (bytes, _, _) = encoding_rs::EUC_KR.encode(concat!(
1082 "<html><head><title>합성함수 미분법 = 연쇄법칙(Chain Rule)</title></head><body>",
1083 "<p>함수 y = f(x) 와 함수 z = g(y)를 합성한 함수의 미분법은 다음과 같다.</p></body></html>",
1084 ));
1085 assert!(decode(&bytes, None, Some("kr")).contains("합성함수 미분법 = 연쇄법칙"));
1086 }
1087
1088 #[test]
1089 fn undeclared_utf8_stays_utf8() {
1090 assert_eq!(
1091 decode("<p>한글 UTF-8 본문</p>".as_bytes(), None, Some("kr")),
1092 "<p>한글 UTF-8 본문</p>"
1093 );
1094 }
1095
1096 #[test]
1097 fn a_body_cut_mid_character_is_still_read_as_utf8() {
1098 let full = "<p>가나다라마바사</p>".repeat(20);
1100 let cut = &full.as_bytes()[..full.len() - 7];
1101 assert!(decode(cut, None, Some("kr")).starts_with("<p>가나다라마바사</p>"));
1102 }
1103
1104 #[test]
1105 fn tld_is_the_last_host_label() {
1106 assert_eq!(tld_of("https://www.math.snu.ac.kr/a.htm").as_deref(), Some("kr"));
1107 assert_eq!(tld_of("not a url"), None);
1108 }
1109
1110 #[test]
1111 fn impersonate_strings_parse() {
1112 assert_eq!(Impersonate::parse("chrome").unwrap(), Impersonate::Chrome);
1113 assert_eq!(Impersonate::parse("none").unwrap(), Impersonate::None);
1114 assert_eq!(
1115 Impersonate::parse("chrome_143").unwrap(),
1116 Impersonate::Named("chrome_143".into())
1117 );
1118 }
1119
1120 #[cfg(feature = "impersonate")]
1121 #[test]
1122 fn named_profiles_resolve_and_reject() {
1123 assert!(named_profile("chrome_143").is_ok());
1124 assert!(named_profile("netscape_2").is_err());
1125 }
1126
1127 #[test]
1128 fn client_builds_with_every_preset() {
1129 for mode in ["chrome", "firefox", "safari", "random", "none"] {
1130 let cfg = FetchConfig {
1131 impersonate: Impersonate::parse(mode).unwrap(),
1132 ..Default::default()
1133 };
1134 assert!(Fetcher::with_config(cfg).is_ok(), "{mode} failed to build");
1135 }
1136 }
1137
1138 #[test]
1139 fn retry_after_is_parsed_and_capped() {
1140 assert_eq!(parse_retry_after("120"), Some(120.0));
1141 assert_eq!(parse_retry_after(" 0 "), Some(0.0));
1142 assert_eq!(parse_retry_after("-5"), Some(0.0));
1143 assert_eq!(parse_retry_after("Wed, 21 Oct 2026 07:28:00 GMT"), None);
1145 assert_eq!(parse_retry_after(""), None);
1146
1147 let e =
1148 Error::Status { url: "https://x.dev/".into(), status: 429, retry_after: Some(30.0) };
1149 assert_eq!(e.retry_after(), Some(Duration::from_secs(30)));
1150 assert!(e.is_retryable());
1151 let none = Error::Status { url: "https://x.dev/".into(), status: 429, retry_after: None };
1152 assert_eq!(none.retry_after(), None);
1153 }
1154
1155 #[test]
1156 fn proxies_build_one_client_each() {
1157 let cfg = FetchConfig {
1158 proxies: vec!["http://127.0.0.1:8080".into(), "socks5://127.0.0.1:1080".into()],
1159 ..Default::default()
1160 };
1161 let fetcher = Fetcher::with_config(cfg).expect("proxied client");
1162 assert_eq!(fetcher.clients.len(), 2);
1163 let first = fetcher.client() as *const _;
1165 let second = fetcher.client() as *const _;
1166 assert_ne!(first, second);
1167 assert_eq!(fetcher.client() as *const _, first);
1168 }
1169
1170 #[test]
1171 fn an_invalid_proxy_is_rejected_at_build_time() {
1172 let cfg = FetchConfig { proxies: vec!["not a proxy".into()], ..Default::default() };
1173 assert!(Fetcher::with_config(cfg).is_err());
1174 }
1175
1176 #[test]
1177 fn cookies_survive_a_round_trip_through_disk() {
1178 let dir = std::env::temp_dir().join(format!("rustai-cookies-{}", std::process::id()));
1179 let path = dir.join("jar.json");
1180 let _ = std::fs::remove_file(&path);
1181
1182 let jar = wreq::cookie::Jar::default();
1183 jar.add("cf_clearance=abc123; Domain=example.com; Path=/", "https://example.com/");
1186 jar.add("session=xyz789; Path=/", "https://example.com/");
1187 let hosts = vec!["example.com".to_string()];
1188 assert_eq!(save_jar(&jar, &hosts, &path).unwrap(), 2);
1189
1190 let reloaded = load_jar(&path);
1191 assert!(reloaded.contains("session", "https://example.com/"), "host-only cookie lost");
1192 assert!(reloaded.contains("cf_clearance", "https://example.com/"), "cookie was lost");
1193 assert_eq!(
1194 reloaded.get("cf_clearance", "https://example.com/").map(|c| c.value().to_string()),
1195 Some("abc123".to_string())
1196 );
1197
1198 assert_eq!(load_jar(&dir.join("nope.json")).len(), 0);
1200 let _ = std::fs::remove_dir_all(&dir);
1201 }
1202
1203 #[cfg(unix)]
1204 #[test]
1205 fn a_saved_jar_is_readable_only_by_its_owner() {
1206 use std::os::unix::fs::PermissionsExt;
1207 let dir = std::env::temp_dir().join(format!("rustai-cookie-perms-{}", std::process::id()));
1208 std::fs::create_dir_all(&dir).unwrap();
1209 let path = dir.join("jar.json");
1210
1211 std::fs::write(&path, "[]").unwrap();
1214 std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap();
1215
1216 let jar = wreq::cookie::Jar::default();
1217 jar.add("cf_clearance=secret; Path=/", "https://example.com/");
1218 save_jar(&jar, &["example.com".to_string()], &path).unwrap();
1219
1220 let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777;
1221 assert_eq!(mode, 0o600, "cookie jar left at {mode:o}");
1222 let _ = std::fs::remove_dir_all(&dir);
1223 }
1224
1225 #[test]
1226 fn robots_is_scoped_to_an_origin_including_its_port() {
1227 let key = |u: &str| robots_origin(&Url::parse(u).unwrap());
1228 assert_eq!(key("http://example.com:8080/a/b"), "http://example.com:8080");
1231 assert_ne!(key("http://example.com:8080/a"), key("http://example.com:9090/a"));
1232 assert_eq!(key("https://example.com/a"), "https://example.com");
1235 assert_eq!(key("https://example.com:443/a"), key("https://example.com/b"));
1236 assert_eq!(key("http://example.com:80/a"), key("http://example.com/b"));
1237 assert_ne!(key("http://example.com/a"), key("https://example.com/a"));
1239 }
1240
1241 struct Fixed(Vec<SocketAddr>);
1243
1244 impl Resolve for Fixed {
1245 fn resolve(&self, _: Name) -> Resolving {
1246 let addrs = self.0.clone();
1247 Box::pin(async move { Ok(Box::new(addrs.into_iter()) as Addrs) })
1248 }
1249 }
1250
1251 fn addr(s: &str) -> SocketAddr {
1252 s.parse().expect("socket addr")
1253 }
1254
1255 #[tokio::test]
1256 async fn resolving_drops_internal_addresses() {
1257 let only_internal = PublicOnlyDns(Fixed(vec![addr("127.0.0.1:80")]));
1260 assert!(
1261 only_internal.resolve(Name::from("rebind.test")).await.is_err(),
1262 "a name with nowhere public to go must not resolve"
1263 );
1264
1265 let mixed = PublicOnlyDns(Fixed(vec![
1268 addr("127.0.0.1:80"),
1269 addr("93.184.216.34:80"),
1270 addr("10.0.0.7:80"),
1271 addr("[::1]:80"),
1272 addr("[2606:4700::1]:80"),
1273 ]));
1274 let kept: Vec<SocketAddr> =
1275 mixed.resolve(Name::from("mixed.test")).await.expect("resolves").collect();
1276 assert_eq!(kept, vec![addr("93.184.216.34:80"), addr("[2606:4700::1]:80")]);
1277 }
1278
1279 #[test]
1280 fn internal_ip_ranges_are_recognised() {
1281 let internal = [
1282 "127.0.0.1",
1283 "127.9.9.9",
1284 "10.0.0.5",
1285 "192.168.1.1",
1286 "172.16.0.1",
1287 "169.254.169.254",
1288 "0.0.0.0",
1289 "255.255.255.255",
1290 "::1",
1291 "::",
1292 "fe80::1",
1293 "fd00::1",
1294 "::ffff:127.0.0.1",
1295 "::ffff:10.1.2.3",
1296 ];
1297 for ip in internal {
1298 let ip: IpAddr = ip.parse().expect(ip);
1299 assert!(is_internal_ip(&ip), "{ip} should be internal");
1300 }
1301
1302 let public = ["8.8.8.8", "93.184.216.34", "172.32.0.1", "2606:4700::1", "::ffff:8.8.8.8"];
1303 for ip in public {
1304 let ip: IpAddr = ip.parse().expect(ip);
1305 assert!(!is_internal_ip(&ip), "{ip} should be reachable");
1306 }
1307 }
1308
1309 #[test]
1310 fn internal_addresses_are_recognised() {
1311 let internal = [
1312 "http://127.0.0.1/",
1313 "http://localhost:3000/",
1314 "http://sub.localhost/",
1315 "http://LOCALHOST./",
1316 "http://169.254.169.254/latest/meta-data/", "http://metadata.google.internal/computeMetadata/v1/",
1318 "http://10.0.0.5/",
1319 "http://192.168.1.1/",
1320 "http://172.16.0.1/",
1321 "http://0.0.0.0/",
1322 "http://[::1]/",
1323 "http://[fe80::1]/",
1324 "http://[fd00::1]/",
1325 "http://[::ffff:127.0.0.1]/",
1326 ];
1327 for u in internal {
1328 assert!(is_internal(&Url::parse(u).unwrap()), "{u} should be internal");
1329 }
1330
1331 let public = [
1332 "https://example.com/",
1333 "https://en.wikipedia.org/wiki/BM25",
1334 "http://8.8.8.8/",
1335 "http://172.32.0.1/", "http://[2606:4700::1]/",
1337 "https://notlocalhost.com/", "https://internal.example.com/",
1339 ];
1340 for u in public {
1341 assert!(!is_internal(&Url::parse(u).unwrap()), "{u} should be reachable");
1342 }
1343 }
1344
1345 #[tokio::test]
1346 async fn fetching_an_internal_address_is_refused_before_the_network() {
1347 let f = Fetcher::new().unwrap();
1348 let err = f.fetch("http://169.254.169.254/latest/meta-data/").await.unwrap_err();
1349 assert!(matches!(err, Error::InvalidUrl(_)), "got {err:?}");
1350 assert!(f.fetch_api("http://127.0.0.1:8080/").await.is_err());
1352
1353 let cfg = FetchConfig { allow_private_hosts: true, ..Default::default() };
1356 let f = Fetcher::with_config(cfg).unwrap();
1357 let err = f.fetch("http://127.0.0.1:9/").await.unwrap_err();
1358 assert!(!matches!(err, Error::InvalidUrl(_)), "gate still firing: {err:?}");
1359 }
1360
1361 #[tokio::test]
1362 async fn every_request_path_shares_one_internal_address_guard() {
1363 let f = Fetcher::new().unwrap();
1364 let internal = Url::parse("http://169.254.169.254/latest/meta-data/").unwrap();
1365 assert!(f.guard(&internal).is_err());
1369 assert!(f.get_with_retry(&internal).await.is_err());
1370 assert!(f.guard(&Url::parse("https://example.com/").unwrap()).is_ok());
1371
1372 let open =
1373 Fetcher::with_config(FetchConfig { allow_private_hosts: true, ..Default::default() })
1374 .unwrap();
1375 assert!(open.guard(&internal).is_ok());
1376 }
1377
1378 #[test]
1379 fn a_meta_refresh_to_an_internal_address_is_still_an_internal_address() {
1380 let base = Url::parse("https://evil.test/post").unwrap();
1383 let body = r#"<!doctype html><meta http-equiv="refresh"
1384 content="0;url=http://169.254.169.254/latest/meta-data/">"#;
1385 let next = detect_html_redirect(body, &base).expect("redirect should parse");
1386 assert!(is_internal(&next), "{next} must be caught by the guard");
1387 }
1388
1389 #[test]
1390 fn proxy_credentials_do_not_reach_debug_output() {
1391 assert_eq!(
1394 mask_proxy("http://user:hunter2@proxy.dev:8080"),
1395 "http://***:***@proxy.dev:8080/"
1396 );
1397 assert_eq!(mask_proxy("socks5://127.0.0.1:1080"), "socks5://127.0.0.1:1080");
1399 assert_eq!(mask_proxy("not a url"), "not a url");
1400
1401 let cfg = FetchConfig {
1402 proxies: vec!["http://bob:s3cr3t@proxy.dev:8080".into()],
1403 ..Default::default()
1404 };
1405 let dumped = format!("{cfg:?}");
1406 assert!(!dumped.contains("s3cr3t"), "password leaked: {dumped}");
1407 assert!(dumped.contains("proxy.dev"), "host should survive: {dumped}");
1408 }
1409
1410 #[test]
1411 fn a_fetcher_without_a_cookie_file_saves_nothing() {
1412 let fetcher = Fetcher::new().unwrap();
1413 assert_eq!(fetcher.save_cookies().unwrap(), 0);
1414 }
1415
1416 #[test]
1417 fn zero_concurrency_is_rejected() {
1418 let cfg = FetchConfig { concurrency: 0, ..Default::default() };
1419 assert!(Fetcher::with_config(cfg).is_err());
1420 }
1421
1422 fn page(body: &str) -> Page {
1423 Page {
1424 url: "https://x.dev/".into(),
1425 final_url: "https://x.dev/".into(),
1426 status: 200,
1427 content_type: Some("text/html".into()),
1428 body: body.into(),
1429 elapsed_ms: 0,
1430 rendered: false,
1431 }
1432 }
1433
1434 #[test]
1435 fn follows_a_meta_refresh_stub() {
1436 let base = Url::parse("https://blog.dev/2024/02/08/Post.html").unwrap();
1437 let body = r#"<!doctype html><title>Redirect</title>
1438 <noscript><meta http-equiv="refresh" content="0; url=https://blog.dev/2024/02/08/Post/"></noscript>"#;
1439 assert_eq!(
1440 detect_html_redirect(body, &base).unwrap().as_str(),
1441 "https://blog.dev/2024/02/08/Post/"
1442 );
1443 }
1444
1445 #[test]
1446 fn follows_a_script_location_stub() {
1447 let base = Url::parse("https://blog.dev/a.html").unwrap();
1448 let body = r#"<!doctype html><title>Redirect</title><script>
1449 const target = "https://blog.dev/a/";
1450 window.location.replace("https://blog.dev/a/");
1451 </script>"#;
1452 assert_eq!(detect_html_redirect(body, &base).unwrap().as_str(), "https://blog.dev/a/");
1453 }
1454
1455 #[test]
1456 fn resolves_a_relative_redirect_target() {
1457 let base = Url::parse("https://blog.dev/x/a.html").unwrap();
1458 let body = r#"<meta http-equiv="refresh" content="0;url=/y/b.html">"#;
1459 assert_eq!(
1460 detect_html_redirect(body, &base).unwrap().as_str(),
1461 "https://blog.dev/y/b.html"
1462 );
1463 }
1464
1465 #[test]
1466 fn ignores_delayed_refreshes_and_self_loops() {
1467 let base = Url::parse("https://blog.dev/a").unwrap();
1468 assert!(
1470 detect_html_redirect(
1471 r#"<meta http-equiv="refresh" content="5; url=https://blog.dev/b">"#,
1472 &base
1473 )
1474 .is_none()
1475 );
1476 assert!(
1478 detect_html_redirect(
1479 r#"<meta http-equiv="refresh" content="0; url=https://blog.dev/a">"#,
1480 &base
1481 )
1482 .is_none()
1483 );
1484 }
1485
1486 #[test]
1487 fn ignores_a_long_page_that_merely_mentions_location() {
1488 let base = Url::parse("https://blog.dev/a").unwrap();
1489 let article = format!(
1490 "<article><p>{}</p><p>You can call location.replace(\"https://evil.dev/\") in JS.</p></article>",
1491 "Real prose. ".repeat(500)
1492 );
1493 assert!(
1494 detect_html_redirect(&article, &base).is_none(),
1495 "a full article was treated as a stub"
1496 );
1497 }
1498
1499 #[test]
1500 fn ignores_non_http_and_unresolvable_targets() {
1501 let base = Url::parse("https://blog.dev/a").unwrap();
1502 assert!(
1503 detect_html_redirect(
1504 r#"<meta http-equiv="refresh" content="0;url=javascript:x()">"#,
1505 &base
1506 )
1507 .is_none()
1508 );
1509 assert!(detect_html_redirect("<p>nothing here</p>", &base).is_none());
1510 assert!(
1512 detect_html_redirect(r#"<script>window.location.replace(target);</script>"#, &base)
1513 .is_none()
1514 );
1515 }
1516
1517 #[test]
1518 fn detects_an_spa_shell() {
1519 let shell = format!(
1520 "<html><body><div id=\"root\"></div><script>{}</script></body></html>",
1521 "x".repeat(2000)
1522 );
1523 assert!(needs_javascript(&page(&shell)));
1524 }
1525
1526 #[test]
1527 fn does_not_escalate_a_real_article() {
1528 let real = format!(
1529 "<html><body><article><h1>T</h1><p>{}</p></article></body></html>",
1530 "Real prose that a reader can actually read. ".repeat(20)
1531 );
1532 assert!(!needs_javascript(&page(&real)));
1533 }
1534}
1535
1536#[cfg(test)]
1537mod js_gate_tests {
1538 use super::*;
1539
1540 fn page(body: String) -> Page {
1541 Page {
1542 url: "https://x.dev/".into(),
1543 final_url: "https://x.dev/".into(),
1544 status: 200,
1545 content_type: Some("text/html".into()),
1546 body,
1547 elapsed_ms: 0,
1548 rendered: false,
1549 }
1550 }
1551
1552 #[test]
1558 fn a_client_rendered_front_page_is_flagged() {
1559 let shell = format!(
1560 "<html><body><nav>{}</nav><main></main><script>{}</script></body></html>",
1561 "<a href=\"/section\">Section link</a>".repeat(20),
1562 "x".repeat(340_000)
1563 );
1564 assert!(needs_javascript(&page(shell)));
1565 }
1566
1567 #[test]
1569 fn a_large_real_article_is_not_flagged() {
1570 let real = format!(
1571 "<html><body><article><h1>T</h1><p>{}</p></article></body></html>",
1572 "Real prose that a reader can actually read and act on. ".repeat(400)
1573 );
1574 assert!(!needs_javascript(&page(real)));
1575 }
1576}