servo_fetch/
fetch.rs

1//! Single-page fetching and rendered content extraction.
2
3use std::collections::HashMap;
4use std::fmt;
5use std::sync::Arc;
6use std::time::Duration;
7
8use serde_json::Value;
9use servo::accesskit::{Node, NodeId};
10
11use crate::error::Error;
12use crate::net::sanitize_user_agent;
13
14/// Rendered page returned by [`crate::fetch()`].
15#[derive(Debug, Clone, Default, serde::Serialize)]
16#[non_exhaustive]
17pub struct Page {
18    /// Fully rendered HTML after JavaScript execution.
19    pub html: String,
20    /// Plain text content (`document.body.innerText`).
21    pub inner_text: String,
22    /// Page title extracted from `<title>` tag.
23    pub title: Option<String>,
24    /// Parsed layout data from the injected CSS heuristics script.
25    #[serde(skip_serializing_if = "Option::is_none")]
26    pub layout_json: Option<String>,
27    /// Per-node visibility flags from the visibility-aware extraction pass.
28    #[serde(skip)]
29    visibility_json: Option<String>,
30    /// Result of JavaScript evaluation, if [`FetchOptions::javascript`] was used.
31    #[serde(skip_serializing_if = "Option::is_none")]
32    pub js_result: Option<String>,
33    /// Browser console messages captured during page load.
34    pub console_messages: Vec<ConsoleMessage>,
35    /// Accessibility tree (AccessKit), serialized as JSON, if requested.
36    #[serde(skip_serializing_if = "Option::is_none")]
37    pub accessibility_tree: Option<String>,
38    /// Structured data extracted via [`FetchOptions::schema`].
39    #[serde(skip_serializing_if = "Option::is_none")]
40    pub extracted: Option<Value>,
41    /// PNG-encoded screenshot bytes — read via [`Page::screenshot_png`].
42    #[serde(skip)]
43    screenshot_png: Option<Vec<u8>>,
44    /// Typed AccessKit tree, shared cheaply across [`Page`] clones.
45    #[serde(skip)]
46    a11y: Option<Arc<HashMap<NodeId, Node>>>,
47    /// Visibility policy that was active when this page was fetched.
48    #[serde(skip)]
49    visibility_policy: crate::visibility::VisibilityPolicy,
50}
51
52impl Page {
53    /// Extract readable Markdown from this page.
54    pub fn markdown(&self) -> crate::error::Result<String> {
55        self.markdown_with_url("")
56    }
57
58    /// Extract readable Markdown, using the original URL for link resolution.
59    pub fn markdown_with_url(&self, url: &str) -> crate::error::Result<String> {
60        Ok(crate::extract::extract_text(&self.extract_input(url, None))?)
61    }
62
63    /// Extract structured JSON from this page.
64    pub fn extract_json(&self) -> crate::error::Result<String> {
65        self.extract_json_with_url("")
66    }
67
68    /// Extract structured JSON, using the original URL for link resolution.
69    pub fn extract_json_with_url(&self, url: &str) -> crate::error::Result<String> {
70        Ok(crate::extract::extract_json(&self.extract_input(url, None))?)
71    }
72
73    /// Extract structured article data, using the original URL for link resolution.
74    pub fn article(&self, url: &str) -> crate::error::Result<crate::extract::ArticleData> {
75        Ok(crate::extract::extract_article(&self.extract_input(url, None))?)
76    }
77
78    /// Extract structured article data from the subtree matched by a CSS selector.
79    pub fn article_with_selector(
80        &self,
81        url: &str,
82        selector: &str,
83    ) -> crate::error::Result<crate::extract::ArticleData> {
84        Ok(crate::extract::extract_article(
85            &self.extract_input(url, Some(selector)),
86        )?)
87    }
88
89    /// Extract readable Markdown from the subtree matched by a CSS selector.
90    pub fn markdown_with_selector(&self, url: &str, selector: &str) -> crate::error::Result<String> {
91        Ok(crate::extract::extract_text(&self.extract_input(url, Some(selector)))?)
92    }
93
94    /// Extract structured JSON from the subtree matched by a CSS selector.
95    pub fn extract_json_with_selector(&self, url: &str, selector: &str) -> crate::error::Result<String> {
96        Ok(crate::extract::extract_json(&self.extract_input(url, Some(selector)))?)
97    }
98
99    /// PNG screenshot bytes, if captured via [`FetchOptions::screenshot`].
100    #[must_use]
101    pub fn screenshot_png(&self) -> Option<&[u8]> {
102        self.screenshot_png.as_deref()
103    }
104
105    fn extract_input<'a>(&'a self, url: &'a str, selector: Option<&'a str>) -> crate::extract::ExtractInput<'a> {
106        crate::extract::ExtractInput::new(&self.html, url)
107            .with_layout_json(self.layout_json.as_deref())
108            .with_visibility_json(self.visibility_json.as_deref())
109            .with_a11y(self.a11y.as_deref())
110            .with_inner_text(Some(&self.inner_text))
111            .with_selector(selector)
112            .with_visibility(self.visibility_policy)
113    }
114
115    pub(crate) fn from_servo(page: crate::bridge::ServoPage) -> Self {
116        let title = {
117            let doc = dom_query::Document::from(page.html.as_str());
118            let t = doc.select("title").text().to_string();
119            if t.is_empty() { None } else { Some(t) }
120        };
121        let screenshot_png = page.screenshot.and_then(|img| {
122            let mut buf = std::io::Cursor::new(Vec::new());
123            img.write_to(&mut buf, image::ImageFormat::Png).ok()?;
124            Some(buf.into_inner())
125        });
126        Self {
127            html: page.html,
128            inner_text: page.inner_text.unwrap_or_default(),
129            title,
130            layout_json: page.layout_json,
131            visibility_json: page.visibility_json,
132            js_result: page.js_result,
133            console_messages: page.console_messages.into_iter().map(ConsoleMessage::from).collect(),
134            screenshot_png,
135            accessibility_tree: page.accessibility_tree,
136            a11y: page.a11y.map(Arc::new),
137            extracted: None,
138            visibility_policy: crate::visibility::VisibilityPolicy::default(),
139        }
140    }
141}
142
143/// Browser console message captured during page load.
144#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
145#[non_exhaustive]
146pub struct ConsoleMessage {
147    /// Severity level.
148    pub level: ConsoleLevel,
149    /// Message text.
150    pub message: String,
151}
152
153/// Console message severity.
154#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
155#[serde(rename_all = "lowercase")]
156#[non_exhaustive]
157pub enum ConsoleLevel {
158    /// General log message.
159    Log,
160    /// Debug-level message.
161    Debug,
162    /// Informational message.
163    Info,
164    /// Warning message.
165    Warn,
166    /// Error message.
167    Error,
168    /// Trace-level message.
169    Trace,
170}
171
172impl ConsoleLevel {
173    /// Returns the string representation of this level.
174    #[must_use]
175    pub fn as_str(&self) -> &'static str {
176        match self {
177            Self::Log => "log",
178            Self::Debug => "debug",
179            Self::Info => "info",
180            Self::Warn => "warn",
181            Self::Error => "error",
182            Self::Trace => "trace",
183        }
184    }
185}
186
187impl fmt::Display for ConsoleLevel {
188    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
189        f.pad(self.as_str())
190    }
191}
192
193impl From<crate::bridge::ConsoleLevel> for ConsoleLevel {
194    fn from(level: crate::bridge::ConsoleLevel) -> Self {
195        use crate::bridge::ConsoleLevel as Bridge;
196        match level {
197            Bridge::Log => Self::Log,
198            Bridge::Debug => Self::Debug,
199            Bridge::Info => Self::Info,
200            Bridge::Warn => Self::Warn,
201            Bridge::Error => Self::Error,
202            Bridge::Trace => Self::Trace,
203        }
204    }
205}
206
207impl From<crate::bridge::ConsoleMessage> for ConsoleMessage {
208    fn from(msg: crate::bridge::ConsoleMessage) -> Self {
209        Self {
210            level: msg.level.into(),
211            message: msg.message,
212        }
213    }
214}
215
216#[derive(Debug, Clone)]
217pub(crate) enum FetchMode {
218    Content { include_a11y: bool },
219    Screenshot { full_page: bool },
220    JavaScript(String),
221}
222
223/// Options for a single page fetch.
224#[must_use = "options do nothing until passed to fetch()"]
225#[derive(Debug, Clone)]
226pub struct FetchOptions {
227    pub(crate) url: String,
228    pub(crate) timeout: Option<Duration>,
229    pub(crate) settle: Option<Duration>,
230    pub(crate) mode: FetchMode,
231    pub(crate) user_agent: Option<String>,
232    pub(crate) extract_schema: Option<crate::schema::ExtractSchema>,
233    pub(crate) visibility: Option<crate::visibility::VisibilityPolicy>,
234    pub(crate) cookies: Vec<crate::cookies::CookieSpec>,
235    pub(crate) headers: http::HeaderMap,
236}
237
238impl FetchOptions {
239    /// Default page-load timeout used when neither `FetchOptions::timeout` nor
240    /// a [`crate::Client`] override is set.
241    pub(crate) const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
242
243    /// Default settle wait used when neither `FetchOptions::settle` nor a
244    /// [`crate::Client`] override is set.
245    pub(crate) const DEFAULT_SETTLE: Duration = Duration::ZERO;
246
247    /// Fetch rendered content (default mode).
248    pub fn new(url: &str) -> Self {
249        Self {
250            url: url.into(),
251            timeout: None,
252            settle: None,
253            mode: FetchMode::Content { include_a11y: false },
254            user_agent: None,
255            extract_schema: None,
256            visibility: None,
257            cookies: Vec::new(),
258            headers: http::HeaderMap::new(),
259        }
260    }
261
262    /// Capture a PNG screenshot.
263    pub fn screenshot(url: &str, full_page: bool) -> Self {
264        Self {
265            mode: FetchMode::Screenshot { full_page },
266            ..Self::new(url)
267        }
268    }
269
270    /// Execute a JavaScript expression and return the result.
271    pub fn javascript(url: &str, expression: impl Into<String>) -> Self {
272        Self {
273            mode: FetchMode::JavaScript(expression.into()),
274            ..Self::new(url)
275        }
276    }
277
278    /// Page load timeout (default: 30s).
279    pub fn timeout(mut self, timeout: Duration) -> Self {
280        self.timeout = Some(timeout);
281        self
282    }
283
284    /// Extra wait after load event for SPA hydration (default: 0).
285    pub fn settle(mut self, settle: Duration) -> Self {
286        self.settle = Some(settle);
287        self
288    }
289
290    /// Override the User-Agent string for this request.
291    pub fn user_agent(mut self, ua: impl Into<String>) -> Self {
292        self.user_agent = Some(sanitize_user_agent(ua.into()));
293        self
294    }
295
296    /// Extract structured data from the rendered page using the given schema.
297    pub fn schema(mut self, schema: crate::schema::ExtractSchema) -> Self {
298        self.extract_schema = Some(schema);
299        self
300    }
301
302    /// Visibility-filtering policy applied during extraction.
303    pub fn visibility(mut self, policy: crate::visibility::VisibilityPolicy) -> Self {
304        self.visibility = Some(policy);
305        self
306    }
307
308    /// Seed session cookies before navigation, scoped to the target's site.
309    pub fn cookies(mut self, cookies: Vec<crate::cookies::CookieSpec>) -> Self {
310        self.cookies = cookies;
311        self
312    }
313
314    /// Custom request headers sent with the navigation request.
315    pub fn headers(mut self, headers: http::HeaderMap) -> Self {
316        self.headers = headers;
317        self
318    }
319
320    /// Capture the page's accessibility tree (read via [`Page::accessibility_tree`]).
321    pub fn accessibility(mut self, on: bool) -> Self {
322        if let FetchMode::Content { include_a11y } = &mut self.mode {
323            *include_a11y = on;
324        }
325        self
326    }
327
328    /// Resolve the effective timeout, falling back to [`Self::DEFAULT_TIMEOUT`].
329    pub(crate) fn effective_timeout(&self) -> Duration {
330        self.timeout.unwrap_or(Self::DEFAULT_TIMEOUT)
331    }
332
333    /// Resolve the effective settle wait, falling back to [`Self::DEFAULT_SETTLE`].
334    pub(crate) fn effective_settle(&self) -> Duration {
335        self.settle.unwrap_or(Self::DEFAULT_SETTLE)
336    }
337
338    /// Resolve the effective visibility policy, falling back to its default.
339    pub(crate) fn effective_visibility(&self) -> crate::visibility::VisibilityPolicy {
340        self.visibility.unwrap_or_default()
341    }
342}
343
344/// Fetch a single page via the embedded Servo engine (blocking).
345pub fn fetch_blocking(opts: &FetchOptions) -> crate::error::Result<Page> {
346    if let Some(pdf_page) = pre_fetch(opts)? {
347        return Ok(pdf_page);
348    }
349    let bridge_opts = build_bridge_options(opts);
350    let servo_page = crate::bridge::fetch_page(bridge_opts).map_err(|e| map_engine_error(e, opts))?;
351    Ok(finalize_page(servo_page, opts))
352}
353
354/// Fetch a single page via the embedded Servo engine.
355pub async fn fetch(opts: &FetchOptions) -> crate::error::Result<Page> {
356    if let Some(pdf_page) = pre_fetch_async(opts).await? {
357        return Ok(pdf_page);
358    }
359    let bridge_opts = build_bridge_options(opts);
360    let servo_page = crate::bridge::fetch_page_async(bridge_opts)
361        .await
362        .map_err(|e| map_engine_error(e, opts))?;
363    Ok(finalize_page(servo_page, opts))
364}
365
366/// Fetch a URL and return readable Markdown (blocking).
367pub fn markdown_blocking(url: &str) -> crate::error::Result<String> {
368    fetch_blocking(&FetchOptions::new(url))?.markdown_with_url(url)
369}
370
371/// Fetch a URL and return readable Markdown.
372pub async fn markdown(url: &str) -> crate::error::Result<String> {
373    fetch(&FetchOptions::new(url)).await?.markdown_with_url(url)
374}
375
376/// Fetch a URL and return structured JSON (blocking).
377pub fn extract_json_blocking(url: &str) -> crate::error::Result<String> {
378    fetch_blocking(&FetchOptions::new(url))?.extract_json_with_url(url)
379}
380
381/// Fetch a URL and return structured JSON.
382pub async fn extract_json(url: &str) -> crate::error::Result<String> {
383    fetch(&FetchOptions::new(url)).await?.extract_json_with_url(url)
384}
385
386/// Fetch a URL and return plain text (`document.body.innerText`) (blocking).
387pub fn text_blocking(url: &str) -> crate::error::Result<String> {
388    Ok(fetch_blocking(&FetchOptions::new(url))?.inner_text)
389}
390
391/// Fetch a URL and return plain text (`document.body.innerText`).
392pub async fn text(url: &str) -> crate::error::Result<String> {
393    Ok(fetch(&FetchOptions::new(url)).await?.inner_text)
394}
395
396fn pre_fetch(opts: &FetchOptions) -> crate::error::Result<Option<Page>> {
397    crate::net::ensure_crypto_provider();
398    crate::net::validate_url(&opts.url)?;
399
400    if matches!(opts.mode, FetchMode::Content { .. })
401        && let Some(bytes) = crate::pdf::probe(&opts.url, opts.effective_timeout().as_secs().max(1))
402    {
403        return Ok(Some(pdf_page(&bytes)));
404    }
405
406    Ok(None)
407}
408
409async fn pre_fetch_async(opts: &FetchOptions) -> crate::error::Result<Option<Page>> {
410    crate::net::ensure_crypto_provider();
411    crate::net::validate_url(&opts.url)?;
412
413    if matches!(opts.mode, FetchMode::Content { .. }) {
414        let url = opts.url.clone();
415        let timeout_secs = opts.effective_timeout().as_secs().max(1);
416        let probe = tokio::task::spawn_blocking(move || crate::pdf::probe(&url, timeout_secs))
417            .await
418            .map_err(|e| Error::engine(anyhow::anyhow!("pdf probe task panicked: {e}"), Some(opts.url.clone())))?;
419        if let Some(bytes) = probe {
420            return Ok(Some(pdf_page(&bytes)));
421        }
422    }
423
424    Ok(None)
425}
426
427fn pdf_page(bytes: &[u8]) -> Page {
428    let text = crate::extract::extract_pdf(bytes);
429    Page {
430        html: String::new(),
431        inner_text: text,
432        ..Page::default()
433    }
434}
435
436fn build_bridge_options(opts: &FetchOptions) -> crate::bridge::FetchOptions<'_> {
437    crate::bridge::FetchOptions {
438        url: &opts.url,
439        timeout_secs: opts.effective_timeout().as_secs().max(1),
440        settle_ms: u64::try_from(opts.effective_settle().as_millis()).unwrap_or(u64::MAX),
441        user_agent: opts.user_agent.as_deref(),
442        cookies: &opts.cookies,
443        headers: &opts.headers,
444        mode: match opts.mode {
445            FetchMode::Content { include_a11y } => crate::bridge::FetchMode::Content { include_a11y },
446            FetchMode::Screenshot { full_page } => crate::bridge::FetchMode::Screenshot { full_page },
447            FetchMode::JavaScript(ref expr) => crate::bridge::FetchMode::ExecuteJs {
448                expression: expr.clone(),
449            },
450        },
451    }
452}
453
454fn finalize_page(servo_page: crate::bridge::ServoPage, opts: &FetchOptions) -> Page {
455    let mut page = Page::from_servo(servo_page);
456    page.visibility_policy = opts.effective_visibility();
457    if let Some(schema) = opts.extract_schema.as_ref() {
458        page.extracted = Some(schema.extract_from(&page.html));
459    }
460    page
461}
462
463fn map_engine_error(e: crate::bridge::EngineError, opts: &FetchOptions) -> Error {
464    match e {
465        crate::bridge::EngineError::Timeout(_) => Error::Timeout {
466            url: opts.url.clone(),
467            timeout: opts.effective_timeout(),
468        },
469        crate::bridge::EngineError::Other(e) => Error::engine(e, Some(opts.url.clone())),
470    }
471}
472
473#[cfg(test)]
474mod tests {
475    use super::*;
476
477    #[test]
478    fn fetch_options_defaults() {
479        let opts = FetchOptions::new("https://example.com");
480        assert_eq!(opts.url, "https://example.com");
481        assert_eq!(opts.timeout, None);
482        assert_eq!(opts.settle, None);
483        assert_eq!(opts.visibility, None);
484        assert!(matches!(opts.mode, FetchMode::Content { .. }));
485    }
486
487    #[test]
488    fn accessibility_flag_threads_into_content_mode() {
489        let off = FetchOptions::new("https://example.com");
490        assert!(matches!(
491            build_bridge_options(&off).mode,
492            crate::bridge::FetchMode::Content { include_a11y: false }
493        ));
494        let on = FetchOptions::new("https://example.com").accessibility(true);
495        assert!(matches!(
496            build_bridge_options(&on).mode,
497            crate::bridge::FetchMode::Content { include_a11y: true }
498        ));
499    }
500
501    #[test]
502    fn fetch_options_effective_defaults() {
503        let opts = FetchOptions::new("https://example.com");
504        assert_eq!(opts.effective_timeout(), Duration::from_secs(30));
505        assert_eq!(opts.effective_settle(), Duration::ZERO);
506    }
507
508    #[test]
509    fn fetch_options_caller_value_preserved() {
510        let opts = FetchOptions::new("https://example.com")
511            .timeout(Duration::from_secs(45))
512            .settle(Duration::from_millis(250));
513        assert_eq!(opts.timeout, Some(Duration::from_secs(45)));
514        assert_eq!(opts.settle, Some(Duration::from_millis(250)));
515        assert_eq!(opts.effective_timeout(), Duration::from_secs(45));
516        assert_eq!(opts.effective_settle(), Duration::from_millis(250));
517    }
518
519    #[test]
520    fn fetch_options_screenshot() {
521        let opts = FetchOptions::screenshot("https://example.com", true);
522        assert!(matches!(opts.mode, FetchMode::Screenshot { full_page: true }));
523    }
524
525    #[test]
526    fn fetch_options_javascript() {
527        let opts = FetchOptions::javascript("https://example.com", "document.title");
528        assert!(matches!(opts.mode, FetchMode::JavaScript(ref e) if e == "document.title"));
529    }
530
531    #[test]
532    fn fetch_options_chaining() {
533        let opts = FetchOptions::new("https://example.com")
534            .timeout(Duration::from_secs(60))
535            .settle(Duration::from_millis(500));
536        assert_eq!(opts.timeout, Some(Duration::from_secs(60)));
537        assert_eq!(opts.settle, Some(Duration::from_millis(500)));
538    }
539
540    #[test]
541    fn fetch_user_agent_set() {
542        let opts = FetchOptions::new("https://example.com").user_agent("MyBot/1.0");
543        assert_eq!(opts.user_agent.as_deref(), Some("MyBot/1.0"));
544    }
545
546    #[test]
547    fn fetch_user_agent_default_is_none() {
548        let opts = FetchOptions::new("https://example.com");
549        assert!(opts.user_agent.is_none());
550    }
551
552    #[test]
553    fn fetch_user_agent_sanitizes_crlf() {
554        let opts = FetchOptions::new("https://example.com").user_agent("Bot\r\nX-Evil: yes");
555        assert_eq!(opts.user_agent.as_deref(), Some("Bot  X-Evil: yes"));
556    }
557
558    #[test]
559    fn fetch_user_agent_sanitizes_null() {
560        let opts = FetchOptions::new("https://example.com").user_agent("Bot\0/1.0");
561        assert_eq!(opts.user_agent.as_deref(), Some("Bot /1.0"));
562    }
563
564    #[test]
565    fn fetch_user_agent_empty_string() {
566        let opts = FetchOptions::new("https://example.com").user_agent("");
567        assert_eq!(opts.user_agent.as_deref(), Some(""));
568    }
569
570    #[test]
571    fn page_markdown_from_html() {
572        let page = Page {
573            html: "<html><head><title>Test</title></head><body><p>hello world</p></body></html>".into(),
574            inner_text: "hello world".into(),
575            ..Page::default()
576        };
577        let md = page.markdown().unwrap();
578        assert!(md.contains("hello world"));
579    }
580
581    #[test]
582    fn page_extract_json_produces_valid_json() {
583        let page = Page {
584            html: "<html><head><title>Test</title></head><body><p>content</p></body></html>".into(),
585            inner_text: "content".into(),
586            ..Page::default()
587        };
588        let json = page.extract_json().unwrap();
589        let _: Value = serde_json::from_str(&json).expect("valid JSON");
590    }
591
592    #[test]
593    fn page_screenshot_png_none_by_default() {
594        let page = Page::default();
595        assert!(page.screenshot_png().is_none());
596    }
597
598    #[test]
599    fn page_markdown_with_selector_scopes_to_subtree() {
600        let page = Page {
601            html: "<html><body><article>keep</article><aside>drop</aside></body></html>".into(),
602            ..Page::default()
603        };
604        let md = page.markdown_with_selector("https://example.com", "article").unwrap();
605        assert!(md.contains("keep"));
606        assert!(!md.contains("drop"));
607    }
608
609    #[test]
610    fn page_extract_json_with_selector_includes_url() {
611        let page = Page {
612            html: "<html><body><article>scoped</article></body></html>".into(),
613            ..Page::default()
614        };
615        let json = page
616            .extract_json_with_selector("https://example.com/page", "article")
617            .unwrap();
618        let parsed: Value = serde_json::from_str(&json).expect("valid JSON");
619        assert_eq!(parsed["url"].as_str(), Some("https://example.com/page"));
620        assert!(parsed["textContent"].as_str().unwrap().contains("scoped"));
621    }
622
623    #[test]
624    fn page_markdown_with_selector_no_match_returns_empty() {
625        let page = Page {
626            html: "<html><body><article>x</article></body></html>".into(),
627            ..Page::default()
628        };
629        let md = page.markdown_with_selector("", ".nonexistent").unwrap();
630        assert!(md.is_empty());
631    }
632
633    #[test]
634    fn page_markdown_with_invalid_selector_returns_error() {
635        let page = Page {
636            html: "<html><body><p>x</p></body></html>".into(),
637            ..Page::default()
638        };
639        let err = page.markdown_with_selector("", "###invalid[[[").unwrap_err();
640        assert!(err.to_string().contains("invalid CSS selector"));
641    }
642
643    #[test]
644    fn page_markdown_with_empty_selector_returns_error() {
645        let page = Page {
646            html: "<html><body><p>x</p></body></html>".into(),
647            ..Page::default()
648        };
649        assert!(page.markdown_with_selector("", "").is_err());
650    }
651
652    #[test]
653    fn fetch_rejects_invalid_url() {
654        let result = fetch_blocking(&FetchOptions::new("not a url"));
655        assert!(result.is_err());
656        let err = result.unwrap_err();
657        assert!(matches!(err, Error::InvalidUrl { .. }));
658    }
659
660    #[test]
661    fn fetch_rejects_private_ip() {
662        let result = fetch_blocking(&FetchOptions::new("http://127.0.0.1/"));
663        assert!(result.is_err());
664    }
665
666    #[test]
667    fn fetch_rejects_file_scheme() {
668        let result = fetch_blocking(&FetchOptions::new("file:///etc/passwd"));
669        assert!(result.is_err());
670    }
671
672    mod page_from_servo {
673        use crate::bridge;
674        use crate::fetch::{ConsoleLevel, Page};
675
676        fn synthetic_image(w: u32, h: u32) -> image::RgbaImage {
677            image::RgbaImage::from_pixel(w, h, image::Rgba([255, 0, 0, 255]))
678        }
679
680        fn empty_servo_page() -> bridge::ServoPage {
681            bridge::ServoPage::default()
682        }
683
684        #[test]
685        fn extracts_title_from_html() {
686            let mut sp = empty_servo_page();
687            sp.html = "<html><head><title>Hello World</title></head></html>".into();
688            let page = Page::from_servo(sp);
689            assert_eq!(page.title.as_deref(), Some("Hello World"));
690        }
691
692        #[test]
693        fn title_is_none_when_tag_missing() {
694            let mut sp = empty_servo_page();
695            sp.html = "<html><body>no title here</body></html>".into();
696            let page = Page::from_servo(sp);
697            assert!(page.title.is_none());
698        }
699
700        #[test]
701        fn title_is_none_when_tag_empty() {
702            let mut sp = empty_servo_page();
703            sp.html = "<html><head><title></title></head></html>".into();
704            let page = Page::from_servo(sp);
705            assert!(page.title.is_none());
706        }
707
708        #[test]
709        fn title_is_none_for_empty_html() {
710            let page = Page::from_servo(empty_servo_page());
711            assert!(page.title.is_none());
712        }
713
714        #[test]
715        fn inner_text_none_becomes_empty_string() {
716            let sp = empty_servo_page();
717            assert!(sp.inner_text.is_none());
718            let page = Page::from_servo(sp);
719            assert_eq!(page.inner_text, "");
720        }
721
722        #[test]
723        fn screenshot_is_encoded_as_png() {
724            let mut sp = empty_servo_page();
725            sp.screenshot = Some(synthetic_image(8, 8));
726            let page = Page::from_servo(sp);
727            let bytes = page.screenshot_png().expect("screenshot encoded");
728            assert_eq!(&bytes[..8], b"\x89PNG\r\n\x1a\n", "PNG magic bytes");
729        }
730
731        #[test]
732        fn console_messages_empty_by_default() {
733            let page = Page::from_servo(empty_servo_page());
734            assert!(page.console_messages.is_empty());
735        }
736
737        #[test]
738        fn console_messages_preserve_all_six_levels() {
739            let cases = [
740                (bridge::ConsoleLevel::Log, ConsoleLevel::Log),
741                (bridge::ConsoleLevel::Debug, ConsoleLevel::Debug),
742                (bridge::ConsoleLevel::Info, ConsoleLevel::Info),
743                (bridge::ConsoleLevel::Warn, ConsoleLevel::Warn),
744                (bridge::ConsoleLevel::Error, ConsoleLevel::Error),
745                (bridge::ConsoleLevel::Trace, ConsoleLevel::Trace),
746            ];
747            for (src, expected) in cases {
748                let mut sp = empty_servo_page();
749                sp.console_messages = vec![bridge::ConsoleMessage {
750                    level: src,
751                    message: "msg".into(),
752                }];
753                let page = Page::from_servo(sp);
754                assert_eq!(
755                    page.console_messages.len(),
756                    1,
757                    "console message lost for source level {src:?}",
758                );
759                assert_eq!(
760                    page.console_messages[0].level, expected,
761                    "level mapping wrong for source {src:?}",
762                );
763            }
764        }
765
766        #[test]
767        fn console_messages_preserve_ordering_across_levels() {
768            let mut sp = empty_servo_page();
769            sp.console_messages = vec![
770                bridge::ConsoleMessage {
771                    level: bridge::ConsoleLevel::Info,
772                    message: "first".into(),
773                },
774                bridge::ConsoleMessage {
775                    level: bridge::ConsoleLevel::Error,
776                    message: "second".into(),
777                },
778                bridge::ConsoleMessage {
779                    level: bridge::ConsoleLevel::Warn,
780                    message: "third".into(),
781                },
782            ];
783            let page = Page::from_servo(sp);
784            assert_eq!(page.console_messages.len(), 3);
785            assert_eq!(page.console_messages[0].message, "first");
786            assert_eq!(page.console_messages[1].message, "second");
787            assert_eq!(page.console_messages[2].message, "third");
788            assert_eq!(page.console_messages[0].level, ConsoleLevel::Info);
789            assert_eq!(page.console_messages[1].level, ConsoleLevel::Error);
790            assert_eq!(page.console_messages[2].level, ConsoleLevel::Warn);
791        }
792
793        #[test]
794        fn extracted_starts_as_none_until_schema_applied() {
795            let page = Page::from_servo(empty_servo_page());
796            assert!(page.extracted.is_none());
797        }
798
799        #[test]
800        fn full_round_trip_preserves_every_field() {
801            let sp = bridge::ServoPage {
802                html: "<html><head><title>T</title></head><body>B</body></html>".into(),
803                inner_text: Some("B".into()),
804                layout_json: Some("[]".into()),
805                visibility_json: Some("[]".into()),
806                screenshot: Some(synthetic_image(2, 2)),
807                js_result: Some("42".into()),
808                accessibility_tree: Some("{}".into()),
809                a11y: None,
810                console_messages: vec![bridge::ConsoleMessage {
811                    level: bridge::ConsoleLevel::Log,
812                    message: "x".into(),
813                }],
814            };
815            let page = Page::from_servo(sp);
816            assert_eq!(page.html, "<html><head><title>T</title></head><body>B</body></html>");
817            assert_eq!(page.inner_text, "B");
818            assert_eq!(page.title.as_deref(), Some("T"));
819            assert_eq!(page.layout_json.as_deref(), Some("[]"));
820            assert_eq!(page.js_result.as_deref(), Some("42"));
821            assert_eq!(page.accessibility_tree.as_deref(), Some("{}"));
822            assert_eq!(page.console_messages.len(), 1);
823            assert!(page.screenshot_png().is_some());
824            assert!(page.extracted.is_none());
825        }
826    }
827}
servo_fetch/fetch.rs

servo_fetch/
fetch.rs
