servo-fetch 0.2.2

A browser engine in a binary. Fetch, render, and extract web content powered by Servo.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

//! Content extraction integration tests β€” validates the full extraction pipeline
//! using HTML fixtures. These tests do NOT require Servo.

use servo_fetch::extract::{self, ExtractInput};

fn fixture(name: &str) -> String {
    std::fs::read_to_string(format!("tests/fixtures/{name}")).expect("fixture exists")
}

fn input(html: &str) -> ExtractInput<'_> {
    ExtractInput::new(html, "https://example.com")
}

#[test]
fn extracts_article_title() {
    let html = fixture("article.html");
    let text = extract::extract_text(&input(&html)).unwrap();
    assert!(text.contains("Test Article Title"));
}

#[test]
fn extracts_article_content() {
    let html = fixture("article.html");
    let text = extract::extract_text(&input(&html)).unwrap();
    assert!(text.contains("main content of the article"));
}

#[test]
fn simple_page_extracts_text() {
    let html = fixture("simple.html");
    let text = extract::extract_text(&input(&html)).unwrap();
    assert!(text.contains("Hello World"));
}

#[test]
fn json_output_is_valid() {
    let html = fixture("article.html");
    let json = extract::extract_json(&input(&html)).unwrap();
    let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON");
    assert!(parsed.get("title").is_some());
    assert!(parsed.get("text_content").is_some());
}

#[test]
fn json_simple_page() {
    let html = fixture("simple.html");
    let json = extract::extract_json(&input(&html)).unwrap();
    let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON");
    assert!(parsed["text_content"].as_str().unwrap().contains("Hello World"));
}

#[test]
fn inner_text_fallback() {
    let mut input = ExtractInput::new("<html><body></body></html>", "https://example.com");
    input.inner_text = Some("fallback text");
    let text = extract::extract_text(&input).unwrap();
    assert!(text.contains("fallback text"));
}

#[test]
fn handles_unicode_content() {
    let html = fixture("unicode.html");
    let text = extract::extract_text(&input(&html)).unwrap();
    assert!(text.contains("見出し"));
}

#[test]
fn strips_script_and_style() {
    let html = fixture("with_scripts.html");
    let text = extract::extract_text(&input(&html)).unwrap();
    assert!(!text.contains("alert"));
    assert!(text.contains("visible content"));
}

#[test]
fn empty_body_without_inner_text_returns_empty() {
    let input = ExtractInput::new("<html><body></body></html>", "https://example.com");
    let text = extract::extract_text(&input).unwrap();
    assert!(text.is_empty());
}

#[test]
fn byline_and_excerpt_rendered_in_markdown() {
    let html = fixture("article_with_meta.html");
    let text = extract::extract_text(&input(&html)).unwrap();
    assert!(text.contains("*Jane Doe*"), "byline should be italic");
    assert!(text.contains("> A short summary"), "excerpt should be blockquote");
}

#[test]
fn selector_extracts_specific_element() {
    let html = fixture("article.html");
    let mut input = input(&html);
    input.selector = Some("article");
    let text = extract::extract_text(&input).unwrap();
    assert!(text.contains("Test Article Title"));
    assert!(!text.contains("Sidebar item"), "sidebar should not be included");
}

#[test]
fn selector_no_match_returns_empty() {
    let html = fixture("simple.html");
    let mut input = input(&html);
    input.selector = Some(".nonexistent");
    let text = extract::extract_text(&input).unwrap();
    assert!(text.is_empty());
}