scraper/html/
mod.rs

1//! HTML documents and fragments.
2
3use ego_tree::iter::Nodes;
4use ego_tree::{NodeId, Tree};
5use html5ever::serialize::SerializeOpts;
6use html5ever::tree_builder::QuirksMode;
7use html5ever::QualName;
8use html5ever::{driver, serialize};
9use tendril::TendrilSink;
10
11use crate::element_ref::ElementRef;
12use crate::node::Node;
13use crate::selector::Selector;
14
15use self::tree_sink::HtmlBuilder;
16
17lazy_static! {
18    static ref HTML_SELECTOR: Selector = Selector::parse("html").unwrap();
19}
20
21/// An HTML tree.
22///
23/// Parsing does not fail hard. Instead, the `quirks_mode` is set and errors are added to the
24/// `errors` field. The `tree` will still be populated as best as possible.
25///
26/// Implements the `TreeSink` trait from the `html5ever` crate, which allows HTML to be parsed.
27#[derive(Debug, Clone)]
28pub struct Html {
29    /// The quirks mode.
30    pub quirks_mode: QuirksMode,
31    /// The node tree.
32    pub tree: Tree<Node>,
33    /// The html language of the document.
34    pub lang: String,
35}
36
37impl Html {
38    /// Creates an empty HTML document.
39    pub fn new_document() -> Self {
40        Html {
41            quirks_mode: QuirksMode::NoQuirks,
42            tree: Tree::new(Node::Document),
43            lang: Default::default(),
44        }
45    }
46
47    /// Creates an empty HTML fragment.
48    pub fn new_fragment() -> Self {
49        Html {
50            quirks_mode: QuirksMode::NoQuirks,
51            tree: Tree::new(Node::Fragment),
52            lang: Default::default(),
53        }
54    }
55
56    /// Parses a string of HTML as a document.
57    ///
58    /// ```
59    /// use scraper::Html;
60    ///
61    /// let html = Html::parse_document("<p>hello</p>");
62    /// assert_eq!(html.root_element().html().is_empty(), false);
63    /// ```
64    pub fn parse_document(document: &str) -> Self {
65        let parser = driver::parse_document(HtmlBuilder::new_document(), Default::default());
66        parser.one(document)
67    }
68
69    /// Parses a string of HTML as a fragment.
70    pub fn parse_fragment(fragment: &str) -> Self {
71        let parser = driver::parse_fragment(
72            HtmlBuilder::new_fragment(),
73            Default::default(),
74            QualName::new(None, ns!(html), local_name!("body")),
75            Vec::new(),
76            false,
77        );
78        parser.one(fragment)
79    }
80
81    /// Returns an iterator over elements matching a selector.
82    pub fn select<'a, 'b>(&'a self, selector: &'b Selector) -> Select<'a, 'b> {
83        Select {
84            inner: self.tree.nodes(),
85            selector,
86        }
87    }
88
89    /// Returns the root `<html>` element.
90    pub fn root_element(&self) -> ElementRef {
91        let root_node = self
92            .tree
93            .root()
94            .children()
95            .find(|child| child.value().is_element())
96            .expect("html node missing");
97        ElementRef::wrap(root_node).unwrap()
98    }
99
100    /// Set the html language of the document by getting the lang attr
101    pub fn set_language(&mut self, lang: String) {
102        self.lang = lang;
103    }
104
105    /// Get the language for the page.
106    pub fn get_lang(&self) -> &str {
107        if self.lang.is_empty() {
108            if let Some(element) = self.select(&HTML_SELECTOR).next() {
109                if let Some(lang) = element.value().attr("lang") {
110                    return lang;
111                }
112            }
113            &self.lang
114        } else {
115            &self.lang
116        }
117    }
118
119    /// Serialize entire document into HTML.
120    pub fn html(&self) -> String {
121        let opts = SerializeOpts {
122            scripting_enabled: false, // It's not clear what this does.
123            traversal_scope: html5ever::serialize::TraversalScope::IncludeNode,
124            create_missing_parent: false,
125        };
126        let mut buf = Vec::new();
127        let _ = serialize(&mut buf, self, opts);
128        auto_encoder::auto_encode_bytes(&buf)
129    }
130
131    /// Find and remove a node
132    pub fn remove_node(&mut self, node_id: NodeId) {
133        if let Some(mut node) = self.tree.get_mut(node_id) {
134            node.detach();
135        }
136    }
137}
138
139/// Iterator over elements matching a selector.
140#[derive(Debug)]
141pub struct Select<'a, 'b> {
142    inner: Nodes<'a, Node>,
143    selector: &'b Selector,
144}
145
146impl<'a, 'b> Iterator for Select<'a, 'b> {
147    type Item = ElementRef<'a>;
148
149    fn next(&mut self) -> Option<ElementRef<'a>> {
150        for node in self.inner.by_ref() {
151            if let Some(element) = ElementRef::wrap(node) {
152                if element.parent().is_some() && self.selector.matches(&element) {
153                    return Some(element);
154                }
155            }
156        }
157        None
158    }
159}
160
161impl<'a, 'b> DoubleEndedIterator for Select<'a, 'b> {
162    fn next_back(&mut self) -> Option<Self::Item> {
163        for node in self.inner.by_ref().rev() {
164            if let Some(element) = ElementRef::wrap(node) {
165                if element.parent().is_some() && self.selector.matches(&element) {
166                    return Some(element);
167                }
168            }
169        }
170        None
171    }
172}
173
174mod serializable;
175mod tree_sink;
176
177#[cfg(test)]
178mod tests {
179    use super::Html;
180    use super::Selector;
181
182    /// Compile-time assertion that the parsed `Html` is `Send`.
183    /// This is the whole point of the spider-html5ever / spider-tendril
184    /// fork swap — `Html` (and the futures that hold it) can now move
185    /// across thread boundaries on a multi-threaded async runtime.
186    ///
187    /// `Sync` is NOT asserted: `Tendril` contains a `Cell<NonZeroUsize>`
188    /// pointer field that is intentionally `!Sync`. Spider_scraper owns
189    /// its tree directly (no `Arc`), so `Send` is the only bound we need
190    /// for cross-thread movement.
191    #[test]
192    fn parsed_html_is_send() {
193        fn assert_send<T: Send>(_: &T) {}
194        let html = Html::parse_document("<p>hi</p>");
195        assert_send(&html);
196    }
197
198    #[test]
199    fn root_element_fragment() {
200        let html = Html::parse_fragment(r#"<a href="http://github.com">1</a>"#);
201        let root_ref = html.root_element();
202        let href = root_ref
203            .select(&Selector::parse("a").unwrap())
204            .next()
205            .unwrap();
206        assert_eq!(href.inner_html(), "1");
207        assert_eq!(href.value().attr("href").unwrap(), "http://github.com");
208    }
209
210    #[test]
211    fn root_element_document_doctype() {
212        let html = Html::parse_document("<!DOCTYPE html>\n<title>abc</title>");
213        let root_ref = html.root_element();
214        let title = root_ref
215            .select(&Selector::parse("title").unwrap())
216            .next()
217            .unwrap();
218        assert_eq!(title.inner_html(), "abc");
219    }
220
221    #[test]
222    fn root_element_document_comment() {
223        let html = Html::parse_document("<!-- comment --><title>abc</title>");
224        let root_ref = html.root_element();
225        let title = root_ref
226            .select(&Selector::parse("title").unwrap())
227            .next()
228            .unwrap();
229        assert_eq!(title.inner_html(), "abc");
230    }
231
232    #[test]
233    fn select_is_reversible() {
234        let html = Html::parse_document("<p>element1</p><p>element2</p><p>element3</p>");
235        let selector = Selector::parse("p").unwrap();
236        let result: Vec<_> = html
237            .select(&selector)
238            .rev()
239            .map(|e| e.inner_html())
240            .collect();
241        assert_eq!(result, vec!["element3", "element2", "element1"]);
242    }
243}
scraper/html/mod.rs

scraper/html/
mod.rs
