Skip to main content

treesearch/parser/
html.rs

1//! Parses visible HTML body content into heading-based document trees.
2
3use crate::document::{Document, Node, SourceType};
4use anyhow::Result;
5use scraper::{Html, Node as HtmlNode, Selector};
6use std::path::Path;
7
8/// Parser for HTML (`.html`, `.htm`) files.
9///
10/// Parses h1-h6 headings into a hierarchical tree structure, same nesting logic
11/// as the markdown parser: h2 nests under h1, h3 under h2, etc.
12/// Text content is extracted by stripping tags and collecting inner text.
13pub struct HtmlParser;
14
15impl super::Parser for HtmlParser {
16    fn extensions(&self) -> &[&str] {
17        &["html", "htm"]
18    }
19
20    fn source_type(&self) -> SourceType {
21        SourceType::Html
22    }
23
24    fn parse(&self, path: &Path, content: &str) -> Result<Document> {
25        let file_name = path
26            .file_name()
27            .map(|n| n.to_string_lossy().to_string())
28            .unwrap_or_default();
29        let doc_id = path.to_string_lossy().to_string();
30        let mut doc = Document::new(&doc_id, &file_name, SourceType::Html);
31
32        if content.trim().is_empty() {
33            doc.assign_node_ids();
34            return Ok(doc);
35        }
36
37        let html = Html::parse_document(content);
38
39        // Extract <title> for doc_description.
40        if let Ok(sel) = Selector::parse("title") {
41            if let Some(title_el) = html.select(&sel).next() {
42                let title_text: String = title_el.text().collect::<Vec<_>>().join(" ");
43                doc.doc_description = title_text.trim().to_string();
44            }
45        }
46
47        // Walk the DOM, building a tree from heading elements.
48        let mut stack: Vec<(u8, Node)> = Vec::new();
49        let mut roots: Vec<Node> = Vec::new();
50        let mut current_text = String::new();
51        let mut saw_heading = false;
52
53        let traversal_nodes: Vec<_> = if let Ok(selector) = Selector::parse("body") {
54            html.select(&selector)
55                .next()
56                .and_then(|body| html.tree.get(body.id()))
57                .map(|body| body.descendants().collect())
58                .unwrap_or_else(|| html.tree.nodes().collect())
59        } else {
60            html.tree.nodes().collect()
61        };
62
63        for node_ref in traversal_nodes {
64            match node_ref.value() {
65                HtmlNode::Element(el) => {
66                    let Some(level) = heading_level(el.name()) else {
67                        continue;
68                    };
69                    saw_heading = true;
70                    // Flush accumulated text to current node.
71                    flush_text(&mut stack, &mut roots, &mut current_text);
72                    // Collapse deeper headings.
73                    collapse_stack(&mut stack, &mut roots, level);
74                    // Extract heading text from child text nodes.
75                    let mut title = String::new();
76                    for child in node_ref.children() {
77                        collect_all_text(child, &mut title);
78                    }
79                    let node = Node::new("", title.trim());
80                    stack.push((level, node));
81                }
82                HtmlNode::Text(text)
83                    if !is_inside_heading(&html, node_ref.id())
84                        && !is_inside_hidden_element(&html, node_ref.id()) =>
85                {
86                    // Skip text that belongs to heading elements (already collected above).
87                    let t = text.trim();
88                    if !t.is_empty() {
89                        if !current_text.is_empty() {
90                            current_text.push(' ');
91                        }
92                        current_text.push_str(t);
93                    }
94                }
95                _ => {}
96            }
97        }
98
99        // Flush remaining text.
100        flush_text(&mut stack, &mut roots, &mut current_text);
101        // Collapse entire stack.
102        collapse_stack(&mut stack, &mut roots, 0);
103
104        if !saw_heading {
105            if let Some(root) = roots.first_mut() {
106                root.title = file_name.clone();
107            }
108        }
109
110        // Fallback: if no headings found, create a single node with all body text.
111        if roots.is_empty() && !content.trim().is_empty() {
112            let body_text = extract_body_text(&html);
113            if !body_text.is_empty() {
114                let mut node = Node::new("", &file_name);
115                node.text = body_text;
116                roots.push(node);
117            }
118        }
119
120        doc.structure = roots;
121        doc.assign_node_ids();
122        Ok(doc)
123    }
124}
125
126// ---------------------------------------------------------------------------
127// Helpers
128// ---------------------------------------------------------------------------
129
130fn heading_level(tag: &str) -> Option<u8> {
131    match tag {
132        "h1" => Some(1),
133        "h2" => Some(2),
134        "h3" => Some(3),
135        "h4" => Some(4),
136        "h5" => Some(5),
137        "h6" => Some(6),
138        _ => None,
139    }
140}
141
142/// Check if a node is a descendant of any heading element.
143fn is_inside_heading(html: &Html, node_id: ego_tree::NodeId) -> bool {
144    let mut current = Some(node_id);
145    while let Some(id) = current {
146        if let Some(node) = html.tree.get(id) {
147            if let HtmlNode::Element(el) = node.value() {
148                if heading_level(el.name()).is_some() {
149                    return true;
150                }
151            }
152            current = node.parent().map(|p| p.id());
153        } else {
154            break;
155        }
156    }
157    false
158}
159
160/// Checks whether a node belongs to non-visible metadata or executable markup.
161fn is_inside_hidden_element(html: &Html, node_id: ego_tree::NodeId) -> bool {
162    let mut current = Some(node_id);
163    while let Some(id) = current {
164        let Some(node) = html.tree.get(id) else {
165            break;
166        };
167        if let HtmlNode::Element(element) = node.value() {
168            if matches!(
169                element.name(),
170                "head" | "script" | "style" | "noscript" | "template" | "svg"
171            ) {
172                return true;
173            }
174        }
175        current = node.parent().map(|parent| parent.id());
176    }
177    false
178}
179
180/// Iteratively collects visible text from a node and its descendants.
181fn collect_all_text(node_ref: ego_tree::NodeRef<'_, scraper::Node>, out: &mut String) {
182    for descendant in node_ref.descendants() {
183        if let HtmlNode::Text(text) = descendant.value() {
184            let text = text.trim();
185            if !text.is_empty() {
186                if !out.is_empty() {
187                    out.push(' ');
188                }
189                out.push_str(text);
190            }
191        }
192    }
193}
194
195/// Extract all visible text from <body>, or from the whole document as fallback.
196fn extract_body_text(html: &Html) -> String {
197    let mut parts = Vec::new();
198    if let Ok(sel) = Selector::parse("body") {
199        if let Some(body) = html.select(&sel).next() {
200            for text in body.text() {
201                let t = text.trim();
202                if !t.is_empty() {
203                    parts.push(t.to_string());
204                }
205            }
206        }
207    }
208    if parts.is_empty() {
209        for node in html.tree.nodes() {
210            if let HtmlNode::Text(t) = node.value() {
211                let t = t.trim();
212                if !t.is_empty() {
213                    parts.push(t.to_string());
214                }
215            }
216        }
217    }
218    parts.join(" ")
219}
220
221fn collapse_stack(stack: &mut Vec<(u8, Node)>, roots: &mut Vec<Node>, target_level: u8) {
222    while let Some(&(lvl, _)) = stack.last() {
223        if lvl >= target_level {
224            let (_, node) = stack.pop().unwrap();
225            if let Some(parent) = stack.last_mut() {
226                parent.1.children.push(node);
227            } else {
228                roots.push(node);
229            }
230        } else {
231            break;
232        }
233    }
234}
235
236fn flush_text(stack: &mut [(u8, Node)], roots: &mut Vec<Node>, text: &mut String) {
237    let trimmed = text.trim().to_string();
238    if trimmed.is_empty() {
239        text.clear();
240        return;
241    }
242    if let Some(top) = stack.last_mut() {
243        if top.1.text.is_empty() {
244            top.1.text = trimmed;
245        } else {
246            top.1.text.push_str("\n\n");
247            top.1.text.push_str(&trimmed);
248        }
249    } else {
250        let mut node = Node::new("", "");
251        node.text = trimmed;
252        roots.push(node);
253    }
254    text.clear();
255}
256
257#[cfg(test)]
258mod tests {
259    use super::*;
260    use crate::parser::Parser;
261
262    fn parse(content: &str) -> Document {
263        let parser = HtmlParser;
264        parser
265            .parse(Path::new("test.html"), content)
266            .expect("parse failed")
267    }
268
269    #[test]
270    fn test_empty() {
271        let doc = parse("");
272        assert!(doc.structure.is_empty());
273    }
274
275    #[test]
276    fn test_single_heading() {
277        let doc = parse("<h1>Hello</h1><p>World</p>");
278        assert_eq!(doc.structure.len(), 1);
279        assert_eq!(doc.structure[0].title, "Hello");
280        assert!(doc.structure[0].text.contains("World"));
281    }
282
283    #[test]
284    fn test_nested_headings() {
285        let html = r#"
286            <h1>Top</h1>
287            <p>Intro</p>
288            <h2>Section A</h2>
289            <p>Text A</p>
290            <h3>Sub A1</h3>
291            <p>Deep</p>
292            <h2>Section B</h2>
293            <p>Text B</p>
294        "#;
295        let doc = parse(html);
296        assert_eq!(doc.structure.len(), 1);
297        let root = &doc.structure[0];
298        assert_eq!(root.title, "Top");
299        assert_eq!(root.children.len(), 2);
300        assert_eq!(root.children[0].title, "Section A");
301        assert_eq!(root.children[1].title, "Section B");
302        assert_eq!(root.children[0].children.len(), 1);
303        assert_eq!(root.children[0].children[0].title, "Sub A1");
304    }
305
306    #[test]
307    fn test_no_headings() {
308        let doc = parse("<p>Just text</p><p>More text</p>");
309        assert_eq!(doc.structure.len(), 1);
310        assert_eq!(doc.structure[0].title, "test.html");
311        assert!(doc.structure[0].text.contains("Just text"));
312    }
313
314    #[test]
315    fn test_node_ids_assigned() {
316        let doc = parse("<h1>A</h1><h2>B</h2><h2>C</h2>");
317        assert_eq!(doc.structure[0].node_id, "0");
318        assert_eq!(doc.structure[0].children[0].node_id, "1");
319        assert_eq!(doc.structure[0].children[1].node_id, "2");
320    }
321
322    #[test]
323    fn test_source_type() {
324        let doc = parse("<h1>Hi</h1>");
325        assert_eq!(doc.source_type, SourceType::Html);
326        assert_eq!(doc.doc_id, "test.html");
327    }
328
329    #[test]
330    fn test_full_html_document() {
331        let html = r#"<!DOCTYPE html>
332<html>
333<head><title>Test Page</title></head>
334<body>
335<h1>Main Title</h1>
336<p>Content here.</p>
337</body>
338</html>"#;
339        let doc = parse(html);
340        assert_eq!(doc.doc_description, "Test Page");
341        // The parser should produce at least one node from the document
342        assert!(!doc.structure.is_empty());
343        assert_eq!(doc.structure[0].title, "Main Title");
344        assert!(!doc.structure[0].text.contains("Test Page"));
345    }
346
347    #[test]
348    fn test_multiple_h1() {
349        let doc = parse("<h1>First</h1><p>A</p><h1>Second</h1><p>B</p>");
350        assert_eq!(doc.structure.len(), 2);
351        assert_eq!(doc.structure[0].title, "First");
352        assert_eq!(doc.structure[1].title, "Second");
353    }
354
355    #[test]
356    fn test_heading_with_nested_tags() {
357        let doc = parse("<h1>Hello <strong>World</strong></h1><p>Body</p>");
358        assert_eq!(doc.structure[0].title, "Hello World");
359    }
360
361    #[test]
362    fn test_hidden_script_and_style_text_is_not_indexed() {
363        let doc = parse(
364            "<html><body><script>secretToken()</script><style>.secret{}</style><h1>Visible</h1><p>Body</p></body></html>",
365        );
366        assert_eq!(doc.structure.len(), 1);
367        assert_eq!(doc.structure[0].title, "Visible");
368        assert_eq!(doc.structure[0].text, "Body");
369    }
370
371    #[test]
372    fn test_heading_level_fn() {
373        assert_eq!(heading_level("h1"), Some(1));
374        assert_eq!(heading_level("h6"), Some(6));
375        assert_eq!(heading_level("p"), None);
376        assert_eq!(heading_level("div"), None);
377    }
378}