treesearch/parser/
html.rs1use crate::document::{Document, Node, SourceType};
4use anyhow::Result;
5use scraper::{Html, Node as HtmlNode, Selector};
6use std::path::Path;
7
8pub struct HtmlParser;
14
15impl super::Parser for HtmlParser {
16 fn extensions(&self) -> &[&str] {
17 &["html", "htm"]
18 }
19
20 fn source_type(&self) -> SourceType {
21 SourceType::Html
22 }
23
24 fn parse(&self, path: &Path, content: &str) -> Result<Document> {
25 let file_name = path
26 .file_name()
27 .map(|n| n.to_string_lossy().to_string())
28 .unwrap_or_default();
29 let doc_id = path.to_string_lossy().to_string();
30 let mut doc = Document::new(&doc_id, &file_name, SourceType::Html);
31
32 if content.trim().is_empty() {
33 doc.assign_node_ids();
34 return Ok(doc);
35 }
36
37 let html = Html::parse_document(content);
38
39 if let Ok(sel) = Selector::parse("title") {
41 if let Some(title_el) = html.select(&sel).next() {
42 let title_text: String = title_el.text().collect::<Vec<_>>().join(" ");
43 doc.doc_description = title_text.trim().to_string();
44 }
45 }
46
47 let mut stack: Vec<(u8, Node)> = Vec::new();
49 let mut roots: Vec<Node> = Vec::new();
50 let mut current_text = String::new();
51 let mut saw_heading = false;
52
53 let traversal_nodes: Vec<_> = if let Ok(selector) = Selector::parse("body") {
54 html.select(&selector)
55 .next()
56 .and_then(|body| html.tree.get(body.id()))
57 .map(|body| body.descendants().collect())
58 .unwrap_or_else(|| html.tree.nodes().collect())
59 } else {
60 html.tree.nodes().collect()
61 };
62
63 for node_ref in traversal_nodes {
64 match node_ref.value() {
65 HtmlNode::Element(el) => {
66 let Some(level) = heading_level(el.name()) else {
67 continue;
68 };
69 saw_heading = true;
70 flush_text(&mut stack, &mut roots, &mut current_text);
72 collapse_stack(&mut stack, &mut roots, level);
74 let mut title = String::new();
76 for child in node_ref.children() {
77 collect_all_text(child, &mut title);
78 }
79 let node = Node::new("", title.trim());
80 stack.push((level, node));
81 }
82 HtmlNode::Text(text)
83 if !is_inside_heading(&html, node_ref.id())
84 && !is_inside_hidden_element(&html, node_ref.id()) =>
85 {
86 let t = text.trim();
88 if !t.is_empty() {
89 if !current_text.is_empty() {
90 current_text.push(' ');
91 }
92 current_text.push_str(t);
93 }
94 }
95 _ => {}
96 }
97 }
98
99 flush_text(&mut stack, &mut roots, &mut current_text);
101 collapse_stack(&mut stack, &mut roots, 0);
103
104 if !saw_heading {
105 if let Some(root) = roots.first_mut() {
106 root.title = file_name.clone();
107 }
108 }
109
110 if roots.is_empty() && !content.trim().is_empty() {
112 let body_text = extract_body_text(&html);
113 if !body_text.is_empty() {
114 let mut node = Node::new("", &file_name);
115 node.text = body_text;
116 roots.push(node);
117 }
118 }
119
120 doc.structure = roots;
121 doc.assign_node_ids();
122 Ok(doc)
123 }
124}
125
126fn heading_level(tag: &str) -> Option<u8> {
131 match tag {
132 "h1" => Some(1),
133 "h2" => Some(2),
134 "h3" => Some(3),
135 "h4" => Some(4),
136 "h5" => Some(5),
137 "h6" => Some(6),
138 _ => None,
139 }
140}
141
142fn is_inside_heading(html: &Html, node_id: ego_tree::NodeId) -> bool {
144 let mut current = Some(node_id);
145 while let Some(id) = current {
146 if let Some(node) = html.tree.get(id) {
147 if let HtmlNode::Element(el) = node.value() {
148 if heading_level(el.name()).is_some() {
149 return true;
150 }
151 }
152 current = node.parent().map(|p| p.id());
153 } else {
154 break;
155 }
156 }
157 false
158}
159
160fn is_inside_hidden_element(html: &Html, node_id: ego_tree::NodeId) -> bool {
162 let mut current = Some(node_id);
163 while let Some(id) = current {
164 let Some(node) = html.tree.get(id) else {
165 break;
166 };
167 if let HtmlNode::Element(element) = node.value() {
168 if matches!(
169 element.name(),
170 "head" | "script" | "style" | "noscript" | "template" | "svg"
171 ) {
172 return true;
173 }
174 }
175 current = node.parent().map(|parent| parent.id());
176 }
177 false
178}
179
180fn collect_all_text(node_ref: ego_tree::NodeRef<'_, scraper::Node>, out: &mut String) {
182 for descendant in node_ref.descendants() {
183 if let HtmlNode::Text(text) = descendant.value() {
184 let text = text.trim();
185 if !text.is_empty() {
186 if !out.is_empty() {
187 out.push(' ');
188 }
189 out.push_str(text);
190 }
191 }
192 }
193}
194
195fn extract_body_text(html: &Html) -> String {
197 let mut parts = Vec::new();
198 if let Ok(sel) = Selector::parse("body") {
199 if let Some(body) = html.select(&sel).next() {
200 for text in body.text() {
201 let t = text.trim();
202 if !t.is_empty() {
203 parts.push(t.to_string());
204 }
205 }
206 }
207 }
208 if parts.is_empty() {
209 for node in html.tree.nodes() {
210 if let HtmlNode::Text(t) = node.value() {
211 let t = t.trim();
212 if !t.is_empty() {
213 parts.push(t.to_string());
214 }
215 }
216 }
217 }
218 parts.join(" ")
219}
220
221fn collapse_stack(stack: &mut Vec<(u8, Node)>, roots: &mut Vec<Node>, target_level: u8) {
222 while let Some(&(lvl, _)) = stack.last() {
223 if lvl >= target_level {
224 let (_, node) = stack.pop().unwrap();
225 if let Some(parent) = stack.last_mut() {
226 parent.1.children.push(node);
227 } else {
228 roots.push(node);
229 }
230 } else {
231 break;
232 }
233 }
234}
235
236fn flush_text(stack: &mut [(u8, Node)], roots: &mut Vec<Node>, text: &mut String) {
237 let trimmed = text.trim().to_string();
238 if trimmed.is_empty() {
239 text.clear();
240 return;
241 }
242 if let Some(top) = stack.last_mut() {
243 if top.1.text.is_empty() {
244 top.1.text = trimmed;
245 } else {
246 top.1.text.push_str("\n\n");
247 top.1.text.push_str(&trimmed);
248 }
249 } else {
250 let mut node = Node::new("", "");
251 node.text = trimmed;
252 roots.push(node);
253 }
254 text.clear();
255}
256
257#[cfg(test)]
258mod tests {
259 use super::*;
260 use crate::parser::Parser;
261
262 fn parse(content: &str) -> Document {
263 let parser = HtmlParser;
264 parser
265 .parse(Path::new("test.html"), content)
266 .expect("parse failed")
267 }
268
269 #[test]
270 fn test_empty() {
271 let doc = parse("");
272 assert!(doc.structure.is_empty());
273 }
274
275 #[test]
276 fn test_single_heading() {
277 let doc = parse("<h1>Hello</h1><p>World</p>");
278 assert_eq!(doc.structure.len(), 1);
279 assert_eq!(doc.structure[0].title, "Hello");
280 assert!(doc.structure[0].text.contains("World"));
281 }
282
283 #[test]
284 fn test_nested_headings() {
285 let html = r#"
286 <h1>Top</h1>
287 <p>Intro</p>
288 <h2>Section A</h2>
289 <p>Text A</p>
290 <h3>Sub A1</h3>
291 <p>Deep</p>
292 <h2>Section B</h2>
293 <p>Text B</p>
294 "#;
295 let doc = parse(html);
296 assert_eq!(doc.structure.len(), 1);
297 let root = &doc.structure[0];
298 assert_eq!(root.title, "Top");
299 assert_eq!(root.children.len(), 2);
300 assert_eq!(root.children[0].title, "Section A");
301 assert_eq!(root.children[1].title, "Section B");
302 assert_eq!(root.children[0].children.len(), 1);
303 assert_eq!(root.children[0].children[0].title, "Sub A1");
304 }
305
306 #[test]
307 fn test_no_headings() {
308 let doc = parse("<p>Just text</p><p>More text</p>");
309 assert_eq!(doc.structure.len(), 1);
310 assert_eq!(doc.structure[0].title, "test.html");
311 assert!(doc.structure[0].text.contains("Just text"));
312 }
313
314 #[test]
315 fn test_node_ids_assigned() {
316 let doc = parse("<h1>A</h1><h2>B</h2><h2>C</h2>");
317 assert_eq!(doc.structure[0].node_id, "0");
318 assert_eq!(doc.structure[0].children[0].node_id, "1");
319 assert_eq!(doc.structure[0].children[1].node_id, "2");
320 }
321
322 #[test]
323 fn test_source_type() {
324 let doc = parse("<h1>Hi</h1>");
325 assert_eq!(doc.source_type, SourceType::Html);
326 assert_eq!(doc.doc_id, "test.html");
327 }
328
329 #[test]
330 fn test_full_html_document() {
331 let html = r#"<!DOCTYPE html>
332<html>
333<head><title>Test Page</title></head>
334<body>
335<h1>Main Title</h1>
336<p>Content here.</p>
337</body>
338</html>"#;
339 let doc = parse(html);
340 assert_eq!(doc.doc_description, "Test Page");
341 assert!(!doc.structure.is_empty());
343 assert_eq!(doc.structure[0].title, "Main Title");
344 assert!(!doc.structure[0].text.contains("Test Page"));
345 }
346
347 #[test]
348 fn test_multiple_h1() {
349 let doc = parse("<h1>First</h1><p>A</p><h1>Second</h1><p>B</p>");
350 assert_eq!(doc.structure.len(), 2);
351 assert_eq!(doc.structure[0].title, "First");
352 assert_eq!(doc.structure[1].title, "Second");
353 }
354
355 #[test]
356 fn test_heading_with_nested_tags() {
357 let doc = parse("<h1>Hello <strong>World</strong></h1><p>Body</p>");
358 assert_eq!(doc.structure[0].title, "Hello World");
359 }
360
361 #[test]
362 fn test_hidden_script_and_style_text_is_not_indexed() {
363 let doc = parse(
364 "<html><body><script>secretToken()</script><style>.secret{}</style><h1>Visible</h1><p>Body</p></body></html>",
365 );
366 assert_eq!(doc.structure.len(), 1);
367 assert_eq!(doc.structure[0].title, "Visible");
368 assert_eq!(doc.structure[0].text, "Body");
369 }
370
371 #[test]
372 fn test_heading_level_fn() {
373 assert_eq!(heading_level("h1"), Some(1));
374 assert_eq!(heading_level("h6"), Some(6));
375 assert_eq!(heading_level("p"), None);
376 assert_eq!(heading_level("div"), None);
377 }
378}