1use tracing::trace;
7
8const STRUCTURAL_PRUNE_TAGS: &[&str] = &[
10 "nav", "footer", "aside", "script", "style", "noscript", "iframe", "svg", "form", "header",
11 "button", "input", "select", "textarea", "picture", "canvas",
12];
13
14const BOILERPLATE_PATTERNS: &[&str] = &[
16 "sidebar",
17 "advertisement",
18 " ad ",
19 "advert",
20 "cookie",
21 "popup",
22 "modal",
23 "banner",
24 "newsletter",
25 "subscribe",
26 "social-share",
27 "share-buttons",
28 "comments",
29 "comment-section",
30 "related-posts",
31 "recommended",
32 "footer",
33 "breadcrumb",
34 "pagination",
35 "navigation",
36 "skip-to",
37 "print-only",
38];
39
40#[derive(Debug, Clone)]
42pub struct TextNode {
43 pub text: String,
45 pub tag_context: String,
47 pub depth: usize,
49 pub estimated_tokens: usize,
51 pub relevance_score: f64,
53}
54
55impl TextNode {
56 pub fn new(text: String, tag_context: String, depth: usize) -> Self {
57 let estimated_tokens = estimate_tokens(&text);
58 Self {
59 text,
60 tag_context,
61 depth,
62 estimated_tokens,
63 relevance_score: 0.0,
64 }
65 }
66
67 pub fn is_heading(&self) -> bool {
69 matches!(
70 self.tag_context.as_str(),
71 "h1" | "h2" | "h3" | "h4" | "h5" | "h6"
72 )
73 }
74
75 pub fn heading_level(&self) -> u8 {
77 match self.tag_context.as_str() {
78 "h1" => 1,
79 "h2" => 2,
80 "h3" => 3,
81 "h4" => 4,
82 "h5" => 5,
83 "h6" => 6,
84 _ => 0,
85 }
86 }
87}
88
89pub fn estimate_tokens(text: &str) -> usize {
91 let chars = text.chars().count();
92 let words = text.split_whitespace().count();
93 ((chars / 4) + (words * 4 / 3)) / 2
95}
96
97pub fn structural_prune(html: &str) -> Vec<TextNode> {
103 let mut nodes = Vec::new();
104 let mut depth = 0usize;
105 let mut current_tag = String::new();
106 let mut skip_depth: Option<usize> = None;
107 let mut in_tag = false;
108 let mut tag_buf = String::new();
109 let mut text_buf = String::new();
110 let mut is_closing = false;
111 let mut char_iter = html.char_indices().peekable();
112
113 while let Some((_, c)) = char_iter.next() {
114 if c == '<' {
115 if !text_buf.trim().is_empty() && skip_depth.is_none() {
117 let trimmed = text_buf.trim().to_string();
118 if trimmed.len() >= 10 {
119 nodes.push(TextNode::new(trimmed, current_tag.clone(), depth));
121 }
122 }
123 text_buf.clear();
124 in_tag = true;
125 tag_buf.clear();
126 is_closing = false;
127 } else if c == '>' && in_tag {
128 in_tag = false;
129 let tag_name = tag_name_from_buf(&tag_buf);
130 let is_self_closing =
133 tag_buf.ends_with('/') || is_void_element(&tag_name) || tag_name.starts_with('!');
134
135 if is_closing {
136 if let Some(sd) = skip_depth {
139 if depth > 0 && depth - 1 == sd {
140 skip_depth = None;
141 trace!("QADD prune: exited skip zone at depth {}", sd);
142 }
143 }
144 depth = depth.saturating_sub(1);
145 if depth == 0 {
146 current_tag.clear();
147 }
148 } else if !is_self_closing {
149 if skip_depth.is_none() && should_prune_tag(&tag_name, &tag_buf) {
151 skip_depth = Some(depth);
152 trace!("QADD prune: skipping <{tag_name}> at depth {depth}");
153 }
154 current_tag = tag_name.clone();
155 depth += 1;
156 }
157 tag_buf.clear();
158 } else if in_tag {
159 if c == '/' && tag_buf.is_empty() {
160 is_closing = true;
161 } else {
162 tag_buf.push(c);
163 }
164 } else if skip_depth.is_none() {
165 if c == '&' {
167 let mut entity = String::from("&");
169 for (_, ec) in char_iter.by_ref() {
170 entity.push(ec);
171 if ec == ';' || entity.len() > 8 {
172 break;
173 }
174 }
175 text_buf.push_str(&decode_entity(&entity));
176 } else {
177 text_buf.push(c);
178 }
179 }
180 }
181
182 if !text_buf.trim().is_empty() && skip_depth.is_none() {
184 let trimmed = text_buf.trim().to_string();
185 if trimmed.len() >= 10 {
186 nodes.push(TextNode::new(trimmed, current_tag, depth));
187 }
188 }
189
190 trace!(
191 "QADD structural_prune: {} text nodes extracted from {} chars",
192 nodes.len(),
193 html.len()
194 );
195 nodes
196}
197
198fn tag_name_from_buf(buf: &str) -> String {
199 buf.split_whitespace()
200 .next()
201 .unwrap_or("")
202 .to_lowercase()
203 .trim_start_matches('/')
204 .to_string()
205}
206
207fn should_prune_tag(tag: &str, attrs: &str) -> bool {
208 if STRUCTURAL_PRUNE_TAGS.contains(&tag) {
209 return true;
210 }
211 let attrs_lower = attrs.to_lowercase();
213 if attrs_lower.contains("role=\"navigation\"") || attrs_lower.contains("role='navigation'") {
214 return true;
215 }
216 BOILERPLATE_PATTERNS.iter().any(|p| attrs_lower.contains(p))
217}
218
219fn is_void_element(tag: &str) -> bool {
220 matches!(
221 tag,
222 "area"
223 | "base"
224 | "br"
225 | "col"
226 | "embed"
227 | "hr"
228 | "img"
229 | "input"
230 | "link"
231 | "meta"
232 | "param"
233 | "source"
234 | "track"
235 | "wbr"
236 )
237}
238
239fn decode_entity(entity: &str) -> String {
240 match entity {
241 "&" => "&".to_string(),
242 "<" => "<".to_string(),
243 ">" => ">".to_string(),
244 """ => "\"".to_string(),
245 "'" | "'" => "'".to_string(),
246 " " => " ".to_string(),
247 _ => entity.to_string(),
248 }
249}
250
251#[cfg(test)]
252mod tests {
253 use super::*;
254
255 #[test]
256 fn prune_removes_nav() {
257 let html = r#"<div><nav>Skip navigation</nav><p>Real content here for testing purposes.</p></div>"#;
258 let nodes = structural_prune(html);
259 assert!(
260 !nodes.iter().any(|n| n.text.contains("Skip navigation")),
261 "nav should be pruned"
262 );
263 assert!(
264 nodes.iter().any(|n| n.text.contains("Real content")),
265 "p content should survive"
266 );
267 }
268
269 #[test]
270 fn prune_removes_script() {
271 let html = r#"<body><script>var x = 1;</script><p>This is real paragraph content for users to read.</p></body>"#;
272 let nodes = structural_prune(html);
273 assert!(!nodes.iter().any(|n| n.text.contains("var x")));
274 }
275
276 #[test]
277 fn prune_keeps_article_content() {
278 let html = r#"<article><h1>Main Title Here</h1><p>This is the main article content with substantial information for testing.</p></article>"#;
279 let nodes = structural_prune(html);
280 assert!(!nodes.is_empty());
281 }
282
283 #[test]
284 fn estimate_tokens_reasonable() {
285 let text = "word ".repeat(100);
287 let tokens = estimate_tokens(&text);
288 assert!(tokens > 50 && tokens < 300, "tokens={tokens}");
289 }
290
291 #[test]
292 fn text_node_heading_detection() {
293 let node = TextNode::new("Title".to_string(), "h2".to_string(), 1);
294 assert!(node.is_heading());
295 assert_eq!(node.heading_level(), 2);
296
297 let node2 = TextNode::new("Para".to_string(), "p".to_string(), 2);
298 assert!(!node2.is_heading());
299 }
300}