Skip to main content

fetchium_core/qadd/
pruning.rs

1//! QADD structural pruning — Step 1 of the 5-step pipeline.
2//!
3//! Removes navigation, footers, scripts, ads, and other boilerplate
4//! BEFORE content scoring, maximizing signal-to-noise ratio.
5
6use tracing::trace;
7
8/// Tags to remove entirely in structural pruning.
9const STRUCTURAL_PRUNE_TAGS: &[&str] = &[
10    "nav", "footer", "aside", "script", "style", "noscript", "iframe", "svg", "form", "header",
11    "button", "input", "select", "textarea", "picture", "canvas",
12];
13
14/// Class/id substrings that indicate boilerplate content.
15const BOILERPLATE_PATTERNS: &[&str] = &[
16    "sidebar",
17    "advertisement",
18    " ad ",
19    "advert",
20    "cookie",
21    "popup",
22    "modal",
23    "banner",
24    "newsletter",
25    "subscribe",
26    "social-share",
27    "share-buttons",
28    "comments",
29    "comment-section",
30    "related-posts",
31    "recommended",
32    "footer",
33    "breadcrumb",
34    "pagination",
35    "navigation",
36    "skip-to",
37    "print-only",
38];
39
40/// A text node extracted from the DOM after structural pruning.
41#[derive(Debug, Clone)]
42pub struct TextNode {
43    /// The text content.
44    pub text: String,
45    /// The containing HTML tag (p, h1, h2, li, td, etc.).
46    pub tag_context: String,
47    /// DOM nesting depth (0 = top level).
48    pub depth: usize,
49    /// Estimated token count (~1.33 tokens/word, or chars/4).
50    pub estimated_tokens: usize,
51    /// BM25 relevance score (set in pipeline Step 2).
52    pub relevance_score: f64,
53}
54
55impl TextNode {
56    pub fn new(text: String, tag_context: String, depth: usize) -> Self {
57        let estimated_tokens = estimate_tokens(&text);
58        Self {
59            text,
60            tag_context,
61            depth,
62            estimated_tokens,
63            relevance_score: 0.0,
64        }
65    }
66
67    /// Returns true if this node is a heading (h1-h6).
68    pub fn is_heading(&self) -> bool {
69        matches!(
70            self.tag_context.as_str(),
71            "h1" | "h2" | "h3" | "h4" | "h5" | "h6"
72        )
73    }
74
75    /// Heading level (1-6 for h1-h6, 0 for non-headings).
76    pub fn heading_level(&self) -> u8 {
77        match self.tag_context.as_str() {
78            "h1" => 1,
79            "h2" => 2,
80            "h3" => 3,
81            "h4" => 4,
82            "h5" => 5,
83            "h6" => 6,
84            _ => 0,
85        }
86    }
87}
88
89/// Estimate token count from text. ~4 chars per token (GPT-style approximation).
90pub fn estimate_tokens(text: &str) -> usize {
91    let chars = text.chars().count();
92    let words = text.split_whitespace().count();
93    // Average of char-based and word-based estimates
94    ((chars / 4) + (words * 4 / 3)) / 2
95}
96
97/// Step 1: Structural pruning — walk the HTML, skip pruned elements,
98/// collect text nodes with context.
99///
100/// This is a fast, regex/state-machine based approach that avoids full DOM
101/// parsing for maximum throughput.
102pub fn structural_prune(html: &str) -> Vec<TextNode> {
103    let mut nodes = Vec::new();
104    let mut depth = 0usize;
105    let mut current_tag = String::new();
106    let mut skip_depth: Option<usize> = None;
107    let mut in_tag = false;
108    let mut tag_buf = String::new();
109    let mut text_buf = String::new();
110    let mut is_closing = false;
111    let mut char_iter = html.char_indices().peekable();
112
113    while let Some((_, c)) = char_iter.next() {
114        if c == '<' {
115            // Flush accumulated text
116            if !text_buf.trim().is_empty() && skip_depth.is_none() {
117                let trimmed = text_buf.trim().to_string();
118                if trimmed.len() >= 10 {
119                    // Skip very short nodes
120                    nodes.push(TextNode::new(trimmed, current_tag.clone(), depth));
121                }
122            }
123            text_buf.clear();
124            in_tag = true;
125            tag_buf.clear();
126            is_closing = false;
127        } else if c == '>' && in_tag {
128            in_tag = false;
129            let tag_name = tag_name_from_buf(&tag_buf);
130            // HTML declarations (<!DOCTYPE>, <!--...-->) and void elements are self-closing.
131            // Treating them as opening tags would shift depth for the entire document.
132            let is_self_closing =
133                tag_buf.ends_with('/') || is_void_element(&tag_name) || tag_name.starts_with('!');
134
135            if is_closing {
136                // Exiting a skip zone: depth was incremented when the pruned element opened,
137                // so the closing tag arrives when depth == skip_depth + 1.
138                if let Some(sd) = skip_depth {
139                    if depth > 0 && depth - 1 == sd {
140                        skip_depth = None;
141                        trace!("QADD prune: exited skip zone at depth {}", sd);
142                    }
143                }
144                depth = depth.saturating_sub(1);
145                if depth == 0 {
146                    current_tag.clear();
147                }
148            } else if !is_self_closing {
149                // Check if we should skip this element
150                if skip_depth.is_none() && should_prune_tag(&tag_name, &tag_buf) {
151                    skip_depth = Some(depth);
152                    trace!("QADD prune: skipping <{tag_name}> at depth {depth}");
153                }
154                current_tag = tag_name.clone();
155                depth += 1;
156            }
157            tag_buf.clear();
158        } else if in_tag {
159            if c == '/' && tag_buf.is_empty() {
160                is_closing = true;
161            } else {
162                tag_buf.push(c);
163            }
164        } else if skip_depth.is_none() {
165            // Decode basic HTML entities on the fly
166            if c == '&' {
167                // Collect entity
168                let mut entity = String::from("&");
169                for (_, ec) in char_iter.by_ref() {
170                    entity.push(ec);
171                    if ec == ';' || entity.len() > 8 {
172                        break;
173                    }
174                }
175                text_buf.push_str(&decode_entity(&entity));
176            } else {
177                text_buf.push(c);
178            }
179        }
180    }
181
182    // Flush any remaining text
183    if !text_buf.trim().is_empty() && skip_depth.is_none() {
184        let trimmed = text_buf.trim().to_string();
185        if trimmed.len() >= 10 {
186            nodes.push(TextNode::new(trimmed, current_tag, depth));
187        }
188    }
189
190    trace!(
191        "QADD structural_prune: {} text nodes extracted from {} chars",
192        nodes.len(),
193        html.len()
194    );
195    nodes
196}
197
198fn tag_name_from_buf(buf: &str) -> String {
199    buf.split_whitespace()
200        .next()
201        .unwrap_or("")
202        .to_lowercase()
203        .trim_start_matches('/')
204        .to_string()
205}
206
207fn should_prune_tag(tag: &str, attrs: &str) -> bool {
208    if STRUCTURAL_PRUNE_TAGS.contains(&tag) {
209        return true;
210    }
211    // Check role/class/id attributes for boilerplate signals
212    let attrs_lower = attrs.to_lowercase();
213    if attrs_lower.contains("role=\"navigation\"") || attrs_lower.contains("role='navigation'") {
214        return true;
215    }
216    BOILERPLATE_PATTERNS.iter().any(|p| attrs_lower.contains(p))
217}
218
219fn is_void_element(tag: &str) -> bool {
220    matches!(
221        tag,
222        "area"
223            | "base"
224            | "br"
225            | "col"
226            | "embed"
227            | "hr"
228            | "img"
229            | "input"
230            | "link"
231            | "meta"
232            | "param"
233            | "source"
234            | "track"
235            | "wbr"
236    )
237}
238
239fn decode_entity(entity: &str) -> String {
240    match entity {
241        "&amp;" => "&".to_string(),
242        "&lt;" => "<".to_string(),
243        "&gt;" => ">".to_string(),
244        "&quot;" => "\"".to_string(),
245        "&#39;" | "&apos;" => "'".to_string(),
246        "&nbsp;" => " ".to_string(),
247        _ => entity.to_string(),
248    }
249}
250
251#[cfg(test)]
252mod tests {
253    use super::*;
254
255    #[test]
256    fn prune_removes_nav() {
257        let html = r#"<div><nav>Skip navigation</nav><p>Real content here for testing purposes.</p></div>"#;
258        let nodes = structural_prune(html);
259        assert!(
260            !nodes.iter().any(|n| n.text.contains("Skip navigation")),
261            "nav should be pruned"
262        );
263        assert!(
264            nodes.iter().any(|n| n.text.contains("Real content")),
265            "p content should survive"
266        );
267    }
268
269    #[test]
270    fn prune_removes_script() {
271        let html = r#"<body><script>var x = 1;</script><p>This is real paragraph content for users to read.</p></body>"#;
272        let nodes = structural_prune(html);
273        assert!(!nodes.iter().any(|n| n.text.contains("var x")));
274    }
275
276    #[test]
277    fn prune_keeps_article_content() {
278        let html = r#"<article><h1>Main Title Here</h1><p>This is the main article content with substantial information for testing.</p></article>"#;
279        let nodes = structural_prune(html);
280        assert!(!nodes.is_empty());
281    }
282
283    #[test]
284    fn estimate_tokens_reasonable() {
285        // 100 words ≈ 133 tokens
286        let text = "word ".repeat(100);
287        let tokens = estimate_tokens(&text);
288        assert!(tokens > 50 && tokens < 300, "tokens={tokens}");
289    }
290
291    #[test]
292    fn text_node_heading_detection() {
293        let node = TextNode::new("Title".to_string(), "h2".to_string(), 1);
294        assert!(node.is_heading());
295        assert_eq!(node.heading_level(), 2);
296
297        let node2 = TextNode::new("Para".to_string(), "p".to_string(), 2);
298        assert!(!node2.is_heading());
299    }
300}