Skip to main content

fetchium_core/extract/
cep_predictor.rs

1//! CEP layer predictor — pure Rust decision tree (Phase 5, PRD §8.3).
2//!
3//! Predicts the required CEP extraction layer (1–5) for a URL *before* attempting
4//! extraction, saving latency by skipping layers that are unlikely to succeed.
5//!
6//! When historical data is available (from PIE), it takes priority.
7//! Otherwise, the decision tree uses URL features (script density, SPA frameworks, etc.).
8
9use tracing::debug;
10
11/// Features extracted from a URL for layer prediction.
12#[derive(Debug, Clone, Default)]
13pub struct CepFeatures {
14    /// Domain is in the known SPA list (React, Vue, Angular, Next.js, etc.).
15    pub domain_is_known_spa: bool,
16    /// URL path contains hash routing (`/#/` or `/#!`).
17    pub path_has_hash_routing: bool,
18    /// Number of `<script>` tags in the first 10 KB of HTML.
19    pub script_tag_count: u32,
20    /// Text-to-HTML character ratio of the first 10 KB (0.0–1.0).
21    pub text_to_html_ratio: f32,
22    /// True if the initial HTML contains framework markers
23    /// (`__next`, `__nuxt`, `ng-app`, `data-reactroot`, `id="__svelte"`).
24    pub has_framework_markers: bool,
25    /// True if a `<noscript>` tag is present (strong SPA signal).
26    pub has_noscript_tag: bool,
27    /// Layer that worked last time for this domain, if known.
28    pub historical_layer: Option<u8>,
29    /// Historical success rate for this domain+layer pair (0.0–1.0).
30    pub historical_success_rate: f32,
31}
32
33/// Known SPA/JavaScript-heavy domains that require Layer 3+.
34const SPA_DOMAINS: &[&str] = &[
35    "twitter.com",
36    "x.com",
37    "facebook.com",
38    "instagram.com",
39    "tiktok.com",
40    "linkedin.com",
41    "discord.com",
42    "slack.com",
43    "notion.so",
44    "figma.com",
45    "airtable.com",
46    "trello.com",
47    "app.github.com",
48    "gitlab.com",
49    "jira.atlassian.net",
50    "app.netlify.com",
51    "vercel.app",
52    "app.render.com",
53    "grafana.com",
54    "kibana.io",
55];
56
57/// Framework markers that indicate a JavaScript SPA.
58const FRAMEWORK_MARKERS: &[&str] = &[
59    "__next",
60    "__nuxt",
61    "ng-app",
62    "data-reactroot",
63    "__svelte",
64    "v-app",
65    "ember-application",
66    "__quasar",
67];
68
69/// Extract `CepFeatures` from a domain + initial HTML peek.
70///
71/// The HTML peek should be the first ~10 KB of the page.
72pub fn extract_features(domain: &str, url: &str, html_peek: &str) -> CepFeatures {
73    CepFeatures {
74        domain_is_known_spa: is_known_spa(domain),
75        path_has_hash_routing: url.contains("/#/") || url.contains("/#!"),
76        script_tag_count: count_script_tags(html_peek),
77        text_to_html_ratio: compute_text_ratio(html_peek),
78        has_framework_markers: has_framework_markers(html_peek),
79        has_noscript_tag: html_peek.to_lowercase().contains("<noscript"),
80        historical_layer: None,
81        historical_success_rate: 1.0,
82    }
83}
84
85/// Predict the optimal starting CEP layer for the given features.
86///
87/// Returns a layer number 1–5:
88/// - 1 = Static HTML (CSS selectors) — fastest
89/// - 2 = HTML + readability — for article pages
90/// - 3 = Headless Chrome — SPAs and heavy JS
91/// - 4 = PDF/document extraction
92/// - 5 = Screenshot OCR — last resort
93pub fn predict_layer(features: &CepFeatures) -> u8 {
94    // Priority 1: Trust historical data when success rate is high
95    if let Some(layer) = features.historical_layer {
96        if features.historical_success_rate > 0.8 {
97            debug!("CEP predictor: using historical layer {layer}");
98            return layer;
99        }
100    }
101
102    // Priority 2: Known SPA domain or framework markers
103    if features.domain_is_known_spa || features.has_framework_markers {
104        if features.path_has_hash_routing || features.has_noscript_tag {
105            debug!("CEP predictor: hash-routed or noscript SPA → Layer 3");
106            return 3;
107        }
108        debug!("CEP predictor: SPA/framework detected → Layer 3");
109        return 3;
110    }
111
112    // Priority 3: Script density heuristic
113    if features.script_tag_count > 20 {
114        debug!(
115            "CEP predictor: heavy JS ({} scripts) → Layer 3",
116            features.script_tag_count
117        );
118        return 3;
119    }
120    if features.script_tag_count > 10 {
121        debug!(
122            "CEP predictor: moderate JS ({} scripts) → Layer 2",
123            features.script_tag_count
124        );
125        return 2;
126    }
127
128    // Priority 4: Text ratio heuristic
129    if features.text_to_html_ratio > 0.3 {
130        debug!("CEP predictor: high text ratio → Layer 1");
131        return 1;
132    }
133    if features.text_to_html_ratio > 0.1 {
134        debug!("CEP predictor: medium text ratio → Layer 2");
135        return 2;
136    }
137
138    // Low text ratio + not SPA = probably a portal/dynamic page
139    debug!("CEP predictor: low text ratio → Layer 2 (default)");
140    2
141}
142
143fn is_known_spa(domain: &str) -> bool {
144    SPA_DOMAINS
145        .iter()
146        .any(|spa| domain == *spa || domain.ends_with(&format!(".{spa}")))
147}
148
149fn count_script_tags(html: &str) -> u32 {
150    let lower = html.to_lowercase();
151    lower.matches("<script").count() as u32
152}
153
154fn compute_text_ratio(html: &str) -> f32 {
155    if html.is_empty() {
156        return 0.0;
157    }
158    // Strip all HTML tags (naive approximation for feature extraction only)
159    let mut in_tag = false;
160    let mut text_len = 0usize;
161    for c in html.chars() {
162        match c {
163            '<' => in_tag = true,
164            '>' => in_tag = false,
165            _ if !in_tag => text_len += 1,
166            _ => {}
167        }
168    }
169    text_len as f32 / html.len() as f32
170}
171
172fn has_framework_markers(html: &str) -> bool {
173    let lower = html.to_lowercase();
174    FRAMEWORK_MARKERS
175        .iter()
176        .any(|marker| lower.contains(marker))
177}
178
179#[cfg(test)]
180mod tests {
181    use super::*;
182
183    #[test]
184    fn predict_layer_static_html() {
185        let features = CepFeatures {
186            text_to_html_ratio: 0.4,
187            script_tag_count: 2,
188            ..Default::default()
189        };
190        assert_eq!(predict_layer(&features), 1);
191    }
192
193    #[test]
194    fn predict_layer_heavy_js() {
195        let features = CepFeatures {
196            script_tag_count: 25,
197            text_to_html_ratio: 0.05,
198            ..Default::default()
199        };
200        assert_eq!(predict_layer(&features), 3);
201    }
202
203    #[test]
204    fn predict_layer_known_spa() {
205        let features = CepFeatures {
206            domain_is_known_spa: true,
207            ..Default::default()
208        };
209        assert_eq!(predict_layer(&features), 3);
210    }
211
212    #[test]
213    fn predict_layer_historical_wins() {
214        let features = CepFeatures {
215            historical_layer: Some(1),
216            historical_success_rate: 0.95,
217            domain_is_known_spa: true, // would normally → Layer 3
218            ..Default::default()
219        };
220        assert_eq!(predict_layer(&features), 1, "historical should override");
221    }
222
223    #[test]
224    fn predict_layer_low_history_success_falls_through() {
225        let features = CepFeatures {
226            historical_layer: Some(1),
227            historical_success_rate: 0.3, // below threshold
228            script_tag_count: 25,
229            ..Default::default()
230        };
231        // Historical below threshold → falls through to script density → Layer 3
232        assert_eq!(predict_layer(&features), 3);
233    }
234
235    #[test]
236    fn is_known_spa_twitter() {
237        assert!(is_known_spa("twitter.com"));
238        assert!(is_known_spa("x.com"));
239        assert!(!is_known_spa("example.com"));
240    }
241
242    #[test]
243    fn count_script_tags_correct() {
244        let html = "<script src='a.js'></script><script>x</script><p>text</p>";
245        assert_eq!(count_script_tags(html), 2);
246    }
247
248    #[test]
249    fn compute_text_ratio_reasonable() {
250        let html = "<p>Hello world</p>";
251        let ratio = compute_text_ratio(html);
252        assert!(ratio > 0.3 && ratio < 1.0, "ratio={ratio}");
253    }
254
255    #[test]
256    fn has_framework_markers_detects_next() {
257        let html = "<div id='__next'></div>";
258        assert!(has_framework_markers(html));
259        assert!(!has_framework_markers("<div>plain html</div>"));
260    }
261
262    #[test]
263    fn extract_features_from_html_peek() {
264        let html =
265            "<html><script src='app.js'></script><div id='__next'><p>content here</p></div></html>";
266        let features = extract_features("example.com", "https://example.com/page", html);
267        assert!(features.has_framework_markers);
268        assert_eq!(features.script_tag_count, 1);
269    }
270}