fetchium_core/extract/
cep_predictor.rs1use tracing::debug;
10
11#[derive(Debug, Clone, Default)]
13pub struct CepFeatures {
14 pub domain_is_known_spa: bool,
16 pub path_has_hash_routing: bool,
18 pub script_tag_count: u32,
20 pub text_to_html_ratio: f32,
22 pub has_framework_markers: bool,
25 pub has_noscript_tag: bool,
27 pub historical_layer: Option<u8>,
29 pub historical_success_rate: f32,
31}
32
33const SPA_DOMAINS: &[&str] = &[
35 "twitter.com",
36 "x.com",
37 "facebook.com",
38 "instagram.com",
39 "tiktok.com",
40 "linkedin.com",
41 "discord.com",
42 "slack.com",
43 "notion.so",
44 "figma.com",
45 "airtable.com",
46 "trello.com",
47 "app.github.com",
48 "gitlab.com",
49 "jira.atlassian.net",
50 "app.netlify.com",
51 "vercel.app",
52 "app.render.com",
53 "grafana.com",
54 "kibana.io",
55];
56
57const FRAMEWORK_MARKERS: &[&str] = &[
59 "__next",
60 "__nuxt",
61 "ng-app",
62 "data-reactroot",
63 "__svelte",
64 "v-app",
65 "ember-application",
66 "__quasar",
67];
68
69pub fn extract_features(domain: &str, url: &str, html_peek: &str) -> CepFeatures {
73 CepFeatures {
74 domain_is_known_spa: is_known_spa(domain),
75 path_has_hash_routing: url.contains("/#/") || url.contains("/#!"),
76 script_tag_count: count_script_tags(html_peek),
77 text_to_html_ratio: compute_text_ratio(html_peek),
78 has_framework_markers: has_framework_markers(html_peek),
79 has_noscript_tag: html_peek.to_lowercase().contains("<noscript"),
80 historical_layer: None,
81 historical_success_rate: 1.0,
82 }
83}
84
85pub fn predict_layer(features: &CepFeatures) -> u8 {
94 if let Some(layer) = features.historical_layer {
96 if features.historical_success_rate > 0.8 {
97 debug!("CEP predictor: using historical layer {layer}");
98 return layer;
99 }
100 }
101
102 if features.domain_is_known_spa || features.has_framework_markers {
104 if features.path_has_hash_routing || features.has_noscript_tag {
105 debug!("CEP predictor: hash-routed or noscript SPA → Layer 3");
106 return 3;
107 }
108 debug!("CEP predictor: SPA/framework detected → Layer 3");
109 return 3;
110 }
111
112 if features.script_tag_count > 20 {
114 debug!(
115 "CEP predictor: heavy JS ({} scripts) → Layer 3",
116 features.script_tag_count
117 );
118 return 3;
119 }
120 if features.script_tag_count > 10 {
121 debug!(
122 "CEP predictor: moderate JS ({} scripts) → Layer 2",
123 features.script_tag_count
124 );
125 return 2;
126 }
127
128 if features.text_to_html_ratio > 0.3 {
130 debug!("CEP predictor: high text ratio → Layer 1");
131 return 1;
132 }
133 if features.text_to_html_ratio > 0.1 {
134 debug!("CEP predictor: medium text ratio → Layer 2");
135 return 2;
136 }
137
138 debug!("CEP predictor: low text ratio → Layer 2 (default)");
140 2
141}
142
143fn is_known_spa(domain: &str) -> bool {
144 SPA_DOMAINS
145 .iter()
146 .any(|spa| domain == *spa || domain.ends_with(&format!(".{spa}")))
147}
148
149fn count_script_tags(html: &str) -> u32 {
150 let lower = html.to_lowercase();
151 lower.matches("<script").count() as u32
152}
153
154fn compute_text_ratio(html: &str) -> f32 {
155 if html.is_empty() {
156 return 0.0;
157 }
158 let mut in_tag = false;
160 let mut text_len = 0usize;
161 for c in html.chars() {
162 match c {
163 '<' => in_tag = true,
164 '>' => in_tag = false,
165 _ if !in_tag => text_len += 1,
166 _ => {}
167 }
168 }
169 text_len as f32 / html.len() as f32
170}
171
172fn has_framework_markers(html: &str) -> bool {
173 let lower = html.to_lowercase();
174 FRAMEWORK_MARKERS
175 .iter()
176 .any(|marker| lower.contains(marker))
177}
178
179#[cfg(test)]
180mod tests {
181 use super::*;
182
183 #[test]
184 fn predict_layer_static_html() {
185 let features = CepFeatures {
186 text_to_html_ratio: 0.4,
187 script_tag_count: 2,
188 ..Default::default()
189 };
190 assert_eq!(predict_layer(&features), 1);
191 }
192
193 #[test]
194 fn predict_layer_heavy_js() {
195 let features = CepFeatures {
196 script_tag_count: 25,
197 text_to_html_ratio: 0.05,
198 ..Default::default()
199 };
200 assert_eq!(predict_layer(&features), 3);
201 }
202
203 #[test]
204 fn predict_layer_known_spa() {
205 let features = CepFeatures {
206 domain_is_known_spa: true,
207 ..Default::default()
208 };
209 assert_eq!(predict_layer(&features), 3);
210 }
211
212 #[test]
213 fn predict_layer_historical_wins() {
214 let features = CepFeatures {
215 historical_layer: Some(1),
216 historical_success_rate: 0.95,
217 domain_is_known_spa: true, ..Default::default()
219 };
220 assert_eq!(predict_layer(&features), 1, "historical should override");
221 }
222
223 #[test]
224 fn predict_layer_low_history_success_falls_through() {
225 let features = CepFeatures {
226 historical_layer: Some(1),
227 historical_success_rate: 0.3, script_tag_count: 25,
229 ..Default::default()
230 };
231 assert_eq!(predict_layer(&features), 3);
233 }
234
235 #[test]
236 fn is_known_spa_twitter() {
237 assert!(is_known_spa("twitter.com"));
238 assert!(is_known_spa("x.com"));
239 assert!(!is_known_spa("example.com"));
240 }
241
242 #[test]
243 fn count_script_tags_correct() {
244 let html = "<script src='a.js'></script><script>x</script><p>text</p>";
245 assert_eq!(count_script_tags(html), 2);
246 }
247
248 #[test]
249 fn compute_text_ratio_reasonable() {
250 let html = "<p>Hello world</p>";
251 let ratio = compute_text_ratio(html);
252 assert!(ratio > 0.3 && ratio < 1.0, "ratio={ratio}");
253 }
254
255 #[test]
256 fn has_framework_markers_detects_next() {
257 let html = "<div id='__next'></div>";
258 assert!(has_framework_markers(html));
259 assert!(!has_framework_markers("<div>plain html</div>"));
260 }
261
262 #[test]
263 fn extract_features_from_html_peek() {
264 let html =
265 "<html><script src='app.js'></script><div id='__next'><p>content here</p></div></html>";
266 let features = extract_features("example.com", "https://example.com/page", html);
267 assert!(features.has_framework_markers);
268 assert_eq!(features.script_tag_count, 1);
269 }
270}