millipede_browser/
detect.rs

1//! Conservative HTTP-response detection for smart browser promotion.
2
3use std::fmt;
4
5use millipede_core::{errors::AntiBotTech, request::Request};
6use millipede_html::{SynchronizedHtml, scraper::Selector};
7
8/// A borrowed snapshot of one successful HTTP/HTML attempt.
9#[non_exhaustive]
10pub struct HttpAttemptSnapshot<'a> {
11    /// Request that produced the response.
12    pub request: &'a Request,
13    /// Final response status.
14    pub status: http::StatusCode,
15    /// Final response headers.
16    pub headers: &'a http::HeaderMap,
17    /// Buffered response body.
18    pub body: &'a [u8],
19    /// Final response URL after redirects.
20    pub final_url: &'a url::Url,
21    /// Parsed HTML, when the response was parsed as HTML.
22    pub html: Option<&'a SynchronizedHtml>,
23}
24
25impl<'a> HttpAttemptSnapshot<'a> {
26    pub(crate) fn new(
27        request: &'a Request,
28        status: http::StatusCode,
29        headers: &'a http::HeaderMap,
30        body: &'a [u8],
31        final_url: &'a url::Url,
32        html: Option<&'a SynchronizedHtml>,
33    ) -> Self {
34        Self {
35            request,
36            status,
37            headers,
38            body,
39            final_url,
40            html,
41        }
42    }
43}
44
45/// Why an HTTP attempt should be repeated through a browser.
46#[derive(Debug, Clone, PartialEq)]
47#[non_exhaustive]
48pub enum PromotionReason {
49    /// A successful HTML document has little visible text but contains JavaScript.
50    EmptyBodyLikelyJs,
51    /// A known anti-bot interstitial was detected.
52    KnownAntiBot(AntiBotTech),
53    /// A caller-required selector was absent from the parsed document.
54    SelectorMissing {
55        /// Selector that did not match any element.
56        selector: String,
57    },
58    /// An HTTP error status was configured for browser promotion.
59    StatusPromoted {
60        /// Numeric HTTP status code.
61        status: u16,
62    },
63    /// A custom detector-specific explanation.
64    Custom(String),
65}
66
67impl fmt::Display for PromotionReason {
68    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
69        match self {
70            Self::EmptyBodyLikelyJs => {
71                formatter.write_str("successful page is likely a JavaScript shell")
72            }
73            Self::KnownAntiBot(tech) => write!(formatter, "known anti-bot interstitial: {tech:?}"),
74            Self::SelectorMissing { selector } => {
75                write!(formatter, "required selector is missing: {selector}")
76            }
77            Self::StatusPromoted { status } => {
78                write!(formatter, "configured HTTP status promoted: {status}")
79            }
80            Self::Custom(reason) => formatter.write_str(reason),
81        }
82    }
83}
84
85/// Decides whether a successful HTTP/HTML attempt needs browser execution.
86pub trait BrowserPromotionDetector: Send + Sync + 'static {
87    /// Returns the promotion reason, or `None` when the HTTP result should be kept.
88    fn should_promote(&self, attempt: &HttpAttemptSnapshot<'_>) -> Option<PromotionReason>;
89}
90
91/// Conservative built-in promotion heuristics.
92///
93/// Anti-bot classification is delegated to the configured core detector.
94#[derive(Debug, Clone)]
95#[must_use = "detector configuration does nothing unless the detector is installed"]
96pub struct DefaultPromotionDetector {
97    required_selector: Option<String>,
98    min_visible_text: usize,
99    anti_bot: std::sync::Arc<dyn millipede_core::antibot::AntiBotDetector>,
100}
101
102impl DefaultPromotionDetector {
103    /// Creates a detector with conservative defaults.
104    pub fn new() -> Self {
105        Self::default()
106    }
107
108    /// Promotes parsed pages that do not contain `selector`.
109    pub fn with_required_selector(mut self, selector: impl Into<String>) -> Self {
110        self.required_selector = Some(selector.into());
111        self
112    }
113
114    /// Sets the minimum visible body-text length for JavaScript-shell detection.
115    pub fn with_min_visible_text(mut self, minimum: usize) -> Self {
116        self.min_visible_text = minimum;
117        self
118    }
119
120    /// Sets the anti-bot detector used to classify response signals.
121    pub fn with_anti_bot_detector(
122        mut self,
123        detector: std::sync::Arc<dyn millipede_core::antibot::AntiBotDetector>,
124    ) -> Self {
125        self.anti_bot = detector;
126        self
127    }
128
129    fn inspect_html(
130        &self,
131        status: http::StatusCode,
132        document: &millipede_html::scraper::Html,
133    ) -> Option<PromotionReason> {
134        if let Some(selector_text) = &self.required_selector {
135            if let Ok(selector) = Selector::parse(selector_text) {
136                if document.select(&selector).next().is_none() {
137                    return Some(PromotionReason::SelectorMissing {
138                        selector: selector_text.clone(),
139                    });
140                }
141            }
142        }
143
144        if status.is_success() {
145            let body_selector = Selector::parse("body").expect("static body selector is valid");
146            let script_selector =
147                Selector::parse("script").expect("static script selector is valid");
148            let visible_text_len = document
149                .select(&body_selector)
150                .next()
151                .map(|body| body.text().collect::<Vec<_>>().join(" ").trim().len())
152                .unwrap_or(0);
153            let has_script = document.select(&script_selector).next().is_some();
154            if visible_text_len < self.min_visible_text && has_script {
155                return Some(PromotionReason::EmptyBodyLikelyJs);
156            }
157        }
158
159        None
160    }
161}
162
163impl Default for DefaultPromotionDetector {
164    fn default() -> Self {
165        Self {
166            required_selector: None,
167            min_visible_text: 40,
168            anti_bot: std::sync::Arc::new(millipede_core::antibot::DefaultAntiBotDetector::new()),
169        }
170    }
171}
172
173impl BrowserPromotionDetector for DefaultPromotionDetector {
174    fn should_promote(&self, attempt: &HttpAttemptSnapshot<'_>) -> Option<PromotionReason> {
175        let signals = millipede_core::antibot::AntiBotSignals::new(
176            attempt.status,
177            attempt.headers,
178            attempt.body,
179            attempt.final_url,
180        );
181        if let Some(tech) = self.anti_bot.detect(&signals) {
182            return Some(PromotionReason::KnownAntiBot(tech));
183        }
184
185        if let Some(html) = attempt.html {
186            if let Some(reason) =
187                html.with_html(|document| self.inspect_html(attempt.status, document))
188            {
189                return Some(reason);
190            }
191        }
192
193        None
194    }
195}
196
197#[cfg(test)]
198mod tests {
199    use super::*;
200    use millipede_core::request::Request;
201
202    fn snapshot<'a>(
203        request: &'a Request,
204        headers: &'a http::HeaderMap,
205        body: &'a [u8],
206        status: http::StatusCode,
207    ) -> HttpAttemptSnapshot<'a> {
208        HttpAttemptSnapshot::new(request, status, headers, body, &request.url, None)
209    }
210
211    fn request() -> Request {
212        Request::get("https://example.com/")
213            .build()
214            .expect("valid test request")
215    }
216
217    fn html(source: &str) -> millipede_html::scraper::Html {
218        millipede_html::scraper::Html::parse_document(source)
219    }
220
221    #[test]
222    fn cloudflare_marker_is_promoted() {
223        let request = request();
224        let headers = http::HeaderMap::new();
225        assert_eq!(
226            DefaultPromotionDetector::new().should_promote(&snapshot(
227                &request,
228                &headers,
229                b"<html><body>Just a moment...</body></html>",
230                http::StatusCode::OK,
231            )),
232            Some(PromotionReason::KnownAntiBot(AntiBotTech::Cloudflare))
233        );
234    }
235
236    #[test]
237    fn contentful_page_stays_http() {
238        let document = html(
239            "<html><body>A long, genuinely contentful page with enough visible text for the conservative detector.</body></html>",
240        );
241        assert_eq!(
242            DefaultPromotionDetector::new().inspect_html(http::StatusCode::OK, &document),
243            None
244        );
245    }
246
247    #[test]
248    fn javascript_shell_is_promoted() {
249        let document = html("<html><body><script src=\"app.js\"></script></body></html>");
250        assert_eq!(
251            DefaultPromotionDetector::new().inspect_html(http::StatusCode::OK, &document),
252            Some(PromotionReason::EmptyBodyLikelyJs)
253        );
254    }
255
256    #[test]
257    fn required_selector_only_promotes_when_missing() {
258        let missing = html("<html><body><main>content</main></body></html>");
259        let present = html("<html><body><main id=\"app\">content</main></body></html>");
260        let detector = DefaultPromotionDetector::new().with_required_selector("#app");
261        assert_eq!(
262            detector.inspect_html(http::StatusCode::OK, &missing),
263            Some(PromotionReason::SelectorMissing {
264                selector: "#app".to_owned(),
265            })
266        );
267        assert_eq!(detector.inspect_html(http::StatusCode::OK, &present), None);
268    }
269
270    #[test]
271    fn markers_after_inspection_cap_are_ignored() {
272        let request = request();
273        let headers = http::HeaderMap::new();
274        // Mirrors the core detector's default inspection window.
275        let mut body =
276            vec![b'x'; millipede_core::antibot::DefaultAntiBotDetector::DEFAULT_INSPECTION_LIMIT];
277        body.extend_from_slice(b"just a moment");
278        assert_eq!(
279            DefaultPromotionDetector::new().should_promote(&snapshot(
280                &request,
281                &headers,
282                &body,
283                http::StatusCode::OK,
284            )),
285            None
286        );
287    }
288
289    #[test]
290    fn perimeterx_fixture_is_promoted() {
291        let body = include_str!(concat!(
292            env!("CARGO_MANIFEST_DIR"),
293            "/../millipede-core/tests/fixtures/antibot/perimeterx.html"
294        ));
295        let request = request();
296        let headers = http::HeaderMap::new();
297        assert_eq!(
298            DefaultPromotionDetector::new().should_promote(&snapshot(
299                &request,
300                &headers,
301                body.as_bytes(),
302                http::StatusCode::OK,
303            )),
304            Some(PromotionReason::KnownAntiBot(AntiBotTech::PerimeterX))
305        );
306    }
307
308    #[test]
309    fn imperva_fixture_is_promoted() {
310        let body = include_str!(concat!(
311            env!("CARGO_MANIFEST_DIR"),
312            "/../millipede-core/tests/fixtures/antibot/imperva.html"
313        ));
314        let request = request();
315        let headers = http::HeaderMap::new();
316        assert_eq!(
317            DefaultPromotionDetector::new().should_promote(&snapshot(
318                &request,
319                &headers,
320                body.as_bytes(),
321                http::StatusCode::OK,
322            )),
323            Some(PromotionReason::KnownAntiBot(AntiBotTech::Imperva))
324        );
325    }
326
327    #[test]
328    fn cloudflare_fixture_is_promoted() {
329        let body = include_str!(concat!(
330            env!("CARGO_MANIFEST_DIR"),
331            "/../millipede-core/tests/fixtures/antibot/cloudflare.html"
332        ));
333        let request = request();
334        let headers = http::HeaderMap::new();
335        assert_eq!(
336            DefaultPromotionDetector::new().should_promote(&snapshot(
337                &request,
338                &headers,
339                body.as_bytes(),
340                http::StatusCode::OK,
341            )),
342            Some(PromotionReason::KnownAntiBot(AntiBotTech::Cloudflare))
343        );
344    }
345
346    macro_rules! shared_vendor_fixture_test {
347        ($name:ident, $fixture:literal, $expected:expr) => {
348            #[test]
349            fn $name() {
350                let body = include_str!(concat!(
351                    env!("CARGO_MANIFEST_DIR"),
352                    "/../millipede-core/tests/fixtures/antibot/",
353                    $fixture
354                ));
355                let request = request();
356                let headers = http::HeaderMap::new();
357                assert_eq!(
358                    DefaultPromotionDetector::new().should_promote(&snapshot(
359                        &request,
360                        &headers,
361                        body.as_bytes(),
362                        http::StatusCode::OK,
363                    )),
364                    Some(PromotionReason::KnownAntiBot($expected))
365                );
366            }
367        };
368    }
369
370    shared_vendor_fixture_test!(
371        datadome_fixture_is_promoted,
372        "datadome.html",
373        AntiBotTech::DataDome
374    );
375    shared_vendor_fixture_test!(
376        kasada_fixture_is_promoted,
377        "kasada.html",
378        AntiBotTech::Kasada
379    );
380    shared_vendor_fixture_test!(
381        akamai_fixture_is_promoted,
382        "akamai.html",
383        AntiBotTech::Akamai
384    );
385    shared_vendor_fixture_test!(
386        unknown_fixture_is_promoted,
387        "unknown.html",
388        AntiBotTech::Unknown
389    );
390
391    #[test]
392    fn benign_contentful_fixture_stays_http() {
393        let body = include_str!(concat!(
394            env!("CARGO_MANIFEST_DIR"),
395            "/../millipede-core/tests/fixtures/antibot/benign_contentful.html"
396        ));
397        let request = request();
398        let headers = http::HeaderMap::new();
399        assert_eq!(
400            DefaultPromotionDetector::new().should_promote(&snapshot(
401                &request,
402                &headers,
403                body.as_bytes(),
404                http::StatusCode::OK,
405            )),
406            None
407        );
408    }
409
410    #[test]
411    fn contentful_forbidden_page_does_not_trip_body_detector() {
412        let document = html(
413            "<html><body>A long forbidden response with meaningful content and no JavaScript shell markers.</body></html>",
414        );
415        assert_eq!(
416            DefaultPromotionDetector::new().inspect_html(http::StatusCode::FORBIDDEN, &document),
417            None
418        );
419    }
420}
millipede_browser/detect.rs

millipede_browser/
detect.rs
