millipede-browser 0.1.0

BrowserCrawler core, BrowserProvider trait, and BrowserPool for the Millipede web crawler.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

//! Browser pre- and post-navigation hook contexts.

use std::sync::Arc;

use futures_util::future::BoxFuture;
use millipede_core::{errors::CrawlError, proxy::ProxyInfo, request::Request, session::Session};

use crate::{BrowserResponse, PageHandle};

/// Context supplied to browser pre-navigation hooks.
///
/// Unlike the INTERFACE §18 sketch, `request` is immutable because execution owns an
/// [`Arc<Request>`]. The proposed `log` field is omitted because no `Log` type exists; use
/// `tracing` macros instead. This context is non-exhaustive so `log` can be added later without a
/// breaking change.
#[non_exhaustive]
pub struct BrowserPreHookCtx<'a> {
    /// Request about to be navigated.
    pub request: &'a Request,
    /// Live page that will perform the navigation.
    pub page: &'a PageHandle,
    /// Session selected for this attempt, when sessions are enabled.
    pub session: Option<&'a Session>,
    /// Proxy assigned to the browser containing the page.
    pub proxy: Option<&'a ProxyInfo>,
}

/// Context supplied to browser post-navigation hooks.
///
/// The proposed INTERFACE §18 `log` field is omitted because no `Log` type exists; use `tracing`
/// macros instead. This context is non-exhaustive so `log` can be added later without a breaking
/// change.
#[non_exhaustive]
pub struct BrowserPostHookCtx<'a> {
    /// Request that was navigated.
    pub request: &'a Request,
    /// Live page after navigation and status classification.
    pub page: &'a PageHandle,
    /// Navigation response metadata, when exposed by the provider.
    pub response: Option<&'a BrowserResponse>,
    /// Session selected for this attempt, when sessions are enabled.
    pub session: Option<&'a Session>,
    /// Proxy assigned to the browser containing the page.
    pub proxy: Option<&'a ProxyInfo>,
}

/// Asynchronous browser hook run after page creation and before navigation.
pub type BrowserPreNavigationHook = Arc<
    dyn for<'a> Fn(BrowserPreHookCtx<'a>) -> BoxFuture<'a, Result<(), CrawlError>> + Send + Sync,
>;

/// Asynchronous browser hook run after navigation and status classification.
pub type BrowserPostNavigationHook = Arc<
    dyn for<'a> Fn(BrowserPostHookCtx<'a>) -> BoxFuture<'a, Result<(), CrawlError>> + Send + Sync,
>;