crawlberg 1.0.1

High-performance web crawling engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

//! Page interaction module for action-based browser automation.
//!
//! The public action types and validation helpers are available regardless of
//! which browser backend is compiled. Runtime execution is selected from the
//! configured [`BrowserBackend`].

/// Page action types and validation limits.
pub mod actions;
#[cfg(feature = "browser-chromiumoxide")]
mod chromiumoxide;
#[cfg(feature = "browser-native")]
mod native;
/// Page action validation helpers.
pub mod validation;

pub use actions::{
    MAX_ACTIONS, MAX_SCRIPT_LEN, MAX_SCROLL_AMOUNT, MAX_SELECTOR_LEN, MAX_SINGLE_WAIT_MS, MAX_TEXT_LEN,
    MAX_TOTAL_WAIT_SECS, PageAction, ScrollDirection,
};
pub use validation::validate_actions;

use crate::engine::CrawlEngine;
use crate::error::CrawlError;
use crate::types::{BrowserBackend, InteractionResult};

/// Execute browser actions on a single page.
pub(crate) async fn run(
    engine: &CrawlEngine,
    url: &str,
    actions: &[PageAction],
) -> Result<InteractionResult, CrawlError> {
    validate_actions(actions)?;
    engine.config.validate()?;

    match engine.config.browser.backend {
        BrowserBackend::Chromiumoxide => run_chromiumoxide(url, actions, &engine.config).await,
        BrowserBackend::Native => run_native(engine, url, actions).await,
    }
}

#[cfg(feature = "browser-chromiumoxide")]
async fn run_chromiumoxide(
    url: &str,
    actions: &[PageAction],
    config: &crate::types::CrawlConfig,
) -> Result<InteractionResult, CrawlError> {
    chromiumoxide::run(url, actions, config).await
}

#[cfg(not(feature = "browser-chromiumoxide"))]
async fn run_chromiumoxide(
    _url: &str,
    _actions: &[PageAction],
    _config: &crate::types::CrawlConfig,
) -> Result<InteractionResult, CrawlError> {
    Err(CrawlError::Unsupported(
        "interact() with BrowserBackend::Chromiumoxide requires the browser-chromiumoxide feature".into(),
    ))
}

#[cfg(feature = "browser-native")]
async fn run_native(engine: &CrawlEngine, url: &str, actions: &[PageAction]) -> Result<InteractionResult, CrawlError> {
    let native_executor = engine.native_browser_executor.as_deref().ok_or_else(|| {
        CrawlError::BrowserError("native browser executor is not available for BrowserBackend::Native".into())
    })?;
    native::run(url, actions, &engine.config, native_executor).await
}

#[cfg(not(feature = "browser-native"))]
async fn run_native(
    _engine: &CrawlEngine,
    _url: &str,
    _actions: &[PageAction],
) -> Result<InteractionResult, CrawlError> {
    Err(CrawlError::Unsupported(
        "interact() with BrowserBackend::Native requires the browser-native feature".into(),
    ))
}