crw-core 0.13.4

Core types, config, and error handling for the CRW web scraper
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

use thiserror::Error;

#[derive(Debug, Error)]
pub enum CrwError {
    #[error("HTTP request failed: {0}")]
    HttpError(String),

    #[error("Target unreachable: {0}")]
    TargetUnreachable(String),

    #[error("URL parse error: {0}")]
    UrlParseError(#[from] url::ParseError),

    #[error("Invalid request: {0}")]
    InvalidRequest(String),

    #[error("Renderer error: {0}")]
    RendererError(String),

    #[error("Extraction error: {0}")]
    ExtractionError(String),

    #[error("Crawl error: {0}")]
    CrawlError(String),

    #[error("Timeout after {0}ms")]
    Timeout(u64),

    #[error("Config error: {0}")]
    ConfigError(String),

    #[error("Not found: {0}")]
    NotFound(String),

    #[error("Rate limited")]
    RateLimited,

    #[error("{0}")]
    SearchDisabled(String),

    #[error("{0}")]
    Internal(String),

    #[error("Renderer pool shutting down")]
    Shutdown,
}

impl CrwError {
    /// Machine-readable error code for API consumers.
    pub fn error_code(&self) -> &'static str {
        match self {
            CrwError::HttpError(_) => "http_error",
            CrwError::TargetUnreachable(_) => "target_unreachable",
            CrwError::UrlParseError(_) => "invalid_url",
            CrwError::InvalidRequest(_) => "invalid_request",
            CrwError::RendererError(_) => "renderer_error",
            CrwError::ExtractionError(_) => "extraction_error",
            CrwError::CrawlError(_) => "crawl_error",
            CrwError::Timeout(_) => "timeout",
            CrwError::ConfigError(_) => "config_error",
            CrwError::NotFound(_) => "not_found",
            CrwError::RateLimited => "rate_limited",
            CrwError::SearchDisabled(_) => "search_disabled",
            CrwError::Internal(_) => "internal_error",
            CrwError::Shutdown => "shutdown",
        }
    }
}

pub type CrwResult<T> = Result<T, CrwError>;