kumo 0.2.1

An async web crawling framework for Rust — Scrapy for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

use thiserror::Error;

#[derive(Debug, Error)]
pub enum KumoError {
    #[error("fetch error: {0}")]
    Fetch(#[from] reqwest::Error),

    #[error("parse error — {context}: {source}")]
    Parse {
        context: String,
        #[source]
        source: Box<dyn std::error::Error + Send + Sync>,
    },

    #[error("store error — {context}: {source}")]
    Store {
        context: String,
        #[source]
        source: Box<dyn std::error::Error + Send + Sync>,
    },

    #[error("invalid URL: {0}")]
    InvalidUrl(String),

    #[error("max crawl depth exceeded")]
    DepthExceeded,

    #[error("domain not allowed: {0}")]
    DomainNotAllowed(String),

    #[error("llm error: {0}")]
    Llm(String),

    #[error("browser error: {0}")]
    Browser(String),

    /// Returned by `StatusRetry` middleware when the response status code matches
    /// the retry set. Triggers the engine's exponential-backoff retry loop.
    #[error("HTTP {status} from {url}")]
    HttpStatus { status: u16, url: String },
}

/// Thin wrapper so plain `String` messages can be boxed as `dyn Error`.
#[derive(Debug)]
struct Msg(String);
impl std::fmt::Display for Msg {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.0)
    }
}
impl std::error::Error for Msg {}

impl KumoError {
    /// Construct a `Parse` variant from a real source error.
    pub fn parse(
        context: impl Into<String>,
        source: impl std::error::Error + Send + Sync + 'static,
    ) -> Self {
        Self::Parse {
            context: context.into(),
            source: Box::new(source),
        }
    }

    /// Construct a `Parse` variant from a plain message (no source).
    pub fn parse_msg(msg: impl Into<String>) -> Self {
        let msg = msg.into();
        Self::Parse {
            context: msg.clone(),
            source: Box::new(Msg(msg)),
        }
    }

    /// Construct a `Store` variant from a real source error.
    pub fn store(
        context: impl Into<String>,
        source: impl std::error::Error + Send + Sync + 'static,
    ) -> Self {
        Self::Store {
            context: context.into(),
            source: Box::new(source),
        }
    }

    /// Construct a `Store` variant from a plain message (no source).
    pub fn store_msg(msg: impl Into<String>) -> Self {
        let msg = msg.into();
        Self::Store {
            context: msg.clone(),
            source: Box::new(Msg(msg)),
        }
    }
}

/// Determines what the engine does when Spider::parse or a fetch fails.
#[derive(Debug, Clone)]
pub enum ErrorPolicy {
    /// Skip this URL and continue crawling. (default)
    Skip,
    /// Abort the entire crawl immediately.
    Abort,
    /// Retry this URL up to N more times (via the frontier).
    Retry(u32),
}