kumo 0.5.0

An async web crawling framework for Rust - Scrapy for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86

pub mod engine;
pub mod error;
pub mod events;
pub mod extract;
pub mod fetch;
pub mod frontier;
#[cfg(feature = "persistence")]
pub use frontier::FileFrontier;
#[cfg(feature = "redis-frontier")]
pub use frontier::RedisFrontier;
pub mod hooks;
pub use hooks::{CrawlHook, HookErrorPolicy};
pub mod llm;
mod logging;
pub mod middleware;
#[cfg(feature = "otel")]
pub mod otel;
pub mod pipeline;
pub mod request;
pub use request::CrawlRequest;
pub mod retry;
pub mod robots;
pub mod scheduler;
pub mod sitemap;
pub mod spider;
pub mod stats;
pub use stats::{CrawlReport, CrawlStats, StopReason};
pub mod store;

/// Convenience re-exports for writing spiders with minimal `use` statements.
///
/// ```rust,ignore
/// use kumo::prelude::*;
/// ```
pub mod prelude {
    pub use crate::engine::{CrawlEngine, CrawlStats, ItemStream};
    pub use crate::error::{ErrorPolicy, KumoError};
    pub use crate::events::{CrawlEvent, ItemDropReason, RequestSkipReason};
    #[cfg(feature = "derive")]
    pub use crate::extract::Extract;
    #[cfg(feature = "jsonpath")]
    pub use crate::extract::JsonPathExtractor;
    pub use crate::extract::{
        CssExtractor, Element, ElementList, ExtractedNode, Extractor, LinkExtractor,
        RegexExtractor, Response, ValueExtractor,
    };
    #[cfg(feature = "browser")]
    pub use crate::fetch::{BrowserConfig, BrowserFetcher};
    pub use crate::fetch::{CachingFetcher, MockFetcher};
    #[cfg(feature = "stealth")]
    pub use crate::fetch::{StealthHttpFetcher, StealthProfile};
    pub use crate::hooks::{CrawlHook, HookErrorPolicy};
    #[cfg(feature = "claude")]
    pub use crate::llm::AnthropicClient;
    #[cfg(feature = "gemini")]
    pub use crate::llm::GeminiClient;
    #[cfg(feature = "ollama")]
    pub use crate::llm::OllamaClient;
    #[cfg(feature = "openai")]
    pub use crate::llm::OpenAiClient;
    #[cfg(feature = "llm")]
    pub use crate::llm::ResponseExtractExt;
    pub use crate::llm::{LlmClient, TokenUsage};
    pub use crate::middleware::{
        AutoThrottle, DefaultHeaders, FetchRequest, Middleware, ProxyRotator, RateLimiter,
        StatusRetry, UserAgentRotator,
    };
    pub use crate::pipeline::{DropDuplicates, FilterPipeline, Pipeline, RequireFields};
    pub use crate::request::CrawlRequest;
    pub use crate::retry::RetryPolicy;
    pub use crate::scheduler::{CrawlScheduler, DomainPolicy, FingerprintPolicy, PolitenessPolicy};
    pub use crate::sitemap::{SitemapEntry, SitemapSpider};
    pub use crate::spider::{Output, Spider};
    pub use crate::stats::CrawlReport;
    pub use crate::stats::StopReason;
    #[cfg(feature = "mysql")]
    pub use crate::store::MySqlStore;
    #[cfg(feature = "postgres")]
    pub use crate::store::PostgresStore;
    #[cfg(feature = "sqlite")]
    pub use crate::store::SqliteStore;
    pub use crate::store::{CsvStore, JsonStore, JsonlStore, StdoutStore};
    #[cfg(feature = "derive")]
    pub use kumo_derive::Extract;
    pub use tokio_stream::StreamExt;
}