Skip to main content

spider_lib/
lib.rs

1#![doc = include_str!("../README.md")]
2
3pub mod builder;
4#[cfg(feature = "checkpoint")]
5pub mod checkpoint;
6pub mod crawler;
7pub mod error;
8pub mod item;
9pub mod prelude;
10pub mod request;
11pub mod response;
12pub mod scheduler;
13pub mod spider;
14pub mod state;
15pub mod utils;
16
17// Conditionally re-export SchedulerCheckpoint for universal access
18#[cfg(feature = "checkpoint")]
19pub use checkpoint::SchedulerCheckpoint;
20#[cfg(not(feature = "checkpoint"))]
21#[derive(Default, Clone)]
22pub struct SchedulerCheckpoint; // Dummy struct when checkpoint feature is disabled
23
24pub mod downloader;
25pub mod downloaders;
26pub mod middleware;
27pub mod middlewares;
28pub mod pipeline;
29pub mod pipelines;
30
31pub use downloader::Downloader;
32pub use middleware::Middleware;
33pub use pipeline::Pipeline;
34
35pub use builder::CrawlerBuilder;
36pub use crawler::Crawler;
37pub use error::{PipelineError, SpiderError};
38pub use item::{ParseOutput, ScrapedItem};
39pub use request::Request;
40pub use response::Response;
41pub use scheduler::Scheduler;
42pub use spider_macro::scraped_item;
43
44pub use async_trait::async_trait;
45pub use dashmap::DashMap;
46pub use spider::Spider;
47pub use tokio;
48
49// Conditionally re-export new middlewares, pipelines, and downloaders
50pub use downloaders::reqwest_client::ReqwestClientDownloader;
51
52#[cfg(feature = "middleware-http-cache")]
53pub use middlewares::http_cache::HttpCacheMiddleware;
54pub use middlewares::rate_limit::RateLimitMiddleware;
55pub use middlewares::referer::RefererMiddleware;
56pub use middlewares::retry::RetryMiddleware;
57#[cfg(feature = "middleware-robots-txt")]
58pub use middlewares::robots_txt::RobotsTxtMiddleware;
59pub use middlewares::user_agent::UserAgentMiddleware;
60
61pub use pipelines::console_writer::ConsoleWriterPipeline;
62pub use pipelines::deduplication::DeduplicationPipeline;
63
64#[cfg(feature = "pipeline-csv")]
65pub use pipelines::csv_exporter::CsvExporterPipeline;
66
67#[cfg(feature = "pipeline-json")]
68pub use pipelines::json_writer::JsonWriterPipeline;
69
70#[cfg(feature = "pipeline-json")]
71pub use pipelines::jsonl_writer::JsonlWriterPipeline;
72
73#[cfg(feature = "pipeline-sqlite")]
74pub use pipelines::sqlite_writer::SqliteWriterPipeline;
75