1#![doc = include_str!("../README.md")]
2
3pub mod builder;
4pub mod checkpoint;
5pub mod crawler;
6pub mod error;
7pub mod item;
8pub mod prelude;
9pub mod request;
10pub mod response;
11pub mod scheduler;
12pub mod spider;
13pub mod state;
14pub mod utils;
15
16pub mod downloader;
17pub mod downloaders;
18pub mod middleware;
19pub mod middlewares;
20pub mod pipeline;
21pub mod pipelines;
22
23pub use downloader::Downloader;
24pub use middleware::Middleware;
25pub use pipeline::Pipeline;
26
27pub use builder::CrawlerBuilder;
28pub use crawler::Crawler;
29pub use error::{PipelineError, SpiderError};
30pub use item::{ParseOutput, ScrapedItem};
31pub use request::Request;
32pub use response::Response;
33pub use scheduler::Scheduler;
34pub use spider_macro::scraped_item;
35
36pub use async_trait::async_trait;
37pub use dashmap::DashMap;
38pub use spider::Spider;
39pub use tokio;
40
41pub use downloaders::reqwest_client::ReqwestClientDownloader;
43
44pub use middlewares::http_cache::HttpCacheMiddleware;
45pub use middlewares::rate_limit::RateLimitMiddleware;
46pub use middlewares::referer::RefererMiddleware;
47pub use middlewares::retry::RetryMiddleware;
48pub use middlewares::robots_txt::RobotsTxtMiddleware;
49pub use middlewares::user_agent::UserAgentMiddleware;
50
51pub use pipelines::console_writer::ConsoleWriterPipeline;
52pub use pipelines::csv_exporter::CsvExporterPipeline;
53pub use pipelines::deduplication::DeduplicationPipeline;
54pub use pipelines::json_writer::JsonWriterPipeline;
55pub use pipelines::jsonl_writer::JsonlWriterPipeline;
56
57#[cfg(feature = "pipeline-sqlite")]
58pub use pipelines::sqlite_writer::SqliteWriterPipeline;
59