Skip to main content

spider_lib/
lib.rs

1#![doc = include_str!("../README.md")]
2
3pub mod builder;
4#[cfg(feature = "checkpoint")]
5pub mod checkpoint;
6pub mod crawler;
7pub mod error;
8pub mod item;
9pub mod prelude;
10pub mod request;
11pub mod response;
12pub mod scheduler;
13pub mod spider;
14pub mod state;
15pub mod stats; // Added
16pub mod utils;
17
18// Conditionally re-export SchedulerCheckpoint for universal access
19#[cfg(feature = "checkpoint")]
20pub use checkpoint::SchedulerCheckpoint;
21#[cfg(not(feature = "checkpoint"))]
22#[derive(Default, Clone)]
23pub struct SchedulerCheckpoint; // Dummy struct when checkpoint feature is disabled
24
25pub mod downloader;
26pub mod downloaders;
27pub mod middleware;
28pub mod middlewares;
29pub mod pipeline;
30pub mod pipelines;
31
32pub use downloader::Downloader;
33pub use middleware::Middleware;
34pub use pipeline::Pipeline;
35
36pub use builder::CrawlerBuilder;
37pub use crawler::Crawler;
38pub use error::{PipelineError, SpiderError};
39pub use item::{ParseOutput, ScrapedItem};
40pub use request::Request;
41pub use response::Response;
42pub use scheduler::Scheduler;
43pub use spider_macro::scraped_item;
44
45pub use async_trait::async_trait;
46pub use dashmap::DashMap;
47pub use spider::Spider;
48pub use tokio;
49
50// Conditionally re-export new middlewares, pipelines, and downloaders
51pub use downloaders::reqwest_client::ReqwestClientDownloader;
52
53#[cfg(feature = "middleware-http-cache")]
54pub use middlewares::http_cache::HttpCacheMiddleware;
55pub use middlewares::rate_limit::RateLimitMiddleware;
56pub use middlewares::referer::RefererMiddleware;
57pub use middlewares::retry::RetryMiddleware;
58#[cfg(feature = "middleware-robots-txt")]
59pub use middlewares::robots_txt::RobotsTxtMiddleware;
60pub use middlewares::user_agent::UserAgentMiddleware;
61
62pub use pipelines::console_writer::ConsoleWriterPipeline;
63pub use pipelines::deduplication::DeduplicationPipeline;
64
65#[cfg(feature = "pipeline-csv")]
66pub use pipelines::csv_exporter::CsvExporterPipeline;
67
68#[cfg(feature = "pipeline-json")]
69pub use pipelines::json_writer::JsonWriterPipeline;
70
71#[cfg(feature = "pipeline-json")]
72pub use pipelines::jsonl_writer::JsonlWriterPipeline;
73
74#[cfg(feature = "pipeline-sqlite")]
75pub use pipelines::sqlite_writer::SqliteWriterPipeline;
76