Skip to main content

spider_lib/
lib.rs

1#![doc = include_str!("../README.md")]
2
3pub mod builder;
4#[cfg(feature = "checkpoint")]
5pub mod checkpoint;
6pub mod crawler;
7pub mod error;
8pub mod item;
9pub mod prelude;
10pub mod request;
11pub mod response;
12pub mod scheduler;
13pub mod spider;
14pub mod state;
15pub mod stats;
16pub mod utils;
17
18// Conditionally re-export SchedulerCheckpoint for universal access
19#[cfg(feature = "checkpoint")]
20pub use checkpoint::SchedulerCheckpoint;
21#[cfg(not(feature = "checkpoint"))]
22#[derive(Default, Clone)]
23pub struct SchedulerCheckpoint; // Dummy struct when checkpoint feature is disabled
24
25pub mod downloader;
26pub mod downloaders;
27pub mod middleware;
28pub mod middlewares;
29pub mod pipeline;
30pub mod pipelines;
31
32pub use downloader::Downloader;
33pub use middleware::Middleware;
34pub use pipeline::Pipeline;
35
36// Conditionally re-export CookieStore for universal access
37#[cfg(feature = "middleware-cookies")]
38pub use cookie_store::CookieStore;
39#[cfg(not(feature = "middleware-cookies"))]
40#[derive(Default, Clone)]
41pub struct CookieStore; // Dummy struct when middleware-cookies feature is disabled
42
43pub use builder::CrawlerBuilder;
44pub use crawler::Crawler;
45pub use error::{PipelineError, SpiderError};
46pub use item::{ParseOutput, ScrapedItem};
47pub use request::Request;
48pub use response::Response;
49pub use scheduler::Scheduler;
50pub use spider_macro::scraped_item;
51
52pub use async_trait::async_trait;
53pub use dashmap::DashMap;
54pub use spider::Spider;
55pub use tokio;
56
57// Conditionally re-export new middlewares, pipelines, and downloaders
58pub use downloaders::reqwest_client::ReqwestClientDownloader;
59
60#[cfg(feature = "middleware-http-cache")]
61pub use middlewares::http_cache::HttpCacheMiddleware;
62pub use middlewares::rate_limit::RateLimitMiddleware;
63pub use middlewares::referer::RefererMiddleware;
64pub use middlewares::retry::RetryMiddleware;
65#[cfg(feature = "middleware-robots-txt")]
66pub use middlewares::robots_txt::RobotsTxtMiddleware;
67pub use middlewares::user_agent::UserAgentMiddleware;
68
69pub use pipelines::console_writer::ConsoleWriterPipeline;
70pub use pipelines::deduplication::DeduplicationPipeline;
71
72#[cfg(feature = "pipeline-csv")]
73pub use pipelines::csv_exporter::CsvExporterPipeline;
74
75#[cfg(feature = "pipeline-json")]
76pub use pipelines::json_writer::JsonWriterPipeline;
77
78#[cfg(feature = "pipeline-json")]
79pub use pipelines::jsonl_writer::JsonlWriterPipeline;
80
81#[cfg(feature = "pipeline-sqlite")]
82pub use pipelines::sqlite_writer::SqliteWriterPipeline;