Crate spider_lib 

Re-exports

pub use downloader::Downloader;

pub use middleware::Middleware;

pub use pipeline::Pipeline;

pub use builder::CrawlerBuilder;

pub use crawler::Crawler;

pub use error::PipelineError;

pub use error::SpiderError;

pub use item::ParseOutput;

pub use item::ScrapedItem;

pub use request::Request;

pub use response::Response;

pub use scheduler::Scheduler;

pub use spider::Spider;

pub use downloaders::reqwest_client::ReqwestClientDownloader;

pub use middlewares::http_cache::HttpCacheMiddleware;

pub use middlewares::rate_limit::RateLimitMiddleware;

pub use middlewares::referer::RefererMiddleware;

pub use middlewares::retry::RetryMiddleware;

pub use middlewares::robots_txt::RobotsTxtMiddleware;

pub use middlewares::user_agent::UserAgentMiddleware;

pub use pipelines::console_writer::ConsoleWriterPipeline;

pub use pipelines::csv_exporter::CsvExporterPipeline;

pub use pipelines::deduplication::DeduplicationPipeline;

pub use pipelines::json_writer::JsonWriterPipeline;

pub use pipelines::jsonl_writer::JsonlWriterPipeline;

pub use tokio;

Modules

builder

checkpoint

crawler

downloader

downloaders

error

item

middleware

middlewares

pipeline

pipelines

prelude

A “prelude” for users of the spider-lib crate.

request

response

scheduler

spider

state

utils

Structs

DashMap

DashMap is an implementation of a concurrent associative array/hashmap in Rust.

Attribute Macros

async_trait

scraped_item

A procedural macro to derive the ScrapedItem trait.