codoseo_crawler/lib.rs
1//! The CodoSEO crawler. No database: the CLI, local MCP and the worker all use it directly.
2
3pub use codoseo_core::output::*;
4
5pub mod crawl;
6pub mod extract;
7pub mod fetch;
8pub mod frontier;
9pub mod guard;
10pub mod politeness;
11pub mod preflight;
12pub(crate) mod record;
13pub mod robots;
14pub mod scope;
15pub mod sitemap;