halldyll_core/crawl/
mod.rs

1//! Crawl - Frontier, normalization, deduplication
2
3pub mod normalize;
4pub mod frontier;
5pub mod dedup;
6pub mod canonical;
7
8pub use normalize::UrlNormalizer;
9pub use frontier::{Frontier, CrawlEntry};
10pub use dedup::{UrlDedup, ContentDedup};
11pub use canonical::CanonicalResolver;