Re-exports§
pub use config::CircuitBreaker;pub use config::CrawlerConfig;pub use config::MemoryMonitor;pub use crawler::crawl_website;pub use filter::matches_pattern;pub use mapper::discover_urls;pub use pagination::PaginationDetector;pub use parallel::ParallelCrawler;pub use prioritizer::PrioritizedUrl;pub use prioritizer::UrlPrioritizer;pub use rate_limiter::DomainRateLimiter;pub use sitemap::SitemapParser;pub use streaming::crawl_website_stream;pub use url_normalization::generate_url_permutations;pub use url_normalization::normalize_url;
Modules§
- config
- crawler
- filter
- mapper
- pagination
- Pagination detection module
- parallel
- prioritizer
- rate_
limiter - sitemap
- streaming
- url_
normalization - URL Normalization and Permutation Generation for Crawl Deduplication