1#![allow(clippy::module_inception)]
2
3pub mod config;
4pub mod crawler;
5pub mod filter;
6pub mod mapper;
7pub mod pagination;
8pub mod parallel;
9pub mod prioritizer;
10pub mod rate_limiter;
11pub mod sitemap;
12pub mod streaming;
13pub mod url_normalization;
14
15pub use config::{CircuitBreaker, CrawlerConfig, MemoryMonitor};
16pub use crawler::crawl_website;
17pub use filter::matches_pattern;
18pub use mapper::discover_urls;
19pub use pagination::PaginationDetector;
20pub use parallel::ParallelCrawler;
21pub use prioritizer::{PrioritizedUrl, UrlPrioritizer};
22pub use rate_limiter::DomainRateLimiter;
23pub use sitemap::SitemapParser;
24pub use streaming::crawl_website_stream;
25pub use url_normalization::{generate_url_permutations, normalize_url};