oratos_html/lib.rs
1//! HTML loading, parsing, and page extraction.
2
3mod extract;
4mod load;
5mod page;
6mod robots;
7mod sitemap;
8
9pub use extract::parse_html;
10pub use load::{load_pages, normalize_url, resolve_internal_path, CrawlOptions, LoadOptions};
11pub use page::{Heading, HtmlPage, ImageInfo, JsonLdBlock, LinkInfo};
12pub use robots::{disallowed_paths, is_disallowed};
13pub use sitemap::urls_from_sitemap_xml;