pub mod types;
pub mod selector;
pub mod metadata;
pub mod text;
pub mod links;
pub mod content;
pub mod parser;
pub mod forms;
pub mod pagination;
pub mod contact;
pub mod feeds;
pub mod fingerprint;
pub mod selectors;
pub use types::{
ParserError, ParserResult,
TextContent,
Heading,
Link, LinkRel, LinkType,
Image, ImageLoading,
ListContent, ListType, ListItem,
TableContent, TableRow, TableCell,
CodeBlock,
Quote,
PageMetadata, OpenGraph, TwitterCard, RobotsMeta, AlternateLink,
StructuredData, StructuredDataFormat,
ParsedContent, ParseStats,
ParserConfig,
normalize_whitespace, clean_text, truncate_text,
};
pub use selector::{
SELECTORS, CachedSelectors,
get_or_create_selector, parse_selector, try_parse_selector,
heading_selector,
CONTENT_SELECTORS, BOILERPLATE_SELECTORS,
attr_selector, class_selector, id_selector,
meta_name_selector, meta_property_selector, link_rel_selector,
};
pub use metadata::{
extract_metadata,
extract_title, extract_charset, extract_language,
extract_meta_content, extract_keywords,
extract_canonical, extract_favicon,
extract_robots,
extract_opengraph, extract_twitter_card,
extract_alternates,
extract_structured_data, extract_json_ld, extract_microdata,
};
pub use text::{
extract_text as extract_text_content,
normalize_text, strip_html_tags,
count_words, count_sentences,
flesch_reading_ease, flesch_kincaid_grade,
detect_language,
is_inline_element,
};
pub use links::{
extract_links, extract_link,
resolve_url, normalize_url,
parse_rel_attribute, is_nofollow, is_sponsored, is_ugc,
filter_internal_links, filter_external_links, filter_followable_links,
get_external_domains, calculate_link_stats, LinkStats,
};
pub use content::{
extract_headings, get_main_heading, build_outline, OutlineItem,
extract_paragraphs,
extract_lists,
extract_tables,
extract_code_blocks,
extract_quotes,
extract_images,
};
pub use parser::{
HtmlParser,
parse, parse_with_url,
get_metadata, get_text, get_links,
};
pub use forms::{
Form, FormField, FormType, FieldType, FormMethod, SelectOption,
extract_forms,
has_forms, has_login_form, has_search_form,
get_login_forms, get_search_forms, get_contact_forms,
};
pub use pagination::{
Pagination, PageUrl, PaginationType,
extract_pagination, has_pagination,
get_next_page, get_prev_page,
};
pub use contact::{
ContactInfo, Email, EmailSource, Phone, PhoneType,
Address, Coordinates, SocialLink, SocialPlatform,
extract_contact_info, extract_emails, extract_phones,
extract_addresses, extract_social_links,
has_contact_info, get_emails, get_phones, get_social_links,
};
pub use feeds::{
FeedInfo, Feed, FeedType, Sitemap, SitemapType, SitemapSource,
extract_feed_info, has_feeds, get_rss_feed, get_atom_feed, get_feed, get_sitemap,
};
pub use fingerprint::{
ContentFingerprint, AmpInfo, CacheHints,
generate_fingerprint, fingerprint_document,
extract_amp_info, extract_cache_hints,
has_content_changed, content_similarity, is_amp_page, get_amp_url, quick_hash,
};
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_basic_parse() {
let html = r#"
<!DOCTYPE html>
<html lang="en">
<head>
<title>Test Page</title>
<meta name="description" content="Test description">
</head>
<body>
<h1>Main Title</h1>
<p>This is a test paragraph with enough content.</p>
<a href="/link">Internal link</a>
</body>
</html>
"#;
let result = parse(html).unwrap();
assert_eq!(result.metadata.title, Some("Test Page".to_string()));
assert_eq!(result.metadata.description, Some("Test description".to_string()));
assert!(!result.headings.is_empty());
assert!(!result.paragraphs.is_empty());
assert!(!result.links.is_empty());
}
#[test]
fn test_parse_with_base_url() {
let html = r#"
<html>
<body>
<a href="/page">Link</a>
<img src="/image.jpg" alt="Image">
</body>
</html>
"#;
let result = parse_with_url(html, "https://example.com").unwrap();
let link = &result.links[0];
assert_eq!(link.url, Some("https://example.com/page".to_string()));
let img = &result.images[0];
assert_eq!(img.url, Some("https://example.com/image.jpg".to_string()));
}
#[test]
fn test_metadata_extraction() {
let html = r#"
<html>
<head>
<title>Title</title>
<meta property="og:title" content="OG Title">
<meta name="twitter:card" content="summary">
<meta name="robots" content="noindex, nofollow">
</head>
</html>
"#;
let metadata = get_metadata(html).unwrap();
assert!(metadata.opengraph.is_present());
assert!(metadata.twitter.is_present());
assert!(!metadata.robots.index);
assert!(!metadata.robots.follow);
}
#[test]
fn test_text_extraction() {
let html = r#"
<html>
<body>
<nav>Skip this navigation</nav>
<article>
<p>This is the main content that should be extracted.</p>
</article>
<footer>Skip this footer</footer>
</body>
</html>
"#;
let text = get_text(html).unwrap();
assert!(text.cleaned_text.contains("main content"));
assert!(text.word_count > 0);
}
#[test]
fn test_link_extraction() {
let html = r#"
<html>
<body>
<a href="https://internal.com/page">Internal</a>
<a href="https://external.com" rel="nofollow">External</a>
</body>
</html>
"#;
let parser = HtmlParser::with_base_url("https://internal.com").unwrap();
let links = parser.extract_links(html).unwrap();
assert_eq!(links.len(), 2);
let internal = links.iter().find(|l| l.text == "Internal").unwrap();
assert_eq!(internal.link_type, LinkType::Internal);
let external = links.iter().find(|l| l.text == "External").unwrap();
assert_eq!(external.link_type, LinkType::External);
assert!(external.is_nofollow);
}
#[test]
fn test_structured_data() {
let html = r#"
<html>
<head>
<script type="application/ld+json">
{
"@context": "https://schema.org",
"@type": "Article",
"headline": "Test Article"
}
</script>
</head>
</html>
"#;
let result = parse(html).unwrap();
assert!(result.has_structured_data());
assert_eq!(result.structured_data[0].schema_type, Some("Article".to_string()));
}
#[test]
fn test_content_extraction() {
let html = r#"
<html>
<body>
<h1 id="main">Main Heading</h1>
<p>Paragraph with enough content to pass filter.</p>
<ul>
<li>Item 1</li>
<li>Item 2</li>
</ul>
<table>
<tr><th>Header</th></tr>
<tr><td>Data</td></tr>
</table>
<pre><code class="language-rust">fn main() {}</code></pre>
<blockquote>A quote</blockquote>
</body>
</html>
"#;
let result = parse(html).unwrap();
assert!(!result.headings.is_empty());
assert_eq!(result.headings[0].id, Some("main".to_string()));
assert!(!result.paragraphs.is_empty());
assert!(!result.lists.is_empty());
assert!(!result.tables.is_empty());
assert!(!result.code_blocks.is_empty());
assert!(!result.quotes.is_empty());
}
#[test]
fn test_html_parser_api() {
let parser = HtmlParser::new();
assert!(!parser.has_base_url());
let mut parser2 = HtmlParser::new();
parser2.set_base_url("https://example.com").unwrap();
assert!(parser2.has_base_url());
let parser3 = HtmlParser::with_config(ParserConfig::minimal());
assert!(!parser3.config().extract_images);
}
#[test]
fn test_selector_utilities() {
let _ = &SELECTORS.h1;
let _ = &SELECTORS.body;
let sel = parse_selector("div.test").unwrap();
assert!(sel.matches(&scraper::Html::parse_fragment("<div class='test'></div>")
.select(&sel).next().unwrap()));
}
#[test]
fn test_readability_scoring() {
let simple = "The cat sat on the mat. The dog ran fast.";
let score = flesch_reading_ease(simple);
assert!(score > 60.0); }
#[test]
fn test_language_detection() {
let english = "The quick brown fox jumps over the lazy dog.";
assert_eq!(detect_language(english), Some("en".to_string()));
let french = "Le chat est sur la table dans la maison.";
assert_eq!(detect_language(french), Some("fr".to_string()));
}
#[test]
fn test_normalize_whitespace() {
let text = " Hello world \n\n test ";
let normalized = normalize_whitespace(text);
assert_eq!(normalized, "Hello world test");
}
#[test]
fn test_parse_stats() {
let html = "<html><body><p>Test</p></body></html>";
let result = parse(html).unwrap();
assert!(result.stats.html_size > 0);
assert!(result.stats.node_count > 0);
assert!(result.stats.parse_time_us > 0);
}
}