Expand description
§crawlkit-parser
HTML 解析与内容提取模块。
基于 scraper（html5ever）和 dom_smoothie（Readability 模式）实现。
§模块
html：原有基础工具（链接提取、文章提取、可读性）types：共享类型定义selector：预编译 CSS 选择器metadata：页面元数据（OG/Twitter/JSON-LD/Robots）text：正文提取、文本规范化、可读性评分、语言检测links：链接提取、URL 规范化、站内外分类content：结构化内容（标题/列表/表格/代码/引用）parser：HtmlParser 统一入口forms：表单检测与分类pagination：翻页检测contact：联系方式提取feeds：Feed/Sitemap 检测rules：URL 正则规则引擎（UrlRule）extractor：基于 DOM 聚类 + 打分的文章链接提取器fingerprint：内容指纹与 AMP 检测
Re-exports§
pub use types::ParserError;pub use types::ParserResult;pub use types::TextContent;pub use types::Heading;pub use types::Link;pub use types::LinkRel;pub use types::LinkType;pub use types::Image;pub use types::ImageLoading;pub use types::ListContent;pub use types::ListType;pub use types::ListItem;pub use types::TableContent;pub use types::TableRow;pub use types::TableCell;pub use types::CodeBlock;pub use types::Quote;pub use types::PageMetadata;pub use types::OpenGraph;pub use types::TwitterCard;pub use types::RobotsMeta;pub use types::AlternateLink;pub use types::StructuredData;pub use types::StructuredDataFormat;pub use types::ParsedContent;pub use types::ParseStats;pub use types::ParserConfig;pub use types::normalize_whitespace;pub use types::clean_text;pub use types::truncate_text;pub use selector::SELECTORS;pub use selector::CachedSelectors;pub use selector::get_or_create_selector;pub use selector::parse_selector;pub use selector::try_parse_selector;pub use selector::heading_selector;pub use selector::CONTENT_SELECTORS;pub use selector::BOILERPLATE_SELECTORS;pub use selector::attr_selector;pub use selector::class_selector;pub use selector::id_selector;pub use selector::meta_name_selector;pub use selector::meta_property_selector;pub use selector::link_rel_selector;pub use metadata::extract_metadata;pub use metadata::extract_title;pub use metadata::extract_charset;pub use metadata::extract_language;pub use metadata::extract_meta_content;pub use metadata::extract_keywords;pub use metadata::extract_canonical;pub use metadata::extract_favicon;pub use metadata::extract_robots;pub use metadata::extract_opengraph;pub use metadata::extract_twitter_card;pub use metadata::extract_alternates;pub use metadata::extract_structured_data;pub use metadata::extract_json_ld;pub use metadata::extract_microdata;pub use text::extract_text as extract_text_content;pub use text::normalize_text;pub use text::count_words;pub use text::count_sentences;pub use text::flesch_reading_ease;pub use text::flesch_kincaid_grade;pub use text::detect_language;pub use text::is_inline_element;pub use html::Article;pub use html::LinkSelectorType;pub use html::extract_absolute_links;pub use html::extract_article;pub use html::extract_attributes;pub use html::extract_content;pub use html::extract_content_by_selector;pub use html::extract_links;pub use html::extract_links_by_selector;pub use html::extract_links_by_xpath;pub use html::extract_readable_content;pub use html::extract_texts;pub use html::resolve_url;pub use html::try_extract_links;pub use html::sanitize_for_xpath;pub use content::extract_headings;pub use content::get_main_heading;pub use content::build_outline;pub use content::extract_paragraphs;pub use content::extract_lists;pub use content::extract_tables;pub use content::extract_code_blocks;pub use content::extract_quotes;pub use links::LinkStats;pub use links::normalize_url;pub use links::parse_rel_attribute;pub use links::is_nofollow;pub use links::is_sponsored;pub use links::is_ugc;pub use links::filter_internal_links;pub use links::filter_external_links;pub use links::filter_followable_links;pub use links::get_external_domains;pub use links::calculate_link_stats;pub use parser::HtmlParser;pub use parser::parse;pub use parser::parse_with_url;pub use parser::get_metadata;pub use parser::get_text;pub use parser::get_links;pub use forms::Form;pub use forms::FormField;pub use forms::FormType;pub use forms::FieldType;pub use forms::FormMethod;pub use forms::SelectOption;pub use forms::extract_forms;pub use forms::has_forms;pub use forms::has_login_form;pub use forms::has_search_form;pub use forms::get_login_forms;pub use forms::get_search_forms;pub use forms::get_contact_forms;pub use pagination::Pagination;pub use pagination::PageUrl;pub use pagination::PaginationType;pub use pagination::extract_pagination;pub use pagination::has_pagination;pub use pagination::get_next_page;pub use pagination::get_prev_page;pub use contact::ContactInfo;pub use contact::Email;pub use contact::EmailSource;pub use contact::Phone;pub use contact::PhoneType;pub use contact::Address;pub use contact::Coordinates;pub use contact::SocialLink;pub use contact::SocialPlatform;pub use contact::extract_contact_info;pub use contact::extract_emails;pub use contact::extract_phones;pub use contact::extract_addresses;pub use contact::has_contact_info;pub use contact::get_emails;pub use contact::get_phones;pub use feeds::FeedInfo;pub use feeds::Feed;pub use feeds::FeedType;pub use feeds::Sitemap;pub use feeds::SitemapType;pub use feeds::SitemapSource;pub use feeds::extract_feed_info;pub use feeds::has_feeds;pub use feeds::get_rss_feed;pub use feeds::get_atom_feed;pub use feeds::get_feed;pub use feeds::get_sitemap;pub use fingerprint::ContentFingerprint;pub use fingerprint::AmpInfo;pub use fingerprint::CacheHints;pub use fingerprint::generate_fingerprint;pub use fingerprint::fingerprint_document;pub use fingerprint::extract_amp_info;pub use fingerprint::extract_cache_hints;pub use fingerprint::has_content_changed;pub use fingerprint::content_similarity;pub use fingerprint::is_amp_page;pub use fingerprint::get_amp_url;pub use fingerprint::quick_hash;pub use rules::UrlRule;pub use extractor::ExtractedLink;pub use extractor::ExtractorConfig;pub use extractor::LinkExtractor;pub use scraper;pub use skyscraper;