crawlkit_parser/
types.rs

1use serde::{Deserialize, Serialize};
2use std::collections::HashMap;
3use thiserror::Error;
4
5// ============================================================================
6// ERROR TYPES
7// ============================================================================
8
9#[derive(Debug, Error)]
10pub enum ParserError {
11    #[error("Failed to parse HTML: {0}")]
12    ParseError(String),
13
14    #[error("无效 CSS 选择器: {0}")]
15    SelectorError(String),
16
17    #[error("URL 错误: {0}")]
18    UrlError(#[from] url::ParseError),
19
20    #[error("IO 错误: {0}")]
21    IoError(#[from] std::io::Error),
22
23    #[error("编码错误: {0}")]
24    EncodingError(String),
25
26    #[error("配置错误: {0}")]
27    ConfigError(String),
28}
29
30pub type ParserResult<T> = Result<T, ParserError>;
31
32// ============================================================================
33// 文本内容
34// ============================================================================
35
36#[derive(Debug, Clone, Serialize, Deserialize, Default)]
37pub struct TextContent {
38    pub raw_text: String,
39    pub cleaned_text: String,
40    pub word_count: usize,
41    pub char_count: usize,
42    pub language: Option<String>,
43    pub readability_score: Option<f64>,
44    pub reading_time_minutes: Option<f64>,
45}
46
47impl TextContent {
48    pub fn from_raw(raw: &str) -> Self {
49        let cleaned = normalize_whitespace(raw);
50        let word_count = cleaned.split_whitespace().count();
51        let char_count = cleaned.chars().count();
52        let reading_time = if word_count > 0 {
53            Some(word_count as f64 / 225.0)
54        } else {
55            None
56        };
57        Self {
58            raw_text: raw.to_string(),
59            cleaned_text: cleaned,
60            word_count,
61            char_count,
62            language: None,
63            readability_score: None,
64            reading_time_minutes: reading_time,
65        }
66    }
67
68    pub fn is_empty(&self) -> bool {
69        self.word_count == 0
70    }
71
72    pub fn is_substantial(&self) -> bool {
73        self.word_count >= 50
74    }
75}
76
77// ============================================================================
78// 标题
79// ============================================================================
80
81#[derive(Debug, Clone, Serialize, Deserialize)]
82pub struct Heading {
83    pub level: u8,
84    pub text: String,
85    pub id: Option<String>,
86    pub classes: Vec<String>,
87}
88
89impl Heading {
90    pub fn new(level: u8, text: impl Into<String>) -> Self {
91        Self {
92            level: level.clamp(1, 6),
93            text: text.into(),
94            id: None,
95            classes: Vec::new(),
96        }
97    }
98
99    pub fn with_id(mut self, id: impl Into<String>) -> Self {
100        self.id = Some(id.into());
101        self
102    }
103}
104
105// ============================================================================
106// 链接
107// ============================================================================
108
109#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
110#[serde(rename_all = "snake_case")]
111pub enum LinkRel {
112    Follow,
113    NoFollow,
114    Ugc,
115    Sponsored,
116    External,
117    NoOpener,
118    NoReferrer,
119    Other,
120}
121
122#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
123#[serde(rename_all = "snake_case")]
124pub enum LinkType {
125    Internal,
126    External,
127    Unknown,
128}
129
130#[derive(Debug, Clone, Serialize, Deserialize)]
131pub struct Link {
132    pub href: String,
133    pub url: Option<String>,
134    pub text: String,
135    pub title: Option<String>,
136    pub rel: Vec<LinkRel>,
137    pub link_type: LinkType,
138    pub is_nofollow: bool,
139    pub target: Option<String>,
140    pub hreflang: Option<String>,
141}
142
143impl Link {
144    pub fn new(href: impl Into<String>, text: impl Into<String>) -> Self {
145        Self {
146            href: href.into(),
147            url: None,
148            text: text.into(),
149            title: None,
150            rel: Vec::new(),
151            link_type: LinkType::Unknown,
152            is_nofollow: false,
153            target: None,
154            hreflang: None,
155        }
156    }
157
158    pub fn should_follow(&self) -> bool {
159        !self.is_nofollow
160            && !self.rel.contains(&LinkRel::Sponsored)
161            && !self.rel.contains(&LinkRel::Ugc)
162    }
163
164    pub fn opens_new_tab(&self) -> bool {
165        self.target.as_deref() == Some("_blank")
166    }
167}
168
169// ============================================================================
170// 图片
171// ============================================================================
172
173#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
174#[serde(rename_all = "snake_case")]
175pub enum ImageLoading {
176    #[default]
177    Eager,
178    Lazy,
179}
180
181#[derive(Debug, Clone, Serialize, Deserialize)]
182pub struct Image {
183    pub src: String,
184    pub url: Option<String>,
185    pub alt: String,
186    pub title: Option<String>,
187    pub width: Option<u32>,
188    pub height: Option<u32>,
189    pub srcset: Option<String>,
190    pub sizes: Option<String>,
191    pub loading: ImageLoading,
192    pub is_decorative: bool,
193}
194
195impl Image {
196    pub fn new(src: impl Into<String>, alt: impl Into<String>) -> Self {
197        let alt_str = alt.into();
198        let is_decorative = alt_str.is_empty();
199        Self {
200            src: src.into(),
201            url: None,
202            alt: alt_str,
203            title: None,
204            width: None,
205            height: None,
206            srcset: None,
207            sizes: None,
208            loading: ImageLoading::default(),
209            is_decorative,
210        }
211    }
212
213    pub fn is_responsive(&self) -> bool {
214        self.srcset.is_some()
215    }
216}
217
218// ============================================================================
219// 列表
220// ============================================================================
221
222#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
223#[serde(rename_all = "snake_case")]
224pub enum ListType {
225    Ordered,
226    Unordered,
227    Definition,
228}
229
230#[derive(Debug, Clone, Serialize, Deserialize)]
231pub struct ListItem {
232    pub text: String,
233    pub nested: Option<Box<ListContent>>,
234}
235
236impl ListItem {
237    pub fn new(text: impl Into<String>) -> Self {
238        Self {
239            text: text.into(),
240            nested: None,
241        }
242    }
243
244    pub fn with_nested(text: impl Into<String>, nested: ListContent) -> Self {
245        Self {
246            text: text.into(),
247            nested: Some(Box::new(nested)),
248        }
249    }
250}
251
252#[derive(Debug, Clone, Serialize, Deserialize)]
253pub struct ListContent {
254    pub list_type: ListType,
255    pub items: Vec<ListItem>,
256    pub total_items: usize,
257}
258
259impl ListContent {
260    pub fn new(list_type: ListType) -> Self {
261        Self {
262            list_type,
263            items: Vec::new(),
264            total_items: 0,
265        }
266    }
267
268    pub fn add_item(&mut self, item: ListItem) {
269        self.total_items += 1;
270        if let Some(ref nested) = item.nested {
271            self.total_items += nested.total_items;
272        }
273        self.items.push(item);
274    }
275
276    pub fn is_empty(&self) -> bool {
277        self.items.is_empty()
278    }
279}
280
281// ============================================================================
282// 表格
283// ============================================================================
284
285#[derive(Debug, Clone, Serialize, Deserialize)]
286pub struct TableCell {
287    pub content: String,
288    pub is_header: bool,
289    pub colspan: u32,
290    pub rowspan: u32,
291}
292
293impl TableCell {
294    pub fn data(content: impl Into<String>) -> Self {
295        Self {
296            content: content.into(),
297            is_header: false,
298            colspan: 1,
299            rowspan: 1,
300        }
301    }
302
303    pub fn header(content: impl Into<String>) -> Self {
304        Self {
305            content: content.into(),
306            is_header: true,
307            colspan: 1,
308            rowspan: 1,
309        }
310    }
311}
312
313#[derive(Debug, Clone, Serialize, Deserialize)]
314pub struct TableRow {
315    pub cells: Vec<TableCell>,
316    pub is_header_row: bool,
317}
318
319impl TableRow {
320    pub fn new(cells: Vec<TableCell>) -> Self {
321        let is_header = cells.iter().all(|c| c.is_header);
322        Self {
323            cells,
324            is_header_row: is_header,
325        }
326    }
327}
328
329#[derive(Debug, Clone, Serialize, Deserialize)]
330pub struct TableContent {
331    pub caption: Option<String>,
332    pub headers: Vec<TableRow>,
333    pub rows: Vec<TableRow>,
334    pub column_count: usize,
335    pub summary: Option<String>,
336}
337
338impl TableContent {
339    pub fn new() -> Self {
340        Self {
341            caption: None,
342            headers: Vec::new(),
343            rows: Vec::new(),
344            column_count: 0,
345            summary: None,
346        }
347    }
348
349    pub fn is_empty(&self) -> bool {
350        self.headers.is_empty() && self.rows.is_empty()
351    }
352
353    pub fn row_count(&self) -> usize {
354        self.headers.len() + self.rows.len()
355    }
356}
357
358impl Default for TableContent {
359    fn default() -> Self {
360        Self::new()
361    }
362}
363
364// ============================================================================
365// 代码块
366// ============================================================================
367
368#[derive(Debug, Clone, Serialize, Deserialize)]
369pub struct CodeBlock {
370    pub code: String,
371    pub language: Option<String>,
372    pub line_count: usize,
373    pub is_inline: bool,
374    pub filename: Option<String>,
375}
376
377impl CodeBlock {
378    pub fn new(code: impl Into<String>) -> Self {
379        let code_str = code.into();
380        let line_count = code_str.lines().count();
381        Self {
382            code: code_str,
383            language: None,
384            line_count,
385            is_inline: false,
386            filename: None,
387        }
388    }
389
390    pub fn with_language(mut self, lang: impl Into<String>) -> Self {
391        self.language = Some(lang.into());
392        self
393    }
394
395    pub fn inline(mut self) -> Self {
396        self.is_inline = true;
397        self
398    }
399}
400
401// ============================================================================
402// 引用
403// ============================================================================
404
405#[derive(Debug, Clone, Serialize, Deserialize)]
406pub struct Quote {
407    pub text: String,
408    pub cite: Option<String>,
409    pub cite_url: Option<String>,
410}
411
412impl Quote {
413    pub fn new(text: impl Into<String>) -> Self {
414        Self {
415            text: text.into(),
416            cite: None,
417            cite_url: None,
418        }
419    }
420
421    pub fn with_cite(mut self, cite: impl Into<String>) -> Self {
422        self.cite = Some(cite.into());
423        self
424    }
425}
426
427// ============================================================================
428// 元数据类型
429// ============================================================================
430
431#[derive(Debug, Clone, Default, Serialize, Deserialize)]
432pub struct OpenGraph {
433    pub title: Option<String>,
434    pub og_type: Option<String>,
435    pub url: Option<String>,
436    pub image: Option<String>,
437    pub description: Option<String>,
438    pub site_name: Option<String>,
439    pub locale: Option<String>,
440    pub video: Option<String>,
441    pub audio: Option<String>,
442    pub extra: HashMap<String, String>,
443}
444
445impl OpenGraph {
446    pub fn is_present(&self) -> bool {
447        self.title.is_some() || self.og_type.is_some() || self.url.is_some()
448    }
449}
450
451#[derive(Debug, Clone, Default, Serialize, Deserialize)]
452pub struct TwitterCard {
453    pub card: Option<String>,
454    pub site: Option<String>,
455    pub creator: Option<String>,
456    pub title: Option<String>,
457    pub description: Option<String>,
458    pub image: Option<String>,
459    pub extra: HashMap<String, String>,
460}
461
462impl TwitterCard {
463    pub fn is_present(&self) -> bool {
464        self.card.is_some() || self.site.is_some()
465    }
466}
467
468#[derive(Debug, Clone, Default, Serialize, Deserialize)]
469pub struct RobotsMeta {
470    pub index: bool,
471    pub follow: bool,
472    pub archive: bool,
473    pub cache: bool,
474    pub snippet: bool,
475    pub max_snippet: i32,
476    pub max_image_preview: Option<String>,
477    pub max_video_preview: i32,
478    pub raw: Option<String>,
479}
480
481impl RobotsMeta {
482    pub fn allowed() -> Self {
483        Self {
484            index: true,
485            follow: true,
486            archive: true,
487            cache: true,
488            snippet: true,
489            max_snippet: -1,
490            max_image_preview: Some("large".to_string()),
491            max_video_preview: -1,
492            raw: None,
493        }
494    }
495
496    pub fn noindex_nofollow() -> Self {
497        Self {
498            index: false,
499            follow: false,
500            ..Self::allowed()
501        }
502    }
503}
504
505#[derive(Debug, Clone, Serialize, Deserialize)]
506pub struct AlternateLink {
507    pub hreflang: String,
508    pub href: String,
509}
510
511#[derive(Debug, Clone, Default, Serialize, Deserialize)]
512pub struct PageMetadata {
513    pub title: Option<String>,
514    pub description: Option<String>,
515    pub keywords: Vec<String>,
516    pub author: Option<String>,
517    pub generator: Option<String>,
518    pub canonical: Option<String>,
519    pub base_url: Option<String>,
520    pub language: Option<String>,
521    pub charset: Option<String>,
522    pub viewport: Option<String>,
523    pub robots: RobotsMeta,
524    pub opengraph: OpenGraph,
525    pub twitter: TwitterCard,
526    pub alternates: Vec<AlternateLink>,
527    pub favicon: Option<String>,
528    pub apple_touch_icon: Option<String>,
529    pub theme_color: Option<String>,
530    pub published_date: Option<String>,
531    pub modified_date: Option<String>,
532    pub schema_type: Option<String>,
533    pub custom: HashMap<String, String>,
534}
535
536impl PageMetadata {
537    pub fn effective_title(&self) -> Option<&str> {
538        self.opengraph.title.as_deref()
539            .or(self.twitter.title.as_deref())
540            .or(self.title.as_deref())
541    }
542
543    pub fn effective_description(&self) -> Option<&str> {
544        self.opengraph.description.as_deref()
545            .or(self.twitter.description.as_deref())
546            .or(self.description.as_deref())
547    }
548
549    pub fn effective_image(&self) -> Option<&str> {
550        self.opengraph.image.as_deref()
551            .or(self.twitter.image.as_deref())
552    }
553
554    pub fn should_index(&self) -> bool {
555        self.robots.index
556    }
557
558    pub fn should_follow(&self) -> bool {
559        self.robots.follow
560    }
561}
562
563// ============================================================================
564// 结构化数据
565// ============================================================================
566
567#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
568#[serde(rename_all = "snake_case")]
569pub enum StructuredDataFormat {
570    JsonLd,
571    Microdata,
572    Rdfa,
573}
574
575#[derive(Debug, Clone, Serialize, Deserialize)]
576pub struct StructuredData {
577    pub format: StructuredDataFormat,
578    pub schema_type: Option<String>,
579    pub raw_json: Option<String>,
580    pub properties: HashMap<String, serde_json::Value>,
581}
582
583impl StructuredData {
584    pub fn json_ld(raw: impl Into<String>) -> Self {
585        Self {
586            format: StructuredDataFormat::JsonLd,
587            schema_type: None,
588            raw_json: Some(raw.into()),
589            properties: HashMap::new(),
590        }
591    }
592
593    pub fn microdata(schema_type: impl Into<String>) -> Self {
594        Self {
595            format: StructuredDataFormat::Microdata,
596            schema_type: Some(schema_type.into()),
597            raw_json: None,
598            properties: HashMap::new(),
599        }
600    }
601}
602
603// ============================================================================
604// 完整解析结果
605// ============================================================================
606
607#[derive(Debug, Clone, Default, Serialize, Deserialize)]
608pub struct ParsedContent {
609    pub metadata: PageMetadata,
610    pub text: TextContent,
611    pub headings: Vec<Heading>,
612    pub paragraphs: Vec<String>,
613    pub links: Vec<Link>,
614    pub images: Vec<Image>,
615    pub lists: Vec<ListContent>,
616    pub tables: Vec<TableContent>,
617    pub code_blocks: Vec<CodeBlock>,
618    pub quotes: Vec<Quote>,
619    pub structured_data: Vec<StructuredData>,
620    pub stats: ParseStats,
621}
622
623impl ParsedContent {
624    pub fn internal_links(&self) -> Vec<&Link> {
625        self.links.iter().filter(|l| l.link_type == LinkType::Internal).collect()
626    }
627
628    pub fn external_links(&self) -> Vec<&Link> {
629        self.links.iter().filter(|l| l.link_type == LinkType::External).collect()
630    }
631
632    pub fn followable_links(&self) -> Vec<&Link> {
633        self.links.iter().filter(|l| l.should_follow()).collect()
634    }
635
636    pub fn has_structured_data(&self) -> bool {
637        !self.structured_data.is_empty()
638    }
639}
640
641#[derive(Debug, Clone, Default, Serialize, Deserialize)]
642pub struct ParseStats {
643    pub html_size: usize,
644    pub parse_time_us: u64,
645    pub node_count: usize,
646    pub element_count: usize,
647    pub text_node_count: usize,
648    pub comment_count: usize,
649    pub errors: Vec<String>,
650    pub warnings: Vec<String>,
651}
652
653impl ParseStats {
654    pub fn has_errors(&self) -> bool {
655        !self.errors.is_empty()
656    }
657
658    pub fn has_warnings(&self) -> bool {
659        !self.warnings.is_empty()
660    }
661}
662
663// ============================================================================
664// 解析器配置
665// ============================================================================
666
667#[derive(Debug, Clone)]
668pub struct ParserConfig {
669    pub base_url: Option<url::Url>,
670    pub max_text_length: usize,
671    pub extract_images: bool,
672    pub extract_links: bool,
673    pub extract_tables: bool,
674    pub extract_code_blocks: bool,
675    pub extract_structured_data: bool,
676    pub compute_readability: bool,
677    pub min_paragraph_length: usize,
678    pub content_selectors: Vec<String>,
679    pub remove_selectors: Vec<String>,
680    pub preserve_whitespace: bool,
681}
682
683impl Default for ParserConfig {
684    fn default() -> Self {
685        Self {
686            base_url: None,
687            max_text_length: 1_000_000,
688            extract_images: true,
689            extract_links: true,
690            extract_tables: true,
691            extract_code_blocks: true,
692            extract_structured_data: true,
693            compute_readability: false,
694            min_paragraph_length: 20,
695            content_selectors: vec![
696                "article".to_string(),
697                "main".to_string(),
698                "[role=main]".to_string(),
699                ".content".to_string(),
700                ".post-content".to_string(),
701                ".entry-content".to_string(),
702            ],
703            remove_selectors: vec![
704                "script".to_string(),
705                "style".to_string(),
706                "noscript".to_string(),
707                "nav".to_string(),
708                "header".to_string(),
709                "footer".to_string(),
710                "aside".to_string(),
711                ".sidebar".to_string(),
712                ".advertisement".to_string(),
713                ".ad".to_string(),
714                ".ads".to_string(),
715                "[role=navigation]".to_string(),
716                "[role=banner]".to_string(),
717                "[role=contentinfo]".to_string(),
718            ],
719            preserve_whitespace: false,
720        }
721    }
722}
723
724impl ParserConfig {
725    pub fn with_base_url(url: impl AsRef<str>) -> Result<Self, url::ParseError> {
726        Ok(Self {
727            base_url: Some(url::Url::parse(url.as_ref())?),
728            ..Default::default()
729        })
730    }
731
732    pub fn minimal() -> Self {
733        Self {
734            extract_images: false,
735            extract_tables: false,
736            extract_code_blocks: false,
737            extract_structured_data: false,
738            compute_readability: false,
739            ..Default::default()
740        }
741    }
742
743    pub fn full() -> Self {
744        Self {
745            compute_readability: true,
746            ..Default::default()
747        }
748    }
749
750    pub fn base_url(mut self, url: url::Url) -> Self {
751        self.base_url = Some(url);
752        self
753    }
754
755    pub fn add_content_selector(mut self, selector: impl Into<String>) -> Self {
756        self.content_selectors.push(selector.into());
757        self
758    }
759
760    pub fn add_remove_selector(mut self, selector: impl Into<String>) -> Self {
761        self.remove_selectors.push(selector.into());
762        self
763    }
764}
765
766// ============================================================================
767// 辅助函数
768// ============================================================================
769
770pub fn normalize_whitespace(text: &str) -> String {
771    let mut result = String::with_capacity(text.len());
772    let mut prev_ws = false;
773    for c in text.chars() {
774        if c.is_whitespace() {
775            if !prev_ws {
776                result.push(' ');
777                prev_ws = true;
778            }
779        } else {
780            result.push(c);
781            prev_ws = false;
782        }
783    }
784    result.trim().to_string()
785}
786
787pub fn clean_text(text: &str) -> String {
788    text.chars()
789        .filter(|c| !c.is_control() || c.is_whitespace())
790        .collect()
791}
792
793pub fn truncate_text(text: &str, max_len: usize) -> String {
794    if text.len() <= max_len {
795        text.to_string()
796    } else {
797        let mut truncated: String = text.chars().take(max_len - 3).collect();
798        truncated.push_str("...");
799        truncated
800    }
801}
802
803#[cfg(test)]
804mod tests {
805    use super::*;
806
807    #[test]
808    fn test_text_content_creation() {
809        let text = TextContent::from_raw("Hello   world,   this is   a test.");
810        assert_eq!(text.cleaned_text, "Hello world, this is a test.");
811        assert_eq!(text.word_count, 6);
812        assert!(!text.is_empty());
813    }
814
815    #[test]
816    fn test_heading_creation() {
817        let h1 = Heading::new(1, "Main Title").with_id("main");
818        assert_eq!(h1.level, 1);
819        assert_eq!(h1.id, Some("main".to_string()));
820    }
821
822    #[test]
823    fn test_link_creation() {
824        let link = Link::new("https://example.com", "Example");
825        assert!(!link.is_nofollow);
826        assert!(link.should_follow());
827    }
828
829    #[test]
830    fn test_image_creation() {
831        let img = Image::new("/img/photo.jpg", "A photo");
832        assert!(!img.is_decorative);
833        let decorative = Image::new("/img/spacer.gif", "");
834        assert!(decorative.is_decorative);
835    }
836
837    #[test]
838    fn test_opengraph() {
839        let og = OpenGraph::default();
840        assert!(!og.is_present());
841        let og2 = OpenGraph {
842            title: Some("Test".to_string()),
843            ..Default::default()
844        };
845        assert!(og2.is_present());
846    }
847
848    #[test]
849    fn test_parser_config() {
850        let config = ParserConfig::default();
851        assert!(config.extract_images);
852        let minimal = ParserConfig::minimal();
853        assert!(!minimal.extract_images);
854    }
855
856    #[test]
857    fn test_normalize_whitespace() {
858        assert_eq!(normalize_whitespace("  hello   world  "), "hello world");
859        assert_eq!(normalize_whitespace("  "), "");
860    }
861
862    #[test]
863    fn test_truncate_text() {
864        assert_eq!(truncate_text("Hello", 10), "Hello");
865        assert_eq!(truncate_text("Hello World", 8), "Hello...");
866    }
867}
crawlkit_parser/types.rs

crawlkit_parser/
types.rs
