Skip to main content

fetchium_core/extract/
mod.rs

1//! Content Extraction Protocol (CEP) — 5-layer extraction system (PRD §16).
2//!
3//! Phase 1: Layers 1-2 (static HTML — CSS selectors + streaming rewriter).
4//! Phase 2: Layers 3-5 (headless Chromium, PDF, Screenshot OCR).
5//!
6//! Layer selection is automatic and progressive:
7//! - Layer 1: HTML + CSS selectors (scraper) — ~85% of pages
8//! - Layer 2: HTML + streaming rewriter (lol_html) — enhanced boilerplate removal
9//! - Layer 3: Headless JS rendering — SPAs, dynamic content [`headless` feature]
10//! - Layer 4: PDF/document extraction — PDF, DOCX, RTF
11//! - Layer 5: Screenshot OCR — last resort for image-heavy/canvas content [`headless` feature]
12
13pub mod boilerplate;
14pub mod cep_predictor;
15pub mod layer1;
16pub mod layer2;
17pub mod layer3;
18pub mod layer4;
19pub mod layer5;
20pub mod pipeline;
21
22use crate::types::CepLayer;
23
24/// Extracted content from a web page.
25#[derive(Debug, Clone)]
26pub struct ExtractedContent {
27    /// Page title.
28    pub title: String,
29    /// Extracted text content (cleaned, boilerplate removed).
30    pub text: String,
31    /// Which CEP layer produced this result.
32    pub layer_used: CepLayer,
33    /// Estimated token count (heuristic: ~4 chars/token).
34    pub tokens: u32,
35    /// Additional metadata (description, author, date, language, content-type).
36    pub metadata: ContentMetadata,
37}
38
39/// Metadata extracted alongside content.
40#[derive(Debug, Clone, Default)]
41pub struct ContentMetadata {
42    pub description: Option<String>,
43    pub author: Option<String>,
44    pub published_date: Option<String>,
45    pub language: Option<String>,
46    pub content_type: String,
47}