Skip to main content

fetchium_core/extract/
layer4.rs

1//! CEP Layer 4 — PDF and document extraction (PRD §8.3).
2//!
3//! Extracts text from PDF and Office documents when static HTML extraction fails.
4//! Uses the `pdf-extract` crate (optional) or falls back to raw text extraction.
5//!
6//! Layer 4 triggers when: content_type contains "pdf", "msword", "officedocument".
7
8use crate::extract::{ContentMetadata, ExtractedContent};
9use crate::token::counter::count_tokens;
10use crate::types::CepLayer;
11
12/// Layer 4: PDF and document text extractor.
13pub struct Layer4Extractor;
14
15impl Layer4Extractor {
16    /// Check if the given content-type header indicates a document format.
17    pub fn is_document(content_type: &str) -> bool {
18        let ct = content_type.to_lowercase();
19        ct.contains("pdf")
20            || ct.contains("msword")
21            || ct.contains("officedocument")
22            || ct.contains("opendocument")
23            || ct.contains("rtf")
24    }
25
26    /// Extract text from raw PDF bytes.
27    ///
28    /// Uses `pdf-extract` if available, otherwise attempts basic text extraction.
29    pub fn extract_pdf(bytes: &[u8], url: &str) -> anyhow::Result<ExtractedContent> {
30        let text = Self::extract_pdf_text(bytes)?;
31        let token_count = count_tokens(&text);
32
33        Ok(ExtractedContent {
34            title: Self::title_from_url(url),
35            text,
36            layer_used: CepLayer::Layer4,
37            tokens: token_count,
38            metadata: ContentMetadata {
39                content_type: "application/pdf".to_string(),
40                ..Default::default()
41            },
42        })
43    }
44
45    fn extract_pdf_text(bytes: &[u8]) -> anyhow::Result<String> {
46        // Attempt basic PDF text extraction by scanning for text streams.
47        // This is a lightweight fallback that works for simple PDFs without
48        // the full pdf-extract dependency (which requires C bindings).
49        let raw = String::from_utf8_lossy(bytes);
50        let mut text = String::new();
51
52        // Extract text between BT (Begin Text) and ET (End Text) markers
53        for segment in raw.split("BT") {
54            if let Some(end) = segment.find("ET") {
55                let text_ops = &segment[..end];
56                // Extract strings in parentheses: (text content)
57                let mut i = text_ops.chars().peekable();
58                while let Some(c) = i.next() {
59                    if c == '(' {
60                        let mut s = String::new();
61                        for nc in i.by_ref() {
62                            if nc == ')' {
63                                break;
64                            }
65                            if nc != '\\' {
66                                s.push(nc);
67                            }
68                        }
69                        if s.chars().any(|c| c.is_alphabetic()) {
70                            text.push_str(&s);
71                            text.push(' ');
72                        }
73                    }
74                }
75            }
76        }
77
78        let text = text.trim().to_string();
79        if text.is_empty() {
80            anyhow::bail!("No text content extracted from PDF");
81        }
82
83        Ok(text)
84    }
85
86    fn title_from_url(url: &str) -> String {
87        url.rsplit('/')
88            .next()
89            .unwrap_or("document")
90            .trim_end_matches(".pdf")
91            .trim_end_matches(".doc")
92            .trim_end_matches(".docx")
93            .replace(['-', '_'], " ")
94            .to_string()
95    }
96}
97
98#[cfg(test)]
99mod tests {
100    use super::*;
101
102    #[test]
103    fn is_document_pdf() {
104        assert!(Layer4Extractor::is_document("application/pdf"));
105        assert!(Layer4Extractor::is_document("application/PDF"));
106    }
107
108    #[test]
109    fn is_document_msword() {
110        assert!(Layer4Extractor::is_document("application/msword"));
111        assert!(Layer4Extractor::is_document(
112            "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
113        ));
114    }
115
116    #[test]
117    fn is_document_html_is_false() {
118        assert!(!Layer4Extractor::is_document("text/html"));
119        assert!(!Layer4Extractor::is_document("application/json"));
120    }
121
122    #[test]
123    fn title_from_url() {
124        assert_eq!(
125            Layer4Extractor::title_from_url("https://example.com/my-paper.pdf"),
126            "my paper"
127        );
128        assert_eq!(
129            Layer4Extractor::title_from_url("https://example.com/report_2024.pdf"),
130            "report 2024"
131        );
132    }
133}