fetchium_core/extract/
layer4.rs1use crate::extract::{ContentMetadata, ExtractedContent};
9use crate::token::counter::count_tokens;
10use crate::types::CepLayer;
11
12pub struct Layer4Extractor;
14
15impl Layer4Extractor {
16 pub fn is_document(content_type: &str) -> bool {
18 let ct = content_type.to_lowercase();
19 ct.contains("pdf")
20 || ct.contains("msword")
21 || ct.contains("officedocument")
22 || ct.contains("opendocument")
23 || ct.contains("rtf")
24 }
25
26 pub fn extract_pdf(bytes: &[u8], url: &str) -> anyhow::Result<ExtractedContent> {
30 let text = Self::extract_pdf_text(bytes)?;
31 let token_count = count_tokens(&text);
32
33 Ok(ExtractedContent {
34 title: Self::title_from_url(url),
35 text,
36 layer_used: CepLayer::Layer4,
37 tokens: token_count,
38 metadata: ContentMetadata {
39 content_type: "application/pdf".to_string(),
40 ..Default::default()
41 },
42 })
43 }
44
45 fn extract_pdf_text(bytes: &[u8]) -> anyhow::Result<String> {
46 let raw = String::from_utf8_lossy(bytes);
50 let mut text = String::new();
51
52 for segment in raw.split("BT") {
54 if let Some(end) = segment.find("ET") {
55 let text_ops = &segment[..end];
56 let mut i = text_ops.chars().peekable();
58 while let Some(c) = i.next() {
59 if c == '(' {
60 let mut s = String::new();
61 for nc in i.by_ref() {
62 if nc == ')' {
63 break;
64 }
65 if nc != '\\' {
66 s.push(nc);
67 }
68 }
69 if s.chars().any(|c| c.is_alphabetic()) {
70 text.push_str(&s);
71 text.push(' ');
72 }
73 }
74 }
75 }
76 }
77
78 let text = text.trim().to_string();
79 if text.is_empty() {
80 anyhow::bail!("No text content extracted from PDF");
81 }
82
83 Ok(text)
84 }
85
86 fn title_from_url(url: &str) -> String {
87 url.rsplit('/')
88 .next()
89 .unwrap_or("document")
90 .trim_end_matches(".pdf")
91 .trim_end_matches(".doc")
92 .trim_end_matches(".docx")
93 .replace(['-', '_'], " ")
94 .to_string()
95 }
96}
97
98#[cfg(test)]
99mod tests {
100 use super::*;
101
102 #[test]
103 fn is_document_pdf() {
104 assert!(Layer4Extractor::is_document("application/pdf"));
105 assert!(Layer4Extractor::is_document("application/PDF"));
106 }
107
108 #[test]
109 fn is_document_msword() {
110 assert!(Layer4Extractor::is_document("application/msword"));
111 assert!(Layer4Extractor::is_document(
112 "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
113 ));
114 }
115
116 #[test]
117 fn is_document_html_is_false() {
118 assert!(!Layer4Extractor::is_document("text/html"));
119 assert!(!Layer4Extractor::is_document("application/json"));
120 }
121
122 #[test]
123 fn title_from_url() {
124 assert_eq!(
125 Layer4Extractor::title_from_url("https://example.com/my-paper.pdf"),
126 "my paper"
127 );
128 assert_eq!(
129 Layer4Extractor::title_from_url("https://example.com/report_2024.pdf"),
130 "report 2024"
131 );
132 }
133}