Skip to main content

kaccy_ai/
document.rs

1//! Document analysis module
2//!
3//! This module provides utilities for parsing and analyzing documents
4//! in various formats including Markdown, HTML, and PDF.
5
6use serde::{Deserialize, Serialize};
7use std::fmt::Write as _;
8
9/// Supported document formats
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
11pub enum DocumentFormat {
12    /// Markdown document
13    Markdown,
14    /// HTML document
15    Html,
16    /// Plain text
17    PlainText,
18    /// PDF document
19    Pdf,
20}
21
22impl DocumentFormat {
23    /// Detect format from content
24    #[must_use]
25    pub fn detect(content: &str) -> Self {
26        let content_lower = content.to_lowercase();
27
28        // Check for HTML markers
29        if content_lower.contains("<!doctype html")
30            || content_lower.contains("<html")
31            || (content_lower.contains("<head") && content_lower.contains("<body"))
32            || content_lower.contains("<div")
33            || content_lower.contains("<p>")
34        {
35            return DocumentFormat::Html;
36        }
37
38        // Check for Markdown markers
39        if content.contains("# ")
40            || content.contains("## ")
41            || content.contains("```")
42            || content.contains("**")
43            || content.contains("__")
44            || content.contains("](") // Markdown link pattern [text](url)
45            || content.contains("![")
46            || content.contains("- [ ]")
47            || content.contains("- [x]")
48        {
49            return DocumentFormat::Markdown;
50        }
51
52        DocumentFormat::PlainText
53    }
54
55    /// Detect format from file extension
56    #[must_use]
57    pub fn from_extension(ext: &str) -> Self {
58        match ext.to_lowercase().as_str() {
59            "md" | "markdown" | "mdown" | "mkd" => DocumentFormat::Markdown,
60            "html" | "htm" | "xhtml" => DocumentFormat::Html,
61            "pdf" => DocumentFormat::Pdf,
62            _ => DocumentFormat::PlainText,
63        }
64    }
65
66    /// Detect format from binary data (for PDF detection)
67    #[must_use]
68    pub fn detect_from_bytes(data: &[u8]) -> Self {
69        // Check for PDF magic bytes (%PDF-)
70        if data.len() >= 5 && &data[0..5] == b"%PDF-" {
71            return DocumentFormat::Pdf;
72        }
73
74        // Fall back to string-based detection
75        if let Ok(content) = std::str::from_utf8(data) {
76            Self::detect(content)
77        } else {
78            // Binary content that's not PDF
79            DocumentFormat::PlainText
80        }
81    }
82}
83
84/// Extracted document structure
85#[derive(Debug, Clone, Serialize, Deserialize)]
86pub struct DocumentStructure {
87    /// Document format
88    pub format: DocumentFormat,
89    /// Title (if detected)
90    pub title: Option<String>,
91    /// Headings with their levels
92    pub headings: Vec<Heading>,
93    /// Extracted links
94    pub links: Vec<Link>,
95    /// Extracted images
96    pub images: Vec<Image>,
97    /// Code blocks
98    pub code_blocks: Vec<CodeBlock>,
99    /// Plain text content (HTML tags stripped)
100    pub plain_text: String,
101    /// Word count
102    pub word_count: usize,
103    /// Character count
104    pub char_count: usize,
105    /// Estimated reading time in minutes
106    pub reading_time_minutes: u32,
107    /// Key statistics
108    pub stats: DocumentStats,
109}
110
111/// Document heading
112#[derive(Debug, Clone, Serialize, Deserialize)]
113pub struct Heading {
114    /// Heading level (1-6)
115    pub level: u8,
116    /// Heading text
117    pub text: String,
118    /// Anchor/ID (if available)
119    pub anchor: Option<String>,
120}
121
122/// Extracted link
123#[derive(Debug, Clone, Serialize, Deserialize)]
124pub struct Link {
125    /// Link URL
126    pub url: String,
127    /// Link text
128    pub text: String,
129    /// Link title (if available)
130    pub title: Option<String>,
131    /// Whether this is an external link
132    pub is_external: bool,
133}
134
135/// Extracted image
136#[derive(Debug, Clone, Serialize, Deserialize)]
137pub struct Image {
138    /// Image URL/path
139    pub src: String,
140    /// Alt text
141    pub alt: String,
142    /// Title (if available)
143    pub title: Option<String>,
144}
145
146/// Code block
147#[derive(Debug, Clone, Serialize, Deserialize)]
148pub struct CodeBlock {
149    /// Programming language (if specified)
150    pub language: Option<String>,
151    /// Code content
152    pub code: String,
153    /// Line count
154    pub line_count: usize,
155}
156
157/// Document statistics
158#[derive(Debug, Clone, Default, Serialize, Deserialize)]
159pub struct DocumentStats {
160    /// Number of headings
161    pub heading_count: usize,
162    /// Number of paragraphs
163    pub paragraph_count: usize,
164    /// Number of lists
165    pub list_count: usize,
166    /// Number of links
167    pub link_count: usize,
168    /// Number of images
169    pub image_count: usize,
170    /// Number of code blocks
171    pub code_block_count: usize,
172    /// Number of tables
173    pub table_count: usize,
174    /// Number of blockquotes
175    pub blockquote_count: usize,
176}
177
178/// Document parser
179pub struct DocumentParser;
180
181impl DocumentParser {
182    /// Parse a document and extract its structure
183    #[must_use]
184    pub fn parse(content: &str) -> DocumentStructure {
185        let format = DocumentFormat::detect(content);
186
187        match format {
188            DocumentFormat::Markdown => Self::parse_markdown(content),
189            DocumentFormat::Html => Self::parse_html(content),
190            DocumentFormat::PlainText => Self::parse_plain_text(content),
191            DocumentFormat::Pdf => Self::parse_plain_text(content), // PDF needs binary data
192        }
193    }
194
195    /// Parse a document with explicit format
196    #[must_use]
197    pub fn parse_with_format(content: &str, format: DocumentFormat) -> DocumentStructure {
198        match format {
199            DocumentFormat::Markdown => Self::parse_markdown(content),
200            DocumentFormat::Html => Self::parse_html(content),
201            DocumentFormat::PlainText => Self::parse_plain_text(content),
202            DocumentFormat::Pdf => Self::parse_plain_text(content), // PDF needs binary data
203        }
204    }
205
206    /// Parse binary PDF data and extract its structure
207    pub fn parse_pdf(data: &[u8]) -> Result<DocumentStructure, PdfParseError> {
208        PdfParser::parse(data)
209    }
210
211    /// Parse binary PDF from a file path
212    pub fn parse_pdf_file(path: &std::path::Path) -> Result<DocumentStructure, PdfParseError> {
213        let data = std::fs::read(path).map_err(|e| PdfParseError::IoError(e.to_string()))?;
214        Self::parse_pdf(&data)
215    }
216
217    /// Parse Markdown document
218    fn parse_markdown(content: &str) -> DocumentStructure {
219        let mut headings = Vec::new();
220        let mut links = Vec::new();
221        let mut images = Vec::new();
222        let mut code_blocks = Vec::new();
223        let mut title = None;
224        let mut stats = DocumentStats::default();
225
226        let mut in_code_block = false;
227        let mut code_block_lang = None;
228        let mut code_block_content = String::new();
229
230        for line in content.lines() {
231            // Handle code blocks
232            if line.starts_with("```") {
233                if in_code_block {
234                    // End of code block
235                    code_blocks.push(CodeBlock {
236                        language: code_block_lang.take(),
237                        line_count: code_block_content.lines().count(),
238                        code: std::mem::take(&mut code_block_content),
239                    });
240                    stats.code_block_count += 1;
241                    in_code_block = false;
242                } else {
243                    // Start of code block
244                    let lang = line.trim_start_matches("```").trim();
245                    code_block_lang = if lang.is_empty() {
246                        None
247                    } else {
248                        Some(lang.to_string())
249                    };
250                    in_code_block = true;
251                }
252                continue;
253            }
254
255            if in_code_block {
256                code_block_content.push_str(line);
257                code_block_content.push('\n');
258                continue;
259            }
260
261            // Parse headings
262            if let Some(heading) = Self::parse_markdown_heading(line) {
263                if title.is_none() && heading.level == 1 {
264                    title = Some(heading.text.clone());
265                }
266                headings.push(heading);
267                stats.heading_count += 1;
268            }
269
270            // Parse links: [text](url) or [text](url "title")
271            Self::extract_markdown_links(line, &mut links);
272
273            // Parse images: ![alt](src) or ![alt](src "title")
274            Self::extract_markdown_images(line, &mut images);
275
276            // Count lists
277            if line.trim_start().starts_with("- ")
278                || line.trim_start().starts_with("* ")
279                || line.trim_start().starts_with("+ ")
280                || line
281                    .trim_start()
282                    .chars()
283                    .next()
284                    .is_some_and(|c| c.is_ascii_digit())
285                    && line.contains(". ")
286            {
287                stats.list_count += 1;
288            }
289
290            // Count blockquotes
291            if line.trim_start().starts_with("> ") {
292                stats.blockquote_count += 1;
293            }
294
295            // Count tables (simple detection)
296            if line.contains('|') && line.trim().starts_with('|') {
297                stats.table_count += 1;
298            }
299        }
300
301        stats.link_count = links.len();
302        stats.image_count = images.len();
303
304        // Calculate plain text
305        let plain_text = Self::markdown_to_plain_text(content);
306        let word_count = plain_text.split_whitespace().count();
307        let char_count = plain_text.chars().count();
308
309        // Count paragraphs (blank line separated blocks)
310        stats.paragraph_count = content
311            .split("\n\n")
312            .filter(|p| !p.trim().is_empty() && !p.trim().starts_with('#'))
313            .count();
314
315        DocumentStructure {
316            format: DocumentFormat::Markdown,
317            title,
318            headings,
319            links,
320            images,
321            code_blocks,
322            plain_text,
323            word_count,
324            char_count,
325            reading_time_minutes: (word_count / 200).max(1) as u32,
326            stats,
327        }
328    }
329
330    /// Parse a markdown heading
331    fn parse_markdown_heading(line: &str) -> Option<Heading> {
332        let trimmed = line.trim();
333        if !trimmed.starts_with('#') {
334            return None;
335        }
336
337        let mut level = 0u8;
338        for c in trimmed.chars() {
339            if c == '#' {
340                level += 1;
341            } else {
342                break;
343            }
344        }
345
346        if level > 6 {
347            return None;
348        }
349
350        let text = trimmed.trim_start_matches('#').trim().to_string();
351        if text.is_empty() {
352            return None;
353        }
354
355        // Generate anchor from text
356        let anchor = text
357            .to_lowercase()
358            .replace(' ', "-")
359            .chars()
360            .filter(|c| c.is_alphanumeric() || *c == '-')
361            .collect::<String>();
362
363        Some(Heading {
364            level,
365            text,
366            anchor: Some(anchor),
367        })
368    }
369
370    /// Extract markdown links from a line
371    fn extract_markdown_links(line: &str, links: &mut Vec<Link>) {
372        let mut remaining = line;
373
374        while let Some(start) = remaining.find('[') {
375            let after_start = &remaining[start + 1..];
376
377            // Find closing bracket
378            if let Some(close) = after_start.find(']') {
379                let text = &after_start[..close];
380                let after_close = &after_start[close + 1..];
381
382                // Check for (url) or (url "title")
383                if after_close.starts_with('(') {
384                    if let Some(paren_close) = after_close.find(')') {
385                        let url_part = &after_close[1..paren_close];
386
387                        // Parse URL and optional title
388                        let (url, title) = if let Some(quote_start) = url_part.find('"') {
389                            let url = url_part[..quote_start].trim().to_string();
390                            let title_part = &url_part[quote_start + 1..];
391                            let title = title_part.trim_end_matches('"').to_string();
392                            (url, Some(title))
393                        } else {
394                            (url_part.trim().to_string(), None)
395                        };
396
397                        // Skip image links (they start with !)
398                        if !remaining[..start].ends_with('!') && !url.is_empty() {
399                            let is_external = url.starts_with("http://")
400                                || url.starts_with("https://")
401                                || url.starts_with("//");
402
403                            links.push(Link {
404                                url,
405                                text: text.to_string(),
406                                title,
407                                is_external,
408                            });
409                        }
410
411                        remaining = &after_close[paren_close + 1..];
412                        continue;
413                    }
414                }
415            }
416
417            remaining = &remaining[start + 1..];
418        }
419    }
420
421    /// Extract markdown images from a line
422    fn extract_markdown_images(line: &str, images: &mut Vec<Image>) {
423        let mut remaining = line;
424
425        while let Some(start) = remaining.find("![") {
426            let after_start = &remaining[start + 2..];
427
428            // Find closing bracket
429            if let Some(close) = after_start.find(']') {
430                let alt = &after_start[..close];
431                let after_close = &after_start[close + 1..];
432
433                // Check for (src) or (src "title")
434                if after_close.starts_with('(') {
435                    if let Some(paren_close) = after_close.find(')') {
436                        let src_part = &after_close[1..paren_close];
437
438                        // Parse src and optional title
439                        let (src, title) = if let Some(quote_start) = src_part.find('"') {
440                            let src = src_part[..quote_start].trim().to_string();
441                            let title_part = &src_part[quote_start + 1..];
442                            let title = title_part.trim_end_matches('"').to_string();
443                            (src, Some(title))
444                        } else {
445                            (src_part.trim().to_string(), None)
446                        };
447
448                        if !src.is_empty() {
449                            images.push(Image {
450                                src,
451                                alt: alt.to_string(),
452                                title,
453                            });
454                        }
455
456                        remaining = &after_close[paren_close + 1..];
457                        continue;
458                    }
459                }
460            }
461
462            remaining = &remaining[start + 2..];
463        }
464    }
465
466    /// Convert markdown to plain text
467    fn markdown_to_plain_text(content: &str) -> String {
468        let mut result = String::new();
469        let mut in_code_block = false;
470
471        for line in content.lines() {
472            if line.starts_with("```") {
473                in_code_block = !in_code_block;
474                continue;
475            }
476
477            if in_code_block {
478                continue;
479            }
480
481            // Remove headings markers
482            let line = if line.starts_with('#') {
483                line.trim_start_matches('#').trim()
484            } else {
485                line
486            };
487
488            // Remove bold/italic markers
489            let line = line
490                .replace("**", "")
491                .replace("__", "")
492                .replace(['*', '_'], "");
493
494            // Remove inline code
495            let line = Self::remove_inline_code(&line);
496
497            // Remove links but keep text
498            let line = Self::remove_markdown_links(&line);
499
500            // Remove images
501            let line = Self::remove_markdown_images(&line);
502
503            if !line.trim().is_empty() {
504                result.push_str(&line);
505                result.push(' ');
506            }
507        }
508
509        result.trim().to_string()
510    }
511
512    /// Remove inline code markers
513    fn remove_inline_code(line: &str) -> String {
514        let mut result = String::new();
515        let mut in_code = false;
516
517        for c in line.chars() {
518            if c == '`' {
519                in_code = !in_code;
520            } else if !in_code {
521                result.push(c);
522            }
523        }
524
525        result
526    }
527
528    /// Remove markdown links but keep text
529    fn remove_markdown_links(line: &str) -> String {
530        let mut result = line.to_string();
531
532        // Simple replacement of [text](url) with text
533        while let Some(start) = result.find('[') {
534            if let Some(close) = result[start..].find(']') {
535                let absolute_close = start + close;
536                if result.len() > absolute_close + 1
537                    && result.as_bytes()[absolute_close + 1] == b'('
538                {
539                    if let Some(paren_close) = result[absolute_close..].find(')') {
540                        let text = &result[start + 1..absolute_close];
541                        let before = &result[..start];
542                        let after = &result[absolute_close + paren_close + 1..];
543                        result = format!("{before}{text}{after}");
544                        continue;
545                    }
546                }
547            }
548            break;
549        }
550
551        result
552    }
553
554    /// Remove markdown images
555    fn remove_markdown_images(line: &str) -> String {
556        let mut result = line.to_string();
557
558        while let Some(start) = result.find("![") {
559            if let Some(close) = result[start..].find(']') {
560                let absolute_close = start + close;
561                if result.len() > absolute_close + 1
562                    && result.as_bytes()[absolute_close + 1] == b'('
563                {
564                    if let Some(paren_close) = result[absolute_close..].find(')') {
565                        let before = &result[..start];
566                        let after = &result[absolute_close + paren_close + 1..];
567                        result = format!("{before}{after}");
568                        continue;
569                    }
570                }
571            }
572            break;
573        }
574
575        result
576    }
577
578    /// Parse HTML document
579    fn parse_html(content: &str) -> DocumentStructure {
580        let mut headings = Vec::new();
581        let mut links = Vec::new();
582        let mut images = Vec::new();
583        let mut code_blocks = Vec::new();
584        let mut title = None;
585        let mut stats = DocumentStats::default();
586
587        // Extract title from <title> tag
588        if let Some(title_text) = Self::extract_html_tag_content(content, "title") {
589            title = Some(title_text);
590        }
591
592        // Extract headings (h1-h6)
593        for level in 1..=6 {
594            let tag = format!("h{level}");
595            for text in Self::extract_all_html_tag_contents(content, &tag) {
596                if title.is_none() && level == 1 {
597                    title = Some(text.clone());
598                }
599                headings.push(Heading {
600                    level: level as u8,
601                    text,
602                    anchor: None,
603                });
604                stats.heading_count += 1;
605            }
606        }
607
608        // Extract links
609        Self::extract_html_links(content, &mut links);
610        stats.link_count = links.len();
611
612        // Extract images
613        Self::extract_html_images(content, &mut images);
614        stats.image_count = images.len();
615
616        // Extract code blocks (<pre><code> or <code>)
617        for code in Self::extract_all_html_tag_contents(content, "code") {
618            code_blocks.push(CodeBlock {
619                language: None,
620                line_count: code.lines().count(),
621                code,
622            });
623            stats.code_block_count += 1;
624        }
625
626        // Count other elements
627        stats.paragraph_count = Self::count_html_tags(content, "p");
628        stats.list_count =
629            Self::count_html_tags(content, "ul") + Self::count_html_tags(content, "ol");
630        stats.table_count = Self::count_html_tags(content, "table");
631        stats.blockquote_count = Self::count_html_tags(content, "blockquote");
632
633        // Get plain text
634        let plain_text = Self::html_to_plain_text(content);
635        let word_count = plain_text.split_whitespace().count();
636        let char_count = plain_text.chars().count();
637
638        DocumentStructure {
639            format: DocumentFormat::Html,
640            title,
641            headings,
642            links,
643            images,
644            code_blocks,
645            plain_text,
646            word_count,
647            char_count,
648            reading_time_minutes: (word_count / 200).max(1) as u32,
649            stats,
650        }
651    }
652
653    /// Extract content from an HTML tag
654    fn extract_html_tag_content(content: &str, tag: &str) -> Option<String> {
655        let open_tag = format!("<{tag}");
656        let close_tag = format!("</{tag}>");
657
658        let start = content.to_lowercase().find(&open_tag)?;
659        let after_open = &content[start..];
660
661        // Find the end of the opening tag
662        let tag_end = after_open.find('>')?;
663        let content_start = start + tag_end + 1;
664
665        let close_pos = content[content_start..].to_lowercase().find(&close_tag)?;
666
667        let text = &content[content_start..content_start + close_pos];
668        Some(Self::html_to_plain_text(text).trim().to_string())
669    }
670
671    /// Extract all contents from HTML tags
672    fn extract_all_html_tag_contents(content: &str, tag: &str) -> Vec<String> {
673        let mut results = Vec::new();
674        let content_lower = content.to_lowercase();
675        let open_tag = format!("<{tag}");
676        let close_tag = format!("</{tag}>");
677
678        let mut search_start = 0;
679        while let Some(start) = content_lower[search_start..].find(&open_tag) {
680            let absolute_start = search_start + start;
681            let after_open = &content[absolute_start..];
682
683            if let Some(tag_end) = after_open.find('>') {
684                let content_start = absolute_start + tag_end + 1;
685
686                if let Some(close_pos) = content_lower[content_start..].find(&close_tag) {
687                    let text = &content[content_start..content_start + close_pos];
688                    let clean_text = Self::html_to_plain_text(text).trim().to_string();
689                    if !clean_text.is_empty() {
690                        results.push(clean_text);
691                    }
692                    search_start = content_start + close_pos + close_tag.len();
693                    continue;
694                }
695            }
696
697            search_start = absolute_start + 1;
698        }
699
700        results
701    }
702
703    /// Count occurrences of an HTML tag
704    fn count_html_tags(content: &str, tag: &str) -> usize {
705        let open_tag = format!("<{tag}");
706        content.to_lowercase().matches(&open_tag).count()
707    }
708
709    /// Extract HTML links
710    fn extract_html_links(content: &str, links: &mut Vec<Link>) {
711        let content_lower = content.to_lowercase();
712        let mut search_start = 0;
713
714        while let Some(start) = content_lower[search_start..].find("<a ") {
715            let absolute_start = search_start + start;
716            let after_open = &content[absolute_start..];
717
718            if let Some(tag_end) = after_open.find('>') {
719                let tag_content = &after_open[..tag_end];
720
721                // Extract href
722                if let Some(href) = Self::extract_html_attribute(tag_content, "href") {
723                    let close_pos = content_lower[absolute_start..].find("</a>");
724
725                    let text = if let Some(close) = close_pos {
726                        let content_start = absolute_start + tag_end + 1;
727                        let content_end = absolute_start + close;
728                        Self::html_to_plain_text(&content[content_start..content_end])
729                            .trim()
730                            .to_string()
731                    } else {
732                        String::new()
733                    };
734
735                    let title = Self::extract_html_attribute(tag_content, "title");
736                    let is_external = href.starts_with("http://")
737                        || href.starts_with("https://")
738                        || href.starts_with("//");
739
740                    links.push(Link {
741                        url: href,
742                        text,
743                        title,
744                        is_external,
745                    });
746                }
747
748                search_start = absolute_start + tag_end;
749            } else {
750                search_start = absolute_start + 1;
751            }
752        }
753    }
754
755    /// Extract HTML images
756    fn extract_html_images(content: &str, images: &mut Vec<Image>) {
757        let content_lower = content.to_lowercase();
758        let mut search_start = 0;
759
760        while let Some(start) = content_lower[search_start..].find("<img ") {
761            let absolute_start = search_start + start;
762            let after_open = &content[absolute_start..];
763
764            if let Some(tag_end) = after_open.find('>').or_else(|| after_open.find("/>")) {
765                let tag_content = &after_open[..tag_end];
766
767                if let Some(src) = Self::extract_html_attribute(tag_content, "src") {
768                    let alt = Self::extract_html_attribute(tag_content, "alt").unwrap_or_default();
769                    let title = Self::extract_html_attribute(tag_content, "title");
770
771                    images.push(Image { src, alt, title });
772                }
773
774                search_start = absolute_start + tag_end;
775            } else {
776                search_start = absolute_start + 1;
777            }
778        }
779    }
780
781    /// Extract an HTML attribute value
782    fn extract_html_attribute(tag_content: &str, attr: &str) -> Option<String> {
783        let attr_pattern = format!("{attr}=");
784        let content_lower = tag_content.to_lowercase();
785
786        let attr_start = content_lower.find(&attr_pattern)?;
787        let after_attr = &tag_content[attr_start + attr_pattern.len()..];
788
789        // Handle quoted attribute values
790        let first_char = after_attr.chars().next()?;
791        if first_char == '"' || first_char == '\'' {
792            let quote = first_char;
793            let value_start = 1;
794            let value_end = after_attr[value_start..].find(quote)?;
795            return Some(after_attr[value_start..value_start + value_end].to_string());
796        }
797
798        // Handle unquoted attribute values
799        let value_end = after_attr.find(|c: char| c.is_whitespace() || c == '>')?;
800        Some(after_attr[..value_end].to_string())
801    }
802
803    /// Convert HTML to plain text
804    fn html_to_plain_text(content: &str) -> String {
805        let mut result = String::new();
806        let mut in_tag = false;
807        let mut in_script = false;
808        let mut in_style = false;
809
810        let content_lower = content.to_lowercase();
811        let chars: Vec<char> = content.chars().collect();
812        let chars_lower: Vec<char> = content_lower.chars().collect();
813
814        let mut i = 0;
815        while i < chars.len() {
816            // Check for script/style tags
817            if i + 7 < chars.len() {
818                let slice: String = chars_lower[i..i + 7].iter().collect();
819                if slice == "<script" {
820                    in_script = true;
821                } else if slice == "</scrip" {
822                    in_script = false;
823                }
824            }
825
826            if i + 6 < chars.len() {
827                let slice: String = chars_lower[i..i + 6].iter().collect();
828                if slice == "<style" {
829                    in_style = true;
830                } else if slice == "</styl" {
831                    in_style = false;
832                }
833            }
834
835            let c = chars[i];
836
837            if c == '<' {
838                in_tag = true;
839            } else if c == '>' {
840                in_tag = false;
841                // Add space after certain tags
842                result.push(' ');
843            } else if !in_tag && !in_script && !in_style {
844                result.push(c);
845            }
846
847            i += 1;
848        }
849
850        // Decode common HTML entities
851        let result = result
852            .replace("&nbsp;", " ")
853            .replace("&amp;", "&")
854            .replace("&lt;", "<")
855            .replace("&gt;", ">")
856            .replace("&quot;", "\"")
857            .replace("&apos;", "'")
858            .replace("&#39;", "'");
859
860        // Normalize whitespace
861        result.split_whitespace().collect::<Vec<_>>().join(" ")
862    }
863
864    /// Parse plain text document
865    fn parse_plain_text(content: &str) -> DocumentStructure {
866        let word_count = content.split_whitespace().count();
867        let char_count = content.chars().count();
868        let paragraph_count = content
869            .split("\n\n")
870            .filter(|p| !p.trim().is_empty())
871            .count();
872
873        DocumentStructure {
874            format: DocumentFormat::PlainText,
875            title: None,
876            headings: Vec::new(),
877            links: Vec::new(),
878            images: Vec::new(),
879            code_blocks: Vec::new(),
880            plain_text: content.to_string(),
881            word_count,
882            char_count,
883            reading_time_minutes: (word_count / 200).max(1) as u32,
884            stats: DocumentStats {
885                paragraph_count,
886                ..Default::default()
887            },
888        }
889    }
890}
891
892/// Document quality analysis
893#[derive(Debug, Clone, Serialize, Deserialize)]
894pub struct DocumentQuality {
895    /// Overall quality score (0-100)
896    pub overall_score: u32,
897    /// Readability score (0-100)
898    pub readability_score: u32,
899    /// Structure score (0-100)
900    pub structure_score: u32,
901    /// Issues found
902    pub issues: Vec<QualityIssue>,
903    /// Suggestions for improvement
904    pub suggestions: Vec<String>,
905}
906
907/// Quality issue
908#[derive(Debug, Clone, Serialize, Deserialize)]
909pub struct QualityIssue {
910    /// Issue severity
911    pub severity: IssueSeverity,
912    /// Issue description
913    pub description: String,
914}
915
916/// Issue severity levels
917#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
918pub enum IssueSeverity {
919    /// Informational note.
920    Info,
921    /// Non-critical issue that should be addressed.
922    Warning,
923    /// Critical issue that must be resolved.
924    Error,
925}
926
927/// Document quality analyzer
928pub struct QualityAnalyzer;
929
930impl QualityAnalyzer {
931    /// Analyze document quality
932    #[must_use]
933    pub fn analyze(structure: &DocumentStructure) -> DocumentQuality {
934        let mut issues = Vec::new();
935        let mut suggestions = Vec::new();
936
937        // Check for title
938        if structure.title.is_none() {
939            issues.push(QualityIssue {
940                severity: IssueSeverity::Warning,
941                description: "Document has no title".to_string(),
942            });
943            suggestions
944                .push("Add a main heading (# Title) at the start of the document".to_string());
945        }
946
947        // Check heading structure
948        let mut prev_level = 0u8;
949        for heading in &structure.headings {
950            if heading.level > prev_level + 1 && prev_level > 0 {
951                issues.push(QualityIssue {
952                    severity: IssueSeverity::Warning,
953                    description: format!(
954                        "Heading level jumps from {} to {}: '{}'",
955                        prev_level, heading.level, heading.text
956                    ),
957                });
958            }
959            prev_level = heading.level;
960        }
961
962        // Check word count
963        if structure.word_count < 100 {
964            issues.push(QualityIssue {
965                severity: IssueSeverity::Info,
966                description: "Document is very short".to_string(),
967            });
968        } else if structure.word_count > 5000 {
969            suggestions.push("Consider breaking long documents into multiple sections".to_string());
970        }
971
972        // Check for broken/empty links
973        for link in &structure.links {
974            if link.url.is_empty() {
975                issues.push(QualityIssue {
976                    severity: IssueSeverity::Error,
977                    description: format!("Empty link URL for text: '{}'", link.text),
978                });
979            }
980            if link.text.is_empty() {
981                issues.push(QualityIssue {
982                    severity: IssueSeverity::Warning,
983                    description: format!("Link has no text: '{}'", link.url),
984                });
985            }
986        }
987
988        // Check for images without alt text
989        for image in &structure.images {
990            if image.alt.is_empty() {
991                issues.push(QualityIssue {
992                    severity: IssueSeverity::Warning,
993                    description: format!("Image missing alt text: '{}'", image.src),
994                });
995            }
996        }
997
998        // Calculate scores
999        let structure_score = Self::calculate_structure_score(structure, &issues);
1000        let readability_score = Self::calculate_readability_score(structure);
1001        let overall_score = u32::midpoint(structure_score, readability_score);
1002
1003        DocumentQuality {
1004            overall_score,
1005            readability_score,
1006            structure_score,
1007            issues,
1008            suggestions,
1009        }
1010    }
1011
1012    /// Calculate structure score
1013    fn calculate_structure_score(structure: &DocumentStructure, issues: &[QualityIssue]) -> u32 {
1014        let mut score = 100u32;
1015
1016        // Deduct for issues
1017        for issue in issues {
1018            match issue.severity {
1019                IssueSeverity::Error => score = score.saturating_sub(15),
1020                IssueSeverity::Warning => score = score.saturating_sub(5),
1021                IssueSeverity::Info => score = score.saturating_sub(2),
1022            }
1023        }
1024
1025        // Bonus for good structure
1026        if structure.title.is_some() {
1027            score = score.saturating_add(5).min(100);
1028        }
1029        if !structure.headings.is_empty() {
1030            score = score.saturating_add(5).min(100);
1031        }
1032
1033        score
1034    }
1035
1036    /// Calculate readability score (simplified Flesch-Kincaid style)
1037    fn calculate_readability_score(structure: &DocumentStructure) -> u32 {
1038        let words = structure.word_count;
1039        if words == 0 {
1040            return 50;
1041        }
1042
1043        // Count sentences (rough estimate)
1044        let sentence_count = structure.plain_text.matches(['.', '!', '?']).count().max(1);
1045
1046        // Average words per sentence
1047        let avg_words_per_sentence = words as f64 / sentence_count as f64;
1048
1049        // Optimal is around 15-20 words per sentence
1050        let score = if avg_words_per_sentence < 10.0 {
1051            70 + ((avg_words_per_sentence / 10.0) * 20.0) as u32
1052        } else if avg_words_per_sentence <= 20.0 {
1053            90 + (10.0 - (avg_words_per_sentence - 15.0).abs()) as u32
1054        } else if avg_words_per_sentence <= 30.0 {
1055            70 - ((avg_words_per_sentence - 20.0) * 2.0) as u32
1056        } else {
1057            50
1058        };
1059
1060        score.min(100)
1061    }
1062}
1063
1064/// Document table of contents generator
1065pub struct TocGenerator;
1066
1067impl TocGenerator {
1068    /// Generate table of contents from document structure
1069    #[must_use]
1070    pub fn generate(structure: &DocumentStructure) -> Vec<TocEntry> {
1071        structure
1072            .headings
1073            .iter()
1074            .map(|h| TocEntry {
1075                level: h.level,
1076                text: h.text.clone(),
1077                anchor: h.anchor.clone(),
1078            })
1079            .collect()
1080    }
1081
1082    /// Generate table of contents as markdown
1083    #[must_use]
1084    pub fn generate_markdown(structure: &DocumentStructure) -> String {
1085        let mut result = String::new();
1086
1087        for heading in &structure.headings {
1088            let indent = "  ".repeat((heading.level - 1) as usize);
1089            let anchor = heading
1090                .anchor
1091                .as_ref()
1092                .map(|a| format!("#{a}"))
1093                .unwrap_or_default();
1094
1095            let _ = writeln!(result, "{}- [{}]({})", indent, heading.text, anchor);
1096        }
1097
1098        result
1099    }
1100}
1101
1102/// Table of contents entry
1103#[derive(Debug, Clone, Serialize, Deserialize)]
1104pub struct TocEntry {
1105    /// Heading level
1106    pub level: u8,
1107    /// Heading text
1108    pub text: String,
1109    /// Anchor link
1110    pub anchor: Option<String>,
1111}
1112
1113/// Document metadata extractor
1114pub struct MetadataExtractor;
1115
1116impl MetadataExtractor {
1117    /// Extract metadata from document
1118    #[must_use]
1119    pub fn extract(content: &str) -> DocumentMetadata {
1120        let structure = DocumentParser::parse(content);
1121        let quality = QualityAnalyzer::analyze(&structure);
1122
1123        DocumentMetadata {
1124            format: structure.format,
1125            title: structure.title,
1126            word_count: structure.word_count,
1127            char_count: structure.char_count,
1128            reading_time_minutes: structure.reading_time_minutes,
1129            heading_count: structure.stats.heading_count,
1130            link_count: structure.stats.link_count,
1131            image_count: structure.stats.image_count,
1132            code_block_count: structure.stats.code_block_count,
1133            quality_score: quality.overall_score,
1134            external_links: structure.links.iter().filter(|l| l.is_external).count(),
1135            internal_links: structure.links.iter().filter(|l| !l.is_external).count(),
1136        }
1137    }
1138}
1139
1140/// Document metadata summary
1141#[derive(Debug, Clone, Serialize, Deserialize)]
1142pub struct DocumentMetadata {
1143    /// Document format
1144    pub format: DocumentFormat,
1145    /// Document title
1146    pub title: Option<String>,
1147    /// Word count
1148    pub word_count: usize,
1149    /// Character count
1150    pub char_count: usize,
1151    /// Estimated reading time
1152    pub reading_time_minutes: u32,
1153    /// Number of headings
1154    pub heading_count: usize,
1155    /// Number of links
1156    pub link_count: usize,
1157    /// Number of images
1158    pub image_count: usize,
1159    /// Number of code blocks
1160    pub code_block_count: usize,
1161    /// Quality score
1162    pub quality_score: u32,
1163    /// External link count
1164    pub external_links: usize,
1165    /// Internal link count
1166    pub internal_links: usize,
1167}
1168
1169/// Error type for PDF parsing
1170#[derive(Debug, Clone)]
1171pub enum PdfParseError {
1172    /// IO error reading PDF
1173    IoError(String),
1174    /// Invalid PDF format
1175    InvalidFormat(String),
1176    /// PDF parsing failed
1177    ParseError(String),
1178    /// Text extraction failed
1179    ExtractionError(String),
1180}
1181
1182impl std::fmt::Display for PdfParseError {
1183    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1184        match self {
1185            PdfParseError::IoError(e) => write!(f, "IO error: {e}"),
1186            PdfParseError::InvalidFormat(e) => write!(f, "Invalid PDF format: {e}"),
1187            PdfParseError::ParseError(e) => write!(f, "Parse error: {e}"),
1188            PdfParseError::ExtractionError(e) => write!(f, "Extraction error: {e}"),
1189        }
1190    }
1191}
1192
1193impl std::error::Error for PdfParseError {}
1194
1195/// PDF document parser
1196pub struct PdfParser;
1197
1198impl PdfParser {
1199    /// Parse a PDF document from binary data
1200    pub fn parse(data: &[u8]) -> Result<DocumentStructure, PdfParseError> {
1201        use lopdf::Document;
1202
1203        let doc = Document::load_mem(data).map_err(|e| PdfParseError::ParseError(e.to_string()))?;
1204
1205        let mut all_text = String::new();
1206        let mut page_count = 0;
1207
1208        // Extract text from all pages
1209        let pages = doc.get_pages();
1210        for (page_num, _) in &pages {
1211            page_count += 1;
1212            if let Ok(text) = Self::extract_page_text(&doc, *page_num) {
1213                all_text.push_str(&text);
1214                all_text.push('\n');
1215            }
1216        }
1217
1218        let plain_text = Self::clean_extracted_text(&all_text);
1219        let word_count = plain_text.split_whitespace().count();
1220        let char_count = plain_text.chars().count();
1221
1222        // Try to extract title from metadata or first heading
1223        let title = Self::extract_title(&doc, &plain_text);
1224
1225        // Extract headings (based on text analysis)
1226        let headings = Self::detect_headings(&plain_text);
1227        let heading_count = headings.len();
1228
1229        // Extract links from PDF
1230        let links = Self::extract_links(&doc);
1231        let link_count = links.len();
1232
1233        Ok(DocumentStructure {
1234            format: DocumentFormat::Pdf,
1235            title,
1236            headings,
1237            links,
1238            images: Vec::new(), // PDF image extraction is complex
1239            code_blocks: Vec::new(),
1240            plain_text,
1241            word_count,
1242            char_count,
1243            reading_time_minutes: (word_count / 200).max(1) as u32,
1244            stats: DocumentStats {
1245                heading_count,
1246                paragraph_count: page_count,
1247                link_count,
1248                ..Default::default()
1249            },
1250        })
1251    }
1252
1253    /// Extract text from a specific page
1254    fn extract_page_text(doc: &lopdf::Document, page_num: u32) -> Result<String, PdfParseError> {
1255        let page_id = doc
1256            .page_iter()
1257            .nth((page_num - 1) as usize)
1258            .ok_or_else(|| PdfParseError::ExtractionError(format!("Page {page_num} not found")))?;
1259
1260        let content = doc
1261            .get_page_content(page_id)
1262            .map_err(|e| PdfParseError::ExtractionError(e.to_string()))?;
1263
1264        // Parse content stream for text
1265        let text = Self::parse_content_stream(&content, doc);
1266        Ok(text)
1267    }
1268
1269    /// Parse PDF content stream to extract text
1270    fn parse_content_stream(content: &[u8], doc: &lopdf::Document) -> String {
1271        use lopdf::content::Content;
1272
1273        let mut text = String::new();
1274
1275        if let Ok(content_obj) = Content::decode(content) {
1276            for operation in content_obj.operations {
1277                match operation.operator.as_str() {
1278                    "Tj" | "TJ" => {
1279                        // Text showing operators
1280                        for operand in &operation.operands {
1281                            Self::extract_text_from_object(operand, doc, &mut text);
1282                        }
1283                    }
1284                    "'" | "\"" => {
1285                        // Text with newline
1286                        text.push('\n');
1287                        for operand in &operation.operands {
1288                            Self::extract_text_from_object(operand, doc, &mut text);
1289                        }
1290                    }
1291                    _ => {}
1292                }
1293            }
1294        }
1295
1296        text
1297    }
1298
1299    /// Extract text from a PDF object
1300    fn extract_text_from_object(obj: &lopdf::Object, _doc: &lopdf::Document, text: &mut String) {
1301        use lopdf::Object;
1302
1303        match obj {
1304            Object::String(bytes, _) => {
1305                // Try UTF-8 first, then PDFDocEncoding (Latin-1)
1306                if let Ok(s) = std::str::from_utf8(bytes) {
1307                    text.push_str(s);
1308                } else {
1309                    // Fall back to Latin-1
1310                    let s: String = bytes.iter().map(|&b| b as char).collect();
1311                    text.push_str(&s);
1312                }
1313            }
1314            Object::Array(arr) => {
1315                for item in arr {
1316                    match item {
1317                        Object::String(bytes, _) => {
1318                            if let Ok(s) = std::str::from_utf8(bytes) {
1319                                text.push_str(s);
1320                            } else {
1321                                let s: String = bytes.iter().map(|&b| b as char).collect();
1322                                text.push_str(&s);
1323                            }
1324                        }
1325                        Object::Integer(n) => {
1326                            // Negative numbers indicate kerning/spacing
1327                            if *n < -100 {
1328                                text.push(' ');
1329                            }
1330                        }
1331                        Object::Real(n) => {
1332                            if *n < -100.0 {
1333                                text.push(' ');
1334                            }
1335                        }
1336                        _ => {}
1337                    }
1338                }
1339            }
1340            _ => {}
1341        }
1342    }
1343
1344    /// Clean extracted text
1345    fn clean_extracted_text(text: &str) -> String {
1346        // Remove excessive whitespace and normalize
1347        let mut result = String::new();
1348        let mut last_was_space = true;
1349        let mut last_was_newline = true;
1350
1351        for c in text.chars() {
1352            if c == '\n' || c == '\r' {
1353                if !last_was_newline {
1354                    result.push('\n');
1355                    last_was_newline = true;
1356                    last_was_space = true;
1357                }
1358            } else if c.is_whitespace() {
1359                if !last_was_space {
1360                    result.push(' ');
1361                    last_was_space = true;
1362                }
1363            } else if c.is_control() {
1364                // Skip control characters
1365            } else {
1366                result.push(c);
1367                last_was_space = false;
1368                last_was_newline = false;
1369            }
1370        }
1371
1372        result.trim().to_string()
1373    }
1374
1375    /// Extract title from PDF metadata or content
1376    fn extract_title(doc: &lopdf::Document, text: &str) -> Option<String> {
1377        // Try to get title from PDF metadata
1378        if let Ok(info) = doc.trailer.get(b"Info") {
1379            if let Ok(lopdf::Object::Dictionary(dict)) = doc.get_object(info.as_reference().ok()?) {
1380                if let Ok(lopdf::Object::String(bytes, _)) = dict.get(b"Title") {
1381                    if let Ok(s) = std::str::from_utf8(bytes) {
1382                        let title = s.trim();
1383                        if !title.is_empty() {
1384                            return Some(title.to_string());
1385                        }
1386                    }
1387                }
1388            }
1389        }
1390
1391        // Fall back to first line that looks like a title
1392        for line in text.lines().take(10) {
1393            let trimmed = line.trim();
1394            if trimmed.len() > 3 && trimmed.len() < 200 {
1395                // Likely a title if it's reasonably sized and not a full paragraph
1396                let word_count = trimmed.split_whitespace().count();
1397                if word_count <= 15 && !trimmed.ends_with('.') {
1398                    return Some(trimmed.to_string());
1399                }
1400            }
1401        }
1402
1403        None
1404    }
1405
1406    /// Detect headings from text structure
1407    fn detect_headings(text: &str) -> Vec<Heading> {
1408        let mut headings = Vec::new();
1409        let lines: Vec<&str> = text.lines().collect();
1410        let numbered_heading = regex::Regex::new(r"^(\d+\.)+\d*\s+[A-Z]").ok();
1411
1412        for (i, line) in lines.iter().enumerate() {
1413            let trimmed = line.trim();
1414
1415            // Skip empty or very long lines
1416            if trimmed.is_empty() || trimmed.len() > 200 {
1417                continue;
1418            }
1419
1420            // Detect numbered headings (e.g., "1. Introduction", "1.2.3 Methods")
1421            if let Some(re) = &numbered_heading {
1422                if re.is_match(trimmed) {
1423                    let depth = trimmed.matches('.').count();
1424                    let level = (depth.min(5) + 1) as u8;
1425                    headings.push(Heading {
1426                        level,
1427                        text: trimmed.to_string(),
1428                        anchor: None,
1429                    });
1430                    continue;
1431                }
1432            }
1433
1434            // Detect ALL CAPS headings (common in PDFs)
1435            let word_count = trimmed.split_whitespace().count();
1436            if (1..=10).contains(&word_count)
1437                && trimmed
1438                    .chars()
1439                    .filter(|c| c.is_alphabetic())
1440                    .all(char::is_uppercase)
1441                && trimmed.chars().any(char::is_alphabetic)
1442            {
1443                headings.push(Heading {
1444                    level: 2,
1445                    text: trimmed.to_string(),
1446                    anchor: None,
1447                });
1448                continue;
1449            }
1450
1451            // Detect headings followed by blank line or significantly shorter
1452            if i + 1 < lines.len() {
1453                let next_line = lines[i + 1].trim();
1454                if next_line.is_empty() && word_count <= 8 && !trimmed.ends_with('.') {
1455                    // Check if it's capitalized like a title
1456                    if trimmed.chars().next().is_some_and(char::is_uppercase) {
1457                        headings.push(Heading {
1458                            level: 3,
1459                            text: trimmed.to_string(),
1460                            anchor: None,
1461                        });
1462                    }
1463                }
1464            }
1465        }
1466
1467        headings
1468    }
1469
1470    /// Extract links from PDF annotations
1471    fn extract_links(doc: &lopdf::Document) -> Vec<Link> {
1472        let mut links = Vec::new();
1473
1474        for (_page_num, page_id) in doc.get_pages() {
1475            if let Ok(lopdf::Object::Dictionary(dict)) = doc.get_object(page_id) {
1476                if let Ok(annots) = dict.get(b"Annots") {
1477                    Self::extract_links_from_annotations(doc, annots, &mut links);
1478                }
1479            }
1480        }
1481
1482        links
1483    }
1484
1485    /// Extract links from annotation array
1486    fn extract_links_from_annotations(
1487        doc: &lopdf::Document,
1488        annots: &lopdf::Object,
1489        links: &mut Vec<Link>,
1490    ) {
1491        let annot_refs = match annots {
1492            lopdf::Object::Array(arr) => arr.clone(),
1493            lopdf::Object::Reference(r) => {
1494                if let Ok(lopdf::Object::Array(arr)) = doc.get_object(*r) {
1495                    arr.clone()
1496                } else {
1497                    return;
1498                }
1499            }
1500            _ => return,
1501        };
1502
1503        for annot_ref in annot_refs {
1504            let annot = match &annot_ref {
1505                lopdf::Object::Reference(r) => doc.get_object(*r).ok().cloned(),
1506                obj => Some(obj.clone()),
1507            };
1508
1509            if let Some(lopdf::Object::Dictionary(dict)) = annot {
1510                // Check if it's a link annotation
1511                if let Ok(lopdf::Object::Name(subtype)) = dict.get(b"Subtype") {
1512                    if subtype == b"Link" {
1513                        // Extract URL from action
1514                        if let Ok(action) = dict.get(b"A") {
1515                            Self::extract_url_from_action(doc, action, links);
1516                        }
1517                    }
1518                }
1519            }
1520        }
1521    }
1522
1523    /// Extract URL from PDF action
1524    fn extract_url_from_action(
1525        doc: &lopdf::Document,
1526        action: &lopdf::Object,
1527        links: &mut Vec<Link>,
1528    ) {
1529        let action_dict = match action {
1530            lopdf::Object::Dictionary(dict) => dict.clone(),
1531            lopdf::Object::Reference(r) => {
1532                if let Ok(lopdf::Object::Dictionary(dict)) = doc.get_object(*r) {
1533                    dict.clone()
1534                } else {
1535                    return;
1536                }
1537            }
1538            _ => return,
1539        };
1540
1541        // Check for URI action
1542        if let Ok(lopdf::Object::Name(s)) = action_dict.get(b"S") {
1543            if s == b"URI" {
1544                if let Ok(lopdf::Object::String(bytes, _)) = action_dict.get(b"URI") {
1545                    if let Ok(url) = std::str::from_utf8(bytes) {
1546                        let is_external = url.starts_with("http://")
1547                            || url.starts_with("https://")
1548                            || url.starts_with("mailto:");
1549                        links.push(Link {
1550                            url: url.to_string(),
1551                            text: String::new(), // PDF links often don't have separate text
1552                            title: None,
1553                            is_external,
1554                        });
1555                    }
1556                }
1557            }
1558        }
1559    }
1560}
1561
1562/// PDF document metadata
1563#[derive(Debug, Clone, Serialize, Deserialize)]
1564pub struct PdfMetadata {
1565    /// PDF version
1566    pub version: String,
1567    /// Page count
1568    pub page_count: usize,
1569    /// Document title
1570    pub title: Option<String>,
1571    /// Document author
1572    pub author: Option<String>,
1573    /// Document subject
1574    pub subject: Option<String>,
1575    /// Document keywords
1576    pub keywords: Option<String>,
1577    /// Creator application
1578    pub creator: Option<String>,
1579    /// Producer application
1580    pub producer: Option<String>,
1581    /// Creation date
1582    pub creation_date: Option<String>,
1583    /// Modification date
1584    pub modification_date: Option<String>,
1585    /// Whether the PDF is encrypted
1586    pub is_encrypted: bool,
1587}
1588
1589impl PdfParser {
1590    /// Extract metadata from PDF
1591    pub fn extract_metadata(data: &[u8]) -> Result<PdfMetadata, PdfParseError> {
1592        use lopdf::Document;
1593
1594        let doc = Document::load_mem(data).map_err(|e| PdfParseError::ParseError(e.to_string()))?;
1595
1596        let page_count = doc.get_pages().len();
1597        let version = doc.version.clone();
1598        let is_encrypted = doc.is_encrypted();
1599
1600        let mut metadata = PdfMetadata {
1601            version,
1602            page_count,
1603            title: None,
1604            author: None,
1605            subject: None,
1606            keywords: None,
1607            creator: None,
1608            producer: None,
1609            creation_date: None,
1610            modification_date: None,
1611            is_encrypted,
1612        };
1613
1614        // Extract info dictionary
1615        if let Ok(info_ref) = doc.trailer.get(b"Info") {
1616            if let Ok(r) = info_ref.as_reference() {
1617                if let Ok(lopdf::Object::Dictionary(dict)) = doc.get_object(r) {
1618                    metadata.title = Self::get_string_from_dict(dict, b"Title");
1619                    metadata.author = Self::get_string_from_dict(dict, b"Author");
1620                    metadata.subject = Self::get_string_from_dict(dict, b"Subject");
1621                    metadata.keywords = Self::get_string_from_dict(dict, b"Keywords");
1622                    metadata.creator = Self::get_string_from_dict(dict, b"Creator");
1623                    metadata.producer = Self::get_string_from_dict(dict, b"Producer");
1624                    metadata.creation_date = Self::get_string_from_dict(dict, b"CreationDate");
1625                    metadata.modification_date = Self::get_string_from_dict(dict, b"ModDate");
1626                }
1627            }
1628        }
1629
1630        Ok(metadata)
1631    }
1632
1633    /// Get string value from dictionary
1634    fn get_string_from_dict(dict: &lopdf::Dictionary, key: &[u8]) -> Option<String> {
1635        if let Ok(lopdf::Object::String(bytes, _)) = dict.get(key) {
1636            if let Ok(s) = std::str::from_utf8(bytes) {
1637                let trimmed = s.trim();
1638                if !trimmed.is_empty() {
1639                    return Some(trimmed.to_string());
1640                }
1641            }
1642        }
1643        None
1644    }
1645}
1646
1647#[cfg(test)]
1648mod tests {
1649    use super::*;
1650
1651    #[test]
1652    fn test_format_detection_markdown() {
1653        let content = "# Hello World\n\nThis is a **test** document.";
1654        assert_eq!(DocumentFormat::detect(content), DocumentFormat::Markdown);
1655    }
1656
1657    #[test]
1658    fn test_format_detection_html() {
1659        let content = "<!DOCTYPE html><html><body><p>Hello</p></body></html>";
1660        assert_eq!(DocumentFormat::detect(content), DocumentFormat::Html);
1661    }
1662
1663    #[test]
1664    fn test_markdown_heading_parsing() {
1665        let content = "# Title\n\n## Section 1\n\n### Subsection\n\nSome text.";
1666        let structure = DocumentParser::parse(content);
1667
1668        assert_eq!(structure.headings.len(), 3);
1669        assert_eq!(structure.headings[0].level, 1);
1670        assert_eq!(structure.headings[0].text, "Title");
1671        assert_eq!(structure.headings[1].level, 2);
1672        assert_eq!(structure.headings[2].level, 3);
1673    }
1674
1675    #[test]
1676    fn test_markdown_link_extraction() {
1677        let content = "Check out [Rust](https://rust-lang.org) and [this](./local.md).";
1678        let structure = DocumentParser::parse(content);
1679
1680        assert_eq!(structure.links.len(), 2);
1681        assert!(structure.links[0].is_external);
1682        assert!(!structure.links[1].is_external);
1683    }
1684
1685    #[test]
1686    fn test_markdown_image_extraction() {
1687        let content = "![Alt text](image.png \"Title\")";
1688        let structure = DocumentParser::parse(content);
1689
1690        assert_eq!(structure.images.len(), 1);
1691        assert_eq!(structure.images[0].alt, "Alt text");
1692        assert_eq!(structure.images[0].src, "image.png");
1693    }
1694
1695    #[test]
1696    fn test_markdown_code_block_extraction() {
1697        let content = "```rust\nfn main() {}\n```";
1698        let structure = DocumentParser::parse(content);
1699
1700        assert_eq!(structure.code_blocks.len(), 1);
1701        assert_eq!(structure.code_blocks[0].language, Some("rust".to_string()));
1702    }
1703
1704    #[test]
1705    fn test_html_to_plain_text() {
1706        let html = "<p>Hello <strong>world</strong>!</p>";
1707        let plain = DocumentParser::html_to_plain_text(html);
1708        assert_eq!(plain, "Hello world !");
1709    }
1710
1711    #[test]
1712    fn test_quality_analysis() {
1713        let content = "# My Document\n\nThis is a test document with some content.\n\n## Section\n\nMore content here.";
1714        let structure = DocumentParser::parse(content);
1715        let quality = QualityAnalyzer::analyze(&structure);
1716
1717        assert!(quality.overall_score > 70);
1718        assert!(
1719            quality.issues.is_empty()
1720                || quality
1721                    .issues
1722                    .iter()
1723                    .all(|i| i.severity != IssueSeverity::Error)
1724        );
1725    }
1726}