xberg-native-pdf 1.3.0

//! Hyphenation-aware text reconstruction for PDF extraction.
//!
//! This module handles the reconstruction of words that have been split across
//! line breaks with hyphens. Per PDF Specification Section 5.3.4, soft hyphens
//! (U+00AD) indicate optional line breaks, while hard hyphens (U+002D) are
//! always present.
//!
//! # Problem
//!
//! PDFs often contain text like:
//! - "Govern-" (line 1) + "ment" (line 2) → Should become "Government"
//! - "content-" (line 1) + "coding" (line 2) → Should remain "content-coding" (compound)
//!
//! # Solution
//!
//! This module provides post-processing to:
//! 1. Detect line-ending hyphens that indicate word continuation
//! 2. Distinguish between soft hyphens (word breaks) and hard hyphens (compound words)
//! 3. Reconstruct complete words while preserving intentional hyphenation
//!
//! # PDF Spec References
//!
//! - Section 5.3.4: String Objects (soft vs hard hyphens)
//! - Section 14.6: Marked Content (ActualText for true character sequences)

/// Hyphenation handler for reconstructing split words.
///
/// Processes text to join words that were split across line breaks with hyphens.
#[derive(Debug, Clone)]
pub struct HyphenationHandler {
    /// Minimum word length for the second part to trigger joining
    /// (prevents joining single letters that might be list markers)
    min_continuation_length: usize,

    /// Whether to preserve compound words (e.g., "content-coding")
    preserve_compounds: bool,
}

impl Default for HyphenationHandler {
    fn default() -> Self {
        Self::new()
    }
}

impl HyphenationHandler {
    /// Create a new hyphenation handler with default settings.
    pub fn new() -> Self {
        Self {
            min_continuation_length: 2,
            preserve_compounds: true,
        }
    }

    /// Set minimum continuation length for word joining.
    pub fn with_min_continuation_length(mut self, len: usize) -> Self {
        self.min_continuation_length = len;
        self
    }

    /// Set whether to preserve compound words.
    pub fn with_preserve_compounds(mut self, preserve: bool) -> Self {
        self.preserve_compounds = preserve;
        self
    }

    /// Check if a line ends with a continuation hyphen.
    ///
    /// A continuation hyphen is:
    /// - A soft hyphen (U+00AD) - ALWAYS indicates word continuation
    /// - A hard hyphen (U+002D) at the end of a line with alphabetic text before
    /// - NOT a hyphen followed by whitespace (e.g., "- " is not continuation)
    ///
    /// Per PDF Specification Section 5.3.4 and Unicode Standard:
    /// - Soft hyphen (U+00AD) marks an optional line break point
    /// - When a word is broken at a soft hyphen, it should be rejoined without the hyphen
    ///
    /// # Arguments
    ///
    /// * `text` - The text to check
    ///
    /// # Returns
    ///
    /// `true` if the text ends with a continuation hyphen
    pub fn is_continuation_hyphen(text: &str) -> bool {
        let trimmed = text.trim_end();
        if trimmed.is_empty() {
            return false;
        }

        if trimmed.ends_with('\u{00AD}') {
            let before_hyphen = &trimmed[..trimmed.len() - '\u{00AD}'.len_utf8()];
            return before_hyphen.chars().last().is_some_and(|c| c.is_alphabetic());
        }

        if !trimmed.ends_with('-') {
            return false;
        }

        // Check that there's at least one letter before the hyphen
        // (avoids matching bullet points like "- item") ~keep
        let before_hyphen = &trimmed[..trimmed.len() - 1];
        before_hyphen.chars().last().is_some_and(|c| c.is_alphabetic())
    }

    /// Check if a word appears to be a compound word that should keep its hyphen.
    ///
    /// Compound words are identified by:
    /// - Common prefixes: non-, self-, re-, pre-, anti-, etc.
    /// - Common technical patterns: content-type, user-agent, etc.
    ///
    /// # Arguments
    ///
    /// * `first_part` - The part before the hyphen
    /// * `second_part` - The part after the hyphen
    ///
    /// # Returns
    ///
    /// `true` if this appears to be a compound word
    fn is_compound_word(first_part: &str, second_part: &str) -> bool {
        let first_lower = first_part.to_lowercase();

        let compound_prefixes = [
            "self", "non", "anti", "pre", "post", "re", "co", "ex", "multi", "semi", "sub", "super", "ultra", "under",
            "over", "cross", "inter", "intra", "counter", "mid", "well", "ill", "all", "half", "high", "low", "full",
            "part", "short", "long", "hard", "soft",
        ];

        if compound_prefixes.contains(&first_lower.as_str()) {
            return true;
        }

        // Technical compounds (common in RFC/technical documents) ~keep
        let technical_patterns = [
            ("content", "type"),
            ("content", "length"),
            ("content", "encoding"),
            ("content", "coding"),
            ("user", "agent"),
            ("cache", "control"),
            ("product", "version"),
            ("media", "type"),
        ];

        for (prefix, suffix) in &technical_patterns {
            if first_lower == *prefix && second_part.to_lowercase().starts_with(suffix) {
                return true;
            }
        }

        // If second part starts with lowercase, likely a compound (not sentence continuation)
        // e.g., "content-coding" vs "Govern-" + "ment" ~keep
        if let Some(first_char) = second_part.chars().next()
            && first_char.is_lowercase()
            && first_part.chars().last().is_some_and(|c| c.is_lowercase())
        {
            // Both parts lowercase - more likely compound
            // But check if the combined form is a common word ~keep
            let combined = format!("{}{}", first_part, second_part);
            if is_common_word(&combined) {
                return false;
            }
            return true;
        }

        false
    }

    /// Process a single line pair to potentially join hyphenated words.
    ///
    /// # Arguments
    ///
    /// * `current_line` - The line that may end with a hyphen
    /// * `next_line` - The line that may continue the word
    ///
    /// # Returns
    ///
    /// A tuple of (processed_text, consumed_next_line) where:
    /// - `processed_text` is the current line (possibly with word joined)
    /// - `consumed_next_line` is true if the next line was consumed (joined)
    pub fn process_line_pair(&self, current_line: &str, next_line: &str) -> (String, bool) {
        let trimmed_current = current_line.trim_end();

        if !Self::is_continuation_hyphen(trimmed_current) {
            return (current_line.to_string(), false);
        }

        let trimmed_next = next_line.trim_start();

        let next_word = trimmed_next.split_whitespace().next().unwrap_or("");

        if next_word.len() < self.min_continuation_length {
            return (current_line.to_string(), false);
        }

        // Get the last word (before hyphen) of current line
        // Handle both soft hyphen (U+00AD, 2 bytes UTF-8) and hard hyphen (U+002D, 1 byte) ~keep
        let without_hyphen = if trimmed_current.ends_with('\u{00AD}') {
            &trimmed_current[..trimmed_current.len() - '\u{00AD}'.len_utf8()]
        } else {
            &trimmed_current[..trimmed_current.len() - 1]
        };
        let last_word = without_hyphen.split_whitespace().next_back().unwrap_or(without_hyphen);

        // Check if this is a compound word that should keep hyphen
        // Note: Soft hyphens are NEVER compound word markers, only hard hyphens ~keep
        let is_soft_hyphen = trimmed_current.ends_with('\u{00AD}');
        if !is_soft_hyphen && self.preserve_compounds && Self::is_compound_word(last_word, next_word) {
            return (current_line.to_string(), false);
        }

        let hyphen_len = if is_soft_hyphen { '\u{00AD}'.len_utf8() } else { 1 };
        let prefix = &trimmed_current[..trimmed_current.len() - last_word.len() - hyphen_len];
        let joined_word = format!("{}{}", last_word, next_word);

        let rest_of_next = trimmed_next[next_word.len()..].trim_start();
        let mut result = if prefix.is_empty() {
            joined_word
        } else {
            format!("{} {}", prefix.trim_end(), joined_word)
        };

        if !rest_of_next.is_empty() {
            result.push(' ');
            result.push_str(rest_of_next);
        }

        (result, true)
    }

    /// Process a complete text block and join hyphenated words.
    ///
    /// # Arguments
    ///
    /// * `text` - The complete text block to process
    ///
    /// # Returns
    ///
    /// The text with hyphenated words joined
    pub fn process_text(&self, text: &str) -> String {
        let lines: Vec<&str> = text.lines().collect();
        if lines.is_empty() {
            return String::new();
        }

        let mut result = Vec::with_capacity(lines.len());
        let mut i = 0;

        while i < lines.len() {
            if i + 1 < lines.len() {
                let (processed, consumed) = self.process_line_pair(lines[i], lines[i + 1]);
                result.push(processed);
                if consumed {
                    i += 2;
                    continue;
                }
            } else {
                result.push(lines[i].to_string());
            }
            i += 1;
        }

        let mut output = result.join("\n");

        if text.ends_with('\n') && !output.ends_with('\n') {
            output.push('\n');
        }

        output
    }
}

/// Check if a word is a common English word (basic dictionary).
///
/// This is used to identify when a hyphenated word should be joined
/// vs when it's a compound word.
fn is_common_word(word: &str) -> bool {
    let common_words = [
        "government",
        "department",
        "information",
        "administration",
        "documentation",
        "implementation",
        "communication",
        "organization",
        "representation",
        "transportation",
        "investigation",
        "determination",
        "consideration",
        "recommendation",
        "responsibility",
        "understanding",
        "international",
        "environmental",
        "constitutional",
        "congressional",
        "agricultural",
        "professional",
        "manufacturing",
        "requirements",
        "development",
        "management",
        "performance",
        "maintenance",
        "compliance",
        "procedures",
        "regulations",
        "activities",
        "operations",
        "provisions",
        "conditions",
        "limitations",
        "applications",
        "publications",
    ];

    let lower = word.to_lowercase();
    common_words.contains(&lower.as_str())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_is_continuation_hyphen_basic() {
        assert!(HyphenationHandler::is_continuation_hyphen("Govern-"));
        assert!(HyphenationHandler::is_continuation_hyphen("content-"));
        assert!(HyphenationHandler::is_continuation_hyphen("word-  "));
    }

    #[test]
    fn test_is_continuation_hyphen_negative() {
        assert!(!HyphenationHandler::is_continuation_hyphen("- bullet"));
        assert!(!HyphenationHandler::is_continuation_hyphen(""));
        assert!(!HyphenationHandler::is_continuation_hyphen("no hyphen"));
        assert!(!HyphenationHandler::is_continuation_hyphen("123-"));
    }

    #[test]
    fn test_is_continuation_hyphen_soft_hyphen() {
        assert!(HyphenationHandler::is_continuation_hyphen("Govern\u{00AD}"));
        assert!(HyphenationHandler::is_continuation_hyphen("busi\u{00AD}"));
        assert!(HyphenationHandler::is_continuation_hyphen("word\u{00AD}  "));
    }

    #[test]
    fn test_is_continuation_hyphen_soft_hyphen_negative() {
        assert!(!HyphenationHandler::is_continuation_hyphen("123\u{00AD}"));
        assert!(!HyphenationHandler::is_continuation_hyphen("\u{00AD}"));
    }

    #[test]
    fn test_is_compound_word() {
        assert!(HyphenationHandler::is_compound_word("self", "regulation"));
        assert!(HyphenationHandler::is_compound_word("non", "linear"));
        assert!(HyphenationHandler::is_compound_word("content", "type"));
    }

    #[test]
    fn test_process_line_pair_join() {
        let handler = HyphenationHandler::new();
        let (result, consumed) = handler.process_line_pair("Govern-", "ment of the");
        assert!(consumed);
        assert_eq!(result, "Government of the");
    }

    #[test]
    fn test_process_line_pair_preserve_compound() {
        let handler = HyphenationHandler::new();
        let (result, consumed) = handler.process_line_pair("self-", "regulation");
        assert!(!consumed);
        assert_eq!(result, "self-");
    }

    #[test]
    fn test_process_line_pair_soft_hyphen() {
        let handler = HyphenationHandler::new();
        // Soft hyphen should ALWAYS join (never a compound word marker) ~keep
        let (result, consumed) = handler.process_line_pair("busi\u{00AD}", "ness today");
        assert!(consumed);
        assert_eq!(result, "business today");
    }

    #[test]
    fn test_process_line_pair_soft_hyphen_always_joins() {
        let handler = HyphenationHandler::new();
        // Even compound-looking words with soft hyphen should join
        // because soft hyphen is ONLY used for optional line breaks ~keep
        let (result, consumed) = handler.process_line_pair("self\u{00AD}", "regulation");
        assert!(consumed);
        assert_eq!(result, "selfregulation");
    }

    #[test]
    fn test_process_text_multiple_lines() {
        let handler = HyphenationHandler::new();
        let text = "The Govern-\nment issued a\nstate-\nment today.";
        let result = handler.process_text(text);
        assert!(result.contains("Government"));
    }

    #[test]
    fn test_process_text_no_hyphen() {
        let handler = HyphenationHandler::new();
        let text = "Normal text\nwith no hyphens\nat line ends.";
        let result = handler.process_text(text);
        assert_eq!(result, text);
    }
}