hypersteeldb 0.5.5

//! Any-doc bridge — lower PDF / DOCX / PPTX / HTML / MD / TXT to plain text, natively in Rust, so the
//! folder ingest can project every readable document into the hypergraph (not just tabular/text files).
//! The extracted text feeds the same `TextEngine` (SPO tagger + SPLADE + gazetteer) as `.txt`.
//!
//! Text-layer extraction only: scanned/image PDFs (no text layer) yield little — OCR (the bundled
//! PP-OCRv6 models) is the follow-up. Gated behind the `docs` feature (implies `onnx`).

use std::io::Read;
use std::path::Path;

/// The document extensions this bridge can lower to text.
pub const DOC_EXTS: &[&str] = &["pdf", "docx", "pptx", "html", "htm", "md", "markdown", "txt", "text"];

pub fn is_doc_ext(ext: &str) -> bool {
    DOC_EXTS.contains(&ext)
}

/// Extract plain text from a document by extension. `Ok(None)` = unsupported type; `Err` = read/parse
/// failure the caller can report and skip.
pub fn extract_text(path: &Path) -> std::io::Result<Option<String>> {
    let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("").to_lowercase();
    let text = match ext.as_str() {
        "txt" | "text" | "md" | "markdown" => Some(std::fs::read_to_string(path)?),
        "html" | "htm" => Some(strip_html(&std::fs::read_to_string(path)?)),
        "docx" => Some(ooxml_text(path, "word/document.xml", &[])?),
        "pptx" => Some(ooxml_text(path, "", &["ppt/slides/slide"])?),
        "pdf" => extract_pdf(path),
        _ => None,
    };
    Ok(text.map(|t| t.trim().to_string()).filter(|t| !t.is_empty()))
}

fn extract_pdf(path: &Path) -> Option<String> {
    // Prefer poppler's `pdftotext` when available: it handles CID/CJK (Identity-H) fonts that the
    // pure-Rust path chokes on. Falls back to pdf-extract for Latin text layers when poppler is absent.
    if let Ok(out) = std::process::Command::new("pdftotext").arg("-q").arg(path).arg("-").output() {
        if out.status.success() {
            let t = String::from_utf8_lossy(&out.stdout).to_string();
            if !t.trim().is_empty() {
                return Some(t);
            }
        }
    }
    // pure-Rust fallback: pdf-extract panics on some encodings — contain AND silence it so one bad file
    // neither crashes nor spams folder ingest.
    let owned = path.to_path_buf();
    let prev = std::panic::take_hook();
    std::panic::set_hook(Box::new(|_| {}));
    let r = std::panic::catch_unwind(move || pdf_extract::extract_text(&owned).ok()).ok().flatten();
    std::panic::set_hook(prev);
    if let Some(t) = &r {
        if !t.trim().is_empty() {
            return r;
        }
    }
    // no text layer → scanned/image PDF: OCR the rasterized pages (PP-OCRv6).
    #[cfg(feature = "ocr")]
    {
        return crate::ocr::ocr_pdf(path);
    }
    #[cfg(not(feature = "ocr"))]
    r
}

/// Read text from an OOXML (zip) document. Either an exact `entry` (docx `word/document.xml`) or every
/// entry whose name starts with one of `prefixes` (pptx `ppt/slides/slideN.xml`), in sorted order.
fn ooxml_text(path: &Path, entry: &str, prefixes: &[&str]) -> std::io::Result<String> {
    let file = std::fs::File::open(path)?;
    let mut zip = zip::ZipArchive::new(file).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;

    let mut names: Vec<String> = (0..zip.len())
        .filter_map(|i| zip.by_index(i).ok().map(|f| f.name().to_string()))
        .filter(|n| (!entry.is_empty() && n == entry) || prefixes.iter().any(|p| n.starts_with(p) && n.ends_with(".xml")))
        .collect();
    names.sort();

    let mut out = String::new();
    for name in names {
        let mut xml = String::new();
        if let Ok(mut f) = zip.by_name(&name) {
            if f.read_to_string(&mut xml).is_ok() {
                out.push_str(&xml_text(&xml));
                out.push('\n');
            }
        }
    }
    Ok(out)
}

/// Pull the visible text out of a WordprocessingML / PresentationML part: the `<w:t>` / `<a:t>` runs
/// (local name `t`), with a newline at each paragraph end (`<w:p>` / `<a:p>`, local name `p`).
fn xml_text(xml: &str) -> String {
    use quick_xml::events::Event;
    use quick_xml::reader::Reader;

    let mut reader = Reader::from_str(xml);
    let mut out = String::new();
    let mut in_t = false;
    let local = |name: &[u8]| -> Vec<u8> { name.rsplit(|&b| b == b':').next().unwrap_or(name).to_vec() };

    loop {
        match reader.read_event() {
            Ok(Event::Start(e)) if local(e.name().as_ref()) == b"t" => in_t = true,
            Ok(Event::End(e)) => match local(e.name().as_ref()).as_slice() {
                b"t" => in_t = false,
                b"p" => out.push('\n'),
                _ => {}
            },
            Ok(Event::Text(t)) if in_t => {
                if let Ok(s) = t.unescape() {
                    out.push_str(&s);
                }
            }
            Ok(Event::Eof) | Err(_) => break,
            _ => {}
        }
    }
    out
}

/// Minimal HTML → text: drop `<script>`/`<style>` blocks, strip tags, decode common entities, collapse
/// whitespace. Enough for retrieval; not a full renderer.
pub fn strip_html(html: &str) -> String {
    let mut s = html.to_string();
    for tag in ["script", "style"] {
        loop {
            let lower = s.to_lowercase();
            let open = format!("<{tag}");
            let close = format!("</{tag}>");
            match (lower.find(&open), lower.find(&close)) {
                (Some(a), Some(b)) if b > a => s.replace_range(a..b + close.len(), " "),
                _ => break,
            }
        }
    }
    // strip tags
    let mut out = String::with_capacity(s.len());
    let mut in_tag = false;
    for ch in s.chars() {
        match ch {
            '<' => in_tag = true,
            '>' => in_tag = false,
            _ if !in_tag => out.push(ch),
            _ => {}
        }
    }
    // decode a handful of entities + collapse whitespace
    let out = out
        .replace("&amp;", "&")
        .replace("&lt;", "<")
        .replace("&gt;", ">")
        .replace("&quot;", "\"")
        .replace("&#39;", "'")
        .replace("&nbsp;", " ");
    out.split_whitespace().collect::<Vec<_>>().join(" ")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn html_stripped() {
        let h = "<html><head><style>a{color:red}</style></head><body><h1>Title</h1><p>Hello &amp; welcome</p><script>x=1</script></body></html>";
        let t = strip_html(h);
        assert!(t.contains("Title"));
        assert!(t.contains("Hello & welcome"));
        assert!(!t.contains("color"));
        assert!(!t.contains("x=1"));
    }

    #[test]
    fn xml_runs_extracted() {
        let x = r#"<w:document><w:body><w:p><w:r><w:t>Hello</w:t></w:r><w:r><w:t xml:space="preserve"> world</w:t></w:r></w:p><w:p><w:r><w:t>Second</w:t></w:r></w:p></w:body></w:document>"#;
        let t = super::xml_text(x);
        assert!(t.contains("Hello world"));
        assert!(t.contains("Second"));
    }
}