mimirs_embed/

lib.rs

//! mimirs-embed: LFM2.5-VL-450M embedding provider
//!
//! Produces 1024-dimensional f32 embeddings from text, images, documents,
//! and media files via the LFM2.5-VL multimodal model.
//!
//! Supported formats:
//!   Text:      .txt, .md, .rst, .log, .rs, .py, .js, .ts, .go, .c, .cpp,
//!              .h, .hpp, .java, .rb, .sh, .toml, .yaml, .yml, .json, .xml,
//!              .html, .htm, .css, .scss, .less, .sql, .php, .swift, .kt,
//!              .scala, .r, .m, .mm, .lua, .pl, .pm, .vim, .el, .lisp, .hs,
//!              .ml, .fs, .fsx, .clj, .ex, .exs, .erl, .hrl
//!   Rich text: .rtf
//!   Word:      .docx, .doc (via antiword)
//!   Excel:     .xlsx, .xls, .csv, .tsv
//!   PowerPoint:.pptx, .ppt (via libreoffice)
//!   OpenDoc:   .odt, .ods, .odp
//!   ePub:      .epub
//!   LaTeX:     .tex, .latex, .bib
//!   PDF:       .pdf
//!   Images:    .png, .jpg, .jpeg, .webp, .bmp, .gif, .tiff, .tif, .svg,
//!              .ico, .heic, .heif, .avif
//!   Audio:     .mp3, .wav, .flac, .ogg, .m4a, .aac, .wma, .opus
//!   Video:     .mp4, .mkv, .avi, .webm, .mov, .wmv, .flv, .m4v, .mpg,
//!              .mpeg, .3gp
//!   Archives:  .zip, .tar, .gz, .bz2, .xz, .7z, .rar (extracts & embeds
//!              contained text files)

use std::fs;
use std::io::Read;

use std::path::{Path, PathBuf};
use std::process::Command;

use ndarray::Array1;
use sha2::{Digest, Sha256};

/// Default embedding dimension for LFM2.5-VL-450M.
///
/// Alias for [`mimirs_core::EMBED_DIM`] — the single source of truth.
pub const EMBED_DIM: usize = mimirs_core::EMBED_DIM;

/// HuggingFace download URL for the LFM2.5-VL-450M GGUF model.
pub const MODEL_URL: &str = "https://huggingface.co/LiquidAI/LFM2.5-VL-450M-GGUF/resolve/main/LFM2.5-VL-450M-Q8_0.gguf?download=true";

/// Model filename after download.
pub const MODEL_FILENAME: &str = "LFM2.5-VL-450M-Q8_0.gguf";

/// Errors that can occur during embedding.
#[derive(Debug, thiserror::Error)]
pub enum EmbedError {
    #[error("IO error: {0}")]
    Io(#[from] std::io::Error),
    #[error("Model not found at {0}")]
    ModelNotFound(PathBuf),
    #[error("llama.cpp binary not found: {0}")]
    LlamaNotFound(String),
    #[error("Unsupported format: {0}")]
    UnsupportedFormat(String),
    #[error("Image decode error: {0}")]
    ImageError(String),
    #[error("PDF extraction error: {0}")]
    PdfError(String),
    #[error("ZIP extraction error: {0}")]
    ZipError(String),
    #[error("Archive error: {0}")]
    ArchiveError(String),
    #[error("XML parse error: {0}")]
    XmlError(String),
    #[error("Fallback embedding (llama.cpp unavailable)")]
    Fallback,
}

/// LFM2.5-VL embedding provider.
///
/// Owns the model path and can produce embeddings from text, images, and
/// multimodal (text+image) inputs. Falls back to deterministic hash-based
/// embeddings when llama.cpp is unavailable.
pub struct Lfm2Embedder {
    model_path: PathBuf,
    cache_dir: PathBuf,
}

impl Default for Lfm2Embedder {
    fn default() -> Self {
        Self::new()
    }
}

impl Lfm2Embedder {
    /// Creates a new embedder. The model will be downloaded on first use.
    pub fn new() -> Self {
        let cache_dir = dirs::cache_dir()
            .unwrap_or_else(|| PathBuf::from("/tmp"))
            .join("mimirswell")
            .join("models");
        let model_path = cache_dir.join(MODEL_FILENAME);
        Self {
            model_path,
            cache_dir,
        }
    }

    /// Ensures the GGUF model is downloaded, downloading if necessary.
    fn ensure_model(&self) -> Result<(), EmbedError> {
        if self.model_path.exists() {
            return Ok(());
        }
        fs::create_dir_all(&self.cache_dir)?;
        eprintln!("[mimirs-embed] Downloading LFM2.5-VL-450M from HuggingFace...");
        let status = Command::new("curl")
            .args([
                "-L",
                "-o",
                self.model_path.to_string_lossy().as_ref(),
                MODEL_URL,
            ])
            .status()?;
        if !status.success() {
            return Err(EmbedError::Io(std::io::Error::other(
                "curl download failed",
            )));
        }
        Ok(())
    }

    /// Returns the path to the llama-embedding binary, or None if not found.
    fn find_llama_embedding(&self) -> Option<PathBuf> {
        which::which("llama-embedding").ok().or_else(|| {
            // Check common build locations
            let candidates = [
                "/usr/local/bin/llama-embedding",
                "/opt/llama.cpp/bin/llama-embedding",
                "/usr/bin/llama-embedding",
            ];
            candidates.iter().map(PathBuf::from).find(|p| p.exists())
        })
    }

    /// Produces an embedding from text via llama.cpp.
    fn embed_via_llama(&self, text: &str) -> Result<Array1<f64>, EmbedError> {
        self.ensure_model()?;
        let bin = self
            .find_llama_embedding()
            .ok_or_else(|| EmbedError::LlamaNotFound("llama-embedding not in PATH".into()))?;
        let output = Command::new(&bin)
            .args([
                "-m",
                self.model_path.to_string_lossy().as_ref(),
                "-c",
                "512",
                "--embd-output-format",
                "array",
                "--embd-separator",
                " ",
                "-p",
                text,
            ])
            .output()?;
        if !output.status.success() {
            return Err(EmbedError::Io(std::io::Error::other(
                String::from_utf8_lossy(&output.stderr).to_string(),
            )));
        }
        parse_embedding_output(&output.stdout)
    }

    /// Produces an embedding from an image via llama.cpp's --image flag.
    fn embed_image_via_llama(&self, image_path: &Path) -> Result<Array1<f64>, EmbedError> {
        self.ensure_model()?;
        let bin = self
            .find_llama_embedding()
            .ok_or_else(|| EmbedError::LlamaNotFound("llama-embedding not in PATH".into()))?;
        let output = Command::new(&bin)
            .args([
                "-m",
                self.model_path.to_string_lossy().as_ref(),
                "-c",
                "512",
                "--embd-output-format",
                "array",
                "--image",
                image_path.to_string_lossy().as_ref(),
            ])
            .output()?;
        if !output.status.success() {
            return Err(EmbedError::Io(std::io::Error::other(
                String::from_utf8_lossy(&output.stderr).to_string(),
            )));
        }
        parse_embedding_output(&output.stdout)
    }

    /// Produces a joint text+image embedding via llama.cpp.
    fn embed_multimodal_via_llama(
        &self,
        text: &str,
        image_path: &Path,
    ) -> Result<Array1<f64>, EmbedError> {
        self.ensure_model()?;
        let bin = self
            .find_llama_embedding()
            .ok_or_else(|| EmbedError::LlamaNotFound("llama-embedding not in PATH".into()))?;
        let output = Command::new(&bin)
            .args([
                "-m",
                self.model_path.to_string_lossy().as_ref(),
                "-c",
                "512",
                "--embd-output-format",
                "array",
                "--image",
                image_path.to_string_lossy().as_ref(),
                "-p",
                text,
            ])
            .output()?;
        if !output.status.success() {
            return Err(EmbedError::Io(std::io::Error::other(
                String::from_utf8_lossy(&output.stderr).to_string(),
            )));
        }
        parse_embedding_output(&output.stdout)
    }

    // ── Public API ──────────────────────────────────────────────────────

    /// Embeds a text string.
    pub fn embed(&self, text: &str) -> Array1<f64> {
        self.embed_via_llama(text)
            .unwrap_or_else(|_| hash_embedding(text.as_bytes()))
    }

    /// Embeds an image file (PNG, JPEG, WebP, BMP, GIF, TIFF, SVG, HEIC, AVIF, ICO).
    pub fn embed_image(&self, path: &Path) -> Array1<f64> {
        // Try to render/convert to pixels first for formats that need it
        let pixel_path = match self.load_image_pixels(path) {
            Ok(p) => p,
            Err(_) => {
                return self
                    .embed_image_via_llama(path)
                    .unwrap_or_else(|_| hash_embedding(&fs::read(path).unwrap_or_default()));
            }
        };
        self.embed_image_via_llama(&pixel_path)
            .unwrap_or_else(|_| hash_embedding(&fs::read(&pixel_path).unwrap_or_default()))
    }

    /// Embeds text and an image jointly.
    pub fn embed_multimodal(&self, text: &str, image_path: &Path) -> Array1<f64> {
        let pixel_path = self
            .load_image_pixels(image_path)
            .unwrap_or_else(|_| image_path.to_path_buf());
        self.embed_multimodal_via_llama(text, &pixel_path)
            .unwrap_or_else(|_| {
                let mut combined = text.as_bytes().to_vec();
                if let Ok(img_bytes) = fs::read(&pixel_path) {
                    combined.extend_from_slice(&img_bytes);
                }
                hash_embedding(&combined)
            })
    }

    /// Embeds a PDF document (text + rendered page images).
    pub fn embed_pdf(&self, path: &Path) -> Array1<f64> {
        let text = self.extract_pdf_text(path);
        if text.trim().is_empty() {
            return hash_embedding(&fs::read(path).unwrap_or_default());
        }
        self.embed(&text)
    }

    /// Embeds a PowerPoint file (.pptx or .ppt).
    pub fn embed_presentation(&self, path: &Path) -> Array1<f64> {
        let ext = path
            .extension()
            .and_then(|e| e.to_str())
            .unwrap_or("")
            .to_lowercase();
        if ext == "pptx" {
            self.embed_pptx(path)
        } else {
            // Legacy .ppt — try libreoffice conversion
            self.embed_via_libreoffice(path)
        }
    }

    /// Embeds a Word document (.docx or .doc).
    pub fn embed_document(&self, path: &Path) -> Array1<f64> {
        let ext = path
            .extension()
            .and_then(|e| e.to_str())
            .unwrap_or("")
            .to_lowercase();
        if ext == "docx" {
            self.embed_docx(path)
        } else if ext == "doc" {
            // Try antiword for legacy .doc
            if let Ok(text) = self.run_command("antiword", &[path.to_string_lossy().as_ref()]) {
                return self.embed(&text);
            }
            self.embed_via_libreoffice(path)
        } else {
            hash_embedding(&fs::read(path).unwrap_or_default())
        }
    }

    /// Embeds a spreadsheet (.xlsx, .xls, .csv, .tsv, .ods).
    pub fn embed_spreadsheet(&self, path: &Path) -> Array1<f64> {
        let ext = path
            .extension()
            .and_then(|e| e.to_str())
            .unwrap_or("")
            .to_lowercase();
        match ext.as_str() {
            "csv" | "tsv" => {
                let text = fs::read_to_string(path).unwrap_or_default();
                self.embed(&text)
            }
            "xlsx" | "xls" | "ods" => {
                if let Ok(text) = self.extract_spreadsheet_text(path, &ext) {
                    return self.embed(&text);
                }
                self.embed_via_libreoffice(path)
            }
            _ => hash_embedding(&fs::read(path).unwrap_or_default()),
        }
    }

    /// Embeds an OpenDocument file (.odt, .ods, .odp).
    pub fn embed_opendocument(&self, path: &Path) -> Array1<f64> {
        let ext = path
            .extension()
            .and_then(|e| e.to_str())
            .unwrap_or("")
            .to_lowercase();
        match ext.as_str() {
            "odt" => {
                if let Ok(text) = self.extract_odt_text(path) {
                    return self.embed(&text);
                }
            }
            "ods" | "odp" => {
                if let Ok(text) = self.extract_opendocument_xml(path) {
                    return self.embed(&text);
                }
            }
            _ => {}
        }
        self.embed_via_libreoffice(path)
    }

    /// Embeds an ePub file.
    pub fn embed_epub(&self, path: &Path) -> Array1<f64> {
        let text = self.extract_epub_text(path);
        if text.trim().is_empty() {
            hash_embedding(&fs::read(path).unwrap_or_default())
        } else {
            self.embed(&text)
        }
    }

    /// Embeds an RTF file.
    pub fn embed_rtf(&self, path: &Path) -> Array1<f64> {
        let raw = fs::read_to_string(path).unwrap_or_default();
        let text = strip_rtf(&raw);
        self.embed(&text)
    }

    /// Embeds a LaTeX file.
    pub fn embed_latex(&self, path: &Path) -> Array1<f64> {
        let text = fs::read_to_string(path).unwrap_or_default();
        self.embed(&text)
    }

    /// Embeds an audio file (extracts metadata, falls back to hash).
    pub fn embed_audio(&self, path: &Path) -> Array1<f64> {
        // Try to extract metadata via symphonia
        if let Ok(text) = self.extract_audio_metadata(path)
            && !text.trim().is_empty()
        {
            return self.embed(&text);
        }
        hash_embedding(&fs::read(path).unwrap_or_default())
    }

    /// Embeds a video file (extracts keyframes as images, embeds each, averages).
    pub fn embed_video(&self, path: &Path) -> Array1<f64> {
        // Try ffmpeg to extract keyframes
        if let Ok(frames) = self.extract_video_keyframes(path)
            && !frames.is_empty()
        {
            let mut sum = Array1::zeros(EMBED_DIM);
            let mut count = 0u32;
            for frame in &frames {
                let e = self
                    .embed_image_via_llama(frame)
                    .unwrap_or_else(|_| hash_embedding(&fs::read(frame).unwrap_or_default()));
                sum = sum + e;
                count += 1;
            }
            if count > 0 {
                return sum / count as f64;
            }
        }
        hash_embedding(&fs::read(path).unwrap_or_default())
    }

    /// Embeds an archive by extracting and embedding contained text files.
    pub fn embed_archive(&self, path: &Path) -> Array1<f64> {
        let ext = path
            .extension()
            .and_then(|e| e.to_str())
            .unwrap_or("")
            .to_lowercase();
        let texts = match ext.as_str() {
            "zip" => self.extract_zip_texts(path),
            "tar" | "gz" | "bz2" | "xz" => self.extract_tar_texts(path),
            "7z" => self.extract_7z_texts(path),
            "rar" => self.extract_rar_texts(path),
            _ => Ok(vec![]),
        };
        match texts {
            Ok(t) if !t.is_empty() => {
                let mut sum = Array1::zeros(EMBED_DIM);
                let mut count = 0u32;
                for text in &t {
                    let e = self.embed(text);
                    sum = sum + e;
                    count += 1;
                }
                if count > 0 {
                    sum / count as f64
                } else {
                    hash_embedding(&[])
                }
            }
            _ => hash_embedding(&fs::read(path).unwrap_or_default()),
        }
    }

    /// Auto-detects the file format and embeds accordingly.
    pub fn embed_file(&self, path: &Path) -> Array1<f64> {
        let ext = path
            .extension()
            .and_then(|e| e.to_str())
            .unwrap_or("")
            .to_lowercase();
        match ext.as_str() {
            // Plain text & code
            "txt" | "md" | "rst" | "log" | "rs" | "py" | "js" | "ts" | "go" | "c" | "cpp" | "h"
            | "hpp" | "java" | "rb" | "sh" | "toml" | "yaml" | "yml" | "json" | "xml" | "html"
            | "htm" | "css" | "scss" | "less" | "sql" | "php" | "swift" | "kt" | "scala" | "r"
            | "m" | "mm" | "lua" | "pl" | "pm" | "vim" | "el" | "lisp" | "hs" | "ml" | "fs"
            | "fsx" | "clj" | "ex" | "exs" | "erl" | "hrl" => {
                let text = fs::read_to_string(path).unwrap_or_default();
                self.embed(&text)
            }
            "rtf" => self.embed_rtf(path),
            "tex" | "latex" | "bib" => self.embed_latex(path),
            "pdf" => self.embed_pdf(path),
            "docx" | "doc" => self.embed_document(path),
            "xlsx" | "xls" | "csv" | "tsv" | "ods" => self.embed_spreadsheet(path),
            "pptx" | "ppt" => self.embed_presentation(path),
            "odt" | "odp" => self.embed_opendocument(path),
            "epub" => self.embed_epub(path),
            // Images
            "png" | "jpg" | "jpeg" | "webp" | "bmp" | "gif" | "tiff" | "tif" | "svg" | "ico"
            | "heic" | "heif" | "avif" => self.embed_image(path),
            // Audio
            "mp3" | "wav" | "flac" | "ogg" | "m4a" | "aac" | "wma" | "opus" => {
                self.embed_audio(path)
            }
            // Video
            "mp4" | "mkv" | "avi" | "webm" | "mov" | "wmv" | "flv" | "m4v" | "mpg" | "mpeg"
            | "3gp" => self.embed_video(path),
            // Archives
            "zip" | "tar" | "gz" | "bz2" | "xz" | "7z" | "rar" => self.embed_archive(path),
            // Unknown — try as text, fall back to hash
            _ => {
                if let Ok(text) = fs::read_to_string(path) {
                    self.embed(&text)
                } else {
                    hash_embedding(&fs::read(path).unwrap_or_default())
                }
            }
        }
    }

    // ── Internal helpers ─────────────────────────────────────────────────

    /// Loads an image, converts to 336x336 RGB PNG, returns the temp path.
    fn load_image_pixels(&self, path: &Path) -> Result<PathBuf, EmbedError> {
        let ext = path
            .extension()
            .and_then(|e| e.to_str())
            .unwrap_or("")
            .to_lowercase();
        let img = if ext == "svg" {
            self.render_svg_to_image(path)?
        } else {
            image::open(path).map_err(|e| EmbedError::ImageError(e.to_string()))?
        };
        let resized = img.resize_exact(336, 336, image::imageops::FilterType::Triangle);
        let mut tmp = tempfile::NamedTempFile::with_suffix(".png")?;
        resized
            .write_to(&mut tmp, image::ImageFormat::Png)
            .map_err(|e| EmbedError::ImageError(e.to_string()))?;
        let path = tmp.into_temp_path().to_path_buf();
        Ok(path)
    }

    fn render_svg_to_image(&self, path: &Path) -> Result<image::DynamicImage, EmbedError> {
        let data = fs::read(path)?;
        let opt = resvg::usvg::Options::default();
        let tree = resvg::usvg::Tree::from_data(&data, &opt)
            .map_err(|e| EmbedError::ImageError(format!("SVG parse: {e}")))?;
        let size = tree.size().to_int_size();
        let mut pixmap = resvg::tiny_skia::Pixmap::new(size.width(), size.height())
            .ok_or_else(|| EmbedError::ImageError("SVG pixmap alloc failed".into()))?;
        resvg::render(
            &tree,
            resvg::tiny_skia::Transform::default(),
            &mut pixmap.as_mut(),
        );
        let img =
            image::RgbaImage::from_raw(pixmap.width(), pixmap.height(), pixmap.data().to_vec())
                .ok_or_else(|| EmbedError::ImageError("SVG image conversion failed".into()))?;
        Ok(image::DynamicImage::ImageRgba8(img))
    }

    fn extract_pdf_text(&self, path: &Path) -> String {
        pdf_extract::extract_text(path).unwrap_or_default()
    }

    fn embed_pptx(&self, path: &Path) -> Array1<f64> {
        let file = match fs::File::open(path) {
            Ok(f) => f,
            Err(_) => return hash_embedding(&[]),
        };
        let mut archive = match zip::ZipArchive::new(file) {
            Ok(a) => a,
            Err(_) => return hash_embedding(&[]),
        };
        let mut texts = Vec::new();
        let mut images: Vec<PathBuf> = Vec::new();
        for i in 0..archive.len() {
            let mut f = match archive.by_index(i) {
                Ok(f) => f,
                Err(_) => continue,
            };
            let name = f.name().to_lowercase();
            if name.starts_with("ppt/slides/slide") && name.ends_with(".xml") {
                let mut content = String::new();
                if f.read_to_string(&mut content).is_ok() {
                    let text = extract_xml_text(&content);
                    if !text.trim().is_empty() {
                        texts.push(text);
                    }
                }
            } else if name.starts_with("ppt/media/") {
                let mut buf = Vec::new();
                if f.read_to_end(&mut buf).is_ok()
                    && let Ok(tmp) = tempfile::NamedTempFile::with_suffix(".img")
                {
                    let tmp_path = tmp.into_temp_path().to_path_buf();
                    if fs::write(&tmp_path, &buf).is_ok() {
                        images.push(tmp_path);
                    }
                }
            }
        }
        let mut sum = Array1::zeros(EMBED_DIM);
        let mut count = 0u32;
        for text in &texts {
            let e = self.embed(text);
            sum = sum + e;
            count += 1;
        }
        for img_path in &images {
            let e = self
                .embed_image_via_llama(img_path)
                .unwrap_or_else(|_| hash_embedding(&[]));
            sum = sum + e;
            count += 1;
        }
        if count > 0 {
            sum / count as f64
        } else {
            hash_embedding(&[])
        }
    }

    fn embed_docx(&self, path: &Path) -> Array1<f64> {
        let file = match fs::File::open(path) {
            Ok(f) => f,
            Err(_) => return hash_embedding(&[]),
        };
        let mut archive = match zip::ZipArchive::new(file) {
            Ok(a) => a,
            Err(_) => return hash_embedding(&[]),
        };
        let mut text = String::new();
        for i in 0..archive.len() {
            let mut f = match archive.by_index(i) {
                Ok(f) => f,
                Err(_) => continue,
            };
            if f.name().to_lowercase() == "word/document.xml" {
                let mut content = String::new();
                if f.read_to_string(&mut content).is_ok() {
                    text = extract_xml_text(&content);
                }
                break;
            }
        }
        if text.trim().is_empty() {
            hash_embedding(&fs::read(path).unwrap_or_default())
        } else {
            self.embed(&text)
        }
    }

    fn extract_spreadsheet_text(&self, path: &Path, ext: &str) -> Result<String, EmbedError> {
        let file = fs::File::open(path)?;
        let mut archive =
            zip::ZipArchive::new(file).map_err(|e| EmbedError::ZipError(e.to_string()))?;
        let mut text = String::new();
        for i in 0..archive.len() {
            let mut f = match archive.by_index(i) {
                Ok(f) => f,
                Err(_) => continue,
            };
            let name = f.name().to_lowercase();
            let is_sheet = (ext == "xlsx"
                && name.starts_with("xl/worksheets/sheet")
                && name.ends_with(".xml"))
                || (ext == "ods" && name == "content.xml");
            if is_sheet {
                let mut content = String::new();
                if f.read_to_string(&mut content).is_ok() {
                    text.push_str(&extract_xml_text(&content));
                    text.push('\n');
                }
            }
        }
        Ok(text)
    }

    fn extract_odt_text(&self, path: &Path) -> Result<String, EmbedError> {
        let file = fs::File::open(path)?;
        let mut archive =
            zip::ZipArchive::new(file).map_err(|e| EmbedError::ZipError(e.to_string()))?;
        for i in 0..archive.len() {
            let mut f = match archive.by_index(i) {
                Ok(f) => f,
                Err(_) => continue,
            };
            if f.name().to_lowercase() == "content.xml" {
                let mut content = String::new();
                f.read_to_string(&mut content)?;
                return Ok(extract_xml_text(&content));
            }
        }
        Err(EmbedError::XmlError("content.xml not found in ODT".into()))
    }

    fn extract_opendocument_xml(&self, path: &Path) -> Result<String, EmbedError> {
        let file = fs::File::open(path)?;
        let mut archive =
            zip::ZipArchive::new(file).map_err(|e| EmbedError::ZipError(e.to_string()))?;
        let mut text = String::new();
        for i in 0..archive.len() {
            let mut f = match archive.by_index(i) {
                Ok(f) => f,
                Err(_) => continue,
            };
            if f.name().to_lowercase() == "content.xml" {
                let mut content = String::new();
                if f.read_to_string(&mut content).is_ok() {
                    text = extract_xml_text(&content);
                }
                break;
            }
        }
        Ok(text)
    }

    fn extract_epub_text(&self, path: &Path) -> String {
        let file = match fs::File::open(path) {
            Ok(f) => f,
            Err(_) => return String::new(),
        };
        let mut archive = match zip::ZipArchive::new(file) {
            Ok(a) => a,
            Err(_) => return String::new(),
        };
        let mut text = String::new();
        for i in 0..archive.len() {
            let mut f = match archive.by_index(i) {
                Ok(f) => f,
                Err(_) => continue,
            };
            let name = f.name().to_lowercase();
            if (name.ends_with(".xhtml") || name.ends_with(".html") || name.ends_with(".htm"))
                && !name.contains("nav")
            {
                let mut content = String::new();
                if f.read_to_string(&mut content).is_ok() {
                    text.push_str(&extract_xml_text(&content));
                    text.push('\n');
                }
            }
        }
        text
    }

    fn extract_audio_metadata(&self, path: &Path) -> Result<String, EmbedError> {
        use symphonia::core::formats::FormatOptions;
        use symphonia::core::io::MediaSourceStream;
        use symphonia::core::meta::MetadataOptions;
        use symphonia::core::probe::Hint;
        let file = std::fs::File::open(path)?;
        let mss = MediaSourceStream::new(Box::new(file), Default::default());
        let hint = Hint::new();
        let format_opts = FormatOptions::default();
        let metadata_opts = MetadataOptions::default();
        let mut probed = symphonia::default::get_probe()
            .format(&hint, mss, &format_opts, &metadata_opts)
            .map_err(|e| EmbedError::Io(std::io::Error::other(e.to_string())))?;
        let mut meta_text = String::new();
        if let Some(metadata) = probed.format.metadata().current() {
            for tag in metadata.tags() {
                meta_text.push_str(&format!("{}: {}\n", tag.key, tag.value));
            }
        }
        Ok(meta_text)
    }

    fn extract_video_keyframes(&self, path: &Path) -> Result<Vec<PathBuf>, EmbedError> {
        let tmp_dir = tempfile::tempdir()?;
        let pattern = tmp_dir.path().join("frame_%03d.png");
        let status = Command::new("ffmpeg")
            .args([
                "-i",
                path.to_string_lossy().as_ref(),
                "-vf",
                "select=eq(pict_type\\,I)",
                "-vsync",
                "vfr",
                "-frames:v",
                "5",
                pattern.to_string_lossy().as_ref(),
            ])
            .stdout(std::process::Stdio::null())
            .stderr(std::process::Stdio::null())
            .status()?;
        if !status.success() {
            return Err(EmbedError::Io(std::io::Error::other("ffmpeg failed")));
        }
        let mut frames = Vec::new();
        for entry in fs::read_dir(tmp_dir.path())? {
            let entry = entry?;
            if entry.path().extension().and_then(|e| e.to_str()) == Some("png") {
                // Copy to a persistent temp file since tmp_dir will be dropped
                let persistent = tempfile::NamedTempFile::with_suffix(".png")?;
                let p = persistent.into_temp_path().to_path_buf();
                fs::copy(entry.path(), &p)?;
                frames.push(p);
            }
        }
        Ok(frames)
    }

    fn extract_zip_texts(&self, path: &Path) -> Result<Vec<String>, EmbedError> {
        let file = fs::File::open(path)?;
        let mut archive =
            zip::ZipArchive::new(file).map_err(|e| EmbedError::ZipError(e.to_string()))?;
        let mut texts = Vec::new();
        for i in 0..archive.len() {
            let mut f = match archive.by_index(i) {
                Ok(f) => f,
                Err(_) => continue,
            };
            let name = f.name().to_lowercase();
            if name.ends_with('/') {
                continue;
            }
            let is_text = name.ends_with(".txt")
                || name.ends_with(".md")
                || name.ends_with(".json")
                || name.ends_with(".xml")
                || name.ends_with(".csv")
                || name.ends_with(".yaml")
                || name.ends_with(".yml")
                || name.ends_with(".toml")
                || name.ends_with(".rs")
                || name.ends_with(".py")
                || name.ends_with(".js")
                || name.ends_with(".ts")
                || name.ends_with(".html")
                || name.ends_with(".css")
                || name.ends_with(".sh")
                || name.ends_with(".log")
                || name.ends_with(".rst");
            if is_text {
                let mut content = String::new();
                if f.read_to_string(&mut content).is_ok() {
                    texts.push(content);
                }
            }
        }
        Ok(texts)
    }

    fn extract_tar_texts(&self, path: &Path) -> Result<Vec<String>, EmbedError> {
        let file = fs::File::open(path)?;
        let mut archive = tar::Archive::new(file);
        let mut texts = Vec::new();
        for entry in archive.entries()? {
            let mut entry = entry?;
            let name = entry.path()?.to_string_lossy().to_lowercase();
            let is_text = name.ends_with(".txt")
                || name.ends_with(".md")
                || name.ends_with(".json")
                || name.ends_with(".xml")
                || name.ends_with(".csv")
                || name.ends_with(".yaml")
                || name.ends_with(".rs")
                || name.ends_with(".py")
                || name.ends_with(".js");
            if is_text {
                let mut content = String::new();
                if entry.read_to_string(&mut content).is_ok() {
                    texts.push(content);
                }
            }
        }
        Ok(texts)
    }

    fn extract_7z_texts(&self, path: &Path) -> Result<Vec<String>, EmbedError> {
        let output = Command::new("7z")
            .args(["l", "-ba", path.to_string_lossy().as_ref()])
            .output()?;
        let _listing = String::from_utf8_lossy(&output.stdout);
        // 7z doesn't easily extract to stdout; use 7z to extract to temp dir
        let tmp_dir = tempfile::tempdir()?;
        let status = Command::new("7z")
            .args([
                "x",
                path.to_string_lossy().as_ref(),
                &format!("-o{}", tmp_dir.path().display()),
                "-y",
            ])
            .stdout(std::process::Stdio::null())
            .stderr(std::process::Stdio::null())
            .status()?;
        if !status.success() {
            return Err(EmbedError::Io(std::io::Error::other(
                "7z extraction failed",
            )));
        }
        let mut texts = Vec::new();
        self.collect_text_files(tmp_dir.path(), &mut texts)?;
        Ok(texts)
    }

    fn extract_rar_texts(&self, path: &Path) -> Result<Vec<String>, EmbedError> {
        let tmp_dir = tempfile::tempdir()?;
        let status = Command::new("unrar")
            .args([
                "x",
                "-y",
                path.to_string_lossy().as_ref(),
                &format!("{}/", tmp_dir.path().display()),
            ])
            .stdout(std::process::Stdio::null())
            .stderr(std::process::Stdio::null())
            .status()?;
        if !status.success() {
            // Try 7z as fallback
            return self.extract_7z_texts(path);
        }
        let mut texts = Vec::new();
        self.collect_text_files(tmp_dir.path(), &mut texts)?;
        Ok(texts)
    }

    fn collect_text_files(&self, dir: &Path, texts: &mut Vec<String>) -> Result<(), EmbedError> {
        if dir.is_dir() {
            for entry in fs::read_dir(dir)? {
                let entry = entry?;
                let path = entry.path();
                if path.is_dir() {
                    self.collect_text_files(&path, texts)?;
                } else if let Ok(content) = fs::read_to_string(&path) {
                    texts.push(content);
                }
            }
        }
        Ok(())
    }

    fn embed_via_libreoffice(&self, path: &Path) -> Array1<f64> {
        let tmp_dir = match tempfile::tempdir() {
            Ok(d) => d,
            Err(_) => return hash_embedding(&[]),
        };
        let status = Command::new("libreoffice")
            .args([
                "--headless",
                "--convert-to",
                "txt:Text",
                path.to_string_lossy().as_ref(),
                "--outdir",
                tmp_dir.path().to_string_lossy().as_ref(),
            ])
            .stdout(std::process::Stdio::null())
            .stderr(std::process::Stdio::null())
            .status();
        match status {
            Ok(s) if s.success() => {
                // Find the converted .txt file
                if let Ok(entries) = fs::read_dir(tmp_dir.path()) {
                    for entry in entries.flatten() {
                        if entry.path().extension().and_then(|e| e.to_str()) == Some("txt")
                            && let Ok(text) = fs::read_to_string(entry.path())
                        {
                            return self.embed(&text);
                        }
                    }
                }
                hash_embedding(&[])
            }
            _ => hash_embedding(&fs::read(path).unwrap_or_default()),
        }
    }

    fn run_command(&self, cmd: &str, args: &[&str]) -> Result<String, EmbedError> {
        let output = Command::new(cmd).args(args).output()?;
        if output.status.success() {
            Ok(String::from_utf8_lossy(&output.stdout).to_string())
        } else {
            Err(EmbedError::Io(std::io::Error::other(
                String::from_utf8_lossy(&output.stderr).to_string(),
            )))
        }
    }
}

// ── Free functions ──────────────────────────────────────────────────────

/// Parses llama.cpp embedding output into an Array1<f64>.
fn parse_embedding_output(stdout: &[u8]) -> Result<Array1<f64>, EmbedError> {
    let text = String::from_utf8_lossy(stdout);
    let values: Vec<f64> = text
        .split_whitespace()
        .filter_map(|s| s.parse::<f64>().ok())
        .collect();
    if values.is_empty() {
        return Err(EmbedError::Io(std::io::Error::other(
            "empty embedding output",
        )));
    }
    Ok(Array1::from(values))
}

/// Deterministic hash-based embedding fallback.
pub fn hash_embedding(data: &[u8]) -> Array1<f64> {
    let mut result = Array1::zeros(EMBED_DIM);
    if data.is_empty() {
        return result;
    }
    // Chunk the data and hash each chunk to fill the embedding
    let chunk_size = (data.len() / EMBED_DIM).max(1);
    for (i, chunk) in data.chunks(chunk_size).enumerate().take(EMBED_DIM) {
        let hash = Sha256::digest(chunk);
        let mut val = 0.0f64;
        for (j, &byte) in hash.iter().take(8).enumerate() {
            val += (byte as f64) / (256.0f64.powi(j as i32 + 1));
        }
        result[i] = val;
    }
    // Normalize
    let norm = result.dot(&result).sqrt();
    if norm > 1e-12 {
        result /= norm;
    }
    result
}

/// Strips RTF formatting to extract plain text.
fn strip_rtf(rtf: &str) -> String {
    let mut result = String::new();
    let mut in_control = false;
    let mut depth: usize = 0;
    let mut control_word = String::new();
    for ch in rtf.chars() {
        match ch {
            '{' => {
                depth += 1;
            }
            '}' => {
                depth = depth.saturating_sub(1);
            }
            '\\' => {
                if in_control {
                    // Escaped backslash
                    result.push('\\');
                    in_control = false;
                } else {
                    in_control = true;
                    control_word.clear();
                }
            }
            ' ' | '\n' | '\r' => {
                if in_control {
                    in_control = false;
                    handle_rtf_control(&control_word, &mut result);
                    control_word.clear();
                } else if depth > 0 {
                    result.push(ch);
                }
            }
            _ => {
                if in_control {
                    if ch.is_ascii_alphabetic() {
                        control_word.push(ch);
                    } else {
                        in_control = false;
                        handle_rtf_control(&control_word, &mut result);
                        control_word.clear();
                        if ch != '\\' {
                            result.push(ch);
                        }
                    }
                } else if depth > 0 {
                    result.push(ch);
                }
            }
        }
    }
    result
}

fn handle_rtf_control(word: &str, output: &mut String) {
    match word {
        "par" | "line" | "row" => output.push('\n'),
        "tab" => output.push('\t'),
        "lquote" => output.push('\u{2018}'),
        "rquote" => output.push('\u{2019}'),
        "ldblquote" => output.push('\u{201C}'),
        "rdblquote" => output.push('\u{201D}'),
        "endash" => output.push('\u{2013}'),
        "emdash" => output.push('\u{2014}'),
        "bullet" => output.push('\u{2022}'),
        "" => {} // skip
        _ => {
            // Unicode escape \uN?
            if word.starts_with('u')
                && word.len() > 1
                && let Ok(n) = word[1..].parse::<i32>()
                && let Some(c) = char::from_u32(n as u32)
            {
                output.push(c);
            }
        }
    }
}

/// Extracts text content from XML by stripping tags.
fn extract_xml_text(xml: &str) -> String {
    let mut text = String::new();
    let mut in_tag = false;
    let mut _in_text = false;
    let mut tag_depth: usize = 0;
    for ch in xml.chars() {
        match ch {
            '<' => {
                in_tag = true;
                tag_depth += 1;
            }
            '>' => {
                in_tag = false;
                tag_depth = tag_depth.saturating_sub(1);
                if tag_depth == 0 {
                    _in_text = false;
                }
            }
            _ => {
                if !in_tag && tag_depth == 0 {
                    text.push(ch);
                }
            }
        }
    }
    // Collapse whitespace
    text.split_whitespace().collect::<Vec<_>>().join(" ")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_embed_dim() {
        assert_eq!(EMBED_DIM, 1024, "LFM2.5-VL-450M hidden_size is 1024");
    }

    #[test]
    fn test_embedder_creation() {
        let _embedder = Lfm2Embedder::new();
    }
}