goosedump 0.12.43

// SPDX-License-Identifier: LGPL-2.1-or-later
// Copyright (C) Jarkko Sakkinen 2026

//! Local text and embedding models.
//!
//! GGUF weights download into the local cache on first use. See
//! `docs/models/` for role, inference, and benchmark notes.

mod bge;
mod expert_store;
#[cfg(feature = "bench")]
mod expert_store_check;
mod gguf;
mod gpt_oss;
mod kernels;
mod profile;
mod tokenizer;
mod types;
mod wordpiece;

use std::cmp::Ordering;
use std::env;
use std::fs::File;
use std::io::Read as _;
use std::path::{Path, PathBuf};
use std::sync::{Mutex, OnceLock};
use std::time::Duration;

use anyhow::{Context as _, bail};
#[cfg(feature = "bench")]
use num_traits::ToPrimitive as _;
use rayon::{ThreadPool, ThreadPoolBuilder};
use sha2::{Digest as _, Sha256};

pub(crate) use self::bge::EMBEDDING_DIMENSIONS;
use self::bge::EmbeddingModel;
use self::gpt_oss::{Generation, GenerationStopReason, PhaseProfile, TextModel};
#[cfg(feature = "bench")]
use crate::alloc_profile::AllocationProfile;

/// Cache subdirectory and Hugging Face source of the fixed text model.
const TEXTGEN_NAME: &str = "gpt-oss-20b-mxfp4-gguf";
const TEXTGEN_WEIGHTS_REPO: &str = "ggml-org/gpt-oss-20b-GGUF";
const TEXTGEN_WEIGHTS_FILE: &str = "gpt-oss-20b-MXFP4.gguf";
const MAX_DECODE_DURATION: Duration = Duration::from_mins(5);

/// Cache identity and pinned Hugging Face source of the sentence embedding model.
pub(crate) const EMBEDDING_NAME: &str = "bge-small-en-v1.5-q8_0-gguf";
pub(crate) const EMBEDDING_WEIGHTS_REPO: &str = "ggml-org/bge-small-en-v1.5-Q8_0-GGUF";
pub(crate) const EMBEDDING_WEIGHTS_FILE: &str = "bge-small-en-v1.5-q8_0.gguf";
pub(crate) const EMBEDDING_WEIGHTS_SHA256: &str =
    "f046db1dc724cf4f6f0a0c5917e922823b73eb1d27b8f9a9c2797f7866974804";
pub(crate) const EMBEDDING_MODEL_ID: &str = "bge-small-en-v1.5-q8_0-pipeline-v1@f046db1dc724cf4f6f0a0c5917e922823b73eb1d27b8f9a9c2797f7866974804";

const TEXTGEN_CONTEXT: u16 = 4_096;

const INFERENCE_THREADS_ENV: &str = "GOOSEDUMP_INFERENCE_THREADS";
const INFERENCE_PROFILE_ENV: &str = "GOOSEDUMP_PROFILE_INFERENCE";
const EMBEDDING_SINGLE_THREADS: usize = 2;

/// Fixed GPT-OSS-20B MXFP4 model for claim extraction and compact candidate filtering.
pub struct TextGen {
    model: TextModel,
    pool: ThreadPool,
}

#[cfg(feature = "bench")]
#[derive(serde::Serialize)]
pub(crate) struct BenchmarkGeneration {
    pub(crate) text: String,
    pub(crate) prompt_tokens: usize,
    pub(crate) generated_tokens: usize,
    pub(crate) decode_tokens: usize,
    pub(crate) token_ids: Vec<u32>,
    pub(crate) stop_reason: &'static str,
    pub(crate) prefill_allocations: AllocationProfile,
    pub(crate) decode_allocations: AllocationProfile,
    pub(crate) prefill_allocation_calls_per_token: f64,
    pub(crate) prefill_requested_bytes_per_token: f64,
    pub(crate) decode_allocation_calls_per_token: Option<f64>,
    pub(crate) decode_requested_bytes_per_token: Option<f64>,
}

#[cfg(feature = "bench")]
pub(crate) fn check_expert_store() -> anyhow::Result<()> {
    expert_store_check::run()
}

/// Fixed BGE-small-en-v1.5 model for semantic claim retrieval.
pub struct Embedder {
    model: EmbeddingModel,
    single_pool: OnceLock<ThreadPool>,
    batch_pool: OnceLock<ThreadPool>,
    pool_init: Mutex<()>,
    max_threads: usize,
}

/// Validated, normalized embedding produced by [`Embedder`].
#[derive(Clone, Debug)]
pub struct Embedding(Vec<f32>);

impl TryFrom<Vec<f32>> for Embedding {
    type Error = anyhow::Error;

    fn try_from(values: Vec<f32>) -> Result<Self, Self::Error> {
        if values.len() != EMBEDDING_DIMENSIONS || !values.iter().all(|value| value.is_finite()) {
            bail!("embedding has an invalid shape or value");
        }
        let norm_squared = values.iter().map(|value| value * value).sum::<f32>();
        if (norm_squared - 1.0).abs() > 1e-3 {
            bail!("embedding is not normalized");
        }
        Ok(Self(values))
    }
}

impl From<&Embedding> for Vec<u8> {
    fn from(value: &Embedding) -> Self {
        value
            .0
            .iter()
            .flat_map(|component| component.to_ne_bytes())
            .collect()
    }
}

/// Finite cosine similarity between two normalized embeddings.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Similarity(f32);

impl Similarity {
    fn from_valid(value: f32) -> Self {
        Self(if value == 0.0 { 0.0 } else { value })
    }
}

impl TryFrom<f32> for Similarity {
    type Error = anyhow::Error;

    fn try_from(value: f32) -> Result<Self, Self::Error> {
        if !value.is_finite() || !(-1.0..=1.0).contains(&value) {
            bail!("embedding similarity is outside the cosine range");
        }
        Ok(Self::from_valid(value))
    }
}

impl Eq for Similarity {}

impl PartialOrd for Similarity {
    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for Similarity {
    fn cmp(&self, other: &Self) -> Ordering {
        self.0.total_cmp(&other.0)
    }
}

impl Embedding {
    pub fn similarity(&self, other: &Self) -> Similarity {
        let value = self
            .0
            .iter()
            .zip(&other.0)
            .map(|(left, right)| left * right)
            .sum::<f32>();
        Similarity::from_valid(value.clamp(-1.0, 1.0))
    }
}

#[cfg(feature = "bench")]
fn allocation_rate(value: u64, tokens: usize) -> anyhow::Result<Option<f64>> {
    if tokens == 0 {
        return Ok(None);
    }
    let value = value
        .to_f64()
        .context("allocation counter exceeds f64 range")?;
    let tokens = tokens.to_f64().context("token count exceeds f64 range")?;
    Ok(Some(value / tokens))
}

impl TextGen {
    /// Download and load the text model from the local model cache.
    ///
    /// # Errors
    /// Returns an error if downloading, loading, or configuring inference fails.
    pub fn load() -> anyhow::Result<Self> {
        let dir = model_cache_dir(TEXTGEN_NAME);
        ensure_files(TEXTGEN_WEIGHTS_REPO, &[TEXTGEN_WEIGHTS_FILE], &dir)?;
        let model = TextModel::load(dir.join(TEXTGEN_WEIGHTS_FILE))?;
        let threads = inference_threads()?;
        let thread_count = usize::try_from(threads).context("text inference thread count")?;
        let pool = ThreadPoolBuilder::new()
            .num_threads(thread_count)
            .build()
            .context("create text inference thread pool")?;
        Ok(Self { model, pool })
    }

    #[cfg(feature = "bench")]
    pub(crate) fn load_for_benchmark(cold: bool) -> anyhow::Result<Self> {
        let dir = model_cache_dir(TEXTGEN_NAME);
        ensure_files(TEXTGEN_WEIGHTS_REPO, &[TEXTGEN_WEIGHTS_FILE], &dir)?;
        if cold {
            evict_text_model_pages()?;
        }
        Self::load()
    }

    /// Return whether the formatted prompt and requested generation fit the context.
    ///
    /// # Errors
    /// Returns an error if tokenization fails.
    pub fn completion_fits(
        &self,
        system: &str,
        user: &str,
        max_tokens: usize,
    ) -> anyhow::Result<bool> {
        let prompt_tokens = self.model.prompt_tokens(&completion_prompt(system, user))?;
        Ok(prompt_tokens
            .checked_add(max_tokens)
            .is_some_and(|tokens| tokens <= usize::from(TEXTGEN_CONTEXT)))
    }

    /// Greedy instruction completion of `system` + `user`, capped at
    /// `max_tokens`.
    ///
    /// # Errors
    /// Returns an error if tokenization or inference fails, or generation is stopped by a safety guard.
    pub fn complete(
        &mut self,
        system: &str,
        user: &str,
        max_tokens: usize,
    ) -> anyhow::Result<String> {
        let generation = self.generate(system, user, max_tokens, Some(MAX_DECODE_DURATION))?;
        print_generation_profile(&generation)?;
        finish_generation(generation)
    }

    /// Greedy instruction completion for background work without a wall-clock decode limit.
    /// Token and repetition guards still bound generation.
    ///
    /// # Errors
    /// Returns an error if tokenization or inference fails, or generation is stopped by the
    /// repetition guard.
    pub fn complete_background(
        &mut self,
        system: &str,
        user: &str,
        max_tokens: usize,
    ) -> anyhow::Result<String> {
        let generation = self.generate(system, user, max_tokens, None)?;
        print_generation_profile(&generation)?;
        finish_generation(generation)
    }

    #[cfg(feature = "bench")]
    pub(crate) fn benchmark_complete(
        &self,
        system: &str,
        user: &str,
        max_tokens: usize,
    ) -> anyhow::Result<BenchmarkGeneration> {
        let generation = self.generate(system, user, max_tokens, Some(MAX_DECODE_DURATION))?;
        print_generation_profile(&generation)?;
        let prefill_allocation_calls_per_token = allocation_rate(
            generation.prefill_allocations.allocation_calls,
            generation.prompt_tokens,
        )?
        .context("prefill has no tokens")?;
        let prefill_requested_bytes_per_token = allocation_rate(
            generation.prefill_allocations.requested_bytes,
            generation.prompt_tokens,
        )?
        .context("prefill has no tokens")?;
        let decode_allocation_calls_per_token = allocation_rate(
            generation.decode_allocations.allocation_calls,
            generation.decode_tokens,
        )?;
        let decode_requested_bytes_per_token = allocation_rate(
            generation.decode_allocations.requested_bytes,
            generation.decode_tokens,
        )?;
        Ok(BenchmarkGeneration {
            text: generation.text,
            prompt_tokens: generation.prompt_tokens,
            generated_tokens: generation.generated_tokens,
            decode_tokens: generation.decode_tokens,
            token_ids: generation.generated_token_ids,
            stop_reason: stop_reason_name(generation.stop_reason),
            prefill_allocations: generation.prefill_allocations,
            decode_allocations: generation.decode_allocations,
            prefill_allocation_calls_per_token,
            prefill_requested_bytes_per_token,
            decode_allocation_calls_per_token,
            decode_requested_bytes_per_token,
        })
    }

    fn generate(
        &self,
        system: &str,
        user: &str,
        max_tokens: usize,
        max_decode_duration: Option<Duration>,
    ) -> anyhow::Result<Generation> {
        let prompt = completion_prompt(system, user);
        let profile_enabled = inference_profile_enabled();
        self.pool.install(|| {
            self.model.generate(
                &prompt,
                max_tokens,
                TEXTGEN_CONTEXT,
                profile_enabled,
                max_decode_duration,
            )
        })
    }
}

fn finish_generation(generation: Generation) -> anyhow::Result<String> {
    match generation.stop_reason {
        GenerationStopReason::EndOfSequence | GenerationStopReason::TokenLimit => {
            Ok(generation.text)
        }
        GenerationStopReason::TokenCycle => bail!(
            "text generation stopped after detecting a repeating token cycle ({} tokens)",
            generation.generated_tokens
        ),
        GenerationStopReason::TimeBudget => bail!(
            "text generation exceeded its decode time budget ({} tokens)",
            generation.generated_tokens
        ),
    }
}

fn inference_profile_enabled() -> bool {
    env::var_os(INFERENCE_PROFILE_ENV).is_some()
        || env::var_os("GOOSEDUMP_PROFILE_COMPACT").is_some()
}

fn print_generation_profile(generation: &Generation) -> anyhow::Result<()> {
    let Some(profile) = generation.profile.as_ref() else {
        return Ok(());
    };
    print_phase_profile(
        "prefill",
        generation.prompt_tokens,
        generation.prefill_duration,
        &profile.prefill,
    )?;
    print_phase_profile(
        "decode",
        generation.decode_tokens,
        generation.decode_duration,
        &profile.decode,
    )
}

fn print_phase_profile(
    name: &str,
    tokens: usize,
    duration: std::time::Duration,
    profile: &PhaseProfile,
) -> anyhow::Result<()> {
    let seconds = duration.as_secs_f64();
    let duration_ms = seconds * 1_000.0;
    let rate = token_rate(tokens, seconds)?;
    if let Some(process) = profile.process {
        eprintln!(
            "goosedump: profile: inference: {name}: tokens={tokens} duration_ms={duration_ms:.1} tokens_per_second={rate:.1} minor_faults={} major_faults={} read_bytes={} rss_kib_start={} rss_kib_end={} peak_rss_kib={}",
            process.minor_faults,
            process.major_faults,
            process.read_bytes,
            process.rss_kib_start,
            process.rss_kib_end,
            process.peak_rss_kib,
        );
    } else {
        eprintln!(
            "goosedump: profile: inference: {name}: tokens={tokens} duration_ms={duration_ms:.1} tokens_per_second={rate:.1} process_metrics=unavailable"
        );
    }

    let selections = profile.expert_routes.selections();
    let unique = profile.expert_routes.unique_layer_experts();
    let repeated = selections.saturating_sub(unique);
    let layer_unique = profile
        .expert_routes
        .layer_unique_counts()
        .into_iter()
        .map(|count| count.to_string())
        .collect::<Vec<_>>()
        .join(",");
    let route_counts = profile
        .expert_routes
        .route_counts()
        .map(|count| count.to_string())
        .collect::<Vec<_>>()
        .join(",");
    eprintln!(
        "goosedump: profile: inference: {name}-experts: selections={selections} unique_layer_experts={unique} repeated_selections={repeated} layer_unique={layer_unique} route_counts={route_counts}"
    );
    Ok(())
}

#[cfg(feature = "bench")]
const fn stop_reason_name(reason: GenerationStopReason) -> &'static str {
    match reason {
        GenerationStopReason::EndOfSequence => "end_of_sequence",
        GenerationStopReason::TokenLimit => "token_limit",
        GenerationStopReason::TokenCycle => "token_cycle",
        GenerationStopReason::TimeBudget => "time_budget",
    }
}

#[cfg(all(feature = "bench", target_os = "linux"))]
fn evict_text_model_pages() -> anyhow::Result<()> {
    use std::os::fd::AsRawFd as _;

    let path = model_cache_dir(TEXTGEN_NAME).join(TEXTGEN_WEIGHTS_FILE);
    let file =
        File::open(&path).with_context(|| format!("open {} for cache eviction", path.display()))?;
    // SAFETY: posix_fadvise does not access Rust memory, and the descriptor remains
    // open for the duration of the call.
    let result = unsafe { libc::posix_fadvise(file.as_raw_fd(), 0, 0, libc::POSIX_FADV_DONTNEED) };
    if result != 0 {
        return Err(std::io::Error::from_raw_os_error(result))
            .with_context(|| format!("evict cached pages for {}", path.display()));
    }
    Ok(())
}

#[cfg(all(feature = "bench", not(target_os = "linux")))]
fn evict_text_model_pages() -> anyhow::Result<()> {
    bail!("cold inference benchmarking is supported only on Linux")
}

impl Embedder {
    /// Download and load the embedding model from the local model cache.
    pub fn load() -> anyhow::Result<Self> {
        let dir = ensure_embedding_model()?;
        Self::load_path(&dir.join(EMBEDDING_WEIGHTS_FILE))
    }

    /// Produce one normalized sentence embedding.
    pub fn embed(&self, text: &str) -> anyhow::Result<Embedding> {
        self.pool(false)?
            .install(|| self.model.embed(text))
            .and_then(Embedding::try_from)
    }

    /// Produce normalized embeddings for an independent batch of sentences.
    pub fn embed_batch(&self, texts: &[&str]) -> anyhow::Result<Vec<Embedding>> {
        if texts.len() <= 1 {
            return texts.first().map_or_else(
                || Ok(Vec::new()),
                |text| self.embed(text).map(|value| vec![value]),
            );
        }
        self.pool(true)?
            .install(|| self.model.embed_batch(texts))?
            .into_iter()
            .map(Embedding::try_from)
            .collect()
    }

    /// Score each candidate by its greatest similarity to any reference.
    pub fn relevance(
        &self,
        references: &[&str],
        candidates: &[&str],
    ) -> anyhow::Result<Vec<Similarity>> {
        if references.is_empty() {
            bail!("relevance has no references");
        }
        let inputs = references
            .iter()
            .chain(candidates)
            .copied()
            .collect::<Vec<_>>();
        let embeddings = self.embed_batch(&inputs)?;
        if embeddings.len() != inputs.len() {
            bail!("relevance embedding count differs");
        }
        let reference_embeddings = embeddings
            .get(..references.len())
            .context("relevance reference range is invalid")?;
        let candidate_embeddings = embeddings
            .get(references.len()..)
            .context("relevance candidate range is invalid")?;
        candidate_embeddings
            .iter()
            .map(|candidate| {
                reference_embeddings
                    .iter()
                    .map(|reference| candidate.similarity(reference))
                    .max()
                    .context("relevance has no reference embedding")
            })
            .collect()
    }

    fn pool(&self, batch: bool) -> anyhow::Result<&ThreadPool> {
        let single_threads = self.max_threads.min(EMBEDDING_SINGLE_THREADS);
        let (pool, threads) = if !batch || self.max_threads == single_threads {
            (&self.single_pool, single_threads)
        } else {
            (&self.batch_pool, self.max_threads)
        };
        if let Some(pool) = pool.get() {
            return Ok(pool);
        }
        let _guard = self
            .pool_init
            .lock()
            .map_err(|_| anyhow::anyhow!("embedding inference pool initialization is poisoned"))?;
        if let Some(pool) = pool.get() {
            return Ok(pool);
        }
        let candidate = ThreadPoolBuilder::new()
            .num_threads(threads)
            .build()
            .context("create embedding inference thread pool")?;
        pool.set(candidate)
            .map_err(|_| anyhow::anyhow!("embedding inference pool initialized twice"))?;
        pool.get()
            .context("initialize embedding inference thread pool")
    }

    fn load_path(path: &Path) -> anyhow::Result<Self> {
        let model = EmbeddingModel::load(path)?;
        let threads = inference_threads()?;
        let max_threads = usize::try_from(threads).context("embedding inference thread count")?;
        Ok(Self {
            model,
            single_pool: OnceLock::new(),
            batch_pool: OnceLock::new(),
            pool_init: Mutex::new(()),
            max_threads,
        })
    }
}

fn completion_prompt(system: &str, user: &str) -> String {
    format!(
        "<|start|>system<|message|>{system}<|end|><|start|>user<|message|>{user}<|end|><|start|>assistant<|channel|>final<|message|>"
    )
}

fn token_rate(tokens: usize, seconds: f64) -> anyhow::Result<f64> {
    if seconds == 0.0 {
        Ok(0.0)
    } else {
        let tokens = u32::try_from(tokens).context("token count exceeds u32")?;
        Ok(f64::from(tokens) / seconds)
    }
}

pub(crate) fn inference_threads() -> anyhow::Result<i32> {
    let available = std::thread::available_parallelism()
        .context("detect available parallelism")?
        .get();
    configured_threads(INFERENCE_THREADS_ENV, available)
}

fn configured_threads(name: &str, default: usize) -> anyhow::Result<i32> {
    let value = match env::var_os(name) {
        Some(value) => {
            let value = value
                .into_string()
                .map_err(|_| anyhow::anyhow!("{name} is not valid UTF-8"))?;
            let threads = value
                .parse::<usize>()
                .with_context(|| format!("parse {name}"))?;
            if threads == 0 {
                bail!("{name} must be greater than zero");
            }
            threads
        }
        None => default,
    };
    i32::try_from(value).with_context(|| format!("{name} exceeds i32"))
}

fn ensure_files(repo_id: &str, files: &[&str], dest: &Path) -> anyhow::Result<()> {
    if files.iter().all(|file| dest.join(file).is_file()) {
        return Ok(());
    }
    pull_files(repo_id, files, dest)
}
/// Fetch and verify the pinned embedding model.
fn ensure_embedding_model() -> anyhow::Result<PathBuf> {
    let dir = model_cache_dir(EMBEDDING_NAME);
    let path = dir.join(EMBEDDING_WEIGHTS_FILE);
    if path.is_file() {
        if verify_sha256(&path, EMBEDDING_WEIGHTS_SHA256).is_ok() {
            return Ok(dir);
        }
        std::fs::remove_file(&path)
            .with_context(|| format!("remove invalid model {}", path.display()))?;
    }
    ensure_files(EMBEDDING_WEIGHTS_REPO, &[EMBEDDING_WEIGHTS_FILE], &dir)?;
    verify_sha256(&path, EMBEDDING_WEIGHTS_SHA256)?;
    Ok(dir)
}

fn verify_sha256(path: &Path, expected: &str) -> anyhow::Result<()> {
    let mut file = File::open(path).with_context(|| format!("open {}", path.display()))?;
    let mut digest = Sha256::new();
    let mut buffer = vec![0_u8; 64 * 1_024];
    loop {
        let read = file
            .read(&mut buffer)
            .with_context(|| format!("read {}", path.display()))?;
        if read == 0 {
            break;
        }
        digest.update(&buffer[..read]);
    }
    let actual = format!("{:x}", digest.finalize());
    if actual != expected {
        bail!(
            "model {} has SHA-256 {actual}, expected {expected}; remove the file and retry",
            path.display()
        );
    }
    Ok(())
}

/// Fetch `files` from the Hugging Face `repo_id` into `dest` (flat layout).
fn pull_files(repo_id: &str, files: &[&str], dest: &Path) -> anyhow::Result<()> {
    std::fs::create_dir_all(dest).with_context(|| format!("create {}", dest.display()))?;
    let api = hf_hub::api::sync::Api::new()?;
    let repo = api.model(repo_id.to_string());
    for file in files {
        let target = dest.join(file);
        if target.is_file() {
            continue;
        }
        let cached = repo
            .get(file)
            .with_context(|| format!("download {repo_id}/{file}"))?;
        let temporary = dest.join(format!(".{file}.{}.part", std::process::id()));
        std::fs::copy(&cached, &temporary)
            .with_context(|| format!("write {}", temporary.display()))?;
        std::fs::rename(&temporary, &target)
            .with_context(|| format!("install {}", target.display()))?;
    }
    Ok(())
}

/// The per-model cache directory (honors `XDG_CACHE_HOME`).
fn model_cache_dir(name: &str) -> PathBuf {
    dirs::cache_dir()
        .unwrap_or_else(|| PathBuf::from("."))
        .join("goosedump")
        .join("models")
        .join(name)
}