hippmem-core 0.3.0

//! Stable hashing and tokenization utilities.
//!
//! Corresponds to ADR-018 (tokenization), ADR-019 (xxh3 64-bit stable hash), 03#1 (signature generation basis).

use jieba_rs::Jieba;
use std::sync::LazyLock;

/// Global Jieba instance (lazy-loaded, shared within the process).
/// Avoids re-initializing the dictionary and HMM data on every tokenization (~200ms → ~1ms).
pub static JIEBA: LazyLock<Jieba> = LazyLock::new(Jieba::new);

/// Computes a stable xxh3 64-bit hash of the text (fixed seed 0).
///
/// **Deterministic**: the same text always yields the same hash across processes and versions.
/// The algorithm and seed are permanently fixed (ADR-019); changing them requires a schema migration + ADR.
pub fn stable_hash64(text: &str) -> u64 {
    xxhash_rust::xxh3::xxh3_64(text.as_bytes())
}

/// SimHash similarity of two 256-bit signatures: fraction of identical bits in [0, 1].
///
/// Deterministic; used for write-time candidate filtering (Semantic dimension hits)
/// and consolidation summary clustering (§8). Same input → 1.0; orthogonal → ~0.0.
pub fn simhash_similarity(a: &[u64; 4], b: &[u64; 4]) -> f32 {
    let same: u32 = a
        .iter()
        .zip(b.iter())
        .map(|(x, y)| 64 - (x ^ y).count_ones())
        .sum();
    same as f32 / 256.0
}

/// Tokenizes by language: Chinese goes through jieba; English/code goes through whitespace + punctuation splitting and lowercasing.
///
/// `language` takes values such as `"zh"`, `"zh-CN"`, `"en"`, `"ja"`, etc.
/// Unknown languages fall back to the English tokenization strategy.
pub fn tokenize(text: &str, language: &str) -> Vec<String> {
    match language {
        "zh" | "zh-CN" | "zh-TW" | "zh-HK" => tokenize_chinese(text),
        _ => tokenize_english(text),
    }
}

/// Tokenizes Chinese text with jieba-rs (HMM enabled).
/// Uses the global JIEBA instance to avoid repeated initialization.
fn tokenize_chinese(text: &str) -> Vec<String> {
    JIEBA
        .cut(text, true) // true = enable HMM new-word discovery
        .into_iter()
        .filter(|s| !s.trim().is_empty())
        .map(|s| s.to_string())
        .collect()
}

/// jieba part-of-speech tagging (uses the global JIEBA instance).
pub fn tag_chinese(text: &str) -> Vec<(String, String)> {
    JIEBA
        .tag(text, true)
        .into_iter()
        .map(|t| (t.word.to_string(), t.tag.to_string()))
        .collect()
}

/// English/code tokenization: splits on whitespace + ASCII punctuation, lowercases, filters empties.
fn tokenize_english(text: &str) -> Vec<String> {
    text.split(|c: char| c.is_whitespace() || c.is_ascii_punctuation())
        .filter(|s| !s.is_empty())
        .map(|s| s.to_lowercase())
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn stable_hash_deterministic() {
        let h1 = stable_hash64("test text");
        let h2 = stable_hash64("test text");
        assert_eq!(h1, h2);
    }

    #[test]
    fn simhash_similarity_identical_and_orthogonal() {
        let a = [0x0123_4567_89AB_CDEF, 1, 2, 3];
        assert_eq!(
            simhash_similarity(&a, &a),
            1.0,
            "identical signatures → 1.0"
        );
        let zeros = [0u64; 4];
        let ones = [u64::MAX; 4];
        assert_eq!(simhash_similarity(&zeros, &ones), 0.0, "orthogonal → 0.0");
        // 一半位相同 → 0.5
        let half = [0x0000_0000_FFFF_FFFFu64; 4];
        assert_eq!(simhash_similarity(&zeros, &half), 0.5);
    }
}