wedb_embed 0.1.0

use rapidhash::RapidHashSet;
use std::mem::take;

/// 默认英文停用词表（对标 Apache Kvrocks 与 RediSearch 默认停用词）
pub const DEFAULT_STOP_WORDS: &[&str] = &[
    "a", "is", "the", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into",
    "it", "no", "not", "of", "on", "or", "such", "that", "their", "then", "there", "these", "they",
    "this", "to", "was", "will", "with",
];

/// 文本分词器（标准小写规范化与标点切分）
#[inline]
pub fn tokenize_text(text: &str) -> Vec<String> {
    tokenize_text_with_stopwords(text, None)
}

/// 文本分词器（支持指定停用词过滤，单次循环高效状态机）
pub fn tokenize_text_with_stopwords(
    text: &str,
    stop_words: Option<&RapidHashSet<String>>,
) -> Vec<String> {
    let mut words = Vec::new();
    let mut cur = String::new();

    for ch in text.chars() {
        if ch.is_alphanumeric() || ch == '_' {
            cur.push(ch.to_ascii_lowercase());
        } else if !cur.is_empty() {
            let word = take(&mut cur);
            if stop_words.is_none_or(|sw| !sw.contains(&word)) {
                words.push(word);
            }
        }
    }
    if !cur.is_empty() && stop_words.is_none_or(|sw| !sw.contains(&cur)) {
        words.push(cur);
    }
    words
}

/// 转义字符反转义处理
pub fn unescape_tag_string(s: &str) -> String {
    let mut res = String::with_capacity(s.len());
    let mut chars = s.chars().peekable();
    while let Some(ch) = chars.next() {
        if ch == '\\' {
            if let Some(next_ch) = chars.next() {
                res.push(next_ch);
            }
        } else {
            res.push(ch);
        }
    }
    res
}

/// 标签字段分割（支持自定义分隔符与大小写敏感选项，支持反转义与引号去除）
pub fn tokenize_tags(text: &str, separator: char, case_sensitive: bool) -> Vec<String> {
    text.split(separator)
        .map(|s| {
            let trimmed = s.trim().trim_matches('"').trim_matches('\'');
            let unescaped = unescape_tag_string(trimmed);
            if case_sensitive {
                unescaped
            } else {
                unescaped.to_lowercase()
            }
        })
        .filter(|s| !s.is_empty())
        .collect()
}

/// 字符串编辑距离（Levenshtein Distance，空间优化为 O(min(M, N))，用于 FUZZY 模糊匹配）
pub fn levenshtein_distance(s1: &str, s2: &str) -> usize {
    if s1 == s2 {
        return 0;
    }
    let s1_chars: Vec<char> = s1.chars().collect();
    let s2_chars: Vec<char> = s2.chars().collect();
    let len1 = s1_chars.len();
    let len2 = s2_chars.len();
    if len1 == 0 {
        return len2;
    }
    if len2 == 0 {
        return len1;
    }

    let (s1_chars, s2_chars, len1, len2) = if len1 < len2 {
        (s2_chars, s1_chars, len2, len1)
    } else {
        (s1_chars, s2_chars, len1, len2)
    };

    let mut prev: Vec<usize> = (0..=len2).collect();
    let mut curr = vec![0; len2 + 1];

    for i in 1..=len1 {
        curr[0] = i;
        for j in 1..=len2 {
            let cost = if s1_chars[i - 1] == s2_chars[j - 1] {
                0
            } else {
                1
            };
            curr[j] = (prev[j] + 1).min(curr[j - 1] + 1).min(prev[j - 1] + cost);
        }
        prev.copy_from_slice(&curr);
    }

    prev[len2]
}