wedb_embed 0.1.1

use rapidhash::RapidHashSet as HashSet;

/// 默认英文停用词表（对标 Apache Kvrocks 与 RediSearch 默认停用词）
pub const DEFAULT_STOP_WORDS: &[&str] = &[
    "a", "is", "the", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into",
    "it", "no", "not", "of", "on", "or", "such", "that", "their", "then", "there", "these", "they",
    "this", "to", "was", "will", "with",
];

/// 栈上编辑距离计算最大长度阈值
const STACK_BUFFER_MAX_LEN: usize = 64;

/// 文本分词器（标准小写规范化与标点切分）
#[inline]
pub fn tokenize_text(text: &str) -> Vec<String> {
    tokenize_text_with_stopwords(text, None)
}

/// 文本分词器（支持指定停用词过滤，单次循环高效状态机）
pub fn tokenize_text_with_stopwords(
    text: &str,
    stop_words: Option<&HashSet<String>>,
) -> Vec<String> {
    let mut words = Vec::new();
    let mut cur = String::with_capacity(16);

    for ch in text.chars() {
        if ch.is_alphanumeric() || ch == '_' {
            cur.push(ch.to_ascii_lowercase());
        } else if !cur.is_empty() {
            if stop_words.is_none_or(|sw| !sw.contains(&cur)) {
                words.push(cur);
                cur = String::with_capacity(16);
            } else {
                cur.clear();
            }
        }
    }
    if !cur.is_empty() && stop_words.is_none_or(|sw| !sw.contains(&cur)) {
        words.push(cur);
    }
    words
}

/// 转义字符反转义处理（针对已提取出的 tag 字符串单次遍历还原，带无转义快速路径）
#[inline]
pub fn unescape_tag_string(s: &str) -> String {
    if !s.contains('\\') {
        return s.to_string();
    }
    let mut res = String::with_capacity(s.len());
    let mut chars = s.chars();
    while let Some(ch) = chars.next() {
        if ch == '\\' {
            if let Some(next_ch) = chars.next() {
                res.push(next_ch);
            }
        } else {
            res.push(ch);
        }
    }
    res
}

/// 标签字段分割（支持自定义分隔符、转义字符、引号去除及大小写规范化）
pub fn tokenize_tags(text: &str, separator: char, case_sensitive: bool) -> Vec<String> {
    let mut tags = Vec::new();
    let mut cur = String::with_capacity(32);
    let mut escaped = false;

    for ch in text.chars() {
        if escaped {
            cur.push('\\');
            cur.push(ch);
            escaped = false;
            continue;
        }
        if ch == '\\' {
            escaped = true;
            continue;
        }
        if ch == separator {
            let trimmed = cur.trim().trim_matches('"').trim_matches('\'');
            if !trimmed.is_empty() {
                let unescaped = unescape_tag_string(trimmed);
                tags.push(if case_sensitive {
                    unescaped
                } else {
                    unescaped.to_lowercase()
                });
            }
            cur.clear();
        } else {
            cur.push(ch);
        }
    }

    if escaped {
        cur.push('\\');
    }

    let trimmed = cur.trim().trim_matches('"').trim_matches('\'');
    if !trimmed.is_empty() {
        let unescaped = unescape_tag_string(trimmed);
        tags.push(if case_sensitive {
            unescaped
        } else {
            unescaped.to_lowercase()
        });
    }

    tags
}

/// 字符串编辑距离（Levenshtein Distance，带栈上零堆分配与 ASCII 极速切片加速）
pub fn levenshtein_distance(s1: &str, s2: &str) -> usize {
    if s1 == s2 {
        return 0;
    }
    if s1.is_empty() {
        return s2.chars().count();
    }
    if s2.is_empty() {
        return s1.chars().count();
    }

    // ASCII 快速路径
    if s1.is_ascii() && s2.is_ascii() {
        let b1 = s1.as_bytes();
        let b2 = s2.as_bytes();
        let (b1, b2) = if b1.len() < b2.len() {
            (b2, b1)
        } else {
            (b1, b2)
        };
        let len1 = b1.len();
        let len2 = b2.len();

        if len2 <= STACK_BUFFER_MAX_LEN {
            let mut prev = [0usize; STACK_BUFFER_MAX_LEN + 1];
            let mut curr = [0usize; STACK_BUFFER_MAX_LEN + 1];
            for (j, item) in prev.iter_mut().enumerate().take(len2 + 1) {
                *item = j;
            }

            for i in 1..=len1 {
                curr[0] = i;
                for j in 1..=len2 {
                    let cost = usize::from(b1[i - 1] != b2[j - 1]);
                    curr[j] = (prev[j] + 1).min(curr[j - 1] + 1).min(prev[j - 1] + cost);
                }
                prev[..=len2].copy_from_slice(&curr[..=len2]);
            }
            return prev[len2];
        }

        let mut prev: Vec<usize> = (0..=len2).collect();
        let mut curr = vec![0; len2 + 1];

        for i in 1..=len1 {
            curr[0] = i;
            for j in 1..=len2 {
                let cost = usize::from(b1[i - 1] != b2[j - 1]);
                curr[j] = (prev[j] + 1).min(curr[j - 1] + 1).min(prev[j - 1] + cost);
            }
            prev.copy_from_slice(&curr);
        }
        return prev[len2];
    }

    // 通用 Unicode 路径
    let s1_chars: Vec<char> = s1.chars().collect();
    let s2_chars: Vec<char> = s2.chars().collect();
    let (s1_chars, s2_chars) = if s1_chars.len() < s2_chars.len() {
        (s2_chars, s1_chars)
    } else {
        (s1_chars, s2_chars)
    };
    let len1 = s1_chars.len();
    let len2 = s2_chars.len();

    if len2 <= STACK_BUFFER_MAX_LEN {
        let mut prev = [0usize; STACK_BUFFER_MAX_LEN + 1];
        let mut curr = [0usize; STACK_BUFFER_MAX_LEN + 1];
        for (j, item) in prev.iter_mut().enumerate().take(len2 + 1) {
            *item = j;
        }

        for i in 1..=len1 {
            curr[0] = i;
            for j in 1..=len2 {
                let cost = usize::from(s1_chars[i - 1] != s2_chars[j - 1]);
                curr[j] = (prev[j] + 1).min(curr[j - 1] + 1).min(prev[j - 1] + cost);
            }
            prev[..=len2].copy_from_slice(&curr[..=len2]);
        }
        return prev[len2];
    }

    let mut prev: Vec<usize> = (0..=len2).collect();
    let mut curr = vec![0; len2 + 1];

    for i in 1..=len1 {
        curr[0] = i;
        for j in 1..=len2 {
            let cost = usize::from(s1_chars[i - 1] != s2_chars[j - 1]);
            curr[j] = (prev[j] + 1).min(curr[j - 1] + 1).min(prev[j - 1] + cost);
        }
        prev.copy_from_slice(&curr);
    }

    prev[len2]
}