edgevane_langdetect 0.1.0

no_std language detection engine (EVLC/EVLD): char/word n-grams, TF-IDF scoring, C API
Documentation
use alloc::vec::Vec;
use core::{fmt, str};

use crate::format::{
    self, SectionId, HEADER_LEN, MAGIC_EVLC, MAGIC_EVLD, SECTBL_OFF, SECTION_COUNT, VERSION,
};
use crate::normalize::CLASS_COUNT as NCLASS;


#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ParseError {
    TooShort,
    BadMagic,
    BadVersion,
    BadSectionTable,
    BadCrc,
    BadLangCode,
    TruncatedSection,
}

impl fmt::Display for ParseError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let s = match self {
            Self::TooShort => "buffer shorter than header",
            Self::BadMagic => "bad magic, not an .evld/.evlc file",
            Self::BadVersion => "unsupported version",
            Self::BadSectionTable => "section offsets out of bounds",
            Self::BadCrc => "CRC32 mismatch (corrupt file)",
            Self::BadLangCode => "lang code is not valid UTF-8",
            Self::TruncatedSection => "section payload truncated",
        };
        f.write_str(s)
    }
}

#[cfg(feature = "std")]
impl std::error::Error for ParseError {}

fn le_u16(b: &[u8]) -> u16 {
    u16::from_le_bytes([b[0], b[1]])
}
fn le_u32(b: &[u8]) -> u32 {
    u32::from_le_bytes([b[0], b[1], b[2], b[3]])
}
fn le_i16(b: &[u8]) -> i16 {
    i16::from_le_bytes([b[0], b[1]])
}
fn le_f32(b: &[u8]) -> f32 {
    f32::from_le_bytes([b[0], b[1], b[2], b[3]])
}
fn le_u64(b: &[u8]) -> u64 {
    u64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]])
}


#[derive(Clone, Debug)]
pub struct Header {
    pub lang: [u8; 8],
    pub flags: u16,
    pub docs: u32,
    pub chars_total: u64,
    pub tokens_total: u64,
    pub avg_doc_len: f32,
    pub bm25_k1: f32,
    pub bm25_b: f32,
    pub oov_logprob_q: i16,

    pub sections: [(u32, u32, u32); SECTION_COUNT],
}

impl Header {
    pub fn lang_code(&self) -> &str {
        let len = self.lang.iter().position(|&b| b == 0).unwrap_or(8);
        str::from_utf8(&self.lang[..len]).unwrap_or("???")
    }
}

fn parse_header(bytes: &[u8], magic: &[u8; 4]) -> Result<Header, ParseError> {
    if bytes.len() < HEADER_LEN {
        return Err(ParseError::TooShort);
    }
    if &bytes[0..4] != magic {
        return Err(ParseError::BadMagic);
    }
    if le_u16(&bytes[4..6]) != VERSION {
        return Err(ParseError::BadVersion);
    }
    if format::crc32(&bytes[0..252]) != le_u32(&bytes[252..256]) {
        return Err(ParseError::BadCrc);
    }
    let mut lang = [0u8; 8];
    lang.copy_from_slice(&bytes[6..14]);
    if str::from_utf8(&lang).is_err() {
        return Err(ParseError::BadLangCode);
    }
    let mut sections = [(0u32, 0u32, 0u32); SECTION_COUNT];
    for i in 0..SECTION_COUNT {
        let o = SECTBL_OFF + i * 12;
        sections[i] = (
            le_u32(&bytes[o..o + 4]),
            le_u32(&bytes[o + 4..o + 8]),
            le_u32(&bytes[o + 8..o + 12]),
        );
        let (off, len, _) = sections[i];
        let end = (off as usize).saturating_add(len as usize);
        if off as usize != 0 && (off as usize) < HEADER_LEN || end > bytes.len() {
            if (off as usize) < HEADER_LEN && len != 0 {
                return Err(ParseError::BadSectionTable);
            }
            if end > bytes.len() {
                return Err(ParseError::BadSectionTable);
            }
        }
    }
    Ok(Header {
        lang,
        flags: le_u16(&bytes[14..16]),
        docs: le_u32(&bytes[16..20]),
        chars_total: le_u64(&bytes[20..28]),
        tokens_total: le_u64(&bytes[28..36]),
        avg_doc_len: le_f32(&bytes[36..40]),
        bm25_k1: le_f32(&bytes[40..44]),
        bm25_b: le_f32(&bytes[44..48]),
        oov_logprob_q: le_i16(&bytes[48..50]),
        sections,
    })
}


#[derive(Clone, Debug)]
pub struct LangDb<'a> {
    bytes: &'a [u8],
    hdr: Header,
}

impl<'a> LangDb<'a> {

    pub fn from_bytes(bytes: &'a [u8]) -> Result<Self, ParseError> {
        let hdr = parse_header(bytes, &MAGIC_EVLD)?;
        let db = Self { bytes, hdr };

        for i in 0..SECTION_COUNT {
            db.section_payload(i)?;
        }
        Ok(db)
    }

    pub fn header(&self) -> &Header {
        &self.hdr
    }

    pub fn lang_code(&self) -> &str {
        self.hdr.lang_code()
    }

    fn section_payload(&self, idx: usize) -> Result<&'a [u8], ParseError> {
        let (off, len, _) = self.hdr.sections[idx];
        if len == 0 {
            return Ok(&[]);
        }
        let off = off as usize;
        let len = len as usize;
        let body = self
            .bytes
            .get(off..off + len)
            .ok_or(ParseError::TruncatedSection)?;
        let crc = self
            .bytes
            .get(off + len..off + len + 4)
            .ok_or(ParseError::TruncatedSection)?;
        if format::crc32(body) != le_u32(crc) {
            return Err(ParseError::BadCrc);
        }
        Ok(body)
    }

    fn section_raw(&self, id: SectionId) -> &'a [u8] {

        let (off, len, _) = self.hdr.sections[id.index()];
        if len == 0 {
            return &[];
        }
        &self.bytes[off as usize..off as usize + len as usize]
    }


    fn lookup_rec(section: &[u8], key: u32) -> Option<i32> {
        let n = section.len() / 6;
        let mut lo = 0usize;
        let mut hi = n;
        while lo < hi {
            let mid = (lo + hi) / 2;
            let o = mid * 6;
            let k = u32::from_le_bytes([
                section[o],
                section[o + 1],
                section[o + 2],
                section[o + 3],
            ]);
            if k < key {
                lo = mid + 1;
            } else if k > key {
                hi = mid;
            } else {
                return Some(i16::from_le_bytes([section[o + 4], section[o + 5]]) as i32);
            }
        }
        None
    }


    pub fn ngram_q(&self, id: SectionId, hash: u32) -> i32 {
        debug_assert!(matches!(
            id,
            SectionId::CharBigram
                | SectionId::CharTrigram
                | SectionId::TokenUnigram
                | SectionId::TokenBigram
                | SectionId::Prefix
                | SectionId::Suffix
        ));
        Self::lookup_rec(self.section_raw(id), hash)
            .unwrap_or(self.hdr.oov_logprob_q as i32)
    }


    pub fn char_unigram_q(&self, codepoint: u32) -> i32 {
        Self::lookup_rec(self.section_raw(SectionId::CharUnigram), codepoint)
            .unwrap_or(self.hdr.oov_logprob_q as i32)
    }


    pub fn token_idf(&self, hash: u32) -> f32 {
        let sec = self.section_raw(SectionId::TokenIdf);
        let n = sec.len() / 6;
        let mut lo = 0usize;
        let mut hi = n;
        while lo < hi {
            let mid = (lo + hi) / 2;
            let o = mid * 6;
            let k = u32::from_le_bytes([sec[o], sec[o + 1], sec[o + 2], sec[o + 3]]);
            if k < hash {
                lo = mid + 1;
            } else if k > hash {
                hi = mid;
            } else {
                let q = u16::from_le_bytes([sec[o + 4], sec[o + 5]]);
                return format::dequantize_idf(q);
            }
        }
        0.0
    }


    pub fn wordlen(&self) -> Vec<f32> {
        let raw = self.section_raw(SectionId::WordLen);
        raw.chunks_exact(2)
            .map(|c| (u16::from_le_bytes([c[0], c[1]]) as f32) / 65535.0)
            .collect()
    }


    pub fn sentlen(&self) -> Vec<f32> {
        let raw = self.section_raw(SectionId::SentLen);
        raw.chunks_exact(2)
            .map(|c| (u16::from_le_bytes([c[0], c[1]]) as f32) / 65535.0)
            .collect()
    }


    pub fn charpos_q(&self, uni_index: Option<usize>, slot: usize) -> i32 {
        let raw = self.section_raw(SectionId::CharPos);
        match uni_index {
            Some(i) => {
                let o = (i * 3 + slot.min(2)) * 2;
                if let Some(c) = raw.get(o..o + 2) {
                    return i16::from_le_bytes([c[0], c[1]]) as i32;
                }
                self.hdr.oov_logprob_q as i32
            }
            None => self.hdr.oov_logprob_q as i32,
        }
    }


    pub fn uni_index(&self, codepoint: u32) -> Option<usize> {
        let sec = self.section_raw(SectionId::CharUnigram);
        let n = sec.len() / 6;
        let mut lo = 0usize;
        let mut hi = n;
        while lo < hi {
            let mid = (lo + hi) / 2;
            let o = mid * 6;
            let k = u32::from_le_bytes([sec[o], sec[o + 1], sec[o + 2], sec[o + 3]]);
            if k < codepoint {
                lo = mid + 1;
            } else if k > codepoint {
                hi = mid;
            } else {
                return Some(mid);
            }
        }
        None
    }


    pub fn class_trans_q(&self, a: usize, b: usize) -> i32 {
        debug_assert!(a < NCLASS && b < NCLASS);
        let raw = self.section_raw(SectionId::ClassTrans);
        let o = (a * NCLASS + b) * 2;
        raw.get(o..o + 2)
            .map(|c| u16::from_le_bytes([c[0], c[1]]) as i32)
            .unwrap_or(0)
    }
}


#[derive(Clone, Debug)]
pub struct CombinedDb<'a> {
    bytes: &'a [u8],
    count: usize,
}

impl<'a> CombinedDb<'a> {
    pub fn from_bytes(bytes: &'a [u8]) -> Result<Self, ParseError> {
        if bytes.len() < HEADER_LEN {
            return Err(ParseError::TooShort);
        }
        if &bytes[0..4] != MAGIC_EVLC {
            return Err(ParseError::BadMagic);
        }
        if le_u16(&bytes[4..6]) != VERSION {
            return Err(ParseError::BadVersion);
        }
        if format::crc32(&bytes[0..252]) != le_u32(&bytes[252..256]) {
            return Err(ParseError::BadCrc);
        }
        let count = le_u16(&bytes[6..8]) as usize;
        let need = HEADER_LEN + count * 16;
        if bytes.len() < need {
            return Err(ParseError::TooShort);
        }

        let db = Self { bytes, count };
        for i in 0..count {
            db.get(i)?;
        }
        Ok(db)
    }

    pub fn len(&self) -> usize {
        self.count
    }

    pub fn is_empty(&self) -> bool {
        self.count == 0
    }

    pub fn get(&self, i: usize) -> Result<LangDb<'a>, ParseError> {
        if i >= self.count {
            return Err(ParseError::TruncatedSection);
        }
        let o = HEADER_LEN + i * 16;
        let off = le_u32(&self.bytes[o + 8..o + 12]) as usize;
        let len = le_u32(&self.bytes[o + 12..o + 16]) as usize;
        let blob = self.bytes.get(off..off + len).ok_or(ParseError::BadSectionTable)?;
        LangDb::from_bytes(blob)
    }


    pub fn find(&self, code: &str) -> Option<LangDb<'a>> {
        (0..self.count).find_map(|i| {
            let db = self.get(i).ok()?;
            (db.lang_code() == code).then_some(db)
        })
    }
}