msime-engine 0.2.0

//! The Chinese administrative divisions `@` mode can offer after the user's own list: provinces, cities, and counties and districts, from modood/Administrative-divisions-of-China (WTFPL) at commit c49d495b40ac73eb1a66f6eeae5f8fd10696f035 (data of 2025-12-27), the pin `resources/dictionary-sources.lock.json` holds under `places/`.
//!
//! `places.tsv` is generated by `msime-dict-build places` and embedded, so the table needs no download and no file on disk. Each line is `name<TAB>key<TAB>parent`: the key is the pinyin `'`-joined as a mention key is, the parent is the province of a city, or the city (the province, for a district of a municipality) of a county or district, and empty for a province. Lines come larger divisions first and each name occurs once.

use std::collections::HashMap;
use std::sync::OnceLock;

const TABLE: &str = include_str!("places.tsv");
// 位图容量由内置表在编译期决定，表增长时不需要手工同步行数上限。
const TABLE_LINE_COUNT: usize = {
    let bytes = TABLE.as_bytes();
    let mut count = 0;
    let mut index = 0;
    while index < bytes.len() {
        if bytes[index] == b'\n' {
            count += 1;
        }
        index += 1;
    }
    if !bytes.is_empty() && bytes[bytes.len() - 1] != b'\n' {
        count += 1;
    }
    count
};
const MATCH_MASK_WORDS: usize = TABLE_LINE_COUNT.div_ceil(u64::BITS as usize);

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Place {
    pub name: &'static str,
    pub key: &'static str,
    pub parent: &'static str,
}

pub struct Places {
    pub places: Vec<Place>,
    /// 每个地点去掉分隔符的完整拼写和各音节首字母，前缀索引借用这两种既有拼写。
    pub spellings: Vec<[String; 2]>,
    /// 拼写编号为地点行号乘 2 加拼写下标，按对应字符串排序，不复制字符串。
    spelling_index: Vec<usize>,
    parents: HashMap<&'static str, &'static str>,
}

impl Places {
    fn parse(table: &'static str) -> Self {
        let mut places = Vec::with_capacity(table.lines().count());
        for line in table.lines() {
            let mut fields = line.split('\t');
            if let (Some(name), Some(key), Some(parent), None) =
                (fields.next(), fields.next(), fields.next(), fields.next())
            {
                places.push(Place { name, key, parent });
            }
        }
        assert!(
            places.len() <= TABLE_LINE_COUNT,
            "地点索引只用于内置表容量范围"
        );
        let spellings: Vec<[String; 2]> = places
            .iter()
            .map(|place| {
                [
                    place.key.replace('\'', ""),
                    place
                        .key
                        .split('\'')
                        .filter_map(|syllable| syllable.chars().next())
                        .collect(),
                ]
            })
            .collect();
        let mut spelling_index: Vec<usize> = (0..spellings.len() * 2).collect();
        spelling_index.sort_unstable_by(|&left, &right| {
            spellings[left / 2][left % 2]
                .cmp(&spellings[right / 2][right % 2])
                .then_with(|| left.cmp(&right))
        });
        let parents = places
            .iter()
            .map(|place| (place.name, place.parent))
            .collect();
        Self {
            places,
            spellings,
            spelling_index,
            parents,
        }
    }

    /// 只提供完整拼写或首字母以 `code` 开头的内置地点，按原表顺序输出，每行一次。
    pub(super) fn matching_entries(
        &self,
        code: &str,
    ) -> impl Iterator<Item = (&Place, &[String; 2])> {
        let start = self
            .spelling_index
            .partition_point(|&index| self.spellings[index / 2][index % 2].as_str() < code);
        let mut masks = [0_u64; MATCH_MASK_WORDS];
        let mut used_words = 0;
        for &index in self.spelling_index[start..]
            .iter()
            .take_while(|&&index| self.spellings[index / 2][index % 2].starts_with(code))
        {
            if self.spellings[index / 2][index % 2].is_empty() {
                continue;
            }
            let row = index / 2;
            let word = row / u64::BITS as usize;
            masks[word] |= 1_u64 << (row % u64::BITS as usize);
            used_words = used_words.max(word + 1);
        }
        // 拼写顺序只用于查找；按行号遍历位图，保留源表优先级和同名过滤的先后关系。
        masks
            .into_iter()
            .take(used_words)
            .enumerate()
            .flat_map(|(word, mut bits)| {
                std::iter::from_fn(move || {
                    if bits == 0 {
                        return None;
                    }
                    let row = word * u64::BITS as usize + bits.trailing_zeros() as usize;
                    bits &= bits - 1;
                    Some(row)
                })
            })
            .map(|row| (&self.places[row], &self.spellings[row]))
    }

    /// The division a place belongs to, empty for a province, `None` when the text is not a place.
    pub fn parent(&self, name: &str) -> Option<&'static str> {
        self.parents.get(name).copied()
    }
}

/// The embedded table, parsed on first use.
pub fn places() -> &'static Places {
    static PLACES: OnceLock<Places> = OnceLock::new();
    PLACES.get_or_init(|| Places::parse(TABLE))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn the_embedded_table_is_well_formed() {
        let table = places();
        assert_eq!(table.places.len(), TABLE.lines().count());
        assert_eq!(table.places.capacity(), TABLE.lines().count());
        assert!(table.places.len() > 3000);
        assert_eq!(table.parents.len(), table.places.len());
        for place in &table.places {
            assert!(!place.name.trim().is_empty(), "{place:?}");
            assert!(
                place.key.split('\'').all(|syllable| !syllable.is_empty()
                    && syllable.bytes().all(|byte| byte.is_ascii_lowercase())),
                "{place:?}"
            );
            assert!(
                place.parent.is_empty() || table.parent(place.parent).is_some(),
                "{place:?}"
            );
        }
    }

    #[test]
    fn places_carry_their_reading_and_parent() {
        let table = places();
        let find = |name: &str| {
            *table
                .places
                .iter()
                .find(|place| place.name == name)
                .unwrap()
        };
        assert_eq!(
            find("重庆市"),
            Place {
                name: "重庆市",
                key: "chong'qing'shi",
                parent: ""
            }
        );
        assert_eq!(find("六安市").key, "lu'an'shi");
        assert_eq!(find("厦门市").key, "xia'men'shi");
        assert_eq!(table.parent("深圳市"), Some("广东省"));
        assert_eq!(table.parent("朝阳区"), Some("北京市"));
        assert_eq!(table.parent("张三"), None);
        assert_eq!(table.places[0].name, "北京市");
    }
}

#[cfg(test)]
#[path = "places/prefix_index_tests.rs"]
mod prefix_index_tests;