msime_engine/
format.rs

1//! 词库格式契约：拼音表的命名规则，以及 `msime-pinyin.db`、`msime-wubi.db`、`msime-others.db` 的文件内版本（`metadata` 表）。dict-builder 和运行时必须对它们取得一致，所以只在这里定义一次。完整规格见 `docs/data-formats.md` §4、§9、§12。
2
3use rusqlite::{Connection, OptionalExtension};
4
5use crate::diagnostics;
6use crate::error::{EngineError, Result};
7
8/// 词库格式版本：发布清单的 `format_version` 与 `engine_compatibility.dictionary_format`，也是 dict-builder 写进三个主库 `metadata` 表的 `format_version`。引擎读得懂这个版本及以下的文件。
9pub const FORMAT_VERSION: i32 = 1;
10
11/// 没有 `metadata` 表的文件按这个版本读：文件内版本出现之前发布的词库（`dict-v2.0.39` 及更早）都是这样，必须照常能用。
12pub const LEGACY_FORMAT_VERSION: i32 = 0;
13
14/// 三个主库的 `metadata` 表，形状与语言词库（`language_dictionary::SCHEMA`）的同名表相同：`name` 是键，`value` 是十进制文本。
15pub const METADATA_SCHEMA: &str =
16    "CREATE TABLE IF NOT EXISTS metadata(name TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;";
17
18/// `metadata` 里存放格式版本的那一行，与语言词库同名。
19pub const METADATA_FORMAT_VERSION: &str = crate::language_dictionary::METADATA_FORMAT_VERSION;
20
21/// Tables are numbered by syllable count up to this; longer keys go to the overflow bucket.
22pub const MAXIMUM_NUMBERED_SYLLABLES: usize = 7;
23/// First letters that have tables. `i`, `u` and `v` start no syllable, so they have none.
24pub const SHIPPED_INITIALS: &str = "abcdefghjklmnopqrstwxyz";
25pub const TABLE_PREFIX: &str = "tbl_";
26pub const OVERFLOW_BUCKET: &str = "others";
27
28/// `tbl_<n>_<initial>`, or `tbl_others_<initial>` past seven syllables. `None` for no syllables or an initial outside `a..=z`. A shipped-set miss (`i`, `u`, `v`) still gets a name; the table simply does not exist, and readers treat that as no rows.
29pub fn quanpin_table(syllables: usize, initial: u8) -> Option<String> {
30    if syllables == 0 || !initial.is_ascii_lowercase() {
31        return None;
32    }
33    let bucket = match syllables {
34        1 => "1",
35        2 => "2",
36        3 => "3",
37        4 => "4",
38        5 => "5",
39        6 => "6",
40        7 => "7",
41        _ => OVERFLOW_BUCKET,
42    };
43    let mut table = String::with_capacity(TABLE_PREFIX.len() + bucket.len() + 2);
44    table.push_str(TABLE_PREFIX);
45    table.push_str(bucket);
46    table.push('_');
47    table.push(initial as char);
48    Some(table)
49}
50
51/// The table for a segmented key: by segment count and the first letter of the first segment.
52pub fn build_table_name(segments: &[String]) -> Option<String> {
53    let first = segments.first()?.as_bytes().first()?;
54    quanpin_table(segments.len(), *first)
55}
56
57/// 读 `connection` 所在文件的格式版本并检查引擎读不读得懂；`file` 是写进错误信息的文件名（只给名字，不带路径）。没有 `metadata` 表时是 [`LEGACY_FORMAT_VERSION`]。有表却读不出 `format_version`（没有这一行，或表的列不对）、值不是非负整数、或比 [`FORMAT_VERSION`] 新时报 `DICTIONARY_FORMAT_UNSUPPORTED`（`EngineError::Failed`），调用方不能把这样的文件当作已知格式去读：新版本的表可能换了列或换了语义，照旧读只会悄悄出错。连 `sqlite_master` 都读不出（不是数据库、被锁住），或读 `metadata` 时遇到被锁、读写出错、文件损坏这类与格式无关的错误时，原样返回 `EngineError::Sqlite`，由调用方按它原来对读不了的文件的处理来办。
58pub fn check_format_version(connection: &Connection, file: &str) -> Result<i32> {
59    let unsupported = |detail: String| {
60        EngineError::failed(format!(
61            "{}{file} {detail}, this engine reads format_version {FORMAT_VERSION} or older",
62            diagnostics::DICTIONARY_FORMAT_UNSUPPORTED
63        ))
64    };
65    let has_metadata = connection.query_row(
66        "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='metadata')",
67        [],
68        |row| row.get::<_, bool>(0),
69    )?;
70    if !has_metadata {
71        return Ok(LEGACY_FORMAT_VERSION);
72    }
73    let value = connection
74        .query_row(
75            "SELECT value FROM metadata WHERE name = ?1",
76            [METADATA_FORMAT_VERSION],
77            |row| row.get::<_, String>(0),
78        )
79        .optional()
80        .map_err(|error| {
81            if is_environmental(&error) {
82                EngineError::Sqlite(error)
83            } else {
84                unsupported(format!("has an unreadable metadata table ({error})"))
85            }
86        })?
87        .ok_or_else(|| unsupported("has a metadata table without format_version".to_owned()))?;
88    match value.parse::<i32>() {
89        Ok(version) if (LEGACY_FORMAT_VERSION..=FORMAT_VERSION).contains(&version) => Ok(version),
90        _ => Err(unsupported(format!("has format_version {value:?}"))),
91    }
92}
93
94/// 读 `metadata` 时的错误里，只有 SQL 层面的形状问题（没有 `value` 列之类，SQLite 报的是笼统的 `SQLITE_ERROR`）和列值转换失败说明这张表不是认得的格式。被锁、读写出错、文件损坏这类错误与格式无关，像读 `sqlite_master` 失败一样原样交给调用方，不能说成格式不受支持。
95fn is_environmental(error: &rusqlite::Error) -> bool {
96    matches!(error, rusqlite::Error::SqliteFailure(failure, _) if failure.code != rusqlite::ErrorCode::Unknown)
97}
98
99#[cfg(test)]
100mod tests {
101    use super::*;
102
103    #[test]
104    fn names_follow_the_contract() {
105        assert_eq!(quanpin_table(2, b'n').as_deref(), Some("tbl_2_n"));
106        assert_eq!(quanpin_table(7, b'a').as_deref(), Some("tbl_7_a"));
107        assert_eq!(quanpin_table(8, b'z').as_deref(), Some("tbl_others_z"));
108        assert_eq!(quanpin_table(2, b'n').unwrap().capacity(), "tbl_2_n".len());
109        assert_eq!(
110            quanpin_table(8, b'z').unwrap().capacity(),
111            "tbl_others_z".len()
112        );
113        assert_eq!(quanpin_table(0, b'a'), None);
114        assert_eq!(quanpin_table(1, b'A'), None);
115        assert_eq!(
116            build_table_name(&["ni".into(), "hao".into()]).as_deref(),
117            Some("tbl_2_n")
118        );
119        assert_eq!(build_table_name(&[]), None);
120        assert_eq!(build_table_name(&[String::new()]), None);
121    }
122
123    fn database(sql: &str) -> Connection {
124        let connection = Connection::open_in_memory().unwrap();
125        connection.execute_batch(sql).unwrap();
126        connection
127    }
128
129    #[test]
130    fn a_file_without_metadata_is_the_legacy_version() {
131        let connection =
132            database("CREATE TABLE tbl_1_a(key TEXT, jp TEXT, value TEXT, weight INTEGER);");
133        assert_eq!(
134            check_format_version(&connection, "msime-pinyin.db").unwrap(),
135            LEGACY_FORMAT_VERSION
136        );
137    }
138
139    #[test]
140    fn a_file_that_is_not_a_database_is_left_to_the_caller() {
141        let directory = tempfile::tempdir().unwrap();
142        let path = directory.path().join("msime-pinyin.db");
143        std::fs::write(&path, b"not a database, not even close to one").unwrap();
144        let connection = Connection::open(&path).unwrap();
145        assert!(matches!(
146            check_format_version(&connection, "msime-pinyin.db"),
147            Err(EngineError::Sqlite(_))
148        ));
149    }
150
151    #[test]
152    fn only_shape_errors_on_the_metadata_table_mean_an_unknown_format() {
153        let failure = |code| {
154            rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(code), Some(String::new()))
155        };
156        for code in [
157            rusqlite::ffi::SQLITE_BUSY,
158            rusqlite::ffi::SQLITE_LOCKED,
159            rusqlite::ffi::SQLITE_IOERR,
160            rusqlite::ffi::SQLITE_CORRUPT,
161            rusqlite::ffi::SQLITE_NOTADB,
162        ] {
163            assert!(is_environmental(&failure(code)), "{code}");
164        }
165        assert!(!is_environmental(&failure(rusqlite::ffi::SQLITE_ERROR)));
166        assert!(!is_environmental(&rusqlite::Error::InvalidColumnType(
167            0,
168            "value".to_owned(),
169            rusqlite::types::Type::Blob
170        )));
171    }
172
173    #[test]
174    fn the_current_and_older_versions_are_read() {
175        for version in [LEGACY_FORMAT_VERSION, FORMAT_VERSION] {
176            let connection = database(&format!(
177                "{METADATA_SCHEMA} INSERT INTO metadata VALUES('format_version', '{version}');"
178            ));
179            assert_eq!(
180                check_format_version(&connection, "msime-wubi.db").unwrap(),
181                version
182            );
183        }
184    }
185
186    #[test]
187    fn a_newer_or_malformed_version_is_refused_with_the_file_name() {
188        let newer = FORMAT_VERSION + 1;
189        for (sql, detail) in [
190            (
191                format!(
192                    "{METADATA_SCHEMA} INSERT INTO metadata VALUES('format_version', '{newer}');"
193                ),
194                format!("has format_version \"{newer}\""),
195            ),
196            (
197                format!("{METADATA_SCHEMA} INSERT INTO metadata VALUES('format_version', 'one');"),
198                "has format_version \"one\"".to_owned(),
199            ),
200            (
201                format!("{METADATA_SCHEMA} INSERT INTO metadata VALUES('format_version', '-1');"),
202                "has format_version \"-1\"".to_owned(),
203            ),
204            (
205                format!("{METADATA_SCHEMA} INSERT INTO metadata VALUES('license', 'MIT');"),
206                "has a metadata table without format_version".to_owned(),
207            ),
208            (
209                "CREATE TABLE metadata(key TEXT, data TEXT);".to_owned(),
210                "has an unreadable metadata table".to_owned(),
211            ),
212        ] {
213            let connection = database(&sql);
214            let error = check_format_version(&connection, "msime-others.db")
215                .unwrap_err()
216                .to_string();
217            assert!(
218                error.starts_with(diagnostics::DICTIONARY_FORMAT_UNSUPPORTED),
219                "{error}"
220            );
221            assert!(error.contains("msime-others.db"), "{error}");
222            assert!(error.contains(&detail), "{error}");
223            assert!(
224                error.ends_with(&format!(
225                    "this engine reads format_version {FORMAT_VERSION} or older"
226                )),
227                "{error}"
228            );
229        }
230    }
231}
msime_engine/format.rs

msime_engine/
format.rs
