msime_engine/user_dictionary/
replay.rs

1//! Replaying the journal into a fresh dictionary generation (user-dictionary.md §12): on install, upgrade and every generation switch, so learning survives a new dictionary release.
2//!
3//! The row writers here (`apply_pinyin`, `apply_simple`, `apply_english`) are also what the personal dictionary edit applies to the working copy, so an edit and its later replay can never disagree about what a journal row means.
4
5use std::path::Path;
6
7use rusqlite::types::{ToSqlOutput, ValueRef};
8use rusqlite::{params, Connection, OpenFlags, Row, ToSql};
9
10use crate::dictionary::english::ensure_english_schema;
11use crate::format;
12use crate::user_dictionary::journal::{open_database, BUSY_TIMEOUT_MS};
13
14#[derive(Debug, Clone, PartialEq, Eq, Default)]
15pub struct ReplayResult {
16    pub applied: i32,
17    pub skipped: i32,
18    pub failed: i32,
19    /// Empty on success.
20    pub error: String,
21}
22
23/// Apply every operation in `(updated_at, rowid)` order to `main_db` and `english_db` (J:1642-1735). Never panics or errors; failures are counted and described.
24pub fn replay(user_db: &Path, main_db: &Path, english_db: &Path) -> ReplayResult {
25    replay_with(user_db, Some(main_db), english_db)
26}
27
28/// 没有 `msime-pinyin.db` 的代次（方案集合不读它，见 `SchemeSet::reads_main_dictionary`）：只把英文行回放进 `english_db`。拼音、五笔和快捷短语的行在这里没有能写的表，计入 `skipped` 而不是 `failed`，所以它们（例如从别的版本恢复来的快照）不会让这个代次被拒；行本身留在日志里不动。
29pub fn replay_english(user_db: &Path, english_db: &Path) -> ReplayResult {
30    replay_with(user_db, None, english_db)
31}
32
33/// 不带 `msime-pinyin.db` 时代替它的连接：一个内存库，`msime-english.db` 和日志照常 attach 上去，写进去的只有 attach 的那几个库。等待锁的时间与 `open_database` 打开的连接相同。
34pub(super) fn open_without_main_dictionary() -> rusqlite::Result<Connection> {
35    let connection = Connection::open_in_memory_with_flags(
36        OpenFlags::SQLITE_OPEN_READ_WRITE
37            | OpenFlags::SQLITE_OPEN_CREATE
38            | OpenFlags::SQLITE_OPEN_FULL_MUTEX,
39    )?;
40    connection.busy_timeout(std::time::Duration::from_millis(BUSY_TIMEOUT_MS))?;
41    Ok(connection)
42}
43
44fn replay_with(user_db: &Path, main_db: Option<&Path>, english_db: &Path) -> ReplayResult {
45    let mut result = ReplayResult::default();
46    // A first install has no journal and nothing to replay; replay never creates one (J:1646-1647).
47    if !user_db.exists() {
48        return result;
49    }
50    let fail = |mut result: ReplayResult, error: &str| {
51        result.error = error.to_owned();
52        result
53    };
54    let Ok(journal) = open_database(user_db, OpenFlags::SQLITE_OPEN_READ_ONLY) else {
55        return fail(result, "cannot open user dictionary database");
56    };
57    if ensure_english_schema(english_db).is_err() {
58        return fail(result, "cannot migrate English dictionary database");
59    }
60    let main = match main_db {
61        Some(main_db) => open_database(main_db, OpenFlags::SQLITE_OPEN_READ_WRITE).ok(),
62        None => open_without_main_dictionary().ok(),
63    };
64    let Some(main) = main else {
65        return fail(result, "cannot open target dictionary database");
66    };
67    if attach(&main, english_db, "replay_english").is_err() {
68        return fail(result, "cannot attach English dictionary database");
69    }
70    // 资源目录不带 `msime-wubi.db`（macOS 发布包改为按需下载）时代次里没有五笔表：五笔行没有能写的表，与没有拼音库的代次一样计入 `skipped`，行留在日志里，码表并进来后的那次回放再写（`generation::merge_downloaded_wubi`）。只要有一张五笔表就照旧逐行写，缺另一张时仍算失败。
71    let wubi_tables = main_db.is_some()
72        && main
73            .query_row(
74                "SELECT EXISTS(SELECT 1 FROM main.sqlite_master WHERE type='table' AND name IN ('wubi86','wubi98'))",
75                [],
76                |row| row.get::<_, bool>(0),
77            )
78            .unwrap_or(true);
79    let Ok(mut rows) = journal.prepare(
80        "SELECT dictionary,key,value,operation,weight,display FROM user_dictionary_operations ORDER BY updated_at,rowid",
81    ) else {
82        return fail(result, "invalid user dictionary database");
83    };
84    // Plain statements rather than a rusqlite transaction: the reference reports whether its ROLLBACK itself succeeded, which a dropped `Transaction` cannot tell.
85    if main.execute_batch("BEGIN IMMEDIATE").is_err() {
86        return fail(result, "cannot start replay transaction");
87    }
88    let read_complete = (|| -> rusqlite::Result<()> {
89        let mut cursor = rows.query([])?;
90        while let Some(row) = cursor.next()? {
91            let operation = JournalRow::read(row)?;
92            if main_db.is_none() && operation.kind != b"english" {
93                result.skipped += 1;
94                continue;
95            }
96            if !wubi_tables && (operation.kind == b"wubi" || operation.kind == b"wubi98") {
97                result.skipped += 1;
98                continue;
99            }
100            if operation.kind == b"pinyin"
101                && operation.operation == b"upsert"
102                && operation.weight < 1
103            {
104                // Rows the old rebalance staircase left below the floor would bury shipped frequencies (J:1695-1699).
105                result.skipped += 1;
106                continue;
107            }
108            // A failed row is counted, not fatal here: the whole replay is rolled back below once any row failed (J:1700-1709).
109            if matches!(operation.apply(&main), Ok(true)) {
110                result.applied += 1;
111            } else {
112                result.failed += 1;
113            }
114        }
115        Ok(())
116    })();
117    if read_complete.is_err() {
118        result.error = if main.execute_batch("ROLLBACK").is_ok() {
119            "cannot read the complete user dictionary journal; changes were rolled back"
120        } else {
121            "cannot read the complete user dictionary journal and rollback also failed"
122        }
123        .to_owned();
124        return result;
125    }
126    if result.failed == 0 {
127        if main.execute_batch("COMMIT").is_err() {
128            result.error = if main.execute_batch("ROLLBACK").is_ok() {
129                "cannot commit replay transaction; changes were rolled back"
130            } else {
131                "cannot commit or roll back replay transaction"
132            }
133            .to_owned();
134        }
135    } else {
136        result.error = if main.execute_batch("ROLLBACK").is_ok() {
137            "one or more operations failed; changes were rolled back"
138        } else {
139            "one or more operations failed and rollback also failed"
140        }
141        .to_owned();
142    }
143    result
144}
145
146/// One `user_dictionary_operations` row as replay reads it. Text columns are raw bytes, as `sqlite3_column_text` gave them to the reference (J:1688-1694), and are written back as the same bytes: a journal an older build wrote with text that is not UTF-8 replays exactly as it did there, rather than rolling back every generation built from it. A NULL, which the reference could not read either, still ends the read as a cursor failure.
147struct JournalRow {
148    kind: Vec<u8>,
149    key: Vec<u8>,
150    value: Vec<u8>,
151    operation: Vec<u8>,
152    weight: i64,
153    display: Vec<u8>,
154}
155
156const MAX_REPLAY_KIND_BYTES: usize = 32;
157const MAX_REPLAY_KEY_BYTES: usize = 512;
158const MAX_REPLAY_VALUE_BYTES: usize = 4 * 1024;
159const MAX_REPLAY_OPERATION_BYTES: usize = 16;
160const MAX_REPLAY_DISPLAY_BYTES: usize = 4 * 1024;
161
162impl JournalRow {
163    fn read(row: &Row<'_>) -> rusqlite::Result<Self> {
164        let text = |index: usize, maximum: usize| -> rusqlite::Result<Vec<u8>> {
165            let bytes = row.get_ref(index)?.as_bytes()?;
166            if bytes.len() > maximum {
167                return Err(rusqlite::Error::InvalidQuery);
168            }
169            Ok(bytes.to_vec())
170        };
171        Ok(Self {
172            kind: text(0, MAX_REPLAY_KIND_BYTES)?,
173            key: text(1, MAX_REPLAY_KEY_BYTES)?,
174            value: text(2, MAX_REPLAY_VALUE_BYTES)?,
175            operation: text(3, MAX_REPLAY_OPERATION_BYTES)?,
176            weight: row.get(4)?,
177            display: text(5, MAX_REPLAY_DISPLAY_BYTES)?,
178        })
179    }
180
181    /// `Ok(false)` for a row that cannot be stored: an unknown dictionary or a malformed pinyin key.
182    fn apply(&self, main: &Connection) -> rusqlite::Result<bool> {
183        let delete = self.operation == b"delete";
184        match self.kind.as_slice() {
185            b"pinyin" => apply_pinyin(main, &self.key, &self.value, delete, self.weight),
186            b"wubi" => apply_simple(main, "wubi86", &self.key, &self.value, delete, self.weight),
187            b"wubi98" => apply_simple(main, "wubi98", &self.key, &self.value, delete, self.weight),
188            b"quick" => apply_simple(
189                main,
190                "quick_parases",
191                &self.key,
192                &self.value,
193                delete,
194                self.weight,
195            ),
196            b"english" => apply_english(
197                main,
198                &self.key,
199                &self.value,
200                delete,
201                self.weight,
202                &self.display,
203            ),
204            _ => Ok(false),
205        }
206    }
207}
208
209/// Bytes bound as TEXT the way `sqlite3_bind_text` bound the reference's `std::string`, whatever their encoding; `&[u8]` alone would bind a BLOB, which never equals a stored text value.
210struct Text<'a>(&'a [u8]);
211
212impl ToSql for Text<'_> {
213    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
214        Ok(ToSqlOutput::Borrowed(ValueRef::Text(self.0)))
215    }
216}
217
218/// `ATTACH DATABASE ?1 AS <schema>`; `schema` is one of this module's fixed aliases, never user input.
219///
220/// iOS 以外日志是 WAL：附上它的事务只在每个文件内原子，崩溃时可能日志已提交而词库回滚，下一次回放补齐，见 `journal::prefer_write_ahead_log`。
221pub(super) fn attach(connection: &Connection, path: &Path, schema: &str) -> rusqlite::Result<()> {
222    let path = crate::paths::sqlite_path_no_follow(path)
223        .map_err(|_| rusqlite::Error::InvalidPath(path.to_owned()))?;
224    let name = path
225        .to_str()
226        .ok_or_else(|| rusqlite::Error::InvalidPath(path.clone()))?;
227    connection.execute(&format!("ATTACH DATABASE ?1 AS {schema}"), [name])?;
228    Ok(())
229}
230
231/// Delete, or update and insert when no row changed, a pinyin row with its jianpin column (J:306-332). A delete that finds nothing still succeeds.
232pub(super) fn apply_pinyin(
233    main: &Connection,
234    key: &[u8],
235    value: &[u8],
236    delete: bool,
237    weight: i64,
238) -> rusqlite::Result<bool> {
239    // The reference's `pinyin_segments` on bytes (J:182-199): an empty segment means the key cannot be stored.
240    let segments: Vec<&[u8]> = key.split(|&byte| byte == b'\'').collect();
241    if segments.iter().any(|segment| segment.is_empty()) {
242        return Ok(false);
243    }
244    // An initial outside `a..=z` names no table: the row cannot be stored.
245    let Some(table) = format::quanpin_table(segments.len(), segments[0][0]) else {
246        return Ok(false);
247    };
248    let (key, value) = (Text(key), Text(value));
249    if delete {
250        main.prepare_cached(&format!(
251            "DELETE FROM \"{table}\" WHERE key=?1 AND value=?2"
252        ))?
253        .execute(params![key, value])?;
254        return Ok(true);
255    }
256    // The first character of each syllable. The reference took the first byte, which is the same for every key a writer accepts (lowercase ASCII); a segment that does not start with valid UTF-8 contributes U+FFFD here rather than a stray byte, since `jp` is only ever matched against typed letters.
257    let jianpin: String = segments
258        .iter()
259        .filter_map(|segment| String::from_utf8_lossy(segment).chars().next())
260        .collect();
261    let changed = main
262        .prepare_cached(&format!(
263            "UPDATE \"{table}\" SET jp=?1,weight=?2 WHERE key=?3 AND value=?4"
264        ))?
265        .execute(params![jianpin, weight, key, value])?;
266    if changed > 0 {
267        return Ok(true);
268    }
269    main.prepare_cached(&format!(
270        "INSERT INTO \"{table}\"(key,jp,value,weight) VALUES(?1,?2,?3,?4)"
271    ))?
272    .execute(params![key, jianpin, value, weight])?;
273    Ok(true)
274}
275
276/// The same for `wubi86` and `quick_parases`, which have no jianpin column (J:334-355).
277pub(super) fn apply_simple(
278    main: &Connection,
279    table: &str,
280    key: &[u8],
281    value: &[u8],
282    delete: bool,
283    weight: i64,
284) -> rusqlite::Result<bool> {
285    let (key, value) = (Text(key), Text(value));
286    if delete {
287        main.prepare_cached(&format!(
288            "DELETE FROM \"{table}\" WHERE \"key\"=?1 AND \"value\"=?2"
289        ))?
290        .execute(params![key, value])?;
291        return Ok(true);
292    }
293    let changed = main
294        .prepare_cached(&format!(
295            "UPDATE \"{table}\" SET weight=?1 WHERE \"key\"=?2 AND \"value\"=?3"
296        ))?
297        .execute(params![weight, key, value])?;
298    if changed > 0 {
299        return Ok(true);
300    }
301    main.prepare_cached(&format!(
302        "INSERT INTO \"{table}\"(\"key\",\"value\",weight) VALUES(?1,?2,?3)"
303    ))?
304    .execute(params![key, value, weight])?;
305    Ok(true)
306}
307
308/// Upsert or delete in the English dictionary attached as `replay_english`, keyed by `(word, display)`; an empty journal display means the value is the display (J:357-371).
309pub(super) fn apply_english(
310    main: &Connection,
311    key: &[u8],
312    value: &[u8],
313    delete: bool,
314    weight: i64,
315    display: &[u8],
316) -> rusqlite::Result<bool> {
317    let display = Text(if display.is_empty() { value } else { display });
318    let key = Text(key);
319    if delete {
320        main.prepare_cached(
321            "DELETE FROM replay_english.english_words WHERE word=?1 AND display=?2",
322        )?
323        .execute(params![key, display])?;
324        return Ok(true);
325    }
326    main.prepare_cached(
327        "INSERT INTO replay_english.english_words(word,display,weight) VALUES(?1,?2,?3) ON CONFLICT(word,display) DO UPDATE SET weight=excluded.weight",
328    )?
329    .execute(params![key, display, weight])?;
330    Ok(true)
331}
332
333#[cfg(test)]
334pub(super) mod tests {
335    use super::*;
336    use std::path::PathBuf;
337
338    /// The v4 journal table as the reference creates it (data-formats.md §6.2), for fixtures that must not depend on the journal module.
339    pub(crate) const OPERATIONS_DDL: &str = "CREATE TABLE IF NOT EXISTS user_dictionary_operations(dictionary TEXT NOT NULL,key TEXT NOT NULL,value TEXT NOT NULL,operation TEXT NOT NULL CHECK(operation IN ('upsert','delete')),weight INTEGER NOT NULL DEFAULT 0,display TEXT NOT NULL DEFAULT '',user_inserted INTEGER NOT NULL DEFAULT 0,updated_at INTEGER NOT NULL DEFAULT(unixepoch()),PRIMARY KEY(dictionary,key,value));";
340
341    pub(crate) fn sql(path: &Path, statements: &str) {
342        Connection::open(path)
343            .unwrap()
344            .execute_batch(statements)
345            .unwrap();
346    }
347
348    pub(crate) fn weight(path: &Path, query: &str) -> Option<i64> {
349        use rusqlite::OptionalExtension;
350        Connection::open(path)
351            .unwrap()
352            .query_row(query, [], |row| row.get(0))
353            .optional()
354            .unwrap()
355    }
356
357    struct Fixture {
358        _root: tempfile::TempDir,
359        journal: PathBuf,
360        main: PathBuf,
361        english: PathBuf,
362    }
363
364    fn fixture() -> Fixture {
365        let root = tempfile::tempdir().unwrap();
366        let journal = root.path().join("msime_user.db");
367        let main = root.path().join("msime-pinyin.db");
368        let english = root.path().join("msime-english.db");
369        sql(
370            &main,
371            "CREATE TABLE tbl_2_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);
372             INSERT INTO tbl_2_n VALUES('ni''hao','nh','你好',100),('ni''hao','xx','您好',100);
373             CREATE TABLE tbl_7_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);
374             CREATE TABLE tbl_others_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);
375             CREATE TABLE wubi86(key TEXT,value TEXT,weight INTEGER);
376             CREATE TABLE wubi98(key TEXT,value TEXT,weight INTEGER);
377             CREATE TABLE quick_parases(key TEXT,value TEXT,weight INTEGER);",
378        );
379        sql(&journal, OPERATIONS_DDL);
380        Fixture {
381            _root: root,
382            journal,
383            main,
384            english,
385        }
386    }
387
388    fn journal_row(fixture: &Fixture, row: &str) {
389        sql(
390            &fixture.journal,
391            &format!("INSERT INTO user_dictionary_operations(dictionary,key,value,operation,weight,display,updated_at) VALUES{row};"),
392        );
393    }
394
395    #[test]
396    fn wubi_and_wubi98_rows_land_in_their_own_tables() {
397        let fixture = fixture();
398        journal_row(&fixture, "('wubi','kl','号','upsert',11,'',1)");
399        journal_row(&fixture, "('wubi98','kg','号','upsert',12,'',2)");
400        replay(&fixture.journal, &fixture.main, &fixture.english);
401        assert_eq!(
402            weight(&fixture.main, "SELECT weight FROM wubi86 WHERE key='kl'"),
403            Some(11)
404        );
405        assert_eq!(
406            weight(&fixture.main, "SELECT weight FROM wubi98 WHERE key='kg'"),
407            Some(12)
408        );
409        assert_eq!(
410            weight(&fixture.main, "SELECT weight FROM wubi86 WHERE key='kg'"),
411            None
412        );
413    }
414
415    /// 代次里没有任何五笔表（资源目录不带 `msime-wubi.db`）时五笔行跳过，其余行照常写；有一张五笔表时，缺的那张仍按失败回滚。
416    #[test]
417    fn wubi_rows_are_skipped_when_the_generation_has_no_wubi_table() {
418        let fixture = fixture();
419        sql(&fixture.main, "DROP TABLE wubi86; DROP TABLE wubi98;");
420        journal_row(&fixture, "('wubi','kl','号','upsert',11,'',1)");
421        journal_row(&fixture, "('wubi98','kg','号','upsert',12,'',2)");
422        journal_row(&fixture, "('pinyin','ni''hao','您好','upsert',500,'',3)");
423        let result = replay(&fixture.journal, &fixture.main, &fixture.english);
424        assert_eq!(
425            (
426                result.applied,
427                result.skipped,
428                result.failed,
429                result.error.as_str()
430            ),
431            (1, 2, 0, "")
432        );
433        assert_eq!(
434            weight(
435                &fixture.main,
436                "SELECT weight FROM tbl_2_n WHERE value='您好'"
437            ),
438            Some(500)
439        );
440
441        let fixture = self::fixture();
442        sql(&fixture.main, "DROP TABLE wubi98;");
443        journal_row(&fixture, "('wubi','kl','号','upsert',11,'',1)");
444        journal_row(&fixture, "('wubi98','kg','号','upsert',12,'',2)");
445        let result = replay(&fixture.journal, &fixture.main, &fixture.english);
446        assert_eq!((result.applied, result.failed), (1, 1));
447        assert_eq!(
448            weight(&fixture.main, "SELECT count(*) FROM wubi86"),
449            Some(0)
450        );
451    }
452
453    #[test]
454    fn a_missing_journal_is_an_empty_success_and_is_not_created() {
455        let fixture = fixture();
456        std::fs::remove_file(&fixture.journal).unwrap();
457        let result = replay(&fixture.journal, &fixture.main, &fixture.english);
458        assert_eq!(result, ReplayResult::default());
459        assert!(!fixture.journal.exists());
460    }
461
462    #[test]
463    fn applies_every_kind_in_journal_order() {
464        let fixture = fixture();
465        journal_row(&fixture, "('pinyin','ni''hao','您好','upsert',500,'',2)");
466        journal_row(&fixture, "('pinyin','ni''hao','拟好','upsert',80,'',3)");
467        journal_row(
468            &fixture,
469            "('pinyin','ni''hao''ni''hao''ni''hao''ni','你好你好你好你','upsert',7,'',1)",
470        );
471        journal_row(
472            &fixture,
473            "('pinyin','ni''ni''ni''ni''ni''ni''ni''ni','你你你你你你你你','upsert',8,'',1)",
474        );
475        journal_row(&fixture, "('wubi','wq','你','upsert',9,'',1)");
476        journal_row(&fixture, "('quick','dh','电话','upsert',10,'',1)");
477        journal_row(
478            &fixture,
479            "('english','hello','Hello','upsert',300,'Hello',1)",
480        );
481        journal_row(&fixture, "('english','world','World','upsert',301,'',1)");
482        journal_row(&fixture, "('pinyin','ni''hao','你好','delete',0,'',4)");
483        journal_row(&fixture, "('pinyin','ni''hao','泥好','upsert',0,'',5)");
484
485        let result = replay(&fixture.journal, &fixture.main, &fixture.english);
486        assert_eq!(
487            result,
488            ReplayResult {
489                applied: 9,
490                skipped: 1,
491                failed: 0,
492                error: String::new()
493            }
494        );
495        assert_eq!(
496            weight(
497                &fixture.main,
498                "SELECT count(*) FROM tbl_2_n WHERE value='你好'"
499            ),
500            Some(0)
501        );
502        // An existing row is updated in place, jianpin included.
503        assert_eq!(
504            weight(
505                &fixture.main,
506                "SELECT weight FROM tbl_2_n WHERE value='您好' AND jp='nh'"
507            ),
508            Some(500)
509        );
510        assert_eq!(
511            weight(
512                &fixture.main,
513                "SELECT weight FROM tbl_2_n WHERE value='拟好' AND jp='nh'"
514            ),
515            Some(80)
516        );
517        assert_eq!(
518            weight(
519                &fixture.main,
520                "SELECT count(*) FROM tbl_2_n WHERE value='泥好'"
521            ),
522            Some(0)
523        );
524        assert_eq!(
525            weight(
526                &fixture.main,
527                "SELECT weight FROM tbl_7_n WHERE jp='nhnhnhn'"
528            ),
529            Some(7)
530        );
531        assert_eq!(
532            weight(
533                &fixture.main,
534                "SELECT weight FROM tbl_others_n WHERE jp='nnnnnnnn'"
535            ),
536            Some(8)
537        );
538        assert_eq!(
539            weight(&fixture.main, "SELECT weight FROM wubi86 WHERE key='wq'"),
540            Some(9)
541        );
542        assert_eq!(
543            weight(
544                &fixture.main,
545                "SELECT weight FROM quick_parases WHERE key='dh'"
546            ),
547            Some(10)
548        );
549        assert_eq!(
550            weight(
551                &fixture.english,
552                "SELECT weight FROM english_words WHERE word='hello' AND display='Hello'"
553            ),
554            Some(300)
555        );
556        assert_eq!(
557            weight(
558                &fixture.english,
559                "SELECT weight FROM english_words WHERE word='world' AND display='World'"
560            ),
561            Some(301)
562        );
563
564        // Replaying again changes nothing: upserts update in place instead of duplicating rows.
565        let again = replay(&fixture.journal, &fixture.main, &fixture.english);
566        assert!(again.error.is_empty() && again.failed == 0);
567        assert_eq!(
568            weight(&fixture.main, "SELECT count(*) FROM tbl_2_n"),
569            Some(2)
570        );
571    }
572
573    #[test]
574    fn a_failed_row_rolls_the_whole_replay_back() {
575        let fixture = fixture();
576        journal_row(&fixture, "('pinyin','ni''hao','您好','upsert',500,'',1)");
577        // `@` has no table, an empty segment cannot be stored, and an unknown dictionary is not applied (runtime_isolation `@` fixture).
578        journal_row(&fixture, "('pinyin','@','无效词','upsert',10,'',2)");
579        journal_row(&fixture, "('pinyin','ni''''hao','无效词','upsert',10,'',3)");
580        journal_row(&fixture, "('japanese','ni','无效词','upsert',10,'',4)");
581        let result = replay(&fixture.journal, &fixture.main, &fixture.english);
582        assert_eq!(result.applied, 1);
583        assert_eq!(result.failed, 3);
584        assert_eq!(
585            result.error,
586            "one or more operations failed; changes were rolled back"
587        );
588        assert_eq!(
589            weight(
590                &fixture.main,
591                "SELECT weight FROM tbl_2_n WHERE value='您好'"
592            ),
593            Some(100)
594        );
595    }
596
597    /// user_dictionary_journal.cpp:1688-1708 copies each column's bytes with no encoding check and binds them back as text, so a legacy row whose text is not UTF-8 replays byte for byte instead of blocking every future generation.
598    #[test]
599    fn a_journal_row_that_is_not_utf8_replays_byte_for_byte() {
600        let fixture = fixture();
601        journal_row(&fixture, "('pinyin','ni''hao','您好','upsert',500,'',1)");
602        journal_row(
603            &fixture,
604            "('pinyin','ni''hao',CAST(x'ff' AS TEXT),'upsert',10,'',2)",
605        );
606        journal_row(
607            &fixture,
608            "('quick',CAST(x'6bff' AS TEXT),'phrase','upsert',5,'',3)",
609        );
610        journal_row(
611            &fixture,
612            "('english','hello','hello','upsert',7,CAST(x'68ff' AS TEXT),4)",
613        );
614        let result = replay(&fixture.journal, &fixture.main, &fixture.english);
615        assert_eq!(result.error, "");
616        assert_eq!(result.applied, 4);
617        assert_eq!(
618            weight(
619                &fixture.main,
620                "SELECT weight FROM tbl_2_n WHERE value='您好'"
621            ),
622            Some(500)
623        );
624        assert_eq!(
625            weight(
626                &fixture.main,
627                "SELECT weight FROM tbl_2_n WHERE key='ni''hao' AND hex(value)='FF' AND jp='nh' AND typeof(value)='text'"
628            ),
629            Some(10)
630        );
631        assert_eq!(
632            weight(
633                &fixture.main,
634                "SELECT weight FROM quick_parases WHERE hex(key)='6BFF' AND value='phrase' AND typeof(key)='text'"
635            ),
636            Some(5)
637        );
638        assert_eq!(
639            weight(
640                &fixture.english,
641                "SELECT weight FROM english_words WHERE word='hello' AND hex(display)='68FF'"
642            ),
643            Some(7)
644        );
645    }
646
647    #[test]
648    fn an_oversized_journal_text_rolls_back_before_copying_it() {
649        let fixture = fixture();
650        let huge = "x".repeat(4097);
651        Connection::open(&fixture.journal)
652            .unwrap()
653            .execute(
654                "INSERT INTO user_dictionary_operations(dictionary,key,value,operation,weight,display,updated_at) VALUES('pinyin','ni''hao',?1,'upsert',10,'',1)",
655                [&huge],
656            )
657            .unwrap();
658
659        let result = replay(&fixture.journal, &fixture.main, &fixture.english);
660        assert_eq!(result.applied, 0);
661        assert!(!result.error.is_empty());
662        assert_eq!(
663            weight(&fixture.main, "SELECT count(*) FROM tbl_2_n"),
664            Some(2)
665        );
666    }
667
668    /// A key that names no table fails in the reference too (pinyin_table yields no name, so the statement does not prepare), which rolls the whole replay back.
669    #[test]
670    fn a_pinyin_key_that_is_not_utf8_and_names_no_table_rolls_back() {
671        let fixture = fixture();
672        journal_row(&fixture, "('pinyin','ni''hao','您好','upsert',500,'',1)");
673        journal_row(
674            &fixture,
675            "('pinyin',CAST(x'ff' AS TEXT),'你','upsert',10,'',2)",
676        );
677        let result = replay(&fixture.journal, &fixture.main, &fixture.english);
678        assert_eq!(result.failed, 1);
679        assert_eq!(
680            result.error,
681            "one or more operations failed; changes were rolled back"
682        );
683        assert_eq!(
684            weight(
685                &fixture.main,
686                "SELECT weight FROM tbl_2_n WHERE value='您好'"
687            ),
688            Some(100)
689        );
690    }
691
692    // test_engine_smoke.cpp:317-403: an unreadable journal, a missing target and a locked target are all reported.
693    #[test]
694    fn open_and_cursor_failures_are_reported() {
695        let fixture = fixture();
696        let unreadable = fixture.journal.with_file_name("msime_user.unreadable.db");
697        sql(
698            &unreadable,
699            "CREATE VIEW user_dictionary_operations AS SELECT 'pinyin' AS dictionary,'ni' AS key,'你' AS value,'upsert' AS operation,100 AS weight,'' AS display,1 AS updated_at UNION ALL SELECT 'pinyin','bad',value,'upsert',1,'',2 FROM json_each('not-json')",
700        );
701        assert!(!replay(&unreadable, &fixture.main, &fixture.english)
702            .error
703            .is_empty());
704
705        let not_a_database = fixture.journal.with_file_name("garbage.db");
706        std::fs::write(&not_a_database, b"synthetic invalid database").unwrap();
707        assert_eq!(
708            replay(&not_a_database, &fixture.main, &fixture.english).error,
709            "invalid user dictionary database"
710        );
711
712        let missing_main = fixture.main.with_file_name("missing.db");
713        assert_eq!(
714            replay(&fixture.journal, &missing_main, &fixture.english).error,
715            "cannot open target dictionary database"
716        );
717        assert!(!missing_main.exists());
718    }
719
720    #[test]
721    fn a_locked_target_is_reported_after_the_busy_timeout() {
722        let fixture = fixture();
723        let locked = Connection::open(&fixture.main).unwrap();
724        locked.execute_batch("BEGIN EXCLUSIVE").unwrap();
725        let result = replay(&fixture.journal, &fixture.main, &fixture.english);
726        locked.execute_batch("ROLLBACK").unwrap();
727        // The exclusive lock already blocks the English attach, which reads the main schema; either way the replay must not report success.
728        assert!(!result.error.is_empty());
729        assert_eq!(result.applied, 0);
730    }
731
732    /// 没有 `msime-pinyin.db` 的代次只回放英文行：拼音、五笔和快捷短语的行跳过，不算失败，也不会去碰（或创建）`msime-pinyin.db`。
733    #[test]
734    fn without_the_main_dictionary_only_english_rows_are_replayed() {
735        let fixture = fixture();
736        std::fs::remove_file(&fixture.main).unwrap();
737        journal_row(&fixture, "('pinyin','ni''hao','您好','upsert',500,'',1)");
738        journal_row(&fixture, "('wubi','wq','你','upsert',9,'',2)");
739        journal_row(&fixture, "('quick','dh','电话','upsert',10,'',3)");
740        journal_row(
741            &fixture,
742            "('english','hello','Hello','upsert',300,'Hello',4)",
743        );
744        journal_row(&fixture, "('english','world','World','upsert',301,'',5)");
745
746        let result = replay_english(&fixture.journal, &fixture.english);
747        assert_eq!(
748            result,
749            ReplayResult {
750                applied: 2,
751                skipped: 3,
752                failed: 0,
753                error: String::new(),
754            }
755        );
756        assert!(!fixture.main.exists());
757        assert_eq!(
758            weight(
759                &fixture.english,
760                "SELECT weight FROM english_words WHERE word='hello' AND display='Hello'"
761            ),
762            Some(300)
763        );
764        assert_eq!(
765            weight(
766                &fixture.english,
767                "SELECT weight FROM english_words WHERE word='world' AND display='World'"
768            ),
769            Some(301)
770        );
771    }
772}
msime_engine/user_dictionary/replay.rs

msime_engine/user_dictionary/
replay.rs
