sqlite_simple_tokenizer/
lib.rs

1#[cfg(feature = "build_extension")]
2mod create_extension;
3mod load_extension;
4mod pinyin;
5pub mod simple_tokenizer;
6mod utils;
7
8use load_extension::create_scalar_functions;
9use load_extension::load_fts5_extension;
10use log::LevelFilter;
11use rusqlite::Connection;
12use rusqlite_ext::error::Error;
13use utils::init_logging;
14
15pub fn load(connection: &Connection) -> Result<(), Error> {
16    load_with_loglevel(connection, LevelFilter::Info)
17}
18
19pub fn load_with_loglevel(connection: &Connection, log_level: LevelFilter) -> Result<(), Error> {
20    // 设置 log
21    init_logging(log_level);
22    // 加载拓展函数
23    create_scalar_functions(connection)?;
24    // 加载 fts5 拓展
25    load_fts5_extension(connection)
26}
27
28#[cfg(test)]
29mod tests {
30    use crate::load;
31    use rusqlite::Connection;
32
33    #[test]
34    fn test_simple_query() {
35        let conn = Connection::open_in_memory().unwrap();
36        load(&conn).unwrap();
37        let mut stmt = conn.prepare("SELECT simple_query('国')").unwrap();
38        let result = stmt
39            .query_map([], |row| Ok(row.get::<_, String>(0).unwrap()))
40            .unwrap();
41        let mut vec = Vec::new();
42        for row in result {
43            let row = row.unwrap();
44            vec.push(row)
45        }
46        assert_eq!(["(g+u+o* OR gu+o* OR guo*)"], vec.as_slice());
47    }
48
49    #[test]
50    fn test_simple() {
51        let conn = Connection::open_in_memory().unwrap();
52        load(&conn).unwrap();
53        // 创建一个测试表
54        conn.execute(
55            "CREATE VIRTUAL TABLE t1 USING fts5(text, tokenize = 'simple');",
56            [],
57        )
58        .unwrap();
59        // 插入数据
60        conn.execute(
61            r#"INSERT INTO t1(text) VALUES ('中华人民共和国国歌'),('静夜思'),('国家'),('举头望明月'),('like'),('liking'),('liked'),('I''m making a sqlite tokenizer'),('I''m learning English');"#,
62            [],
63        )
64            .unwrap();
65        let mut stmt = conn
66            .prepare("SELECT * FROM t1 WHERE text MATCH simple_query('国');")
67            .unwrap();
68        let result = stmt
69            .query_map([], |row| Ok(row.get::<_, String>(0).unwrap()))
70            .unwrap();
71        let mut vec = Vec::new();
72        for row in result {
73            let row = row.unwrap();
74            vec.push(row)
75        }
76        assert_eq!(["中华人民共和国国歌", "国家"], vec.as_slice());
77    }
78}
sqlite_simple_tokenizer/lib.rs

sqlite_simple_tokenizer/
lib.rs
