all_seg/
all_seg.rs

1use char_trie::Trie;
2use std::fs::File;
3use std::io::{BufRead, BufReader};
4
5pub fn main() {
6    // let mut trie = Trie::default();
7    let mut trie = Trie::new_big();
8
9    let start = std::time::Instant::now();
10
11    for line in BufReader::new(File::open("dict/default.dic").unwrap()).lines() {
12        let line = line.unwrap();
13        let parts: Vec<&str> = line.split('\t').collect();
14        if parts.len() == 3 {
15            trie.insert(
16                parts[0],
17                (parts[1].to_string(), parts[2].parse::<i32>().unwrap()),
18            );
19        }
20    }
21
22    trie.insert("中国人", (String::from("ud"), 10000));
23
24    println!("load dict use {:?}", start.elapsed());
25
26    let text = "我爱北京天安门，天安门上太阳升。我是中国人，我爱中国。";
27
28    for token in trie.iter_all(text) {
29        println!("{:?}", token);
30    }
31
32    let c: Vec<_> = trie.iter_all(text).map(|t| t.0).collect();
33    println!("{:?}", c);
34}
all_seg/all_seg.rs

all_seg/
all_seg.rs
