kaiji 0.1.0

High-performance CJK fuzzy search and text normalization engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

//! Normalize CJK text line by line from stdin to stdout.
//!
//! Usage:
//!   echo "齋藤一郎" | cargo run --example normalize_cli
//!   cat names.csv | cargo run --example normalize_cli
//!   cargo run --example normalize_cli -- --width   # also convert fullwidth ASCII / halfwidth kana

use std::env;
use std::io::{self, BufRead, Write};

use kaiji::Normalizer;

fn main() {
    let args: Vec<String> = env::args().collect();
    let width = args.iter().any(|a| a == "--width" || a == "-w");

    let normalizer = Normalizer::builder()
        .fold_variants(true)
        .strip_ivs(true)
        .width_normalization(width)
        .build();

    let stdin = io::stdin();
    let stdout = io::stdout();
    let mut out = io::BufWriter::new(stdout.lock());

    for line in stdin.lock().lines() {
        let line = match line {
            Ok(l) => l,
            Err(e) => {
                eprintln!("read error: {e}");
                std::process::exit(1);
            }
        };

        match normalizer.normalize(&line) {
            Ok(normalized) => {
                if let Err(e) = writeln!(out, "{normalized}") {
                    eprintln!("write error: {e}");
                    std::process::exit(1);
                }
            }
            Err(e) => {
                eprintln!("normalize error: {e}");
                std::process::exit(1);
            }
        }
    }
}