khmercut 0.1.5

A blazingly fast Khmer word segmentation tool written in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

use clap::Parser;
use crfs::Model;
use khmercut;
use std::io::{stdin, stdout, BufRead, Result, Write};

static MODEL_FILE: &'static [u8] = std::include_bytes!("crf_ner_10000.crfsuite");

#[derive(Parser, Debug)]
#[command(author = "Seanghay Yath", version, about = "Fast Khmer word segmentation tool", long_about = None)]
struct Args {
    #[arg(short, long, default_value = " ")]
    delimiter: String,
}

fn main() -> Result<()> {
    let args = Args::parse();
    let model = Model::new(MODEL_FILE).unwrap();
    for line in stdin().lock().lines() {
        let input_str = line.unwrap();
        stdout()
            .write_all(
                khmercut::tokenize(&model, &input_str)
                    .join(&args.delimiter)
                    .as_bytes(),
            )
            .unwrap();
    }
    Ok(())
}