use clap::Parser;
use rsmarisa::{Keyset, Trie};
use std::fs::File;
use std::io::{self, BufRead, BufReader};
use std::path::PathBuf;
use std::process;
#[derive(Parser)]
#[command(name = "rsmarisa-build")]
#[command(about = "Build a MARISA trie dictionary from text input")]
#[command(version)]
struct Args {
#[arg(short = 'n', long, value_name = "N", default_value = "3")]
num_tries: u32,
#[arg(short = 't', long, conflicts_with = "binary_tail")]
text_tail: bool,
#[arg(short = 'b', long)]
binary_tail: bool,
#[arg(short = 'w', long, conflicts_with = "label_order")]
weight_order: bool,
#[arg(short = 'l', long)]
label_order: bool,
#[arg(short = 'c', long, value_name = "N", default_value = "3")]
cache_level: u32,
#[arg(short = 'o', long, value_name = "FILE")]
output: Option<PathBuf>,
files: Vec<PathBuf>,
}
fn read_keys<R: BufRead>(input: R, keyset: &mut Keyset) -> io::Result<()> {
for line in input.lines() {
let line = line?;
if let Some(delim_pos) = line.rfind('\t') {
let key = &line[..delim_pos];
let weight_str = &line[delim_pos + 1..];
if let Ok(weight) = weight_str.parse::<f32>() {
keyset.push_back_bytes(key.as_bytes(), weight)?;
continue;
}
}
keyset.push_back_str(&line)?;
}
Ok(())
}
fn build_config(args: &Args) -> i32 {
use rsmarisa::base::{CacheLevel, NodeOrder, TailMode};
let mut config = 0i32;
config |= (args.num_tries & 0x7F) as i32;
let cache = match args.cache_level {
1 => CacheLevel::Tiny,
2 => CacheLevel::Small,
3 => CacheLevel::Normal,
4 => CacheLevel::Large,
5 => CacheLevel::Huge,
_ => {
eprintln!("error: cache level must be 1-5");
process::exit(1);
}
};
config |= cache as i32;
let tail_mode = if args.binary_tail {
TailMode::BinaryTail
} else {
TailMode::TextTail
};
config |= tail_mode as i32;
let node_order = if args.label_order {
NodeOrder::Label
} else {
NodeOrder::Weight
};
config |= node_order as i32;
config
}
fn main() {
let args = Args::parse();
if args.num_tries < 1 || args.num_tries > 127 {
eprintln!("error: num-tries must be in range [1, 127]");
process::exit(1);
}
let mut keyset = Keyset::new();
if args.files.is_empty() {
let stdin = io::stdin();
let reader = BufReader::new(stdin.lock());
if let Err(e) = read_keys(reader, &mut keyset) {
eprintln!("error: failed to read keys from stdin: {}", e);
process::exit(10);
}
} else {
for path in &args.files {
match File::open(path) {
Ok(file) => {
let reader = BufReader::new(file);
if let Err(e) = read_keys(reader, &mut keyset) {
eprintln!("error: failed to read keys from {}: {}", path.display(), e);
process::exit(11);
}
}
Err(e) => {
eprintln!("error: failed to open {}: {}", path.display(), e);
process::exit(12);
}
}
}
}
let mut trie = Trie::new();
let config = build_config(&args);
trie.build(&mut keyset, config);
eprintln!("#keys: {}", trie.num_keys());
eprintln!("#nodes: {}", trie.num_nodes());
eprintln!("size: {}", trie.io_size());
if let Some(output_path) = args.output {
if let Err(e) = trie.save(output_path.to_str().unwrap()) {
eprintln!(
"error: failed to save dictionary to {}: {}",
output_path.display(),
e
);
process::exit(30);
}
} else {
use rsmarisa::grimoire::io::Writer;
let _stdout = io::stdout();
let _writer = Writer::new();
eprintln!("error: stdout output not yet supported, please use -o option");
process::exit(31);
}
}