ainu-utils 0.4.0

A collection of utilities for the Ainu language
Documentation 
1
2
3
4
5
6
7
8
9
10
11

use ainu_utils::tokenizer::tokenize;
use std::env;

fn main() {
    let args: Vec<String> = env::args().collect();
    let text = &args[1];

    let tokens = tokenize(text, false);

    println!("{:?}", tokens);
}