strizer

strizer is a minimal and fast library for text tokenization.

Usage

Install

Add this to your Cargo.toml:

[dependencies]
strizer = "0.1.0"

StreamTokenizer

use std::fs::File;
use std::io::BufReader;
use strizer::{StreamTokenizer, Token, TokenKind};

fn main() -> std::io::Result<()> {
  // read contest to a reader buffer
  let file = File::open("log.txt")?;
  let mut reader = BufReader::new(file);

  // tokenize BufRead, and count number of "ERROR" words
  let error_count = StreamTokenizer::new(&mut reader, &[])
    .filter(|(_, _, slice)| slice == "ERROR")
    .count();

  println!("number of error logs: {}", error_count);
  Ok(())
}

StringTokenizer

use strizer::StringTokenizer;

fn main() -> std::io::Result<()> {
  // tokenize input string and count the amount of words
  let token_count = StringTokenizer::new("hello world", &[]).count();

  println!("number of words: {}", token_count);
  Ok(())
}

License

MIT