alpino_tokenizer/lib.rs
1//! Wrapper for the Alpino tokenizer for Dutch.
2//!
3//! This crate provides a wrapper around the Alpino tokenizer for
4//! Dutch. Since the tokenizer itself is included through the
5//! `alpino-tokenizer-sys` crate, this crate can be used without
6//! any external dependencies.
7//!
8//! This crate exposes a single function `tokenize`, that takes a
9//! single paragraph as a string and returns a `Vec<Vec<String>>`
10//! holding the sentences and tokens. For example:
11//!
12//! ```
13//! use std::fs::File;
14//! use std::io::BufReader;
15//!
16//! use alpino_tokenizer::{AlpinoTokenizer, Tokenizer};
17//!
18//! let read = BufReader::new(File::open("testdata/toy.proto").unwrap());
19//! let tokenizer = AlpinoTokenizer::from_buf_read(read).unwrap();
20//!
21//! assert_eq!(
22//! tokenizer.tokenize("Groningen is een Hanzestad. Groningen heeft veel bezienswaardigheden.").unwrap(),
23//! vec![vec!["Groningen", "is", "een", "Hanzestad", "."],
24//! vec!["Groningen", "heeft", "veel", "bezienswaardigheden", "."]]);
25//! ```
26
27mod alpino;
28pub use alpino::AlpinoTokenizer;
29
30mod fst;
31pub use fst::FiniteStateTokenizer;
32
33mod preproc;
34
35mod postproc;
36
37mod small_string;
38
39mod tokenizer;
40pub use tokenizer::{Tokenizer, TokenizerError};
41
42mod util;