vaporetto/
lib.rs

1//! # Vaporetto
2//!
3//! Vaporetto is a fast and lightweight pointwise prediction based tokenizer.
4//!
5//!
6#![cfg_attr(
7    all(feature = "std", feature = "tag-prediction"),
8    doc = "
9## Examples
10
11```
12# fn main() -> Result<(), Box<dyn std::error::Error>> {
13use std::fs::File;
14
15use vaporetto::{Model, Predictor, Sentence};
16
17let f = File::open(\"../resources/model.bin\")?;
18let model = Model::read(f)?;
19let predictor = Predictor::new(model, true)?;
20
21let mut buf = String::new();
22
23let mut s = Sentence::default();
24
25s.update_raw(\"まぁ社長は火星猫だ\")?;
26predictor.predict(&mut s);
27s.fill_tags();
28s.write_tokenized_text(&mut buf);
29assert_eq!(
30    \"まぁ/名詞/マー 社長/名詞/シャチョー は/助詞/ワ 火星/名詞/カセー 猫/名詞/ネコ だ/助動詞/ダ\",
31    buf,
32);
33
34s.update_raw(\"まぁ良いだろう\")?;
35predictor.predict(&mut s);
36s.fill_tags();
37s.write_tokenized_text(&mut buf);
38assert_eq!(
39    \"まぁ/副詞/マー 良い/形容詞/ヨイ だろう/助動詞/ダロー\",
40    buf,
41);
42# Ok(())
43# }
44```
45"
46)]
47//!
48//! Tag prediction requires **crate feature** `tag-prediction`.
49//!
50//! Training requires **crate feature** `train`. For more details, see [`Trainer`].
51
52#![deny(missing_docs)]
53#![cfg_attr(docsrs, feature(doc_cfg))]
54#![cfg_attr(not(feature = "std"), no_std)]
55#![cfg_attr(feature = "portable-simd", feature(portable_simd))]
56
57#[cfg(not(feature = "alloc"))]
58compile_error!("`alloc` feature is currently required to build this crate");
59
60#[macro_use]
61extern crate alloc;
62
63mod char_scorer;
64mod dict_model;
65mod model;
66mod ngram_model;
67mod predictor;
68mod sentence;
69mod type_scorer;
70mod utils;
71
72pub mod errors;
73
74#[cfg(feature = "train")]
75mod tag_trainer;
76#[cfg(feature = "train")]
77mod trainer;
78
79#[cfg(feature = "kytea")]
80mod kytea_model;
81
82pub use dict_model::WordWeightRecord;
83pub use model::Model;
84pub use predictor::Predictor;
85pub use sentence::{CharacterBoundary, CharacterType, Sentence, Token, TokenIterator};
86
87#[cfg(feature = "train")]
88pub use trainer::{SolverType, Trainer};
89
90#[cfg(feature = "kytea")]
91pub use kytea_model::KyteaModel;
92
93/// Version number of this library.
94pub const VERSION: &str = env!("CARGO_PKG_VERSION");