segtok/
lib.rs

1//! A rule-based sentence segmenter (splitter) and a word tokenizer using orthographic features.
2//! Ported from the [python package](https://github.com/fnl/segtok) (not maintained anymore),
3//! and fixes the [contractions bug](https://github.com/fnl/segtok/issues/26).
4//!
5//! ```rust
6//! use segtok::{segmenter::*, tokenizer::*};
7//!
8//! let input = include_str!("../tests/test_google.txt");
9//!
10//! let sentences: Vec<Vec<_>> = split_multi(input, SegmentConfig::default())
11//!     .into_iter()
12//!     .map(|span| split_contractions(web_tokenizer(&span)).collect())
13//!     .collect();
14//! ```
15
16use std::ops::Deref;
17
18pub(crate) mod regex;
19pub mod segmenter;
20pub mod tokenizer;
21
22/// Can be used in benchmarks.
23#[doc(hidden)]
24pub fn init() {
25    let _ = segmenter::dates::MONTH.deref();
26    let _ = segmenter::dates::ENDS_IN_DATE_DIGITS.deref();
27    let _ = segmenter::BEFORE_LOWER.deref();
28    let _ = segmenter::LOWER_WORD.deref();
29    let _ = segmenter::MIDDLE_INITIAL_END.deref();
30    let _ = segmenter::UPPER_WORD_START.deref();
31    let _ = segmenter::LONE_WORD.deref();
32    let _ = segmenter::UPPER_CASE_END.deref();
33    let _ = segmenter::UPPER_CASE_START.deref();
34    let _ = segmenter::DO_NOT_CROSS_LINES.deref();
35    let _ = segmenter::MAY_CROSS_ONE_LINE.deref();
36    let _ = segmenter::ABBREVIATIONS.deref();
37    let _ = segmenter::CONTINUATIONS.deref();
38
39    let _ = tokenizer::HYPHENATED_LINEBREAK.deref();
40    let _ = tokenizer::IS_CONTRACTION.deref();
41    let _ = tokenizer::IS_POSSESSIVE.deref();
42    let _ = tokenizer::SYMBOLIC.deref();
43    let _ = tokenizer::URI_OR_MAIL.deref();
44    let _ = tokenizer::WORD_BITS.deref();
45}
segtok/lib.rs

segtok/
lib.rs
