1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
//! Demonstrates sentence and word tokenization.
//!
//! Run with:
//! ```
//! cargo run --example tokenize
//! ```
use rustam::{SentenceTokenizer, WordTokenizer, WordTokenizerConfig};
fn main() {
// -----------------------------------------------------------------------
// Sentence tokenizer
// -----------------------------------------------------------------------
let text = "سلام. حال شما چطور است؟ امیدوارم خوب باشید!";
let sent_tok = SentenceTokenizer::new();
let sentences = sent_tok.tokenize(text);
println!("=== Sentence Tokenizer ===");
println!("Input: {text}");
for (i, s) in sentences.iter().enumerate() {
println!(" [{i}] {s}");
}
// -----------------------------------------------------------------------
// Word tokenizer — default (joins multi-part verbs with _)
// -----------------------------------------------------------------------
let sentence = "او رفته است و ما خواهیم رفت";
let word_tok = WordTokenizer::new();
println!("\n=== Word Tokenizer (verb joining ON) ===");
println!("Input: {sentence}");
println!("Tokens: {:?}", word_tok.tokenize(sentence));
// -----------------------------------------------------------------------
// Word tokenizer — verb joining disabled
// -----------------------------------------------------------------------
let no_join = WordTokenizer::without_verb_joining();
println!("\n=== Word Tokenizer (verb joining OFF) ===");
println!("Input: {sentence}");
println!("Tokens: {:?}", no_join.tokenize(sentence));
// -----------------------------------------------------------------------
// Word tokenizer — custom config
// -----------------------------------------------------------------------
let custom = WordTokenizer::with_config(WordTokenizerConfig {
join_verb_parts: true,
replace_emails: true,
replace_links: true,
replace_ids: false,
replace_numbers: false,
replace_hashtags: false,
});
let mixed = "ایمیل من test@example.com است و وبسایت https://example.com را ببینید";
println!("\n=== Word Tokenizer (email + URL masking) ===");
println!("Input: {mixed}");
println!("Tokens: {:?}", custom.tokenize(mixed));
// -----------------------------------------------------------------------
// Punctuation and parentheses
// -----------------------------------------------------------------------
let punct = "این جمله (خیلی) پیچیده نیست!!!";
println!("\n=== Punctuation handling ===");
println!("Input: {punct}");
println!("Tokens: {:?}", word_tok.tokenize(punct));
}