normalizer-tr 0.3.0

Bounded Turkish text-to-speech normalization with original-source diagnostics
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

//! Explicit reader precedence and centrally sealed source claims.
use normalizer_tr::{
    Hint, HintKind, NormalizeError, NormalizeOptions, Normalizer, SegmentKind, SourceRange,
};

#[test]
fn structured_claims_cannot_be_cut_by_numeric_hints() {
    let n = Normalizer::new().unwrap();
    for (text, end) in [
        ("3 + 4", 1),
        ("5 MG", 1),
        ("25 TL", 2),
        ("10-15 kişi", 2),
        ("AB12 25 TL", 2),
    ] {
        assert_eq!(
            n.normalize(
                text,
                &NormalizeOptions {
                    hints: vec![Hint::new(SourceRange::new(0, end), HintKind::Cardinal)],
                    ..Default::default()
                }
            ),
            Err(NormalizeError::InvalidHint),
            "{text}"
        );
    }
    let input = "0850 222 33 44";
    let r = n
        .normalize(
            input,
            &NormalizeOptions {
                hints: vec![Hint::new(
                    SourceRange::new(0, input.len()),
                    HintKind::Digits,
                )],
                ..Default::default()
            },
        )
        .unwrap();
    assert_eq!(r.segments()[0].kind(), SegmentKind::Digits);
    assert_eq!(
        r.normalized_text(),
        "sıfır sekiz beş sıfır iki iki iki üç üç dört dört"
    );
}

#[test]
fn addresses_and_quantities_win_before_component_protection() {
    let n = Normalizer::new().unwrap();
    let input = "https://ornek.com/a@b?x=2&y=3";
    let r = n.normalize(input, &NormalizeOptions::default()).unwrap();
    assert!(r.complete());
    assert_eq!(r.segments()[0].kind(), SegmentKind::Electronic);
    assert_eq!(r.segments()[0].range(), SourceRange::new(0, input.len()));
    let r = n
        .normalize("5 kg + 2 kg", &NormalizeOptions::default())
        .unwrap();
    assert!(!r.complete());
    assert_eq!(r.normalized_text(), "5 kg + 2 kg");
    assert_eq!(r.segments()[0].range(), SourceRange::new(0, 11));
    let r = n
        .normalize("tarih 14.03.2026 kg", &NormalizeOptions::default())
        .unwrap();
    assert!(!r.complete());
    assert_eq!(r.normalized_text(), "tarih 14.03.2026 kg");
}

#[test]
fn explicit_context_roles_are_carried_into_preserved_words() {
    let r = Normalizer::new()
        .unwrap()
        .normalize(
            "II. DÜNYA SAVAŞI ve XXI. YÜZYIL",
            &NormalizeOptions::default(),
        )
        .unwrap();
    assert!(r.complete());
    assert_eq!(
        r.normalized_text(),
        "ikinci DÜNYA SAVAŞI ve yirmi birinci YÜZYIL"
    );
}