normalizer-tr 0.4.0

Bounded Turkish text-to-speech normalization with original-source diagnostics
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

use crate::{
    FallbackClass, IssueCategory,
    model::{Clock, Date, TemporalFailure, TemporalPreference, Value},
    morphology::Inflection,
    verbalize::{date_spoken, time_spoken},
};

use super::{
    context::{DATE_CUES, TIME_CUES, cue_key, is_clock_word},
    scan::{Token, whitespace_between},
};
use crate::domain::numeric::split_suffix;

pub(super) fn date(text: &str, permitted: bool) -> Result<Value, TemporalFailure> {
    let (base, suffix) = split_suffix(text).ok_or(IssueCategory::InvalidExpression)?;
    let date = Date::parse(base).ok_or(IssueCategory::InvalidExpression)?;
    if suffix.is_some() && !date.dotted() {
        return Err(IssueCategory::Unsupported.into());
    }
    if let Some(suffix) = suffix
        && date_spoken(date).source_suffix(Inflection::Locative) != suffix
    {
        return Err(IssueCategory::InvalidExpression.into());
    }
    if !permitted {
        return Err(TemporalFailure {
            category: IssueCategory::Ambiguous,
            preference: Some(TemporalPreference::Date(date, suffix.is_some())),
        });
    }
    Ok(Value::Date(date, suffix.is_some()))
}

pub(super) fn time(text: &str, permitted: bool) -> Result<Value, TemporalFailure> {
    let (base, suffix) = split_suffix(text).ok_or(IssueCategory::InvalidExpression)?;
    let time = Clock::parse(base).ok_or(IssueCategory::InvalidExpression)?;
    if let Some(suffix) = suffix
        && time_spoken(time).source_suffix(Inflection::Locative) != suffix
    {
        return Err(IssueCategory::InvalidExpression.into());
    }
    if !permitted {
        return Err(TemporalFailure {
            category: IssueCategory::Ambiguous,
            preference: Some(TemporalPreference::Time(time, suffix.is_some())),
        });
    }
    Ok(Value::Time(time, suffix.is_some()))
}

fn cue(text: &str, tokens: &[Token<'_>], index: usize, allowed: &[&str]) -> bool {
    let Some(mut previous_index) = index.checked_sub(1) else {
        return false;
    };
    let adjacent = |start, end| start == end || whitespace_between(text, start, end);
    if !adjacent(tokens[previous_index].range.end, tokens[index].range.start) {
        return false;
    }
    if tokens[previous_index].text == ":" {
        let Some(word_index) = previous_index.checked_sub(1) else {
            return false;
        };
        if !whitespace_between(
            text,
            tokens[word_index].range.end,
            tokens[previous_index].range.start,
        ) || tokens[word_index].text.contains(':')
        {
            return false;
        }
        previous_index = word_index;
    }
    allowed.contains(&cue_key(tokens[previous_index].text).as_str())
}

fn frame(text: &str, tokens: &[Token<'_>], index: usize) -> bool {
    let (Some(date_token), Some(cue_token), Some(time_token)) = (
        tokens.get(index),
        tokens.get(index + 1),
        tokens.get(index + 2),
    ) else {
        return false;
    };
    matches!(date(date_token.text, true), Ok(Value::Date(date, true)) if date.dotted())
        && is_clock_word(cue_token.text)
        && matches!(time(time_token.text, true), Ok(Value::Time(_, true)))
        && whitespace_between(text, date_token.range.end, cue_token.range.start)
        && whitespace_between(text, cue_token.range.end, time_token.range.start)
}

pub(super) fn recognize(
    text: &str,
    tokens: &[Token<'_>],
    index: usize,
) -> Option<(Result<Value, TemporalFailure>, FallbackClass)> {
    let token = tokens[index].text;
    let (base, _) = split_suffix(token)?;
    let dots = base.bytes().filter(|b| *b == b'.').count();
    let hyphens = base.bytes().filter(|b| *b == b'-').count();
    if dots == 2 || (hyphens == 2 && base.len() >= 8) {
        if dots == 2
            && crate::numerals::Number::parse(base).is_some_and(|n| n.grouped())
            && !cue(text, tokens, index, DATE_CUES)
        {
            return None;
        }
        return Some((
            date(
                token,
                frame(text, tokens, index) || cue(text, tokens, index, DATE_CUES),
            ),
            FallbackClass::Date,
        ));
    }
    if base.contains(':') || (dots == 1 && cue(text, tokens, index, TIME_CUES)) {
        return Some((
            time(token, cue(text, tokens, index, TIME_CUES)),
            FallbackClass::Time,
        ));
    }
    None
}