Skip to main content

smooth_reading/
tokenize.rs

1//! Tokenization: words with their fixation split, and separators (SPEC §3, §4).
2
3use crate::fixation::fixation_length;
4use crate::options::Options;
5use crate::segment::Segmenter;
6
7/// A piece of the input: a word or a separator.
8///
9/// Tokens borrow from the input. Concatenating [`Token::text`] over the
10/// slice returned by [`tokenize`] gives the input back unchanged, and for a
11/// word `fixation_text` + `rest_text` == `text`.
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
13pub enum Token<'a> {
14    /// A word, with its fixation split.
15    Word {
16        /// The exact slice of the input this word covers.
17        text: &'a str,
18        /// Number of emphasised leading grapheme clusters; `0` when the word
19        /// gets no fixation (suppressed by a rule, or off the saccade beat).
20        fixation: usize,
21        /// The emphasised prefix (empty when `fixation` is `0`).
22        fixation_text: &'a str,
23        /// The rest of the word (the whole word when `fixation` is `0`).
24        rest_text: &'a str,
25    },
26    /// Whitespace, punctuation, emoji or other non-word text between words.
27    Separator {
28        /// The exact slice of the input this separator covers.
29        text: &'a str,
30    },
31}
32
33impl<'a> Token<'a> {
34    /// The exact slice of the input this token covers.
35    #[must_use]
36    pub const fn text(&self) -> &'a str {
37        match self {
38            Self::Word { text, .. } | Self::Separator { text } => text,
39        }
40    }
41
42    /// `true` for [`Token::Word`], `false` for [`Token::Separator`].
43    #[must_use]
44    pub const fn is_word(&self) -> bool {
45        matches!(self, Self::Word { .. })
46    }
47
48    /// A word that receives no emphasis: `fixation` `0`, all text in `rest_text`.
49    const fn plain(text: &'a str) -> Self {
50        Self::Word {
51            text,
52            fixation: 0,
53            fixation_text: "",
54            rest_text: text,
55        }
56    }
57}
58
59/// Saccade bookkeeping shared across several [`tokenize_with_state`] calls,
60/// so [`to_html`](crate::to_html) keeps counting words across markup (SPEC §4).
61#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
62#[non_exhaustive]
63pub struct TokenizeState {
64    /// Index of the next word token; the first word of the input is `0`.
65    pub word_index: usize,
66}
67
68/// Splits `text` into words and separators and computes each word's fixation
69/// prefix (SPEC §3, §4).
70///
71/// ```
72/// use smooth_reading::{Options, Token, tokenize};
73///
74/// let tokens = tokenize("Smooth reading", &Options::new());
75/// assert_eq!(
76///     tokens,
77///     [
78///         Token::Word { text: "Smooth", fixation: 3, fixation_text: "Smo", rest_text: "oth" },
79///         Token::Separator { text: " " },
80///         Token::Word { text: "reading", fixation: 4, fixation_text: "read", rest_text: "ing" },
81///     ]
82/// );
83/// ```
84#[must_use]
85pub fn tokenize<'a>(text: &'a str, options: &Options) -> Vec<Token<'a>> {
86    tokenize_with_state(text, options, &mut TokenizeState::default())
87}
88
89/// [`tokenize`] with an externally owned saccade counter.
90///
91/// Every word token advances `state.word_index`, so tokenizing one text in
92/// several pieces keeps counting words across the pieces.
93///
94/// ```
95/// use smooth_reading::{Options, Token, TokenizeState, tokenize_with_state};
96///
97/// let options = Options::new().saccade(2);
98/// let mut state = TokenizeState::default();
99/// let first = tokenize_with_state("one two", &options, &mut state);
100/// let second = tokenize_with_state("three four", &options, &mut state);
101/// assert_eq!(state.word_index, 4);
102/// assert!(matches!(first[0], Token::Word { fixation: 2, .. })); // "one", index 0
103/// assert!(matches!(first[2], Token::Word { fixation: 0, .. })); // "two", index 1, off the beat
104/// assert!(matches!(second[0], Token::Word { fixation: 3, .. })); // "three", index 2, on the beat
105/// assert!(matches!(second[2], Token::Word { fixation: 0, .. })); // "four", index 3, off the beat
106/// ```
107#[must_use]
108pub fn tokenize_with_state<'a>(
109    text: &'a str,
110    options: &Options,
111    state: &mut TokenizeState,
112) -> Vec<Token<'a>> {
113    let segmenter = options.segmenter_impl();
114    let mut tokens: Vec<Token<'a>> = Vec::new();
115    for segment in segmenter.segment_words(text, options.get_locale()) {
116        if segment.text.is_empty() {
117            continue;
118        }
119        if segment.is_word {
120            tokens.push(word_token(segment.text, options, segmenter, state));
121            continue;
122        }
123        // Consecutive separators merge into one token so every segmenter
124        // produces the same token stream.
125        if let Some(Token::Separator { text: last }) = tokens.last_mut() {
126            if let Some(merged) = merge_adjacent(text, last, segment.text) {
127                *last = merged;
128                continue;
129            }
130        }
131        tokens.push(Token::Separator { text: segment.text });
132    }
133    tokens
134}
135
136/// The slice of `text` covering `first` immediately followed by `second`, when
137/// both are adjacent slices of `text` (which every well-behaved segmenter
138/// returns); `None` otherwise.
139fn merge_adjacent<'a>(text: &'a str, first: &str, second: &str) -> Option<&'a str> {
140    let base = text.as_ptr().addr();
141    let first_start = first.as_ptr().addr();
142    let second_start = second.as_ptr().addr();
143    if first_start < base
144        || first_start + first.len() != second_start
145        || second_start + second.len() > base + text.len()
146    {
147        return None;
148    }
149    let start = first_start - base;
150    text.get(start..start + first.len() + second.len())
151}
152
153fn word_token<'a>(
154    word: &'a str,
155    options: &Options,
156    segmenter: &dyn Segmenter,
157    state: &mut TokenizeState,
158) -> Token<'a> {
159    // Every word token consumes a saccade index, including numbers and words
160    // too short to be emphasised (SPEC §4).
161    let on_saccade = state.word_index % options.get_saccade() == 0;
162    state.word_index += 1;
163    if !on_saccade {
164        return Token::plain(word);
165    }
166
167    let graphemes = segmenter.graphemes(word, options.get_locale());
168    let fixation = fixation_length(word, graphemes.len(), options).min(graphemes.len());
169    if fixation == 0 {
170        return Token::plain(word);
171    }
172    let split: usize = graphemes[..fixation].iter().map(|g| g.len()).sum();
173    match (word.get(..split), word.get(split..)) {
174        (Some(fixation_text), Some(rest_text)) => Token::Word {
175            text: word,
176            fixation,
177            fixation_text,
178            rest_text,
179        },
180        // Only reachable with a segmenter whose clusters are not slices of the
181        // word; the word is then emitted unemphasised rather than mis-split.
182        _ => Token::plain(word),
183    }
184}