smooth_reading/tokenize.rs
1//! Tokenization: words with their fixation split, and separators (SPEC §3, §4).
2
3use crate::fixation::fixation_length;
4use crate::options::Options;
5use crate::segment::Segmenter;
6
7/// A piece of the input: a word or a separator.
8///
9/// Tokens borrow from the input. Concatenating [`Token::text`] over the
10/// slice returned by [`tokenize`] gives the input back unchanged, and for a
11/// word `fixation_text` + `rest_text` == `text`.
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
13pub enum Token<'a> {
14 /// A word, with its fixation split.
15 Word {
16 /// The exact slice of the input this word covers.
17 text: &'a str,
18 /// Number of emphasised leading grapheme clusters; `0` when the word
19 /// gets no fixation (suppressed by a rule, or off the saccade beat).
20 fixation: usize,
21 /// The emphasised prefix (empty when `fixation` is `0`).
22 fixation_text: &'a str,
23 /// The rest of the word (the whole word when `fixation` is `0`).
24 rest_text: &'a str,
25 },
26 /// Whitespace, punctuation, emoji or other non-word text between words.
27 Separator {
28 /// The exact slice of the input this separator covers.
29 text: &'a str,
30 },
31}
32
33impl<'a> Token<'a> {
34 /// The exact slice of the input this token covers.
35 #[must_use]
36 pub const fn text(&self) -> &'a str {
37 match self {
38 Self::Word { text, .. } | Self::Separator { text } => text,
39 }
40 }
41
42 /// `true` for [`Token::Word`], `false` for [`Token::Separator`].
43 #[must_use]
44 pub const fn is_word(&self) -> bool {
45 matches!(self, Self::Word { .. })
46 }
47
48 /// A word that receives no emphasis: `fixation` `0`, all text in `rest_text`.
49 const fn plain(text: &'a str) -> Self {
50 Self::Word {
51 text,
52 fixation: 0,
53 fixation_text: "",
54 rest_text: text,
55 }
56 }
57}
58
59/// Saccade bookkeeping shared across several [`tokenize_with_state`] calls,
60/// so [`to_html`](crate::to_html) keeps counting words across markup (SPEC §4).
61#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
62#[non_exhaustive]
63pub struct TokenizeState {
64 /// Index of the next word token; the first word of the input is `0`.
65 pub word_index: usize,
66}
67
68/// Splits `text` into words and separators and computes each word's fixation
69/// prefix (SPEC §3, §4).
70///
71/// ```
72/// use smooth_reading::{Options, Token, tokenize};
73///
74/// let tokens = tokenize("Smooth reading", &Options::new());
75/// assert_eq!(
76/// tokens,
77/// [
78/// Token::Word { text: "Smooth", fixation: 3, fixation_text: "Smo", rest_text: "oth" },
79/// Token::Separator { text: " " },
80/// Token::Word { text: "reading", fixation: 4, fixation_text: "read", rest_text: "ing" },
81/// ]
82/// );
83/// ```
84#[must_use]
85pub fn tokenize<'a>(text: &'a str, options: &Options) -> Vec<Token<'a>> {
86 tokenize_with_state(text, options, &mut TokenizeState::default())
87}
88
89/// [`tokenize`] with an externally owned saccade counter.
90///
91/// Every word token advances `state.word_index`, so tokenizing one text in
92/// several pieces keeps counting words across the pieces.
93///
94/// ```
95/// use smooth_reading::{Options, Token, TokenizeState, tokenize_with_state};
96///
97/// let options = Options::new().saccade(2);
98/// let mut state = TokenizeState::default();
99/// let first = tokenize_with_state("one two", &options, &mut state);
100/// let second = tokenize_with_state("three four", &options, &mut state);
101/// assert_eq!(state.word_index, 4);
102/// assert!(matches!(first[0], Token::Word { fixation: 2, .. })); // "one", index 0
103/// assert!(matches!(first[2], Token::Word { fixation: 0, .. })); // "two", index 1, off the beat
104/// assert!(matches!(second[0], Token::Word { fixation: 3, .. })); // "three", index 2, on the beat
105/// assert!(matches!(second[2], Token::Word { fixation: 0, .. })); // "four", index 3, off the beat
106/// ```
107#[must_use]
108pub fn tokenize_with_state<'a>(
109 text: &'a str,
110 options: &Options,
111 state: &mut TokenizeState,
112) -> Vec<Token<'a>> {
113 let segmenter = options.segmenter_impl();
114 let mut tokens: Vec<Token<'a>> = Vec::new();
115 for segment in segmenter.segment_words(text, options.get_locale()) {
116 if segment.text.is_empty() {
117 continue;
118 }
119 if segment.is_word {
120 tokens.push(word_token(segment.text, options, segmenter, state));
121 continue;
122 }
123 // Consecutive separators merge into one token so every segmenter
124 // produces the same token stream.
125 if let Some(Token::Separator { text: last }) = tokens.last_mut() {
126 if let Some(merged) = merge_adjacent(text, last, segment.text) {
127 *last = merged;
128 continue;
129 }
130 }
131 tokens.push(Token::Separator { text: segment.text });
132 }
133 tokens
134}
135
136/// The slice of `text` covering `first` immediately followed by `second`, when
137/// both are adjacent slices of `text` (which every well-behaved segmenter
138/// returns); `None` otherwise.
139fn merge_adjacent<'a>(text: &'a str, first: &str, second: &str) -> Option<&'a str> {
140 let base = text.as_ptr().addr();
141 let first_start = first.as_ptr().addr();
142 let second_start = second.as_ptr().addr();
143 if first_start < base
144 || first_start + first.len() != second_start
145 || second_start + second.len() > base + text.len()
146 {
147 return None;
148 }
149 let start = first_start - base;
150 text.get(start..start + first.len() + second.len())
151}
152
153fn word_token<'a>(
154 word: &'a str,
155 options: &Options,
156 segmenter: &dyn Segmenter,
157 state: &mut TokenizeState,
158) -> Token<'a> {
159 // Every word token consumes a saccade index, including numbers and words
160 // too short to be emphasised (SPEC §4).
161 let on_saccade = state.word_index % options.get_saccade() == 0;
162 state.word_index += 1;
163 if !on_saccade {
164 return Token::plain(word);
165 }
166
167 let graphemes = segmenter.graphemes(word, options.get_locale());
168 let fixation = fixation_length(word, graphemes.len(), options).min(graphemes.len());
169 if fixation == 0 {
170 return Token::plain(word);
171 }
172 let split: usize = graphemes[..fixation].iter().map(|g| g.len()).sum();
173 match (word.get(..split), word.get(split..)) {
174 (Some(fixation_text), Some(rest_text)) => Token::Word {
175 text: word,
176 fixation,
177 fixation_text,
178 rest_text,
179 },
180 // Only reachable with a segmenter whose clusters are not slices of the
181 // word; the word is then emitted unemphasised rather than mis-split.
182 _ => Token::plain(word),
183 }
184}