use crate::fixation::fixation_length;
use crate::options::Options;
use crate::segment::Segmenter;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Token<'a> {
Word {
text: &'a str,
fixation: usize,
fixation_text: &'a str,
rest_text: &'a str,
},
Separator {
text: &'a str,
},
}
impl<'a> Token<'a> {
#[must_use]
pub const fn text(&self) -> &'a str {
match self {
Self::Word { text, .. } | Self::Separator { text } => text,
}
}
#[must_use]
pub const fn is_word(&self) -> bool {
matches!(self, Self::Word { .. })
}
const fn plain(text: &'a str) -> Self {
Self::Word {
text,
fixation: 0,
fixation_text: "",
rest_text: text,
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub struct TokenizeState {
pub word_index: usize,
}
#[must_use]
pub fn tokenize<'a>(text: &'a str, options: &Options) -> Vec<Token<'a>> {
tokenize_with_state(text, options, &mut TokenizeState::default())
}
#[must_use]
pub fn tokenize_with_state<'a>(
text: &'a str,
options: &Options,
state: &mut TokenizeState,
) -> Vec<Token<'a>> {
let segmenter = options.segmenter_impl();
let mut tokens: Vec<Token<'a>> = Vec::new();
for segment in segmenter.segment_words(text, options.get_locale()) {
if segment.text.is_empty() {
continue;
}
if segment.is_word {
tokens.push(word_token(segment.text, options, segmenter, state));
continue;
}
if let Some(Token::Separator { text: last }) = tokens.last_mut() {
if let Some(merged) = merge_adjacent(text, last, segment.text) {
*last = merged;
continue;
}
}
tokens.push(Token::Separator { text: segment.text });
}
tokens
}
fn merge_adjacent<'a>(text: &'a str, first: &str, second: &str) -> Option<&'a str> {
let base = text.as_ptr().addr();
let first_start = first.as_ptr().addr();
let second_start = second.as_ptr().addr();
if first_start < base
|| first_start + first.len() != second_start
|| second_start + second.len() > base + text.len()
{
return None;
}
let start = first_start - base;
text.get(start..start + first.len() + second.len())
}
fn word_token<'a>(
word: &'a str,
options: &Options,
segmenter: &dyn Segmenter,
state: &mut TokenizeState,
) -> Token<'a> {
let on_saccade = state.word_index % options.get_saccade() == 0;
state.word_index += 1;
if !on_saccade {
return Token::plain(word);
}
let graphemes = segmenter.graphemes(word, options.get_locale());
let fixation = fixation_length(word, graphemes.len(), options).min(graphemes.len());
if fixation == 0 {
return Token::plain(word);
}
let split: usize = graphemes[..fixation].iter().map(|g| g.len()).sum();
match (word.get(..split), word.get(split..)) {
(Some(fixation_text), Some(rest_text)) => Token::Word {
text: word,
fixation,
fixation_text,
rest_text,
},
_ => Token::plain(word),
}
}