morpheme 0.4.0

Fast, pure-Rust subword tokenization (BPE, WordPiece, Unigram) compatible with Hugging Face tokenizers.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

//! Run several pre-tokenizers in order.

use serde::{Deserialize, Serialize};

use super::PreTokenizerWrapper;
use crate::error::Result;
use crate::pre_tokenized_string::PreTokenizedString;
use crate::traits::PreTokenizer;

/// Applies each pre-tokenizer in turn.
///
/// # Example
///
/// ```
/// use morpheme::{OffsetType, PreTokenizedString, PreTokenizer};
///
/// fn split(pt: &impl PreTokenizer, text: &str) -> Vec<String> {
///     let mut s = PreTokenizedString::from(text);
///     pt.pre_tokenize(&mut s).unwrap();
///     s.get_splits(OffsetType::Byte).into_iter().map(|(w, _, _)| w.to_owned()).collect()
/// }
///
/// use morpheme::pre_tokenizers::{Digits, Sequence, WhitespaceSplit};
///
/// let pt = Sequence::new(vec![WhitespaceSplit.into(), Digits::new(true).into()]);
/// assert_eq!(split(&pt, "abc12 x"), ["abc", "1", "2", "x"]);
/// ```
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Sequence {
    pretokenizers: Vec<PreTokenizerWrapper>,
}

impl Sequence {
    /// Build a sequence.
    pub fn new(pretokenizers: Vec<PreTokenizerWrapper>) -> Self {
        Self { pretokenizers }
    }

    /// The pre-tokenizers, in order.
    pub fn pretokenizers(&self) -> &[PreTokenizerWrapper] {
        &self.pretokenizers
    }

    /// Mutable access to the pre-tokenizers.
    pub fn pretokenizers_mut(&mut self) -> &mut Vec<PreTokenizerWrapper> {
        &mut self.pretokenizers
    }
}

impl PreTokenizer for Sequence {
    fn pre_tokenize(&self, pretokenized: &mut PreTokenizedString) -> Result<()> {
        for p in &self.pretokenizers {
            p.pre_tokenize(pretokenized)?;
        }
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::pre_tokenizers::tests::splits;
    use crate::pre_tokenizers::{Punctuation, WhitespaceSplit};

    #[test]
    fn basic() {
        let seq = Sequence::new(vec![WhitespaceSplit.into(), Punctuation::default().into()]);
        assert_eq!(
            splits(&seq, "Hey friend!     How are you?!?"),
            vec![
                ("Hey", (0, 3)),
                ("friend", (4, 10)),
                ("!", (10, 11)),
                ("How", (16, 19)),
                ("are", (20, 23)),
                ("you", (24, 27)),
                ("?", (27, 28)),
                ("!", (28, 29)),
                ("?", (29, 30)),
            ]
        );
    }
}