morpheme 0.4.0

Fast, pure-Rust subword tokenization (BPE, WordPiece, Unigram) compatible with Hugging Face tokenizers.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

//! Split numbers out of the text.

use serde::{Deserialize, Serialize};

use crate::error::Result;
use crate::normalized_string::SplitDelimiterBehavior;
use crate::pre_tokenized_string::PreTokenizedString;
use crate::traits::PreTokenizer;

/// Isolates numeric chars: each digit on its own
/// (`individual_digits: true`) or runs of digits together.
///
/// # Example
///
/// ```
/// use morpheme::{OffsetType, PreTokenizedString, PreTokenizer};
///
/// fn split(pt: &impl PreTokenizer, text: &str) -> Vec<String> {
///     let mut s = PreTokenizedString::from(text);
///     pt.pre_tokenize(&mut s).unwrap();
///     s.get_splits(OffsetType::Byte).into_iter().map(|(w, _, _)| w.to_owned()).collect()
/// }
///
/// use morpheme::pre_tokenizers::Digits;
///
/// assert_eq!(split(&Digits::new(true), "abc123"), ["abc", "1", "2", "3"]);
/// assert_eq!(split(&Digits::new(false), "abc123"), ["abc", "123"]);
/// ```
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Digits {
    /// Split every digit into its own piece.
    #[serde(default)]
    pub individual_digits: bool,
}

impl Digits {
    /// Build a `Digits` pre-tokenizer.
    pub fn new(individual_digits: bool) -> Self {
        Self { individual_digits }
    }
}

impl PreTokenizer for Digits {
    fn pre_tokenize(&self, pretokenized: &mut PreTokenizedString) -> Result<()> {
        let behavior = if self.individual_digits {
            SplitDelimiterBehavior::Isolated
        } else {
            SplitDelimiterBehavior::Contiguous
        };
        pretokenized.split(|_, s| s.split(char::is_numeric, behavior))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::pre_tokenizers::tests::splits;

    #[test]
    fn numbers() {
        assert_eq!(
            splits(&Digits::new(false), "Hey 123 friend!"),
            vec![("Hey ", (0, 4)), ("123", (4, 7)), (" friend!", (7, 15))]
        );
    }

    #[test]
    fn individual_digits() {
        assert_eq!(
            splits(&Digits::new(true), "Hey 123 friend!"),
            vec![
                ("Hey ", (0, 4)),
                ("1", (4, 5)),
                ("2", (5, 6)),
                ("3", (6, 7)),
                (" friend!", (7, 15))
            ]
        );
    }
}