morpheme 0.4.0

Fast, pure-Rust subword tokenization (BPE, WordPiece, Unigram) compatible with Hugging Face tokenizers.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86

//! Split on punctuation.

use serde::{Deserialize, Serialize};

use super::bert::is_bert_punc;
use crate::error::Result;
use crate::normalized_string::SplitDelimiterBehavior;
use crate::pre_tokenized_string::PreTokenizedString;
use crate::traits::PreTokenizer;

fn default_behavior() -> SplitDelimiterBehavior {
    SplitDelimiterBehavior::Isolated
}

/// Splits on punctuation chars (ASCII punctuation plus Unicode `P*`).
///
/// # Example
///
/// ```
/// use morpheme::{OffsetType, PreTokenizedString, PreTokenizer};
///
/// fn split(pt: &impl PreTokenizer, text: &str) -> Vec<String> {
///     let mut s = PreTokenizedString::from(text);
///     pt.pre_tokenize(&mut s).unwrap();
///     s.get_splits(OffsetType::Byte).into_iter().map(|(w, _, _)| w.to_owned()).collect()
/// }
///
/// use morpheme::pre_tokenizers::Punctuation;
/// use morpheme::SplitDelimiterBehavior;
///
/// let p = Punctuation::new(SplitDelimiterBehavior::Isolated);
/// assert_eq!(split(&p, "Hey, you!"), ["Hey", ",", " you", "!"]);
/// ```
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Punctuation {
    /// What to do with the punctuation (default `Isolated`).
    #[serde(default = "default_behavior")]
    pub behavior: SplitDelimiterBehavior,
}

impl Default for Punctuation {
    fn default() -> Self {
        Self::new(default_behavior())
    }
}

impl Punctuation {
    /// Build a `Punctuation` pre-tokenizer.
    pub fn new(behavior: SplitDelimiterBehavior) -> Self {
        Self { behavior }
    }
}

impl PreTokenizer for Punctuation {
    fn pre_tokenize(&self, pretokenized: &mut PreTokenizedString) -> Result<()> {
        pretokenized.split(|_, s| s.split(is_bert_punc, self.behavior))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::pre_tokenizers::tests::splits;

    #[test]
    fn basic() {
        assert_eq!(
            splits(&Punctuation::default(), "Hey friend!     How are you?!?"),
            vec![
                ("Hey friend", (0, 10)),
                ("!", (10, 11)),
                ("     How are you", (11, 27)),
                ("?", (27, 28)),
                ("!", (28, 29)),
                ("?", (29, 30)),
            ]
        );
    }

    #[test]
    fn default_behavior_when_missing() {
        let p: Punctuation = serde_json::from_str("{}").unwrap();
        assert_eq!(p, Punctuation::default());
    }
}