morpheme 0.4.0

Fast, pure-Rust subword tokenization (BPE, WordPiece, Unigram) compatible with Hugging Face tokenizers.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

//! Strip a char from the edges of every token.

use serde::{Deserialize, Serialize};

use crate::error::Result;
use crate::traits::Decoder;

/// Removes up to `start` leading and `stop` trailing occurrences of
/// `content` from each token.
///
/// # Example
///
/// ```
/// use morpheme::decoders::Strip;
/// use morpheme::Decoder;
///
/// // Remove one `' '` from the start of each token.
/// assert_eq!(Strip::new(' ', 1, 0).decode_chain(vec![" hi".to_string(), "  there".to_string()])?, ["hi", " there"]);
/// # Ok::<(), morpheme::Error>(())
/// ```
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[non_exhaustive]
pub struct Strip {
    /// The char to strip.
    pub content: char,
    /// Max occurrences to strip at the start.
    pub start: usize,
    /// Max occurrences to strip at the end.
    pub stop: usize,
}

impl Strip {
    /// Build a `Strip` decoder.
    pub fn new(content: char, start: usize, stop: usize) -> Self {
        Self {
            content,
            start,
            stop,
        }
    }
}

impl Decoder for Strip {
    fn decode_chain(&self, tokens: Vec<String>) -> Result<Vec<String>> {
        Ok(tokens
            .into_iter()
            .map(|token| {
                let chars: Vec<char> = token.chars().collect();
                let start = chars
                    .iter()
                    .take(self.start)
                    .take_while(|&&c| c == self.content)
                    .count();
                let stop = chars[start..]
                    .iter()
                    .rev()
                    .take(self.stop)
                    .take_while(|&&c| c == self.content)
                    .count();
                chars[start..chars.len() - stop].iter().collect()
            })
            .collect())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn strips() {
        let d = Strip::new('H', 1, 0);
        assert_eq!(
            d.decode_chain(vec!["Hey".into(), " friend!".into(), "HHH".into()])
                .unwrap(),
            vec!["ey", " friend!", "HH"]
        );
        let d = Strip::new('y', 0, 1);
        assert_eq!(
            d.decode_chain(vec!["Hey".into(), "yyy".into(), "".into()])
                .unwrap(),
            vec!["He", "yy", ""]
        );
    }
}