morpheme 0.4.0

Fast, pure-Rust subword tokenization (BPE, WordPiece, Unigram) compatible with Hugging Face tokenizers.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

//! Decode `<0xNN>` byte tokens (SentencePiece / Llama byte fallback).

use crate::error::Result;
use crate::traits::Decoder;

/// Turns runs of `<0xNN>` tokens back into the UTF-8 text they encode.
/// Invalid byte sequences become one `�` per byte.
///
/// # Example
///
/// ```
/// use morpheme::decoders::ByteFallback;
/// use morpheme::Decoder;
///
/// // `<0xNN>` byte tokens are reassembled into UTF-8.
/// assert_eq!(ByteFallback::new().decode(vec!["caf".to_string(), "<0xC3>".to_string(), "<0xA9>".to_string()])?, "café");
/// # Ok::<(), morpheme::Error>(())
/// ```
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ByteFallback;

crate::pre_tokenizers::impl_unit_serde!(ByteFallback);

impl ByteFallback {
    /// Build a `ByteFallback` decoder.
    pub fn new() -> Self {
        Self
    }
}

fn parse_byte(token: &str) -> Option<u8> {
    if token.len() == 6 && token.starts_with("<0x") && token.ends_with('>') {
        u8::from_str_radix(&token[3..5], 16).ok()
    } else {
        None
    }
}

fn flush(bytes: &mut Vec<u8>, out: &mut Vec<String>) {
    if bytes.is_empty() {
        return;
    }
    match String::from_utf8(std::mem::take(bytes)) {
        Ok(s) => out.push(s),
        Err(e) => out.extend(std::iter::repeat_n("�".to_string(), e.as_bytes().len())),
    }
}

impl Decoder for ByteFallback {
    fn decode_chain(&self, tokens: Vec<String>) -> Result<Vec<String>> {
        let mut out = Vec::with_capacity(tokens.len());
        let mut bytes = Vec::new();
        for token in tokens {
            match parse_byte(&token) {
                Some(b) => bytes.push(b),
                None => {
                    flush(&mut bytes, &mut out);
                    out.push(token);
                }
            }
        }
        flush(&mut bytes, &mut out);
        Ok(out)
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn run(tokens: &[&str]) -> Vec<String> {
        ByteFallback
            .decode_chain(tokens.iter().map(|s| s.to_string()).collect())
            .unwrap()
    }

    #[test]
    fn decodes_bytes() {
        assert_eq!(run(&["Hey", "friend!"]), vec!["Hey", "friend!"]);
        assert_eq!(run(&["<0x61>"]), vec!["a"]);
        assert_eq!(run(&["<0xE5>"]), vec!["�"]);
        assert_eq!(run(&["<0xE5>", "<0x8f>"]), vec!["�", "�"]);
        assert_eq!(run(&["<0xE5>", "<0x8f>", "<0xab>"]), vec!["叫"]);
        assert_eq!(run(&["<0xE5>", "<0x8f>", "<0xab>", "a"]), vec!["叫", "a"]);
        assert_eq!(run(&["<0xE5>", "<0x8f>", "a"]), vec!["�", "�", "a"]);
    }
}