morpheme 0.4.0

Fast, pure-Rust subword tokenization (BPE, WordPiece, Unigram) compatible with Hugging Face tokenizers.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

//! Join all tokens into one.

use crate::error::Result;
use crate::traits::Decoder;

/// Concatenates every token into a single one.
///
/// # Example
///
/// ```
/// use morpheme::decoders::Fuse;
/// use morpheme::Decoder;
///
/// assert_eq!(Fuse::new().decode_chain(vec!["hel".to_string(), "lo".to_string()])?, ["hello"]);
/// # Ok::<(), morpheme::Error>(())
/// ```
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Fuse;

crate::pre_tokenizers::impl_unit_serde!(Fuse);

impl Fuse {
    /// Build a `Fuse` decoder.
    pub fn new() -> Self {
        Self
    }
}

impl Decoder for Fuse {
    fn decode_chain(&self, tokens: Vec<String>) -> Result<Vec<String>> {
        Ok(vec![tokens.concat()])
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn fuses() {
        let out = Fuse
            .decode_chain(vec!["Hey".into(), " friend".into()])
            .unwrap();
        assert_eq!(out, vec!["Hey friend"]);
    }
}