liendl_tokenizer 0.1.0

A simple BPE tokenizer for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

use crate::shared::vocabulary::{ReadOnly, Vocabulary};

pub struct Decoder {
    vocabulary: Vocabulary<ReadOnly>,
}

impl Decoder {
    pub fn new(vocabulary: Vocabulary<ReadOnly>) -> Self {
        Decoder { vocabulary }
    }

    pub fn decode(&self, tokens: Vec<usize>) -> String {
        let mut result = String::new();

        for token in tokens {
            match self.vocabulary.get_tokens().iter().find(|x| x.get_token_id() == token) {
                Some(tkn) => {
                    result.push_str(&tkn.get_token());
                }
                None => {
                    result.push_str("<unknown>");
                }
            }
        }

        result.to_string()
    }
}