tokenizers/decoders/
fuse.rs

1use crate::tokenizer::{Decoder, Result};
2use monostate::MustBe;
3use serde::{Deserialize, Serialize};
4
5#[derive(Clone, Debug, Serialize, Deserialize, Default)]
6/// Fuse simply fuses all tokens into one big string.
7/// It's usually the last decoding step anyway, but this
8/// decoder exists incase some decoders need to happen after that
9/// step
10#[non_exhaustive]
11pub struct Fuse {
12    #[serde(rename = "type")]
13    type_: MustBe!("Fuse"),
14}
15
16impl Fuse {
17    pub fn new() -> Self {
18        Self {
19            type_: MustBe!("Fuse"),
20        }
21    }
22}
23
24impl Decoder for Fuse {
25    fn decode_chain(&self, tokens: Vec<String>) -> Result<Vec<String>> {
26        let new_string = tokens.join("");
27        Ok(vec![new_string])
28    }
29}
30
31#[cfg(test)]
32mod tests {
33    use super::*;
34
35    #[test]
36    fn decode() {
37        let decoder = Fuse::new();
38        let res = decoder
39            .decode_chain(vec!["Hey".into(), " friend!".into()])
40            .unwrap();
41        assert_eq!(res, vec!["Hey friend!"]);
42    }
43}