hermes-tokenizer 1.8.101

Stable-Rust byte-level BPE tokenization for Hermes
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

//! Once we have a document, we can pretokenize it (potentially in parallel)

use std::ops::Deref;

#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
pub struct Pretoken<'a>(pub &'a [u8]);

impl AsRef<[u8]> for Pretoken<'_> {
    fn as_ref(&self) -> &[u8] {
        self.0
    }
}

impl<'a> Deref for Pretoken<'a> {
    type Target = &'a [u8];

    fn deref(&self) -> &Self::Target {
        &self.0
    }
}