piecer 2.0.1

Unicode codepoint BPE tokenizer featuring byte fallback and optional byte-level-only tokenization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

use crate::Token;

#[derive(Debug, Eq)]
pub(crate) struct PairOrd(pub u32, pub (Token, Token));
impl PairOrd {
    fn len(&self) -> usize {
        self.0 as usize
    }
}

impl PartialEq for PairOrd {
    fn eq(&self, other: &Self) -> bool {
        self.1 == other.1
    }
}

impl PartialOrd for PairOrd {
    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
        Some(self.cmp(other))
    }
}

impl Ord for PairOrd {
    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
        self.len()
            .cmp(&other.len())
            .then_with(|| self.1.cmp(&other.1))
    }
}