piecer 2.0.1

Unicode codepoint BPE tokenizer featuring byte fallback and optional byte-level-only tokenization
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

use std::borrow::Borrow;

use crate::Token;

pub(crate) struct ArenaNode {
    pub value: Token,
    prev: Option<usize>,
    next: Option<usize>,
}

pub(crate) struct ArenaList(Vec<ArenaNode>);
impl ArenaList {
    fn last_mut(&mut self) -> Option<&mut ArenaNode> {
        self.0.last_mut()
    }

    pub fn raw_pairs(&self) -> std::slice::Windows<'_, ArenaNode> {
        self.0.windows(2)
    }

    pub fn drop(&mut self, index: usize) -> bool {
        if index > self.0.len() - 1 {
            false
        } else {
            let (p, n) = (self.0[index].prev, self.0[index].next);
            self.0[index].prev = None;
            self.0[index].next = None;
            if let Some(prev_index) = p {
                self.0[prev_index].next = n
            }
            if let Some(next_index) = n {
                self.0[next_index].prev = p
            }
            true
        }
    }

    pub fn pair_at(&self, first: usize) -> Option<(Token, Token)> {
        if first >= self.0.len() - 1 {
            None
        } else {
            self.0[first]
                .next
                .map(|second| (self.0[first].value, self.0[second].value))
        }
    }

    pub fn prev_pair_pos(&self, second: usize) -> Option<((Token, Token), usize)> {
        if second > self.0.len() - 1 {
            None
        } else {
            self.0[second]
                .prev
                .map(|first| ((self.0[first].value, self.0[second].value), first))
        }
    }

    pub fn next_pair_pos(&self, proto: usize) -> Option<((Token, Token), usize)> {
        if proto >= self.0.len() - 1 {
            None
        } else {
            match self.0[proto].next {
                Some(first) => self.0[first]
                    .next
                    .map(|second| ((self.0[first].value, self.0[second].value), first)),
                None => None,
            }
        }
    }

    pub fn fuse_into(
        &mut self,
        index: usize,
        tok: Token,
    ) -> (
        Option<((Token, Token), usize)>,
        Option<((Token, Token), usize)>,
    ) {
        match self.0[index].next {
            Some(second) => {
                self.0[index].value = tok;
                self.drop(second);
                (
                    self.prev_pair_pos(index),
                    self.pair_at(index).map(|p| (p, index)),
                )
            }
            None => panic!("pair fusion failed"),
        }
    }
}

impl<I> FromIterator<I> for ArenaList
where
    I: Borrow<Token>,
{
    fn from_iter<T: IntoIterator<Item = I>>(iter: T) -> Self {
        let mut al = ArenaList(Vec::new());
        for (i, value) in iter.into_iter().enumerate() {
            al.0.push(ArenaNode {
                value: *value.borrow(),
                prev: if i > 0 { Some(i - 1) } else { None },
                next: Some(i + 1),
            });
        }
        al.last_mut().unwrap().next = None;
        al
    }
}