roketok 0.3.1

This crate provides a way to simply set up a tokenizer and use it. Not recommended for simple tokenizers as this crate adds a bunch of stuff to support many if not all kinds of tokenizers.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

use crate::record::Record;

#[derive(Clone, Copy)]
pub struct StreamIterator<'str> {
    contents: &'str [u8],
    pos: usize,
    record: Record,
}

impl<'str> StreamIterator<'str> {
    pub(crate) fn new(stream: &'str str) -> Self {
        Self {
            contents: stream.as_bytes(),
            pos: 0,
            record: Record {
                pos: (1, 1)
            }
        }
    }
    
    #[must_use]
    #[inline(always)]
    pub fn peek(&self) -> Option<char> {
        match self.contents.get(self.pos) {
            Some(byte) => Some(*byte as char),
            None => None,
        }
    }
    
    #[must_use]
    #[inline(always)]
    pub fn last(&self) -> Option<char> {
        match self.contents.get(self.pos - 1) {
            Some(byte) => Some(*byte as char),
            None => None,
        }
    }
    
    #[must_use]
    #[inline(always)]
    pub fn next(&mut self) -> Option<char> {
        match self.contents.get(self.pos) {
            Some(byte) => {
                let char = *byte as char;
                if char == '\n' {
                    self.record.pos.0 += 1;
                    self.record.pos.1 = 0; 
                } else {
                    self.record.pos.1 += 1;
                }
                self.pos += 1;
                Some(char)
            },
            None => None,
        }
    }
    
    #[must_use]
    #[inline(always)]
    pub fn position(&self) -> usize {
        self.pos as usize
    }
    
    #[must_use]
    pub fn grab<Idx: Iterator<Item=usize>>(&self, idx: Idx) -> String {
        let mut string = String::new();
        for i in idx {
            string.push(self.contents[i] as char);
        }
        string
    }
    
    #[must_use]
    pub fn record(&self) -> &Record {
        &self.record
    }
}