simple-tokenizer 0.4.2

A tiny no_std tokenizer with line & column tracking.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

use crate::{AsTokens, Offset, Position, Tokens};
use core::{ops::Deref, str::FromStr};

/// A type holding the byte offset as well as position info.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Location {
    offset: Offset,
    pos: Position,
}

impl Location {
    /// Get the position info.
    #[inline]
    pub fn position(&self) -> Position {
        self.pos
    }
}

impl yap::TokenLocation for Location {
    #[inline]
    fn offset(&self) -> usize {
        self.offset.0
    }
}

/// Wrapper around `Tokens` that implements `yap::Tokens`.
///
/// This is a separate struct to avoid confusion, since both have a few similar (or outright identical)
/// method names that may not work as identically as their names might suggest.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct YapTokens<'s> {
    /// Underlying `Tokens` object.
    pub simple_tokens: Tokens<'s>,
}

impl<'s> From<Tokens<'s>> for YapTokens<'s> {
    #[inline]
    fn from(simple_tokens: Tokens<'s>) -> Self {
        YapTokens { simple_tokens }
    }
}

impl<'s> yap::Tokens for YapTokens<'s> {
    type Item = char;
    type Location = Location;

    #[inline]
    fn next(&mut self) -> Option<Self::Item> {
        self.simple_tokens.char()
    }

    #[inline]
    fn location(&self) -> Self::Location {
        Location {
            offset: Offset(self.simple_tokens.offset),
            pos: self.simple_tokens.pos,
        }
    }

    #[inline]
    fn set_location(&mut self, location: Self::Location) {
        self.simple_tokens.set_offset(location.offset);
    }

    #[inline]
    fn is_at_location(&self, location: &Self::Location) -> bool {
        self.simple_tokens.offset == location.offset.0 && self.simple_tokens.pos == location.pos
    }

    #[inline]
    fn parse<Out, Buf>(&mut self) -> Result<Out, <Out as FromStr>::Err>
    where
        Out: FromStr,
        Buf: FromIterator<Self::Item> + Deref<Target = str>,
    {
        let result = self.simple_tokens.remaining_input.parse()?;

        self.simple_tokens.consume_all();

        Ok(result)
    }

    #[inline]
    fn peek(&mut self) -> Option<Self::Item> {
        self.simple_tokens.peek()
    }
}

impl<'s> yap::IntoTokens<char> for YapTokens<'s> {
    type Tokens = Self;

    #[inline]
    fn into_tokens(self) -> Self::Tokens {
        self
    }
}

impl<'s> AsTokens for YapTokens<'s> {
    #[inline]
    fn as_tokens(&self) -> Tokens<'_> {
        self.simple_tokens.as_tokens()
    }
}