lexer-rs 0.0.4

Lexical anaylzer framework for context-free text parsing into tokens
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

//a Imports
use crate::{PosnInCharStream, UserPosn};

//a StreamCharPos
//tp StreamCharPos
/// This provides the byte offset of a character within a stream, with
/// an associated position that might also accurately provide line and
/// column numbers of the position
///
/// This can be used as a [PosnInCharStream] implementation for a
/// lexer; if [crate::LineColumn] is used for the generic argument
/// then the lexer will correctly track the line and column of the
/// position of tokens.
#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Hash)]
pub struct StreamCharPos<P>
where
    P: UserPosn,
{
    byte_ofs: usize,
    pos: P,
}

//ip StreamCharPos
impl<P> StreamCharPos<P>
where
    P: UserPosn,
{
    /// Get the descriptive position that [Self] includes; if that
    /// type provides them accurately, this can give the line number
    /// and column number of the position
    pub fn pos(&self) -> P {
        self.pos
    }
}

//ip UserPosn for StreamCharPos
impl<P> UserPosn for StreamCharPos<P>
where
    P: UserPosn,
{
    fn advance_cols(mut self, num_bytes: usize, num_chars: usize) -> Self {
        self.byte_ofs += num_bytes;
        self.pos = self.pos.advance_cols(num_bytes, num_chars);
        self
    }
    fn advance_line(mut self, num_bytes: usize) -> Self {
        self.byte_ofs += num_bytes;
        self.pos = self.pos.advance_line(num_bytes);
        self
    }
    fn line(&self) -> usize {
        self.pos.line()
    }

    fn column(&self) -> usize {
        self.pos.column()
    }
    fn error_fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
        self.pos.error_fmt(fmt)
    }
}

//ip PosnInCharStream for StreamCharPos
impl<P> PosnInCharStream for StreamCharPos<P>
where
    P: UserPosn,
{
    fn byte_ofs(&self) -> usize {
        self.byte_ofs
    }
}

//ip Display for StreamCharPos
impl<P> std::fmt::Display for StreamCharPos<P>
where
    P: UserPosn,
{
    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
        self.pos.error_fmt(fmt)
    }
}