lexington 0.3.0

A very simple library for lexing / parsing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

use std::ops::Range;
use crate::util::{Region};

/// A token constitutes a label and identifying region within the
/// original sequence.
#[derive(Clone,Copy,Debug,PartialEq)]
pub struct Token<T>
{
    /// Type of the token
    pub kind : T,
    /// Identifies the (half open) region in the sequence.
    pub region: Region
}

impl<T> Token<T>  {
    pub fn new(kind: T, range: Range<usize>) -> Self {
        Self { kind, region: Region::from(range) }
    }

    /// Get first index of this token.
    pub fn start(&self) -> usize {
	self.region.start
    }

    /// Get end of this token (that is one past its last character).
    pub fn end(&self) -> usize {
	self.region.end
    }

    /// Get the length (in chars) of this token.
    pub fn len(&self) -> usize {
        self.region.end - self.region.start
    }

    /// Extract the underlying region covered by this span as a
    /// `Range`.  This is really just for convenience.
    pub fn range(&self) -> Range<usize> { self.start() .. self.end() }

    /// Shift the span to a different position in the underlying
    /// sequence.  The position is taken as a delta from the current
    /// position (e.g. `delta==1` means we shift one up the sequence).
    pub fn shift(&mut self, delta: usize) {
        self.region.shift(delta);
    }
}

impl<T:PartialEq> PartialEq<(T,Range<usize>)> for Token<T> {
    //
    fn eq(&self, other: &(T,Range<usize>)) -> bool {
        self.kind == other.0 && self.range() == other.1
    }
}

// =============================================================================
// TokenStr
// =============================================================================

/// A token constitutes a label and identifying region within the
/// original sequence.
#[derive(Clone,Copy,Debug,PartialEq)]
pub struct TokenStr<'a, T>
{
    /// Type of the token
    pub kind : T,
    /// Identifies the text representing this token
    pub text: &'a str
}

impl<'a, T> TokenStr<'a, T>  {

    pub fn new(kind: T, text: &'a str) -> Self {
        Self { kind, text }
    }
    
    pub fn from_token(input: &'a str, tok: Token<T>) -> Self {
        let t = &input[tok.range()];
        Self { kind: tok.kind, text: t }
    }
    
    /// Get first index of this token.
    pub fn start(&self) -> usize {
	//self.region.start
        todo!()
    }

    /// Get end of this token (that is one past its last character).
    pub fn end(&self) -> usize {
        todo!()
	    //self.region.end
    }

    /// Get the length (in chars) of this token.
    pub fn len(&self) -> usize {
        //self.region.end - self.region.start
        self.text.len()
    }
}