forth-lexer 0.2.1

Simple but complete lexer for the forth language
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

use std::{fmt::Display, ops::RangeBounds};

#[derive(Debug, PartialEq, Default, Copy, Clone)]
pub struct Data<'a> {
    pub start: usize,
    pub end: usize,
    pub value: &'a str,
}

impl<'a> Data<'a> {
    pub fn new(start: usize, end: usize, value: &'a str) -> Data<'a> {
        Data { start, end, value }
    }
}

impl<'a> RangeBounds<usize> for &Data<'a> {
    fn start_bound(&self) -> std::ops::Bound<&usize> {
        std::ops::Bound::Included(&self.start)
    }

    fn end_bound(&self) -> std::ops::Bound<&usize> {
        std::ops::Bound::Excluded(&self.end)
    }
}

#[derive(Debug, PartialEq, Clone)]
pub enum Token<'a> {
    Illegal(Data<'a>),
    Eof(Data<'a>),
    Colon(Data<'a>),
    Semicolon(Data<'a>),
    Word(Data<'a>),
    Number(Data<'a>),
    Comment(Data<'a>),
    StackComment(Data<'a>),
}

impl<'a> Token<'a> {
    pub fn get_data(&self) -> &Data<'a> {
        match self {
            Token::Illegal(dat) => dat,
            Token::Eof(dat) => dat,
            Token::Colon(dat) => dat,
            Token::Semicolon(dat) => dat,
            Token::Word(dat) => dat,
            Token::Number(dat) => dat,
            Token::Comment(dat) => dat,
            Token::StackComment(dat) => dat,
        }
    }
}

impl<'a> Display for Token<'a> {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Token::Illegal(_) => write!(f, ""),
            Token::Eof(_) => write!(f, "\0"),
            Token::Word(value)
            | Token::Number(value)
            | Token::StackComment(value)
            | Token::Comment(value) => write!(f, "{value:?}"),
            Token::Colon(_) => write!(f, ":"),
            Token::Semicolon(_) => write!(f, ";"),
        }
    }
}

impl<'a> From<Data<'a>> for Token<'a> {
    fn from(value: Data<'a>) -> Self {
        match value.value {
            ";" => Self::Semicolon(value),
            ":" => Self::Colon(value),
            "\0" => Self::Eof(value),
            _ => {
                if value.value.chars().all(|b| b.is_ascii_digit()) {
                    Self::Number(value)
                } else {
                    Self::Word(value)
                }
            }
        }
    }
}