forth_lexer/
token.rs

1use std::{fmt::Display, ops::RangeBounds};
2
3#[derive(Debug, PartialEq, Default, Copy, Clone)]
4pub struct Data<T> {
5    pub start: usize,
6    pub end: usize,
7    pub value: T,
8}
9
10impl<T> Data<T> {
11    pub fn new(start: usize, end: usize, value: T) -> Data<T> {
12        Data::<T> { start, end, value }
13    }
14}
15
16impl<T> RangeBounds<usize> for &Data<T> {
17    fn start_bound(&self) -> std::ops::Bound<&usize> {
18        std::ops::Bound::Included(&self.start)
19    }
20
21    fn end_bound(&self) -> std::ops::Bound<&usize> {
22        std::ops::Bound::Excluded(&self.end)
23    }
24}
25
26#[derive(Debug, PartialEq, Clone)]
27pub enum Token {
28    Illegal(Data<char>),
29    Eof(Data<char>),
30    Colon(Data<char>),
31    Semicolon(Data<char>),
32    Word(Data<String>),
33    Number(Data<String>),
34    Comment(Data<String>),
35    StackComment(Data<String>),
36}
37
38impl Display for Token {
39    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
40        match self {
41            Token::Illegal(_) => write!(f, ""),
42            Token::Eof(_) => write!(f, "\0"),
43            Token::Word(value)
44            | Token::Number(value)
45            | Token::StackComment(value)
46            | Token::Comment(value) => write!(f, "{value:?}"),
47            Token::Colon(_) => write!(f, ":"),
48            Token::Semicolon(_) => write!(f, ";"),
49        }
50    }
51}
52
53impl From<Data<char>> for Token {
54    fn from(ch: Data<char>) -> Self {
55        match ch.value {
56            ';' => Self::Semicolon(ch),
57            ':' => Self::Colon(ch),
58            '\0' => Self::Eof(ch),
59            _ => Self::Illegal(ch),
60        }
61    }
62}
63
64impl From<Data<String>> for Token {
65    fn from(value: Data<String>) -> Self {
66        if value.value.chars().all(|b| b.is_ascii_digit()) {
67            Self::Number(value)
68        } else {
69            Self::Word(value)
70        }
71    }
72}
forth_lexer/token.rs

forth_lexer/
token.rs
