ebnf_parser/
token.rs

1use std::fmt::{Debug, Display};
2
3use crate::span::Span;
4
5#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6pub enum TokenKind<'src> {
7    Identifier(&'src str),
8    Terminal(&'src str),
9    Comment(&'src str),
10    SpecialSeq(&'src str),
11    Integer(usize),
12
13    LBrace,
14    RBrace,
15    LBracket,
16    RBracket,
17    LParen,
18    RParen,
19    Pipe,
20    Comma,
21    Semicolon,
22    Equal,
23    Star,
24    Dash,
25}
26
27impl Display for TokenKind<'_> {
28    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
29        match self {
30            TokenKind::Identifier(name) => write!(f, "{name}"),
31            TokenKind::Terminal(text) => write!(f, "\"{text}\""),
32            TokenKind::Comment(text) => write!(f, "(* {text} *)"),
33            TokenKind::SpecialSeq(text) => write!(f, "? {text} ?"),
34            TokenKind::Integer(num) => write!(f, "{num}"),
35            TokenKind::LBrace => write!(f, "{{"),
36            TokenKind::RBrace => write!(f, "}}"),
37            TokenKind::LBracket => write!(f, "["),
38            TokenKind::RBracket => write!(f, "]"),
39            TokenKind::LParen => write!(f, "("),
40            TokenKind::RParen => write!(f, ")"),
41            TokenKind::Pipe => write!(f, "|"),
42            TokenKind::Comma => write!(f, ","),
43            TokenKind::Semicolon => write!(f, ";"),
44            TokenKind::Equal => write!(f, "="),
45            TokenKind::Star => write!(f, "*"),
46            TokenKind::Dash => write!(f, "-"),
47        }
48    }
49}
50
51/// A token with positional information
52#[derive(Clone)]
53pub struct Token<'src> {
54    pub kind: TokenKind<'src>,
55    pub span: Span,
56}
57
58impl<'src> Token<'src> {
59    pub(crate) fn new(kind: TokenKind<'src>, span: Span) -> Self {
60        Self { kind, span }
61    }
62}
63
64impl Debug for Token<'_> {
65    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
66        write!(f, "({:?} @ {:?})", self.kind, self.span)
67    }
68}
ebnf_parser/token.rs

ebnf_parser/
token.rs
