Skip to main content

mirascript_core/lexer/
token.rs

1use std::{
2    fmt::Display,
3    ops::{Deref, DerefMut},
4};
5
6use super::prelude::*;
7
8#[derive(Debug, Clone)]
9pub struct Token<'s> {
10    pub kind: TokenKind<'s>,
11    pub range: SourceRange,
12    pub leading_trivia: TriviaList<'s>,
13    pub tailing_trivia: TriviaList<'s>,
14}
15
16impl winnow::stream::Location for Token<'_> {
17    fn current_token_start(&self) -> usize {
18        self.range.start
19    }
20
21    fn previous_token_end(&self) -> usize {
22        self.range.end
23    }
24}
25
26impl<'s> Deref for Token<'s> {
27    type Target = TokenKind<'s>;
28
29    fn deref(&self) -> &Self::Target {
30        &self.kind
31    }
32}
33
34impl DerefMut for Token<'_> {
35    fn deref_mut(&mut self) -> &mut Self::Target {
36        &mut self.kind
37    }
38}
39
40impl<'s> Token<'s> {
41    pub(crate) fn new(kind: TokenKind<'s>, range: SourceRange) -> Self {
42        Token {
43            kind,
44            range,
45            leading_trivia: Default::default(),
46            tailing_trivia: Default::default(),
47        }
48    }
49
50    pub(crate) fn empty(pos: usize) -> Self {
51        Self::new(TokenKind::Empty, pos..pos)
52    }
53    pub(crate) fn empty_range(range: SourceRange) -> Self {
54        Self::new(TokenKind::Empty, range)
55    }
56
57    pub(crate) fn unknown<R: Into<TokenKind<'s>>>(
58        range: SourceRange,
59        recovered: R,
60        error: DiagnosticCode,
61    ) -> Self {
62        Self::new(TokenKind::unknown(recovered, range.clone(), error), range)
63    }
64
65    pub(crate) fn unknown_at<R: Into<TokenKind<'s>>>(
66        pos: usize,
67        recovered: R,
68        error: DiagnosticCode,
69    ) -> Self {
70        Self::unknown(pos..pos, recovered, error)
71    }
72
73    pub(crate) fn wrap_as_unknown(&mut self, error: DiagnosticCode) {
74        let kind = std::mem::replace(&mut self.kind, TokenKind::Empty);
75        self.kind = TokenKind::unknown(kind, self.range.clone(), error);
76    }
77}
78
79impl PartialEq for Token<'_> {
80    fn eq(&self, other: &Self) -> bool {
81        self.kind == other.kind
82    }
83}
84
85impl Display for Token<'_> {
86    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
87        self.kind.fmt(f)
88    }
89}