Skip to main content

mirascript_core/lexer/
token.rs

1use std::{
2    fmt::Display,
3    ops::{Deref, DerefMut},
4};
5
6use super::prelude::*;
7
8#[derive(Debug, Clone)]
9pub struct Token<'s> {
10    pub kind: TokenKind<'s>,
11    pub range: SourceRange,
12    pub leading_trivia: TriviaList<'s>,
13    pub tailing_trivia: TriviaList<'s>,
14}
15
16impl winnow::stream::Location for Token<'_> {
17    fn current_token_start(&self) -> usize {
18        self.range.start
19    }
20
21    fn previous_token_end(&self) -> usize {
22        self.range.end
23    }
24}
25
26impl<'s> Deref for Token<'s> {
27    type Target = TokenKind<'s>;
28
29    fn deref(&self) -> &Self::Target {
30        &self.kind
31    }
32}
33
34impl DerefMut for Token<'_> {
35    fn deref_mut(&mut self) -> &mut Self::Target {
36        &mut self.kind
37    }
38}
39
40impl<'s> Token<'s> {
41    #[cfg(feature = "formatter")]
42    pub(crate) fn full_range(&self) -> SourceRange {
43        let start = self
44            .leading_trivia
45            .first()
46            .map_or(self.range.start, |trivia| trivia.range().start);
47        let end = self
48            .tailing_trivia
49            .last()
50            .map_or(self.range.end, |trivia| trivia.range().end);
51        start..end
52    }
53
54    pub(crate) fn new(kind: TokenKind<'s>, range: SourceRange) -> Self {
55        Token {
56            kind,
57            range,
58            leading_trivia: Default::default(),
59            tailing_trivia: Default::default(),
60        }
61    }
62
63    pub(crate) fn empty(pos: usize) -> Self {
64        Self::new(TokenKind::Empty, pos..pos)
65    }
66    pub(crate) fn empty_range(range: SourceRange) -> Self {
67        Self::new(TokenKind::Empty, range)
68    }
69
70    pub(crate) fn unknown<R: Into<TokenKind<'s>>>(
71        range: SourceRange,
72        recovered: R,
73        error: DiagnosticCode,
74    ) -> Self {
75        Self::new(TokenKind::unknown(recovered, range.clone(), error), range)
76    }
77
78    pub(crate) fn unknown_at<R: Into<TokenKind<'s>>>(
79        pos: usize,
80        recovered: R,
81        error: DiagnosticCode,
82    ) -> Self {
83        Self::unknown(pos..pos, recovered, error)
84    }
85
86    pub(crate) fn wrap_as_unknown(&mut self, error: DiagnosticCode) {
87        let kind = std::mem::replace(&mut self.kind, TokenKind::Empty);
88        self.kind = TokenKind::unknown(kind, self.range.clone(), error);
89    }
90}
91
92impl PartialEq for Token<'_> {
93    fn eq(&self, other: &Self) -> bool {
94        self.kind == other.kind
95    }
96}
97
98impl Display for Token<'_> {
99    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
100        self.kind.fmt(f)
101    }
102}