mirascript_core/lexer/
token.rs1use std::{
2 fmt::Display,
3 ops::{Deref, DerefMut},
4};
5
6use super::prelude::*;
7
8#[derive(Debug, Clone)]
9pub struct Token<'s> {
10 pub kind: TokenKind<'s>,
11 pub range: SourceRange,
12 pub leading_trivia: TriviaList<'s>,
13 pub tailing_trivia: TriviaList<'s>,
14}
15
16impl winnow::stream::Location for Token<'_> {
17 fn current_token_start(&self) -> usize {
18 self.range.start
19 }
20
21 fn previous_token_end(&self) -> usize {
22 self.range.end
23 }
24}
25
26impl<'s> Deref for Token<'s> {
27 type Target = TokenKind<'s>;
28
29 fn deref(&self) -> &Self::Target {
30 &self.kind
31 }
32}
33
34impl DerefMut for Token<'_> {
35 fn deref_mut(&mut self) -> &mut Self::Target {
36 &mut self.kind
37 }
38}
39
40impl<'s> Token<'s> {
41 pub(crate) fn new(kind: TokenKind<'s>, range: SourceRange) -> Self {
42 Token {
43 kind,
44 range,
45 leading_trivia: Default::default(),
46 tailing_trivia: Default::default(),
47 }
48 }
49
50 pub(crate) fn empty(pos: usize) -> Self {
51 Self::new(TokenKind::Empty, pos..pos)
52 }
53 pub(crate) fn empty_range(range: SourceRange) -> Self {
54 Self::new(TokenKind::Empty, range)
55 }
56
57 pub(crate) fn unknown<R: Into<TokenKind<'s>>>(
58 range: SourceRange,
59 recovered: R,
60 error: DiagnosticCode,
61 ) -> Self {
62 Self::new(TokenKind::unknown(recovered, range.clone(), error), range)
63 }
64
65 pub(crate) fn unknown_at<R: Into<TokenKind<'s>>>(
66 pos: usize,
67 recovered: R,
68 error: DiagnosticCode,
69 ) -> Self {
70 Self::unknown(pos..pos, recovered, error)
71 }
72
73 pub(crate) fn wrap_as_unknown(&mut self, error: DiagnosticCode) {
74 let kind = std::mem::replace(&mut self.kind, TokenKind::Empty);
75 self.kind = TokenKind::unknown(kind, self.range.clone(), error);
76 }
77}
78
79impl PartialEq for Token<'_> {
80 fn eq(&self, other: &Self) -> bool {
81 self.kind == other.kind
82 }
83}
84
85impl Display for Token<'_> {
86 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
87 self.kind.fmt(f)
88 }
89}