device_driver_lexer/
semantic_token_object.rs

1use device_driver_common::{
2    span::{Span, SpanExt, Spanned},
3    specifiers::{Access, AddressMode, BaseType, ByteOrder, Integer},
4};
5
6use crate::{Token, TokenModifier, TokenType};
7
8#[derive(Debug, Clone, Copy, PartialEq)]
9pub struct SemanticToken<'src> {
10    pub token: Token<'src>,
11    pub semantic_type: Option<TokenType>,
12    pub modifiers: u32,
13}
14
15impl<'src> Token<'src> {
16    pub fn without_semantics(self) -> SemanticToken<'src> {
17        SemanticToken {
18            token: self,
19            semantic_type: None,
20            modifiers: 0,
21        }
22    }
23
24    pub fn with_semantics(
25        self,
26        semantic_type: TokenType,
27        modifiers: &[TokenModifier],
28    ) -> SemanticToken<'src> {
29        SemanticToken {
30            token: self,
31            semantic_type: Some(semantic_type),
32            modifiers: modifiers.iter().map(|modifier| *modifier as u32).sum(),
33        }
34    }
35}
36
37/// An object that can be turned into semantic tokens
38pub trait SemanticTokenObject {
39    type Context;
40
41    fn to_tokens_in(&self, ctx: &Self::Context, tokens: &mut Vec<SemanticToken<'_>>);
42    fn to_tokens(&self, ctx: &Self::Context) -> Vec<SemanticToken<'_>> {
43        let mut tokens = Vec::new();
44        self.to_tokens_in(ctx, &mut tokens);
45        tokens
46    }
47    fn to_respanned_tokens<'src>(
48        &self,
49        source: &'src str,
50        object_span: Span,
51        ctx: &Self::Context,
52    ) -> Vec<Spanned<SemanticToken<'src>>> {
53        let self_tokens = self.to_tokens(ctx);
54        let mut self_index = 0;
55
56        let source_tokens = super::lex(&source[std::ops::Range::from(object_span)]);
57        let mut semantic_source_tokens = Vec::with_capacity(source_tokens.len());
58
59        let ignore = |token: &Token| matches!(token, Token::Comma);
60
61        for mut source_token in source_tokens.into_iter() {
62            source_token.span.start += object_span.start;
63            source_token.span.end += object_span.start;
64
65            if !ignore(&source_token) {
66                while let Some(self_token) = self_tokens.get(self_index)
67                    && ignore(&self_token.token)
68                {
69                    self_index += 1;
70                }
71
72                let semantic_token = if let Some(self_token) = self_tokens.get(self_index) {
73                    SemanticToken {
74                        token: source_token.value,
75                        semantic_type: self_token.semantic_type,
76                        modifiers: self_token.modifiers,
77                    }
78                    .with_span(source_token.span)
79                } else {
80                    source_token
81                        .without_semantics()
82                        .with_span(source_token.span)
83                };
84
85                semantic_source_tokens.push(semantic_token);
86
87                self_index += 1;
88            } else {
89                semantic_source_tokens.push(
90                    source_token
91                        .value
92                        .without_semantics()
93                        .with_span(source_token.span),
94                );
95            }
96        }
97
98        semantic_source_tokens
99    }
100}
101
102impl SemanticTokenObject for Access {
103    type Context = ();
104
105    fn to_tokens_in(&self, _ctx: &Self::Context, tokens: &mut Vec<SemanticToken<'_>>) {
106        tokens.push(Token::Access(*self).with_semantics(TokenType::Keyword, &[]));
107    }
108}
109
110impl SemanticTokenObject for ByteOrder {
111    type Context = ();
112
113    fn to_tokens_in(&self, _ctx: &Self::Context, tokens: &mut Vec<SemanticToken<'_>>) {
114        tokens.push(Token::ByteOrder(*self).with_semantics(TokenType::Keyword, &[]));
115    }
116}
117
118impl SemanticTokenObject for BaseType {
119    type Context = ();
120
121    fn to_tokens_in(&self, _ctx: &Self::Context, tokens: &mut Vec<SemanticToken<'_>>) {
122        tokens.push(
123            Token::BaseType(*self)
124                .with_semantics(TokenType::Type, &[TokenModifier::DefaultLibrary]),
125        );
126    }
127}
128
129impl SemanticTokenObject for Integer {
130    type Context = ();
131
132    fn to_tokens_in(&self, _ctx: &Self::Context, tokens: &mut Vec<SemanticToken<'_>>) {
133        tokens.push(
134            Token::Integer(*self).with_semantics(TokenType::Type, &[TokenModifier::DefaultLibrary]),
135        );
136    }
137}
138
139impl SemanticTokenObject for AddressMode {
140    type Context = ();
141
142    fn to_tokens_in(&self, _ctx: &Self::Context, tokens: &mut Vec<SemanticToken<'_>>) {
143        tokens.push(Token::AddressMode(*self).with_semantics(TokenType::Keyword, &[]));
144    }
145}
device_driver_lexer/semantic_token_object.rs

device_driver_lexer/
semantic_token_object.rs
