1use crate::doc_comment_table;
2use crate::resource_table::{self, PathId, StrId, TokenId};
3use crate::text_table::{self, TextId};
4use crate::veryl_grammar_trait::*;
5use once_cell::sync::Lazy;
6use paste::paste;
7use regex::Regex;
8use serde::{Deserialize, Serialize};
9use std::fmt;
10
11#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
12pub enum TokenSource {
13 File { path: PathId, text: TextId },
14 Builtin,
15 External,
16 Generated(PathId),
17}
18
19impl fmt::Display for TokenSource {
20 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
21 let text = match self {
22 TokenSource::File { path, .. } => path.to_string(),
23 TokenSource::Builtin => "builtin".to_string(),
24 TokenSource::External => "external".to_string(),
25 TokenSource::Generated(_) => "generated".to_string(),
26 };
27 text.fmt(f)
28 }
29}
30
31impl PartialEq<PathId> for TokenSource {
32 fn eq(&self, other: &PathId) -> bool {
33 match self {
34 TokenSource::File { path, .. } => path == other,
35 TokenSource::Generated(x) => x == other,
36 _ => false,
37 }
38 }
39}
40
41impl PartialEq<Option<PathId>> for TokenSource {
42 fn eq(&self, other: &Option<PathId>) -> bool {
43 match self {
44 TokenSource::File { path, .. } => Some(path) == other.as_ref(),
45 TokenSource::Generated(x) => Some(x) == other.as_ref(),
46 _ => false,
47 }
48 }
49}
50
51impl PartialOrd<PathId> for TokenSource {
52 fn partial_cmp(&self, other: &PathId) -> Option<std::cmp::Ordering> {
53 match self {
54 TokenSource::File { path, .. } => Some(path.cmp(other)),
55 TokenSource::Generated(x) => Some(x.cmp(other)),
56 _ => None,
57 }
58 }
59}
60
61impl TokenSource {
62 pub fn get_text(&self) -> String {
63 if let TokenSource::File { text, .. } = self {
64 if let Some(x) = text_table::get(*text) {
65 x.text
66 } else {
67 String::new()
68 }
69 } else {
70 String::new()
71 }
72 }
73
74 pub fn get_path(&self) -> Option<PathId> {
75 match self {
76 TokenSource::File { path, .. } => Some(*path),
77 TokenSource::Generated(x) => Some(*x),
78 _ => None,
79 }
80 }
81}
82
83#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
84pub struct Token {
85 pub id: TokenId,
86 pub text: StrId,
87 pub line: u32,
88 pub column: u32,
89 pub length: u32,
90 pub pos: u32,
91 pub source: TokenSource,
92}
93
94impl Token {
95 pub fn new(
96 text: &str,
97 line: u32,
98 column: u32,
99 length: u32,
100 pos: u32,
101 source: TokenSource,
102 ) -> Self {
103 let id = resource_table::new_token_id();
104 let text = resource_table::insert_str(text);
105 Token {
106 id,
107 text,
108 line,
109 column,
110 length,
111 pos,
112 source,
113 }
114 }
115
116 pub fn generate(text: StrId, path: PathId) -> Self {
117 let id = resource_table::new_token_id();
118 Token {
119 id,
120 text,
121 line: 0,
122 column: 0,
123 length: 0,
124 pos: 0,
125 source: TokenSource::Generated(path),
126 }
127 }
128
129 pub fn end_line(&self) -> u32 {
130 let text = self.to_string();
131 self.line + text.matches('\n').count() as u32
132 }
133
134 pub fn end_column(&self) -> u32 {
135 let text = self.to_string();
138 if text.matches('\n').count() > 0 {
139 text.split('\n')
140 .next_back()
141 .map(|x| x.chars().count() as u32)
142 .unwrap()
143 } else {
144 self.column + text.chars().count() as u32 - 1
145 }
146 }
147}
148
149pub fn is_anonymous_text(text: StrId) -> bool {
150 let anonymous_id = resource_table::insert_str("_");
151 text == anonymous_id
152}
153
154pub fn is_anonymous_token(token: &Token) -> bool {
155 is_anonymous_text(token.text)
156}
157
158impl Default for Token {
159 fn default() -> Self {
160 Self::generate(StrId::default(), PathId::default())
161 }
162}
163
164impl fmt::Display for Token {
165 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
166 let text = format!("{}", self.text);
167 text.fmt(f)
168 }
169}
170
171impl<'t> TryFrom<&parol_runtime::lexer::Token<'t>> for Token {
172 type Error = anyhow::Error;
173 fn try_from(x: &parol_runtime::lexer::Token<'t>) -> Result<Self, anyhow::Error> {
174 let id = resource_table::new_token_id();
175 let text = resource_table::insert_str(x.text());
176 let pos = x.location.start;
177 let source = TokenSource::File {
178 path: resource_table::insert_path(&x.location.file_name),
179 text: text_table::get_current_text(),
180 };
181 Ok(Token {
182 id,
183 text,
184 line: x.location.start_line,
185 column: x.location.start_column,
186 length: x.location.len() as u32,
187 pos,
188 source,
189 })
190 }
191}
192
193impl From<&Token> for miette::SourceSpan {
194 fn from(x: &Token) -> Self {
195 (x.pos as usize, x.length as usize).into()
196 }
197}
198
199impl From<Token> for miette::SourceSpan {
200 fn from(x: Token) -> Self {
201 (x.pos as usize, x.length as usize).into()
202 }
203}
204
205#[derive(Debug, Clone, Serialize, Deserialize)]
206pub struct VerylToken {
207 pub token: Token,
208 pub comments: Vec<Token>,
209}
210
211impl VerylToken {
212 pub fn new(token: Token) -> Self {
213 Self {
214 token,
215 comments: vec![],
216 }
217 }
218
219 pub fn replace(&self, text: &str) -> Self {
220 let length = text.len();
221 let text = resource_table::insert_str(text);
222 let mut ret = self.clone();
223 ret.token.text = text;
224 ret.token.length = length as u32;
225 ret
226 }
227
228 pub fn append(&self, prefix: &Option<String>, suffix: &Option<String>) -> Self {
229 let prefix_str = if let Some(x) = prefix { x.as_str() } else { "" };
230 let suffix_str = if let Some(x) = suffix { x.as_str() } else { "" };
231 let text = format!("{}{}{}", prefix_str, self.token.text, suffix_str);
232 let length = text.len();
233 let text = resource_table::insert_str(&text);
234 let mut ret = self.clone();
235 ret.token.text = text;
236 ret.token.length = length as u32;
237 ret
238 }
239
240 pub fn strip_prefix(&self, prefix: &str) -> Self {
241 let text = self.token.text.to_string();
242 if let Some(text) = text.strip_prefix(prefix) {
243 let length = text.len();
244 let text = resource_table::insert_str(text);
245 let mut ret = self.clone();
246 ret.token.text = text;
247 ret.token.length = length as u32;
248 ret
249 } else {
250 self.clone()
251 }
252 }
253}
254
255impl fmt::Display for VerylToken {
256 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
257 let text = format!("{}", self.token);
258 text.fmt(f)
259 }
260}
261
262impl ScopedIdentifier {
263 pub fn identifier(&self) -> &VerylToken {
264 match &*self.scoped_identifier_group {
265 ScopedIdentifierGroup::IdentifierScopedIdentifierOpt(x) => {
266 &x.identifier.identifier_token
267 }
268 ScopedIdentifierGroup::DollarIdentifier(x) => {
269 &x.dollar_identifier.dollar_identifier_token
270 }
271 }
272 }
273}
274
275impl ExpressionIdentifier {
276 pub fn identifier(&self) -> &VerylToken {
277 self.scoped_identifier.identifier()
278 }
279}
280
281static COMMENT_REGEX: Lazy<Regex> =
282 Lazy::new(|| Regex::new(r"((?://.*(?:\r\n|\r|\n|$))|(?:(?ms)/\u{2a}.*?\u{2a}/))").unwrap());
283
284fn split_comment_token(token: Token) -> Vec<Token> {
285 let mut line = token.line;
286 let mut column = token.column;
287 let text = resource_table::get_str_value(token.text).unwrap();
288
289 let mut prev_pos = 0;
290 let mut ret = Vec::new();
291 for cap in COMMENT_REGEX.captures_iter(&text) {
292 let cap = cap.get(0).unwrap();
293 let pos = cap.start();
294 let length = (cap.end() - pos) as u32;
295
296 let prev_text = &text[prev_pos..(pos)];
297 let n_lines = prev_text.matches('\n').count() as u32;
298 line += n_lines;
299
300 column = if n_lines == 0 {
301 column + prev_text.len() as u32
302 } else {
303 (prev_text.len() - prev_text.rfind('\n').unwrap_or(0)) as u32
304 };
305
306 prev_pos = pos;
307
308 let id = resource_table::new_token_id();
309 let text = &text[pos..pos + length as usize];
310 let is_doc_comment = text.starts_with("///");
311 let text = resource_table::insert_str(text);
312
313 if is_doc_comment && let TokenSource::File { path, .. } = token.source {
314 doc_comment_table::insert(path, line, text);
315 }
316
317 let token = Token {
318 id,
319 text,
320 line,
321 column,
322 length,
323 pos: pos as u32 + length,
324 source: token.source,
325 };
326 ret.push(token);
327 }
328 ret
329}
330
331impl TryFrom<&StartToken> for VerylToken {
332 type Error = anyhow::Error;
333
334 fn try_from(x: &StartToken) -> Result<Self, anyhow::Error> {
335 let mut comments = Vec::new();
336 if let Some(ref x) = x.comments.comments_opt {
337 let mut tokens = split_comment_token(x.comments_term.comments_term);
338 comments.append(&mut tokens)
339 }
340 let id = resource_table::new_token_id();
341 let text = resource_table::insert_str("");
342 let source = TokenSource::Builtin;
343 let token = Token {
344 id,
345 text,
346 line: 1,
347 column: 1,
348 length: 0,
349 pos: 0,
350 source,
351 };
352 Ok(VerylToken { token, comments })
353 }
354}
355
356macro_rules! token_with_comments {
357 ($x:ident) => {
358 paste! {
359 impl TryFrom<&[<$x Token>]> for VerylToken {
360 type Error = anyhow::Error;
361
362 fn try_from(x: &[<$x Token>]) -> Result<Self, anyhow::Error> {
363 let mut comments = Vec::new();
364 if let Some(ref x) = x.comments.comments_opt {
365 let mut tokens = split_comment_token(x.comments_term.comments_term);
366 comments.append(&mut tokens)
367 }
368 Ok(VerylToken {
369 token: x.[<$x:snake _term>].clone(),
370 comments,
371 })
372 }
373 }
374 impl TryFrom<&[<$x Term>]> for Token {
375 type Error = anyhow::Error;
376
377 fn try_from(x: &[<$x Term>]) -> Result<Self, anyhow::Error> {
378 Ok(Token {
379 id: x.[<$x:snake _term>].id,
380 text: x.[<$x:snake _term>].text,
381 line: x.[<$x:snake _term>].line,
382 column: x.[<$x:snake _term>].column,
383 length: x.[<$x:snake _term>].length,
384 pos: x.[<$x:snake _term>].pos,
385 source: x.[<$x:snake _term>].source,
386 })
387 }
388 }
389 }
390 };
391}
392
393macro_rules! token_without_comments {
394 ($x:ident, $y:ident) => {
395 paste! {
396 impl TryFrom<&[<$x Token>]> for VerylToken {
397 type Error = anyhow::Error;
398
399 fn try_from(x: &[<$x Token>]) -> Result<Self, anyhow::Error> {
400 Ok(VerylToken {
401 token: x.[<$y:snake _term>].clone(),
402 comments: Vec::new(),
403 })
404 }
405 }
406 }
407 };
408 ($x:ident) => {
409 paste! {
410 impl TryFrom<&[<$x Token>]> for VerylToken {
411 type Error = anyhow::Error;
412
413 fn try_from(x: &[<$x Token>]) -> Result<Self, anyhow::Error> {
414 Ok(VerylToken {
415 token: x.[<$x:snake _term>].clone(),
416 comments: Vec::new(),
417 })
418 }
419 }
420 impl TryFrom<&[<$x Term>]> for Token {
421 type Error = anyhow::Error;
422
423 fn try_from(x: &[<$x Term>]) -> Result<Self, anyhow::Error> {
424 Ok(Token {
425 id: x.[<$x:snake _term>].id,
426 text: x.[<$x:snake _term>].text,
427 line: x.[<$x:snake _term>].line,
428 column: x.[<$x:snake _term>].column,
429 length: x.[<$x:snake _term>].length,
430 pos: x.[<$x:snake _term>].pos,
431 source: x.[<$x:snake _term>].source,
432 })
433 }
434 }
435 }
436 };
437}
438
439token_with_comments!(StringLiteral);
440
441token_with_comments!(FixedPoint);
442token_with_comments!(Exponent);
443token_with_comments!(Based);
444token_with_comments!(BaseLess);
445token_with_comments!(AllBit);
446
447token_with_comments!(Colon);
448token_with_comments!(ColonColon);
449token_with_comments!(ColonColonLAngle);
450token_with_comments!(Comma);
451token_with_comments!(DotDot);
452token_with_comments!(DotDotEqu);
453token_with_comments!(Dot);
454token_with_comments!(Equ);
455token_with_comments!(HashLBracket);
456token_with_comments!(Hash);
457token_with_comments!(Question);
458token_with_comments!(Quote);
459token_with_comments!(QuoteLBrace);
460token_with_comments!(LAngle);
461token_without_comments!(EmbedLBrace, LBrace);
462token_without_comments!(EscapedLBrace);
463token_without_comments!(TripleLBrace);
464token_with_comments!(LBrace);
465token_with_comments!(LBracket);
466token_with_comments!(LParen);
467token_with_comments!(LTMinus);
468token_with_comments!(MinusColon);
469token_with_comments!(MinusGT);
470token_with_comments!(PlusColon);
471token_with_comments!(RAngle);
472token_without_comments!(EmbedRBrace, RBrace);
473token_without_comments!(EscapedRBrace);
474token_with_comments!(TripleRBrace);
475token_with_comments!(RBrace);
476token_with_comments!(RBracket);
477token_with_comments!(RParen);
478token_with_comments!(Semicolon);
479token_with_comments!(Star);
480
481token_with_comments!(AssignmentOperator);
482token_with_comments!(DiamondOperator);
483token_with_comments!(Operator01);
484token_with_comments!(Operator02);
485token_with_comments!(Operator03);
486token_with_comments!(Operator04);
487token_with_comments!(Operator05);
488token_with_comments!(Operator06);
489token_with_comments!(Operator07);
490token_with_comments!(Operator08);
491token_with_comments!(UnaryOperator);
492
493token_with_comments!(Alias);
494token_with_comments!(AlwaysComb);
495token_with_comments!(AlwaysFf);
496token_with_comments!(As);
497token_with_comments!(Assign);
498token_with_comments!(Bind);
499token_with_comments!(Bit);
500token_with_comments!(Block);
501token_with_comments!(BBool);
502token_with_comments!(LBool);
503token_with_comments!(Break);
504token_with_comments!(Case);
505token_with_comments!(Clock);
506token_with_comments!(ClockPosedge);
507token_with_comments!(ClockNegedge);
508token_with_comments!(Connect);
509token_with_comments!(Const);
510token_with_comments!(Converse);
511token_with_comments!(Default);
512token_with_comments!(Else);
513token_with_comments!(Embed);
514token_with_comments!(Enum);
515token_with_comments!(F32);
516token_with_comments!(F64);
517token_with_comments!(False);
518token_with_comments!(Final);
519token_with_comments!(For);
520token_with_comments!(Function);
521token_with_comments!(Gen);
522token_with_comments!(I8);
523token_with_comments!(I16);
524token_with_comments!(I32);
525token_with_comments!(I64);
526token_with_comments!(If);
527token_with_comments!(IfReset);
528token_with_comments!(Import);
529token_with_comments!(Include);
530token_with_comments!(Initial);
531token_with_comments!(Inout);
532token_with_comments!(Input);
533token_with_comments!(Inside);
534token_with_comments!(Inst);
535token_with_comments!(Interface);
536token_with_comments!(In);
537token_with_comments!(Let);
538token_with_comments!(Logic);
539token_with_comments!(Lsb);
540token_with_comments!(Modport);
541token_with_comments!(Module);
542token_with_comments!(Msb);
543token_with_comments!(Output);
544token_with_comments!(Outside);
545token_with_comments!(Package);
546token_with_comments!(Param);
547token_with_comments!(Proto);
548token_with_comments!(Pub);
549token_with_comments!(Repeat);
550token_with_comments!(Reset);
551token_with_comments!(ResetAsyncHigh);
552token_with_comments!(ResetAsyncLow);
553token_with_comments!(ResetSyncHigh);
554token_with_comments!(ResetSyncLow);
555token_with_comments!(Return);
556token_with_comments!(Rev);
557token_with_comments!(Same);
558token_with_comments!(Signed);
559token_with_comments!(Step);
560token_with_comments!(String);
561token_with_comments!(Struct);
562token_with_comments!(Switch);
563token_with_comments!(Tri);
564token_with_comments!(True);
565token_with_comments!(Type);
566token_with_comments!(P8);
567token_with_comments!(P16);
568token_with_comments!(P32);
569token_with_comments!(P64);
570token_with_comments!(U8);
571token_with_comments!(U16);
572token_with_comments!(U32);
573token_with_comments!(U64);
574token_with_comments!(Union);
575token_with_comments!(Unsafe);
576token_with_comments!(Var);
577
578token_with_comments!(DollarIdentifier);
579token_with_comments!(Identifier);
580
581token_without_comments!(Any);