veryl_parser/
veryl_token.rs

1use crate::doc_comment_table;
2use crate::resource_table::{self, PathId, StrId, TokenId};
3use crate::text_table::{self, TextId};
4use crate::veryl_grammar_trait::*;
5use once_cell::sync::Lazy;
6use paste::paste;
7use regex::Regex;
8use serde::{Deserialize, Serialize};
9use std::fmt;
10
11#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
12pub enum TokenSource {
13    File { path: PathId, text: TextId },
14    Builtin,
15    External,
16    Generated(PathId),
17}
18
19impl fmt::Display for TokenSource {
20    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
21        let text = match self {
22            TokenSource::File { path, .. } => path.to_string(),
23            TokenSource::Builtin => "builtin".to_string(),
24            TokenSource::External => "external".to_string(),
25            TokenSource::Generated(_) => "generated".to_string(),
26        };
27        text.fmt(f)
28    }
29}
30
31impl PartialEq<PathId> for TokenSource {
32    fn eq(&self, other: &PathId) -> bool {
33        match self {
34            TokenSource::File { path, .. } => path == other,
35            TokenSource::Generated(x) => x == other,
36            _ => false,
37        }
38    }
39}
40
41impl PartialEq<Option<PathId>> for TokenSource {
42    fn eq(&self, other: &Option<PathId>) -> bool {
43        match self {
44            TokenSource::File { path, .. } => Some(path) == other.as_ref(),
45            TokenSource::Generated(x) => Some(x) == other.as_ref(),
46            _ => false,
47        }
48    }
49}
50
51impl PartialOrd<PathId> for TokenSource {
52    fn partial_cmp(&self, other: &PathId) -> Option<std::cmp::Ordering> {
53        match self {
54            TokenSource::File { path, .. } => Some(path.cmp(other)),
55            TokenSource::Generated(x) => Some(x.cmp(other)),
56            _ => None,
57        }
58    }
59}
60
61impl TokenSource {
62    pub fn get_text(&self) -> String {
63        if let TokenSource::File { text, .. } = self {
64            if let Some(x) = text_table::get(*text) {
65                x.text
66            } else {
67                String::new()
68            }
69        } else {
70            String::new()
71        }
72    }
73
74    pub fn get_path(&self) -> Option<PathId> {
75        match self {
76            TokenSource::File { path, .. } => Some(*path),
77            TokenSource::Generated(x) => Some(*x),
78            _ => None,
79        }
80    }
81}
82
83#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
84pub struct Token {
85    pub id: TokenId,
86    pub text: StrId,
87    pub line: u32,
88    pub column: u32,
89    pub length: u32,
90    pub pos: u32,
91    pub source: TokenSource,
92}
93
94impl Token {
95    pub fn new(
96        text: &str,
97        line: u32,
98        column: u32,
99        length: u32,
100        pos: u32,
101        source: TokenSource,
102    ) -> Self {
103        let id = resource_table::new_token_id();
104        let text = resource_table::insert_str(text);
105        Token {
106            id,
107            text,
108            line,
109            column,
110            length,
111            pos,
112            source,
113        }
114    }
115
116    pub fn from_external_text(text: &str) -> Self {
117        Self::new(text, 0, 0, 0, 0, TokenSource::External)
118    }
119
120    pub fn builtin_text(text: &str) -> Self {
121        Self::new(text, 0, 0, 0, 0, TokenSource::Builtin)
122    }
123
124    pub fn generate(text: StrId, path: PathId) -> Self {
125        let id = resource_table::new_token_id();
126        Token {
127            id,
128            text,
129            line: 0,
130            column: 0,
131            length: 0,
132            pos: 0,
133            source: TokenSource::Generated(path),
134        }
135    }
136
137    pub fn end_line(&self) -> u32 {
138        let text = self.to_string();
139        self.line + text.matches('\n').count() as u32
140    }
141
142    pub fn end_column(&self) -> u32 {
143        // Columns count CHARACTERS (parol), while `length` is the byte
144        // length — multi-byte UTF-8 text must not overshoot.
145        let text = self.to_string();
146        if text.matches('\n').count() > 0 {
147            text.split('\n')
148                .next_back()
149                .map(|x| x.chars().count() as u32)
150                .unwrap()
151        } else {
152            self.column + text.chars().count() as u32 - 1
153        }
154    }
155}
156
157pub fn is_anonymous_text(text: StrId) -> bool {
158    let anonymous_id = resource_table::insert_str("_");
159    text == anonymous_id
160}
161
162pub fn is_anonymous_token(token: &Token) -> bool {
163    is_anonymous_text(token.text)
164}
165
166impl Default for Token {
167    fn default() -> Self {
168        Self::generate(StrId::default(), PathId::default())
169    }
170}
171
172impl fmt::Display for Token {
173    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
174        let text = format!("{}", self.text);
175        text.fmt(f)
176    }
177}
178
179impl<'t> TryFrom<&parol_runtime::lexer::Token<'t>> for Token {
180    type Error = anyhow::Error;
181    fn try_from(x: &parol_runtime::lexer::Token<'t>) -> Result<Self, anyhow::Error> {
182        let id = resource_table::new_token_id();
183        let text = resource_table::insert_str(x.text());
184        let pos = x.location.start;
185        let source = TokenSource::File {
186            path: resource_table::insert_path(&x.location.file_name),
187            text: text_table::get_current_text(),
188        };
189        Ok(Token {
190            id,
191            text,
192            line: x.location.start_line,
193            column: x.location.start_column,
194            length: x.location.len() as u32,
195            pos,
196            source,
197        })
198    }
199}
200
201impl From<&Token> for miette::SourceSpan {
202    fn from(x: &Token) -> Self {
203        (x.pos as usize, x.length as usize).into()
204    }
205}
206
207impl From<Token> for miette::SourceSpan {
208    fn from(x: Token) -> Self {
209        (x.pos as usize, x.length as usize).into()
210    }
211}
212
213#[derive(Debug, Clone, Serialize, Deserialize)]
214pub struct VerylToken {
215    pub token: Token,
216    pub comments: Vec<Token>,
217}
218
219impl VerylToken {
220    pub fn new(token: Token) -> Self {
221        Self {
222            token,
223            comments: vec![],
224        }
225    }
226
227    pub fn from_external_text(text: &str) -> Self {
228        Self {
229            token: Token::from_external_text(text),
230            comments: vec![],
231        }
232    }
233
234    pub fn replace(&self, text: &str) -> Self {
235        let length = text.len();
236        let text = resource_table::insert_str(text);
237        let mut ret = self.clone();
238        ret.token.text = text;
239        ret.token.length = length as u32;
240        ret
241    }
242
243    pub fn append(&self, prefix: &Option<String>, suffix: &Option<String>) -> Self {
244        let prefix_str = if let Some(x) = prefix { x.as_str() } else { "" };
245        let suffix_str = if let Some(x) = suffix { x.as_str() } else { "" };
246        let text = format!("{}{}{}", prefix_str, self.token.text, suffix_str);
247        let length = text.len();
248        let text = resource_table::insert_str(&text);
249        let mut ret = self.clone();
250        ret.token.text = text;
251        ret.token.length = length as u32;
252        ret
253    }
254
255    pub fn strip_prefix(&self, prefix: &str) -> Self {
256        let text = self.token.text.to_string();
257        if let Some(text) = text.strip_prefix(prefix) {
258            let length = text.len();
259            let text = resource_table::insert_str(text);
260            let mut ret = self.clone();
261            ret.token.text = text;
262            ret.token.length = length as u32;
263            ret
264        } else {
265            self.clone()
266        }
267    }
268}
269
270impl fmt::Display for VerylToken {
271    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
272        let text = format!("{}", self.token);
273        text.fmt(f)
274    }
275}
276
277impl ScopedIdentifier {
278    pub fn identifier(&self) -> &VerylToken {
279        match &*self.scoped_identifier_group {
280            ScopedIdentifierGroup::IdentifierScopedIdentifierOpt(x) => {
281                &x.identifier.identifier_token
282            }
283            ScopedIdentifierGroup::DollarIdentifier(x) => {
284                &x.dollar_identifier.dollar_identifier_token
285            }
286        }
287    }
288}
289
290impl ExpressionIdentifier {
291    pub fn identifier(&self) -> &VerylToken {
292        self.scoped_identifier.identifier()
293    }
294}
295
296// The block-comment part must match the lexer's CommentsTerm (veryl.par)
297// exactly, or text between matches is silently dropped.
298static COMMENT_REGEX: Lazy<Regex> = Lazy::new(|| {
299    Regex::new(r"((?://.*(?:\r\n|\r|\n|$))|(?:/\u{2a}(?:[^\u{2a}]|\u{2a}+[^\u{2a}/])*\u{2a}+/))")
300        .unwrap()
301});
302
303fn split_comment_token(token: Token) -> Vec<Token> {
304    let mut line = token.line;
305    let mut column = token.column;
306    let text = resource_table::get_str_value(token.text).unwrap();
307
308    let mut prev_pos = 0;
309    let mut ret = Vec::new();
310    for cap in COMMENT_REGEX.captures_iter(&text) {
311        let cap = cap.get(0).unwrap();
312        let pos = cap.start();
313        let length = (cap.end() - pos) as u32;
314
315        let prev_text = &text[prev_pos..(pos)];
316        let n_lines = prev_text.matches('\n').count() as u32;
317        line += n_lines;
318
319        column = if n_lines == 0 {
320            column + prev_text.len() as u32
321        } else {
322            (prev_text.len() - prev_text.rfind('\n').unwrap_or(0)) as u32
323        };
324
325        prev_pos = pos;
326
327        let id = resource_table::new_token_id();
328        let text = &text[pos..pos + length as usize];
329        let is_doc_comment = text.starts_with("///");
330        let text = resource_table::insert_str(text);
331
332        if is_doc_comment && let TokenSource::File { path, .. } = token.source {
333            doc_comment_table::insert(path, line, text);
334        }
335
336        let token = Token {
337            id,
338            text,
339            line,
340            column,
341            length,
342            pos: pos as u32 + length,
343            source: token.source,
344        };
345        ret.push(token);
346    }
347    ret
348}
349
350impl TryFrom<&StartToken> for VerylToken {
351    type Error = anyhow::Error;
352
353    fn try_from(x: &StartToken) -> Result<Self, anyhow::Error> {
354        let mut comments = Vec::new();
355        if let Some(ref x) = x.comments.comments_opt {
356            let mut tokens = split_comment_token(x.comments_term.comments_term);
357            comments.append(&mut tokens)
358        }
359        let id = resource_table::new_token_id();
360        let text = resource_table::insert_str("");
361        let source = TokenSource::Builtin;
362        let token = Token {
363            id,
364            text,
365            line: 1,
366            column: 1,
367            length: 0,
368            pos: 0,
369            source,
370        };
371        Ok(VerylToken { token, comments })
372    }
373}
374
375macro_rules! token_with_comments {
376    ($x:ident) => {
377        paste! {
378            impl TryFrom<&[<$x Token>]> for VerylToken {
379                type Error = anyhow::Error;
380
381                fn try_from(x: &[<$x Token>]) -> Result<Self, anyhow::Error> {
382                    let mut comments = Vec::new();
383                    if let Some(ref x) = x.comments.comments_opt {
384                        let mut tokens = split_comment_token(x.comments_term.comments_term);
385                        comments.append(&mut tokens)
386                    }
387                    Ok(VerylToken {
388                        token: x.[<$x:snake _term>].clone(),
389                        comments,
390                    })
391                }
392            }
393            impl TryFrom<&[<$x Term>]> for Token {
394                type Error = anyhow::Error;
395
396                fn try_from(x: &[<$x Term>]) -> Result<Self, anyhow::Error> {
397                    Ok(Token {
398                        id: x.[<$x:snake _term>].id,
399                        text: x.[<$x:snake _term>].text,
400                        line: x.[<$x:snake _term>].line,
401                        column: x.[<$x:snake _term>].column,
402                        length: x.[<$x:snake _term>].length,
403                        pos: x.[<$x:snake _term>].pos,
404                        source: x.[<$x:snake _term>].source,
405                    })
406                }
407            }
408        }
409    };
410}
411
412macro_rules! token_without_comments {
413    ($x:ident, $y:ident) => {
414        paste! {
415            impl TryFrom<&[<$x Token>]> for VerylToken {
416                type Error = anyhow::Error;
417
418                fn try_from(x: &[<$x Token>]) -> Result<Self, anyhow::Error> {
419                    Ok(VerylToken {
420                        token: x.[<$y:snake _term>].clone(),
421                        comments: Vec::new(),
422                    })
423                }
424            }
425        }
426    };
427    ($x:ident) => {
428        paste! {
429            impl TryFrom<&[<$x Token>]> for VerylToken {
430                type Error = anyhow::Error;
431
432                fn try_from(x: &[<$x Token>]) -> Result<Self, anyhow::Error> {
433                    Ok(VerylToken {
434                        token: x.[<$x:snake _term>].clone(),
435                        comments: Vec::new(),
436                    })
437                }
438            }
439            impl TryFrom<&[<$x Term>]> for Token {
440                type Error = anyhow::Error;
441
442                fn try_from(x: &[<$x Term>]) -> Result<Self, anyhow::Error> {
443                    Ok(Token {
444                        id: x.[<$x:snake _term>].id,
445                        text: x.[<$x:snake _term>].text,
446                        line: x.[<$x:snake _term>].line,
447                        column: x.[<$x:snake _term>].column,
448                        length: x.[<$x:snake _term>].length,
449                        pos: x.[<$x:snake _term>].pos,
450                        source: x.[<$x:snake _term>].source,
451                    })
452                }
453            }
454        }
455    };
456}
457
458token_with_comments!(StringLiteral);
459
460token_with_comments!(FixedPoint);
461token_with_comments!(Exponent);
462token_with_comments!(Based);
463token_with_comments!(BaseLess);
464token_with_comments!(AllBit);
465
466token_with_comments!(Colon);
467token_with_comments!(ColonColon);
468token_with_comments!(ColonColonLAngle);
469token_with_comments!(Comma);
470token_with_comments!(DotDot);
471token_with_comments!(DotDotEqu);
472token_with_comments!(Dot);
473token_with_comments!(Equ);
474token_with_comments!(HashLBracket);
475token_with_comments!(Hash);
476token_with_comments!(Question);
477token_with_comments!(Quote);
478token_with_comments!(QuoteLBrace);
479token_with_comments!(LAngle);
480token_without_comments!(EmbedLBrace, LBrace);
481token_without_comments!(EscapedLBrace);
482token_without_comments!(TripleLBrace);
483token_with_comments!(LBrace);
484token_with_comments!(LBracket);
485token_with_comments!(LParen);
486token_with_comments!(LTMinus);
487token_with_comments!(MinusColon);
488token_with_comments!(MinusGT);
489token_with_comments!(PlusColon);
490token_with_comments!(RAngle);
491token_without_comments!(EmbedRBrace, RBrace);
492token_without_comments!(EscapedRBrace);
493token_with_comments!(TripleRBrace);
494token_with_comments!(RBrace);
495token_with_comments!(RBracket);
496token_with_comments!(RParen);
497token_with_comments!(Semicolon);
498token_with_comments!(Star);
499
500token_with_comments!(AssignmentOperator);
501token_with_comments!(DiamondOperator);
502token_with_comments!(Operator01);
503token_with_comments!(Operator02);
504token_with_comments!(Operator03);
505token_with_comments!(Operator04);
506token_with_comments!(Operator05);
507token_with_comments!(Operator06);
508token_with_comments!(Operator07);
509token_with_comments!(Operator08);
510token_with_comments!(UnaryOperator);
511
512token_with_comments!(Alias);
513token_with_comments!(AlwaysComb);
514token_with_comments!(AlwaysFf);
515token_with_comments!(As);
516token_with_comments!(Assign);
517token_with_comments!(Bind);
518token_with_comments!(Bit);
519token_with_comments!(Block);
520token_with_comments!(BBool);
521token_with_comments!(LBool);
522token_with_comments!(Break);
523token_with_comments!(Case);
524token_with_comments!(Clock);
525token_with_comments!(ClockPosedge);
526token_with_comments!(ClockNegedge);
527token_with_comments!(Connect);
528token_with_comments!(Const);
529token_with_comments!(Converse);
530token_with_comments!(Default);
531token_with_comments!(Else);
532token_with_comments!(Embed);
533token_with_comments!(Enum);
534token_with_comments!(F32);
535token_with_comments!(F64);
536token_with_comments!(False);
537token_with_comments!(Final);
538token_with_comments!(For);
539token_with_comments!(Function);
540token_with_comments!(Gen);
541token_with_comments!(I8);
542token_with_comments!(I16);
543token_with_comments!(I32);
544token_with_comments!(I64);
545token_with_comments!(If);
546token_with_comments!(IfReset);
547token_with_comments!(Import);
548token_with_comments!(Include);
549token_with_comments!(Initial);
550token_with_comments!(Inout);
551token_with_comments!(Input);
552token_with_comments!(Inside);
553token_with_comments!(Inst);
554token_with_comments!(Interface);
555token_with_comments!(In);
556token_with_comments!(Let);
557token_with_comments!(Logic);
558token_with_comments!(Lsb);
559token_with_comments!(Mixin);
560token_with_comments!(Modport);
561token_with_comments!(Module);
562token_with_comments!(Msb);
563token_with_comments!(Output);
564token_with_comments!(Outside);
565token_with_comments!(Package);
566token_with_comments!(Param);
567token_with_comments!(Proto);
568token_with_comments!(Pub);
569token_with_comments!(Repeat);
570token_with_comments!(Reset);
571token_with_comments!(ResetAsyncHigh);
572token_with_comments!(ResetAsyncLow);
573token_with_comments!(ResetSyncHigh);
574token_with_comments!(ResetSyncLow);
575token_with_comments!(Return);
576token_with_comments!(Rev);
577token_with_comments!(Same);
578token_with_comments!(Signed);
579token_with_comments!(Step);
580token_with_comments!(String);
581token_with_comments!(Struct);
582token_with_comments!(Switch);
583token_with_comments!(Tri);
584token_with_comments!(True);
585token_with_comments!(Type);
586token_with_comments!(P8);
587token_with_comments!(P16);
588token_with_comments!(P32);
589token_with_comments!(P64);
590token_with_comments!(U8);
591token_with_comments!(U16);
592token_with_comments!(U32);
593token_with_comments!(U64);
594token_with_comments!(Union);
595token_with_comments!(Unsafe);
596token_with_comments!(Var);
597
598token_with_comments!(DollarIdentifier);
599token_with_comments!(Identifier);
600
601token_without_comments!(Any);
veryl_parser/veryl_token.rs

veryl_parser/
veryl_token.rs
