llvm_ir_parser/
lexer.rs

1//! Hand-rolled LLVM IR lexer.
2
3use std::fmt;
4
5// ---------------------------------------------------------------------------
6// Keyword enum
7// ---------------------------------------------------------------------------
8
9/// Public API for `Keyword`.
10#[derive(Clone, Debug, PartialEq)]
11pub enum Keyword {
12    // Directives
13    /// `Source` variant.
14    Source,     // source_filename
15    /// `Target` variant.
16    Target,     // target
17    /// `Triple` variant.
18    Triple,     // triple
19    /// `Datalayout` variant.
20    Datalayout, // datalayout
21    /// `Define` variant.
22    Define,
23    /// `Declare` variant.
24    Declare,
25    /// `Type` variant.
26    Type, // "type" keyword after %Foo =
27
28    // Linkage
29    /// `Private` variant.
30    Private,
31    /// `Internal` variant.
32    Internal,
33    /// `External` variant.
34    External,
35    /// `Weak` variant.
36    Weak,
37    /// `WeakOdr` variant.
38    WeakOdr,
39    /// `Linkonce` variant.
40    Linkonce,
41    /// `LinkonceOdr` variant.
42    LinkonceOdr,
43    /// `Common` variant.
44    Common,
45    /// `AvailableExternally` variant.
46    AvailableExternally,
47
48    // Type keywords
49    /// `Void` variant.
50    Void,
51    /// `Half` variant.
52    Half,
53    /// `Bfloat` variant.
54    Bfloat,
55    /// `Float` variant.
56    Float,
57    /// `Double` variant.
58    Double,
59    /// `Fp128` variant.
60    Fp128,
61    /// `X86Fp80` variant.
62    X86Fp80,
63    /// `Label` variant.
64    Label,
65    /// `Metadata` variant.
66    Metadata,
67    /// `Ptr` variant.
68    Ptr,
69
70    // Storage
71    /// `Global` variant.
72    Global,
73    /// `Constant` variant.
74    Constant,
75
76    // Modifiers
77    /// `Inbounds` variant.
78    Inbounds,
79    /// `Exact` variant.
80    Exact,
81    /// `Nuw` variant.
82    Nuw,
83    /// `Nsw` variant.
84    Nsw,
85    /// `Volatile` variant.
86    Volatile,
87    /// `Tail` variant.
88    Tail,
89    /// `Musttail` variant.
90    Musttail,
91    /// `Notail` variant.
92    Notail,
93    /// `Fast` variant.
94    Fast,
95    /// `Nnan` variant.
96    Nnan,
97    /// `Ninf` variant.
98    Ninf,
99    /// `Nsz` variant.
100    Nsz,
101    /// `Arcp` variant.
102    Arcp,
103    /// `Contract` variant.
104    Contract,
105    /// `Afn` variant.
106    Afn,
107    /// `Reassoc` variant.
108    Reassoc,
109
110    // Opcodes
111    /// `Add` variant.
112    Add,
113    /// `Sub` variant.
114    Sub,
115    /// `Mul` variant.
116    Mul,
117    /// `Udiv` variant.
118    Udiv,
119    /// `Sdiv` variant.
120    Sdiv,
121    /// `Urem` variant.
122    Urem,
123    /// `Srem` variant.
124    Srem,
125    /// `And` variant.
126    And,
127    /// `Or` variant.
128    Or,
129    /// `Xor` variant.
130    Xor,
131    /// `Shl` variant.
132    Shl,
133    /// `Lshr` variant.
134    Lshr,
135    /// `Ashr` variant.
136    Ashr,
137    /// `Fadd` variant.
138    Fadd,
139    /// `Fsub` variant.
140    Fsub,
141    /// `Fmul` variant.
142    Fmul,
143    /// `Fdiv` variant.
144    Fdiv,
145    /// `Frem` variant.
146    Frem,
147    /// `Fneg` variant.
148    Fneg,
149    /// `Icmp` variant.
150    Icmp,
151    /// `Fcmp` variant.
152    Fcmp,
153    /// `Alloca` variant.
154    Alloca,
155    /// `Load` variant.
156    Load,
157    /// `Store` variant.
158    Store,
159    /// `Getelementptr` variant.
160    Getelementptr,
161    /// `Trunc` variant.
162    Trunc,
163    /// `Zext` variant.
164    Zext,
165    /// `Sext` variant.
166    Sext,
167    /// `Fptrunc` variant.
168    Fptrunc,
169    /// `Fpext` variant.
170    Fpext,
171    /// `Fptoui` variant.
172    Fptoui,
173    /// `Fptosi` variant.
174    Fptosi,
175    /// `Uitofp` variant.
176    Uitofp,
177    /// `Sitofp` variant.
178    Sitofp,
179    /// `Ptrtoint` variant.
180    Ptrtoint,
181    /// `Inttoptr` variant.
182    Inttoptr,
183    /// `Bitcast` variant.
184    Bitcast,
185    /// `Addrspacecast` variant.
186    Addrspacecast,
187    /// `Freeze` variant.
188    Freeze,
189    /// `Select` variant.
190    Select,
191    /// `Phi` variant.
192    Phi,
193    /// `Extractvalue` variant.
194    Extractvalue,
195    /// `Insertvalue` variant.
196    Insertvalue,
197    /// `Extractelement` variant.
198    Extractelement,
199    /// `Insertelement` variant.
200    Insertelement,
201    /// `Shufflevector` variant.
202    Shufflevector,
203    /// `Call` variant.
204    Call,
205    /// `Ret` variant.
206    Ret,
207    /// `Br` variant.
208    Br,
209    /// `Switch` variant.
210    Switch,
211    /// `Unreachable` variant.
212    Unreachable,
213
214    // ICmp predicates
215    /// `Eq` variant.
216    Eq,
217    /// `Ne` variant.
218    Ne,
219    /// `Ugt` variant.
220    Ugt,
221    /// `Uge` variant.
222    Uge,
223    /// `Ult` variant.
224    Ult,
225    /// `Ule` variant.
226    Ule,
227    /// `Sgt` variant.
228    Sgt,
229    /// `Sge` variant.
230    Sge,
231    /// `Slt` variant.
232    Slt,
233    /// `Sle` variant.
234    Sle,
235    // FCmp predicates
236    /// `False` variant.
237    False,
238    /// `Oeq` variant.
239    Oeq,
240    /// `Ogt` variant.
241    Ogt,
242    /// `Oge` variant.
243    Oge,
244    /// `Olt` variant.
245    Olt,
246    /// `Ole` variant.
247    Ole,
248    /// `One` variant.
249    One,
250    /// `Ord` variant.
251    Ord,
252    /// `Uno` variant.
253    Uno,
254    /// `Ueq` variant.
255    Ueq,
256    /// `Une` variant.
257    Une,
258    /// `True` variant.
259    True,
260
261    // Aggregate/misc constants
262    /// `Zeroinitializer` variant.
263    Zeroinitializer,
264    /// `Undef` variant.
265    Undef,
266    /// `Poison` variant.
267    Poison,
268    /// `Null` variant.
269    Null,
270    /// `Align` variant.
271    Align,
272    /// `To` variant.
273    To,
274    /// `X` variant.
275    X,      // "x" in vector / array size
276    /// `Vscale` variant.
277    Vscale, // "vscale" before "x" in scalable vector
278}
279
280// ---------------------------------------------------------------------------
281// Token
282// ---------------------------------------------------------------------------
283
284/// Public API for `Token`.
285#[derive(Clone, Debug, PartialEq)]
286pub enum Token {
287    /// %name  (local identifier)
288    LocalIdent(String),
289    /// @name  (global identifier)
290    GlobalIdent(String),
291    /// iNN   (integer type, dynamic width)
292    IntType(u32),
293    /// Integer literal (signed)
294    IntLit(i64),
295    /// Unsigned integer literal (when value doesn't fit i64)
296    UIntLit(u64),
297    /// Float literal
298    FloatLit(f64),
299    /// String literal (inside double quotes)
300    StringLit(String),
301    /// Keyword
302    Kw(Keyword),
303    /// `=`
304    Equal,
305    /// `,`
306    Comma,
307    /// `:`
308    Colon,
309    /// `*`
310    Star,
311    /// `(`
312    LParen,
313    /// `)`
314    RParen,
315    /// `[`
316    LBracket,
317    /// `]`
318    RBracket,
319    /// `{`
320    LBrace,
321    /// `}`
322    RBrace,
323    /// `<`
324    LAngle,
325    /// `>`
326    RAngle,
327    /// `...`
328    Ellipsis,
329    /// `!`
330    Bang,
331    /// `#`
332    Hash,
333    /// End of input
334    Eof,
335}
336
337// ---------------------------------------------------------------------------
338// Lex error
339// ---------------------------------------------------------------------------
340
341/// Public API for `LexError`.
342#[derive(Clone, Debug)]
343pub struct LexError {
344    /// Public API for `line`.
345    pub line: usize,
346    /// Public API for `col`.
347    pub col: usize,
348    /// Public API for `message`.
349    pub message: String,
350}
351
352impl fmt::Display for LexError {
353    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
354        write!(
355            f,
356            "lex error at {}:{}: {}",
357            self.line, self.col, self.message
358        )
359    }
360}
361
362// ---------------------------------------------------------------------------
363// Lexer
364// ---------------------------------------------------------------------------
365
366/// Public API for `Lexer`.
367pub struct Lexer<'src> {
368    // `src` field.
369    src: &'src [u8],
370    // `pos` field.
371    pos: usize,
372    // `line` field.
373    line: usize,
374    // `col` field.
375    col: usize,
376    /// One-token lookahead.
377    peeked: Option<Result<Token, LexError>>,
378}
379
380impl<'src> Lexer<'src> {
381    /// Public API for `new`.
382    pub fn new(src: &'src str) -> Self {
383        Lexer {
384            src: src.as_bytes(),
385            pos: 0,
386            line: 1,
387            col: 1,
388            peeked: None,
389        }
390    }
391
392    // -----------------------------------------------------------------------
393    // Character-level helpers
394    // -----------------------------------------------------------------------
395
396    fn peek_ch(&self) -> Option<u8> {
397        self.src.get(self.pos).copied()
398    }
399
400    fn peek_ch2(&self) -> Option<u8> {
401        self.src.get(self.pos + 1).copied()
402    }
403
404    fn advance(&mut self) -> Option<u8> {
405        let ch = self.src.get(self.pos).copied()?;
406        self.pos += 1;
407        if ch == b'\n' {
408            self.line += 1;
409            self.col = 1;
410        } else {
411            self.col += 1;
412        }
413        Some(ch)
414    }
415
416    fn skip_whitespace_and_comments(&mut self) {
417        loop {
418            while matches!(self.peek_ch(), Some(b' ' | b'\t' | b'\r' | b'\n')) {
419                self.advance();
420            }
421            if self.peek_ch() == Some(b';') {
422                while !matches!(self.peek_ch(), None | Some(b'\n')) {
423                    self.advance();
424                }
425            } else {
426                break;
427            }
428        }
429    }
430
431    fn make_err(&self, msg: impl Into<String>) -> LexError {
432        LexError {
433            line: self.line,
434            col: self.col,
435            message: msg.into(),
436        }
437    }
438
439    // -----------------------------------------------------------------------
440    // Public interface
441    // -----------------------------------------------------------------------
442
443    /// Peek at the next token without consuming it.
444    pub fn peek(&mut self) -> Result<&Token, &LexError> {
445        if self.peeked.is_none() {
446            self.peeked = Some(self.next_token());
447        }
448        match self.peeked.as_ref().unwrap() {
449            Ok(t) => Ok(t),
450            Err(e) => Err(e),
451        }
452    }
453
454    /// Consume and return the next token.
455    #[allow(clippy::should_implement_trait)]
456    pub fn next(&mut self) -> Result<Token, LexError> {
457        if let Some(t) = self.peeked.take() {
458            return t;
459        }
460        self.next_token()
461    }
462
463    /// Consume if next token matches; otherwise leave it in the peek buffer.
464    pub fn eat(&mut self, expected: &Token) -> bool {
465        match self.peek() {
466            Ok(t) if t == expected => {
467                let _ = self.next();
468                true
469            }
470            _ => false,
471        }
472    }
473
474    /// Public API for `eat_kw`.
475    pub fn eat_kw(&mut self, kw: Keyword) -> bool {
476        self.eat(&Token::Kw(kw))
477    }
478
479    /// Consume a token and verify it matches expected; return an error otherwise.
480    pub fn expect(&mut self, expected: &Token) -> Result<(), LexError> {
481        let tok = self.next()?;
482        if tok == *expected {
483            Ok(())
484        } else {
485            Err(self.make_err(format!("expected {:?}, got {:?}", expected, tok)))
486        }
487    }
488
489    /// Public API for `expect_kw`.
490    pub fn expect_kw(&mut self, kw: &Keyword) -> Result<(), LexError> {
491        let tok = self.next()?;
492        if tok == Token::Kw(kw.clone()) {
493            Ok(())
494        } else {
495            Err(self.make_err(format!("expected keyword {:?}, got {:?}", kw, tok)))
496        }
497    }
498
499    /// Public API for `expect_local_ident`.
500    pub fn expect_local_ident(&mut self) -> Result<String, LexError> {
501        match self.next()? {
502            Token::LocalIdent(s) => Ok(s),
503            t => Err(self.make_err(format!("expected local ident, got {:?}", t))),
504        }
505    }
506
507    /// Public API for `expect_global_ident`.
508    pub fn expect_global_ident(&mut self) -> Result<String, LexError> {
509        match self.next()? {
510            Token::GlobalIdent(s) => Ok(s),
511            t => Err(self.make_err(format!("expected global ident, got {:?}", t))),
512        }
513    }
514
515    /// Public API for `expect_int_lit`.
516    pub fn expect_int_lit(&mut self) -> Result<i64, LexError> {
517        match self.next()? {
518            Token::IntLit(n) => Ok(n),
519            Token::UIntLit(n) => Ok(n as i64),
520            t => Err(self.make_err(format!("expected integer literal, got {:?}", t))),
521        }
522    }
523
524    /// Public API for `expect_uint_lit`.
525    pub fn expect_uint_lit(&mut self) -> Result<u64, LexError> {
526        match self.next()? {
527            Token::IntLit(n) => Ok(n as u64),
528            Token::UIntLit(n) => Ok(n),
529            t => Err(self.make_err(format!("expected integer literal, got {:?}", t))),
530        }
531    }
532
533    /// Public API for `expect_string_lit`.
534    pub fn expect_string_lit(&mut self) -> Result<String, LexError> {
535        match self.next()? {
536            Token::StringLit(s) => Ok(s),
537            t => Err(self.make_err(format!("expected string literal, got {:?}", t))),
538        }
539    }
540
541    /// Public API for `current_line`.
542    pub fn current_line(&self) -> usize {
543        self.line
544    }
545    /// Public API for `current_col`.
546    pub fn current_col(&self) -> usize {
547        self.col
548    }
549
550    // -----------------------------------------------------------------------
551    // Core tokenizer
552    // -----------------------------------------------------------------------
553
554    fn next_token(&mut self) -> Result<Token, LexError> {
555        self.skip_whitespace_and_comments();
556
557        let start_line = self.line;
558        let start_col = self.col;
559
560        let ch = match self.peek_ch() {
561            None => return Ok(Token::Eof),
562            Some(c) => c,
563        };
564
565        match ch {
566            b'%' => {
567                self.advance();
568                let name = self.read_ident_or_int()?;
569                Ok(Token::LocalIdent(name))
570            }
571            b'@' => {
572                self.advance();
573                let name = self.read_ident_or_int()?;
574                Ok(Token::GlobalIdent(name))
575            }
576            b'"' => {
577                self.advance();
578                let s = self.read_string_literal()?;
579                Ok(Token::StringLit(s))
580            }
581            b'-' | b'0'..=b'9' => self.lex_number(),
582            b'=' => {
583                self.advance();
584                Ok(Token::Equal)
585            }
586            b',' => {
587                self.advance();
588                Ok(Token::Comma)
589            }
590            b':' => {
591                self.advance();
592                Ok(Token::Colon)
593            }
594            b'*' => {
595                self.advance();
596                Ok(Token::Star)
597            }
598            b'(' => {
599                self.advance();
600                Ok(Token::LParen)
601            }
602            b')' => {
603                self.advance();
604                Ok(Token::RParen)
605            }
606            b'[' => {
607                self.advance();
608                Ok(Token::LBracket)
609            }
610            b']' => {
611                self.advance();
612                Ok(Token::RBracket)
613            }
614            b'{' => {
615                self.advance();
616                Ok(Token::LBrace)
617            }
618            b'}' => {
619                self.advance();
620                Ok(Token::RBrace)
621            }
622            b'<' => {
623                self.advance();
624                Ok(Token::LAngle)
625            }
626            b'>' => {
627                self.advance();
628                Ok(Token::RAngle)
629            }
630            b'!' => {
631                self.advance();
632                Ok(Token::Bang)
633            }
634            b'#' => {
635                self.advance();
636                Ok(Token::Hash)
637            }
638            b'.' => {
639                if self.src.get(self.pos + 1) == Some(&b'.')
640                    && self.src.get(self.pos + 2) == Some(&b'.')
641                {
642                    self.advance();
643                    self.advance();
644                    self.advance();
645                    Ok(Token::Ellipsis)
646                } else {
647                    Err(LexError {
648                        line: start_line,
649                        col: start_col,
650                        message: "unexpected '.'".into(),
651                    })
652                }
653            }
654            _ if ch.is_ascii_alphabetic() || ch == b'_' || ch == b'$' => {
655                let word = self.read_word();
656                Ok(self.classify_word(&word))
657            }
658            _ => {
659                self.advance();
660                Err(LexError {
661                    line: start_line,
662                    col: start_col,
663                    message: format!("unexpected character {:?}", ch as char),
664                })
665            }
666        }
667    }
668
669    // -----------------------------------------------------------------------
670    // Identifier / keyword scanning
671    // -----------------------------------------------------------------------
672
673    fn read_word(&mut self) -> String {
674        let mut s = String::new();
675        while let Some(c) = self.peek_ch() {
676            if c.is_ascii_alphanumeric() || c == b'_' || c == b'.' || c == b'$' {
677                self.advance();
678                s.push(c as char);
679            } else {
680                break;
681            }
682        }
683        s
684    }
685
686    /// After `%` or `@`, read an identifier (possibly quoted or numeric).
687    fn read_ident_or_int(&mut self) -> Result<String, LexError> {
688        if self.peek_ch() == Some(b'"') {
689            self.advance();
690            self.read_string_literal()
691        } else if self.peek_ch().map_or(false, |c| c.is_ascii_digit()) {
692            let mut s = String::new();
693            while let Some(c) = self.peek_ch() {
694                if c.is_ascii_digit() {
695                    self.advance();
696                    s.push(c as char);
697                } else {
698                    break;
699                }
700            }
701            Ok(s)
702        } else {
703            let mut s = String::new();
704            while let Some(c) = self.peek_ch() {
705                if c.is_ascii_alphanumeric() || c == b'_' || c == b'.' || c == b'$' || c == b'-' {
706                    self.advance();
707                    s.push(c as char);
708                } else {
709                    break;
710                }
711            }
712            Ok(s)
713        }
714    }
715
716    fn read_string_literal(&mut self) -> Result<String, LexError> {
717        let mut s = String::new();
718        loop {
719            match self.peek_ch() {
720                None => return Err(self.make_err("unterminated string")),
721                Some(b'"') => {
722                    self.advance();
723                    break;
724                }
725                Some(b'\\') => {
726                    self.advance();
727                    let h1 = self.advance().ok_or_else(|| self.make_err("bad escape"))?;
728                    let h2 = self.advance().ok_or_else(|| self.make_err("bad escape"))?;
729                    let hex_str = format!("{}{}", h1 as char, h2 as char);
730                    let byte = u8::from_str_radix(&hex_str, 16)
731                        .map_err(|_| self.make_err(format!("invalid hex escape \\{}", hex_str)))?;
732                    s.push(byte as char);
733                }
734                Some(c) => {
735                    self.advance();
736                    s.push(c as char);
737                }
738            }
739        }
740        Ok(s)
741    }
742
743    // -----------------------------------------------------------------------
744    // Number lexing
745    // -----------------------------------------------------------------------
746
747    fn lex_number(&mut self) -> Result<Token, LexError> {
748        let negative = self.peek_ch() == Some(b'-');
749        if negative {
750            self.advance();
751        }
752
753        // Hex float "0x..."
754        if self.peek_ch() == Some(b'0') && matches!(self.peek_ch2(), Some(b'x') | Some(b'X')) {
755            self.advance(); // '0'
756            self.advance(); // 'x'
757            let mut hex = String::new();
758            while let Some(c) = self.peek_ch() {
759                if c.is_ascii_hexdigit() {
760                    self.advance();
761                    hex.push(c as char);
762                } else {
763                    break;
764                }
765            }
766            let bits = u64::from_str_radix(&hex, 16).map_err(|_| self.make_err("bad hex float"))?;
767            let f = f64::from_bits(bits);
768            return Ok(Token::FloatLit(if negative { -f } else { f }));
769        }
770
771        // Read digits.
772        let mut digits = String::new();
773        while let Some(c) = self.peek_ch() {
774            if c.is_ascii_digit() {
775                self.advance();
776                digits.push(c as char);
777            } else {
778                break;
779            }
780        }
781
782        // Float if decimal point or exponent follows.
783        let is_float = matches!(self.peek_ch(), Some(b'.') | Some(b'e') | Some(b'E'));
784        if is_float {
785            let mut s = if negative {
786                format!("-{}", digits)
787            } else {
788                digits
789            };
790            if self.peek_ch() == Some(b'.') {
791                self.advance();
792                s.push('.');
793                while let Some(c) = self.peek_ch() {
794                    if c.is_ascii_digit() {
795                        self.advance();
796                        s.push(c as char);
797                    } else {
798                        break;
799                    }
800                }
801            }
802            if matches!(self.peek_ch(), Some(b'e') | Some(b'E')) {
803                self.advance();
804                s.push('e');
805                if matches!(self.peek_ch(), Some(b'+') | Some(b'-')) {
806                    let sign = self.advance().unwrap();
807                    s.push(sign as char);
808                }
809                while let Some(c) = self.peek_ch() {
810                    if c.is_ascii_digit() {
811                        self.advance();
812                        s.push(c as char);
813                    } else {
814                        break;
815                    }
816                }
817            }
818            let f: f64 = s
819                .parse()
820                .map_err(|_| self.make_err(format!("bad float: {}", s)))?;
821            Ok(Token::FloatLit(f))
822        } else {
823            // Integer.
824            let n: u64 = digits
825                .parse()
826                .map_err(|_| self.make_err(format!("bad int: {}", digits)))?;
827            if negative {
828                // i64::MIN is -(2^63) = 9223372036854775808; values beyond that don't fit.
829                if n > (i64::MAX as u64) + 1 {
830                    return Err(self.make_err(format!("integer -{} out of i64 range", n)));
831                }
832                Ok(Token::IntLit((n as i64).wrapping_neg()))
833            } else if n <= i64::MAX as u64 {
834                Ok(Token::IntLit(n as i64))
835            } else {
836                Ok(Token::UIntLit(n))
837            }
838        }
839    }
840
841    // -----------------------------------------------------------------------
842    // Keyword classification
843    // -----------------------------------------------------------------------
844
845    fn classify_word(&self, word: &str) -> Token {
846        // iNN integer type?
847        if let Some(bits_str) = word.strip_prefix('i') {
848            if !bits_str.is_empty() && bits_str.chars().all(|c| c.is_ascii_digit()) {
849                if let Ok(bits) = bits_str.parse::<u32>() {
850                    return Token::IntType(bits);
851                }
852            }
853        }
854
855        let kw = match word {
856            "source_filename" => Keyword::Source,
857            "target" => Keyword::Target,
858            "triple" => Keyword::Triple,
859            "datalayout" => Keyword::Datalayout,
860            "define" => Keyword::Define,
861            "declare" => Keyword::Declare,
862            "type" => Keyword::Type,
863            "private" => Keyword::Private,
864            "internal" => Keyword::Internal,
865            "external" => Keyword::External,
866            "weak" => Keyword::Weak,
867            "weak_odr" => Keyword::WeakOdr,
868            "linkonce" => Keyword::Linkonce,
869            "linkonce_odr" => Keyword::LinkonceOdr,
870            "common" => Keyword::Common,
871            "available_externally" => Keyword::AvailableExternally,
872            "void" => Keyword::Void,
873            "half" => Keyword::Half,
874            "bfloat" => Keyword::Bfloat,
875            "float" => Keyword::Float,
876            "double" => Keyword::Double,
877            "fp128" => Keyword::Fp128,
878            "x86_fp80" => Keyword::X86Fp80,
879            "label" => Keyword::Label,
880            "metadata" => Keyword::Metadata,
881            "ptr" => Keyword::Ptr,
882            "global" => Keyword::Global,
883            "constant" => Keyword::Constant,
884            "inbounds" => Keyword::Inbounds,
885            "exact" => Keyword::Exact,
886            "nuw" => Keyword::Nuw,
887            "nsw" => Keyword::Nsw,
888            "volatile" => Keyword::Volatile,
889            "tail" => Keyword::Tail,
890            "musttail" => Keyword::Musttail,
891            "notail" => Keyword::Notail,
892            "fast" => Keyword::Fast,
893            "nnan" => Keyword::Nnan,
894            "ninf" => Keyword::Ninf,
895            "nsz" => Keyword::Nsz,
896            "arcp" => Keyword::Arcp,
897            "contract" => Keyword::Contract,
898            "afn" => Keyword::Afn,
899            "reassoc" => Keyword::Reassoc,
900            "add" => Keyword::Add,
901            "sub" => Keyword::Sub,
902            "mul" => Keyword::Mul,
903            "udiv" => Keyword::Udiv,
904            "sdiv" => Keyword::Sdiv,
905            "urem" => Keyword::Urem,
906            "srem" => Keyword::Srem,
907            "and" => Keyword::And,
908            "or" => Keyword::Or,
909            "xor" => Keyword::Xor,
910            "shl" => Keyword::Shl,
911            "lshr" => Keyword::Lshr,
912            "ashr" => Keyword::Ashr,
913            "fadd" => Keyword::Fadd,
914            "fsub" => Keyword::Fsub,
915            "fmul" => Keyword::Fmul,
916            "fdiv" => Keyword::Fdiv,
917            "frem" => Keyword::Frem,
918            "fneg" => Keyword::Fneg,
919            "icmp" => Keyword::Icmp,
920            "fcmp" => Keyword::Fcmp,
921            "alloca" => Keyword::Alloca,
922            "load" => Keyword::Load,
923            "store" => Keyword::Store,
924            "getelementptr" => Keyword::Getelementptr,
925            "trunc" => Keyword::Trunc,
926            "zext" => Keyword::Zext,
927            "sext" => Keyword::Sext,
928            "fptrunc" => Keyword::Fptrunc,
929            "fpext" => Keyword::Fpext,
930            "fptoui" => Keyword::Fptoui,
931            "fptosi" => Keyword::Fptosi,
932            "uitofp" => Keyword::Uitofp,
933            "sitofp" => Keyword::Sitofp,
934            "ptrtoint" => Keyword::Ptrtoint,
935            "inttoptr" => Keyword::Inttoptr,
936            "bitcast" => Keyword::Bitcast,
937            "addrspacecast" => Keyword::Addrspacecast,
938            "freeze" => Keyword::Freeze,
939            "select" => Keyword::Select,
940            "phi" => Keyword::Phi,
941            "extractvalue" => Keyword::Extractvalue,
942            "insertvalue" => Keyword::Insertvalue,
943            "extractelement" => Keyword::Extractelement,
944            "insertelement" => Keyword::Insertelement,
945            "shufflevector" => Keyword::Shufflevector,
946            "call" => Keyword::Call,
947            "ret" => Keyword::Ret,
948            "br" => Keyword::Br,
949            "switch" => Keyword::Switch,
950            "unreachable" => Keyword::Unreachable,
951            "eq" => Keyword::Eq,
952            "ne" => Keyword::Ne,
953            "ugt" => Keyword::Ugt,
954            "uge" => Keyword::Uge,
955            "ult" => Keyword::Ult,
956            "ule" => Keyword::Ule,
957            "sgt" => Keyword::Sgt,
958            "sge" => Keyword::Sge,
959            "slt" => Keyword::Slt,
960            "sle" => Keyword::Sle,
961            "false" => Keyword::False,
962            "oeq" => Keyword::Oeq,
963            "ogt" => Keyword::Ogt,
964            "oge" => Keyword::Oge,
965            "olt" => Keyword::Olt,
966            "ole" => Keyword::Ole,
967            "one" => Keyword::One,
968            "ord" => Keyword::Ord,
969            "uno" => Keyword::Uno,
970            "ueq" => Keyword::Ueq,
971            "une" => Keyword::Une,
972            "true" => Keyword::True,
973            "zeroinitializer" => Keyword::Zeroinitializer,
974            "undef" => Keyword::Undef,
975            "poison" => Keyword::Poison,
976            "null" => Keyword::Null,
977            "align" => Keyword::Align,
978            "to" => Keyword::To,
979            "x" => Keyword::X,
980            "vscale" => Keyword::Vscale,
981            // Unknown words become bare local identifiers (shouldn't normally happen at module level).
982            other => return Token::LocalIdent(other.to_string()),
983        };
984        Token::Kw(kw)
985    }
986}
987
988#[cfg(test)]
989mod tests {
990    use super::*;
991
992    fn lex_all(src: &str) -> Vec<Token> {
993        let mut lex = Lexer::new(src);
994        let mut toks = Vec::new();
995        loop {
996            let t = lex.next().unwrap();
997            if t == Token::Eof {
998                break;
999            }
1000            toks.push(t);
1001        }
1002        toks
1003    }
1004
1005    #[test]
1006    fn lex_basic_tokens() {
1007        let toks = lex_all("define i32 @foo(i32 %x) {");
1008        assert_eq!(toks[0], Token::Kw(Keyword::Define));
1009        assert_eq!(toks[1], Token::IntType(32));
1010        assert_eq!(toks[2], Token::GlobalIdent("foo".to_string()));
1011        assert_eq!(toks[3], Token::LParen);
1012        assert_eq!(toks[4], Token::IntType(32));
1013        assert_eq!(toks[5], Token::LocalIdent("x".to_string()));
1014        assert_eq!(toks[6], Token::RParen);
1015        assert_eq!(toks[7], Token::LBrace);
1016    }
1017
1018    #[test]
1019    fn lex_integer_literals() {
1020        let toks = lex_all("42 -7 0");
1021        assert_eq!(toks[0], Token::IntLit(42));
1022        assert_eq!(toks[1], Token::IntLit(-7));
1023        assert_eq!(toks[2], Token::IntLit(0));
1024    }
1025
1026    #[test]
1027    fn lex_hex_float() {
1028        let toks = lex_all("0x4000000000000000");
1029        assert_eq!(toks[0], Token::FloatLit(2.0));
1030    }
1031
1032    #[test]
1033    fn lex_comment_skipped() {
1034        let toks = lex_all("; this is a comment\ni32");
1035        assert_eq!(toks[0], Token::IntType(32));
1036    }
1037
1038    #[test]
1039    fn lex_string_literal() {
1040        let toks = lex_all("\"hello\"");
1041        assert_eq!(toks[0], Token::StringLit("hello".to_string()));
1042    }
1043
1044    #[test]
1045    fn lex_ellipsis() {
1046        let toks = lex_all("...");
1047        assert_eq!(toks[0], Token::Ellipsis);
1048    }
1049
1050    #[test]
1051    fn lex_keywords() {
1052        let toks = lex_all("ret void add nsw");
1053        assert_eq!(toks[0], Token::Kw(Keyword::Ret));
1054        assert_eq!(toks[1], Token::Kw(Keyword::Void));
1055        assert_eq!(toks[2], Token::Kw(Keyword::Add));
1056        assert_eq!(toks[3], Token::Kw(Keyword::Nsw));
1057    }
1058}
llvm_ir_parser/lexer.rs

llvm_ir_parser/
lexer.rs
