miden_assembly_syntax/parser/
token.rs

1use alloc::string::String;
2use core::fmt;
3
4use miden_core::{
5    Felt,
6    field::{PrimeCharacteristicRing, PrimeField64},
7    serde::{ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable},
8};
9#[cfg(feature = "serde")]
10use serde::{Deserialize, Serialize};
11
12// DOCUMENTATION TYPE
13// ================================================================================================
14
15/// Represents the scope of a given documentation comment
16#[derive(Debug, Clone)]
17pub enum DocumentationType {
18    Module(String),
19    Form(String),
20}
21
22impl From<DocumentationType> for String {
23    fn from(doc: DocumentationType) -> Self {
24        match doc {
25            DocumentationType::Module(s) => s,
26            DocumentationType::Form(s) => s,
27        }
28    }
29}
30
31impl core::ops::Deref for DocumentationType {
32    type Target = String;
33    fn deref(&self) -> &Self::Target {
34        match self {
35            Self::Module(s) => s,
36            Self::Form(s) => s,
37        }
38    }
39}
40
41// PUSH VALUE
42// ================================================================================================
43
44#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45pub enum PushValue {
46    Int(IntValue),
47    Word(WordValue),
48}
49
50impl From<u8> for PushValue {
51    fn from(value: u8) -> Self {
52        Self::Int(value.into())
53    }
54}
55
56impl From<u16> for PushValue {
57    fn from(value: u16) -> Self {
58        Self::Int(value.into())
59    }
60}
61
62impl From<u32> for PushValue {
63    fn from(value: u32) -> Self {
64        Self::Int(value.into())
65    }
66}
67
68impl From<Felt> for PushValue {
69    fn from(value: Felt) -> Self {
70        Self::Int(value.into())
71    }
72}
73
74impl From<IntValue> for PushValue {
75    fn from(value: IntValue) -> Self {
76        Self::Int(value)
77    }
78}
79
80impl From<WordValue> for PushValue {
81    fn from(value: WordValue) -> Self {
82        Self::Word(value)
83    }
84}
85
86impl fmt::Display for PushValue {
87    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
88        match self {
89            Self::Int(value) => fmt::Display::fmt(value, f),
90            Self::Word(value) => fmt::Display::fmt(value, f),
91        }
92    }
93}
94
95impl crate::prettier::PrettyPrint for PushValue {
96    fn render(&self) -> crate::prettier::Document {
97        match self {
98            Self::Int(value) => value.render(),
99            Self::Word(value) => value.render(),
100        }
101    }
102}
103
104// WORD VALUE
105// ================================================================================================
106
107#[derive(Debug, Clone, Copy, PartialEq, Eq)]
108#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
109#[cfg_attr(feature = "serde", serde(transparent))]
110#[cfg_attr(
111    all(feature = "arbitrary", test),
112    miden_test_serde_macros::serde_test(binary_serde(true))
113)]
114pub struct WordValue(pub [Felt; 4]);
115
116impl fmt::Display for WordValue {
117    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
118        let mut builder = f.debug_list();
119        for value in self.0 {
120            builder.entry(&value.as_canonical_u64());
121        }
122        builder.finish()
123    }
124}
125
126impl crate::prettier::PrettyPrint for WordValue {
127    fn render(&self) -> crate::prettier::Document {
128        use crate::prettier::*;
129
130        const_text("[")
131            + self
132                .0
133                .iter()
134                .copied()
135                .map(display)
136                .reduce(|acc, doc| acc + const_text(",") + doc)
137                .unwrap_or_default()
138            + const_text("]")
139    }
140}
141
142impl PartialOrd for WordValue {
143    fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
144        Some(self.cmp(other))
145    }
146}
147impl Ord for WordValue {
148    fn cmp(&self, other: &Self) -> core::cmp::Ordering {
149        let (WordValue([l0, l1, l2, l3]), WordValue([r0, r1, r2, r3])) = (self, other);
150        l0.as_canonical_u64()
151            .cmp(&r0.as_canonical_u64())
152            .then_with(|| l1.as_canonical_u64().cmp(&r1.as_canonical_u64()))
153            .then_with(|| l2.as_canonical_u64().cmp(&r2.as_canonical_u64()))
154            .then_with(|| l3.as_canonical_u64().cmp(&r3.as_canonical_u64()))
155    }
156}
157
158impl core::hash::Hash for WordValue {
159    fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
160        let WordValue([a, b, c, d]) = self;
161        [
162            a.as_canonical_u64(),
163            b.as_canonical_u64(),
164            c.as_canonical_u64(),
165            d.as_canonical_u64(),
166        ]
167        .hash(state)
168    }
169}
170
171#[cfg(feature = "arbitrary")]
172impl proptest::arbitrary::Arbitrary for WordValue {
173    type Parameters = ();
174
175    fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy {
176        use proptest::{array::uniform4, strategy::Strategy};
177        uniform4((0..crate::FIELD_MODULUS).prop_map(Felt::new))
178            .prop_map(WordValue)
179            .no_shrink()  // Pure random values, no meaningful shrinking pattern
180            .boxed()
181    }
182
183    type Strategy = proptest::prelude::BoxedStrategy<Self>;
184}
185
186impl Serializable for WordValue {
187    fn write_into<W: ByteWriter>(&self, target: &mut W) {
188        self.0[0].write_into(target);
189        self.0[1].write_into(target);
190        self.0[2].write_into(target);
191        self.0[3].write_into(target);
192    }
193}
194
195impl Deserializable for WordValue {
196    fn read_from<R: ByteReader>(source: &mut R) -> Result<Self, DeserializationError> {
197        let a = Felt::read_from(source)?;
198        let b = Felt::read_from(source)?;
199        let c = Felt::read_from(source)?;
200        let d = Felt::read_from(source)?;
201        Ok(Self([a, b, c, d]))
202    }
203}
204
205// INT VALUE
206// ================================================================================================
207
208/// Represents one of the various types of values that have a hex-encoded representation in Miden
209/// Assembly source files.
210#[derive(Debug, Copy, Clone, PartialEq, Eq)]
211#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
212#[cfg_attr(feature = "serde", serde(untagged))]
213#[cfg_attr(
214    all(feature = "arbitrary", test),
215    miden_test_serde_macros::serde_test(binary_serde(true))
216)]
217pub enum IntValue {
218    /// A tiny value
219    U8(u8),
220    /// A small value
221    U16(u16),
222    /// A u32 constant, typically represents a memory address
223    U32(u32),
224    /// A single field element, 8 bytes, encoded as 16 hex digits
225    Felt(Felt),
226}
227
228impl From<u8> for IntValue {
229    fn from(value: u8) -> Self {
230        Self::U8(value)
231    }
232}
233
234impl From<u16> for IntValue {
235    fn from(value: u16) -> Self {
236        Self::U16(value)
237    }
238}
239
240impl From<u32> for IntValue {
241    fn from(value: u32) -> Self {
242        Self::U32(value)
243    }
244}
245
246impl From<Felt> for IntValue {
247    fn from(value: Felt) -> Self {
248        Self::Felt(value)
249    }
250}
251
252impl IntValue {
253    pub fn as_int(&self) -> u64 {
254        match self {
255            Self::U8(value) => *value as u64,
256            Self::U16(value) => *value as u64,
257            Self::U32(value) => *value as u64,
258            Self::Felt(value) => value.as_canonical_u64(),
259        }
260    }
261
262    /// Returns the value as a `u64`.
263    ///
264    /// This is an alias for [`as_int`](Self::as_int) that matches the `Felt` API,
265    /// allowing the generated grammar code to use a consistent method name.
266    pub fn as_canonical_u64(&self) -> u64 {
267        self.as_int()
268    }
269
270    pub fn checked_add(&self, rhs: Self) -> Option<Self> {
271        self.as_int().checked_add(rhs.as_int()).map(super::lexer::shrink_u64_hex)
272    }
273
274    pub fn checked_sub(&self, rhs: Self) -> Option<Self> {
275        self.as_int().checked_sub(rhs.as_int()).map(super::lexer::shrink_u64_hex)
276    }
277
278    pub fn checked_mul(&self, rhs: Self) -> Option<Self> {
279        self.as_int().checked_mul(rhs.as_int()).map(super::lexer::shrink_u64_hex)
280    }
281
282    pub fn checked_div(&self, rhs: Self) -> Option<Self> {
283        self.as_int().checked_div(rhs.as_int()).map(super::lexer::shrink_u64_hex)
284    }
285}
286
287impl core::ops::Add<IntValue> for IntValue {
288    type Output = IntValue;
289
290    fn add(self, rhs: IntValue) -> Self::Output {
291        super::lexer::shrink_u64_hex(self.as_int() + rhs.as_int())
292    }
293}
294
295impl core::ops::Sub<IntValue> for IntValue {
296    type Output = IntValue;
297
298    fn sub(self, rhs: IntValue) -> Self::Output {
299        super::lexer::shrink_u64_hex(self.as_int() - rhs.as_int())
300    }
301}
302
303impl core::ops::Mul<IntValue> for IntValue {
304    type Output = IntValue;
305
306    fn mul(self, rhs: IntValue) -> Self::Output {
307        super::lexer::shrink_u64_hex(self.as_int() * rhs.as_int())
308    }
309}
310
311impl core::ops::Div<IntValue> for IntValue {
312    type Output = IntValue;
313
314    fn div(self, rhs: IntValue) -> Self::Output {
315        super::lexer::shrink_u64_hex(self.as_int() / rhs.as_int())
316    }
317}
318
319impl PartialEq<Felt> for IntValue {
320    fn eq(&self, other: &Felt) -> bool {
321        match self {
322            Self::U8(lhs) => (*lhs as u64) == other.as_canonical_u64(),
323            Self::U16(lhs) => (*lhs as u64) == other.as_canonical_u64(),
324            Self::U32(lhs) => (*lhs as u64) == other.as_canonical_u64(),
325            Self::Felt(lhs) => lhs == other,
326        }
327    }
328}
329
330impl fmt::Display for IntValue {
331    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
332        match self {
333            Self::U8(value) => write!(f, "{value}"),
334            Self::U16(value) => write!(f, "{value}"),
335            Self::U32(value) => write!(f, "{value:#04x}"),
336            Self::Felt(value) => write!(f, "{:#08x}", &value.as_canonical_u64().to_be()),
337        }
338    }
339}
340
341impl crate::prettier::PrettyPrint for IntValue {
342    fn render(&self) -> crate::prettier::Document {
343        match self {
344            Self::U8(v) => v.render(),
345            Self::U16(v) => v.render(),
346            Self::U32(v) => v.render(),
347            Self::Felt(v) => v.as_canonical_u64().render(),
348        }
349    }
350}
351
352impl PartialOrd for IntValue {
353    fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
354        Some(self.cmp(other))
355    }
356}
357
358impl Ord for IntValue {
359    fn cmp(&self, other: &Self) -> core::cmp::Ordering {
360        use core::cmp::Ordering;
361        match (self, other) {
362            (Self::U8(l), Self::U8(r)) => l.cmp(r),
363            (Self::U8(_), _) => Ordering::Less,
364            (Self::U16(_), Self::U8(_)) => Ordering::Greater,
365            (Self::U16(l), Self::U16(r)) => l.cmp(r),
366            (Self::U16(_), _) => Ordering::Less,
367            (Self::U32(_), Self::U8(_) | Self::U16(_)) => Ordering::Greater,
368            (Self::U32(l), Self::U32(r)) => l.cmp(r),
369            (Self::U32(_), _) => Ordering::Less,
370            (Self::Felt(_), Self::U8(_) | Self::U16(_) | Self::U32(_)) => Ordering::Greater,
371            (Self::Felt(l), Self::Felt(r)) => l.as_canonical_u64().cmp(&r.as_canonical_u64()),
372        }
373    }
374}
375
376impl core::hash::Hash for IntValue {
377    fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
378        core::mem::discriminant(self).hash(state);
379        match self {
380            Self::U8(value) => value.hash(state),
381            Self::U16(value) => value.hash(state),
382            Self::U32(value) => value.hash(state),
383            Self::Felt(value) => value.as_canonical_u64().hash(state),
384        }
385    }
386}
387
388impl Serializable for IntValue {
389    fn write_into<W: ByteWriter>(&self, target: &mut W) {
390        self.as_int().write_into(target)
391    }
392}
393
394impl Deserializable for IntValue {
395    fn read_from<R: ByteReader>(source: &mut R) -> Result<Self, DeserializationError> {
396        let raw = source.read_u64()?;
397        if raw >= Felt::ORDER_U64 {
398            Err(DeserializationError::InvalidValue(
399                "int value is greater than field modulus".into(),
400            ))
401        } else {
402            Ok(super::lexer::shrink_u64_hex(raw))
403        }
404    }
405}
406
407#[cfg(feature = "arbitrary")]
408impl proptest::arbitrary::Arbitrary for IntValue {
409    type Parameters = ();
410
411    fn arbitrary_with(_args: Self::Parameters) -> Self::Strategy {
412        use proptest::{num, prop_oneof, strategy::Strategy};
413        prop_oneof![
414            // U8 values - full range
415            num::u8::ANY.prop_map(IntValue::U8),
416            // U16 values that don't overlap with U8 to preserve variant during serialization
417            (u8::MAX as u16 + 1..=u16::MAX).prop_map(IntValue::U16),
418            // U32 values that don't overlap with U8/U16 to preserve variant during serialization
419            (u16::MAX as u32 + 1..=u32::MAX).prop_map(IntValue::U32),
420            // Felt values - values that don't fit in u32 but are within field modulus
421            (num::u64::ANY)
422                .prop_filter_map("valid felt value", |n| {
423                    if n > u32::MAX as u64 && n < crate::FIELD_MODULUS {
424                        Some(IntValue::Felt(Felt::new(n)))
425                    } else {
426                        None
427                    }
428                }),
429        ]
430        .no_shrink()  // Pure random values, no meaningful shrinking pattern
431        .boxed()
432    }
433
434    type Strategy = proptest::prelude::BoxedStrategy<Self>;
435}
436
437// BINARY ENCODED VALUE
438// ================================================================================================
439
440/// Represents one of the various types of values that have a hex-encoded representation in Miden
441/// Assembly source files.
442#[derive(Debug, Copy, Clone, PartialEq, Eq)]
443pub enum BinEncodedValue {
444    /// A tiny value
445    U8(u8),
446    /// A small value
447    U16(u16),
448    /// A u32 constant, typically represents a memory address
449    U32(u32),
450}
451
452// TOKEN
453// ================================================================================================
454
455/// The token type produced by [crate::parser::Lexer], and consumed by the parser.
456#[derive(Debug, Clone)]
457pub enum Token<'input> {
458    Add,
459    Addrspace,
460    Adv,
461    AdvMap,
462    InsertHdword,
463    InsertHdwordWithDomain,
464    InsertHqword,
465    InsertHperm,
466    InsertMem,
467    AdvLoadw,
468    AdvPipe,
469    AdvPush,
470    AdvStack,
471    PushMapval,
472    PushMapvalCount,
473    PushMapvaln,
474    PushMtnode,
475    And,
476    Assert,
477    Assertz,
478    AssertEq,
479    AssertEqw,
480    EvalCircuit,
481    Begin,
482    Byte,
483    Caller,
484    Call,
485    Cdrop,
486    Cdropw,
487    Clk,
488    Const,
489    CryptoStream,
490    Cswap,
491    Cswapw,
492    Debug,
493    Div,
494    Drop,
495    Dropw,
496    Dup,
497    Dupw,
498    Dynexec,
499    Dyncall,
500    Else,
501    Emit,
502    End,
503    Enum,
504    Eq,
505    Eqw,
506    Ext2Add,
507    Ext2Div,
508    Ext2Inv,
509    Ext2Mul,
510    Ext2Neg,
511    Ext2Sub,
512    Err,
513    Exec,
514    Export,
515    Exp,
516    ExpU,
517    False,
518    Felt,
519    FriExt2Fold4,
520    Gt,
521    Gte,
522    Hash,
523    HasMapkey,
524    HornerBase,
525    HornerExt,
526    LogPrecompile,
527    Hperm,
528    Hmerge,
529    I1,
530    I8,
531    I16,
532    I32,
533    I64,
534    I128,
535    If,
536    ILog2,
537    Inv,
538    IsOdd,
539    Local,
540    Locaddr,
541    LocLoad,
542    LocLoadw,
543    LocLoadwBe,
544    LocLoadwLe,
545    LocStore,
546    LocStorew,
547    LocStorewBe,
548    LocStorewLe,
549    Lt,
550    Lte,
551    Mem,
552    MemLoad,
553    MemLoadw,
554    MemLoadwBe,
555    MemLoadwLe,
556    MemStore,
557    MemStorew,
558    MemStorewBe,
559    MemStorewLe,
560    MemStream,
561    Movdn,
562    Movdnw,
563    Movup,
564    Movupw,
565    MtreeGet,
566    MtreeMerge,
567    MtreeSet,
568    MtreeVerify,
569    Mul,
570    Neg,
571    Neq,
572    Not,
573    Nop,
574    Or,
575    Padw,
576    Pow2,
577    Proc,
578    Procref,
579    Ptr,
580    Pub,
581    Push,
582    Repeat,
583    Reversew,
584    Reversedw,
585    Range,
586    Sdepth,
587    Stack,
588    Struct,
589    Sub,
590    Swap,
591    Swapw,
592    Swapdw,
593    Syscall,
594    Trace,
595    True,
596    Type,
597    Use,
598    U8,
599    U16,
600    U32,
601    U32And,
602    U32Assert,
603    U32Assert2,
604    U32Assertw,
605    U32Cast,
606    U32Div,
607    U32Divmod,
608    U32Gt,
609    U32Gte,
610    U32Lt,
611    U32Lte,
612    U32Max,
613    U32Min,
614    U32Mod,
615    U32Not,
616    U32Or,
617    U32OverflowingAdd,
618    U32OverflowingAdd3,
619    U32WideningAdd,
620    U32WideningAdd3,
621    U32WideningMadd,
622    U32WideningMul,
623    U32OverflowingSub,
624    U32Popcnt,
625    U32Clz,
626    U32Ctz,
627    U32Clo,
628    U32Cto,
629    U32Rotl,
630    U32Rotr,
631    U32Shl,
632    U32Shr,
633    U32Split,
634    U32Test,
635    U32Testw,
636    U32WrappingAdd,
637    U32WrappingAdd3,
638    U32WrappingMadd,
639    U32WrappingMul,
640    U32WrappingSub,
641    U32Xor,
642    U64,
643    U128,
644    While,
645    Word,
646    Event,
647    Xor,
648    At,
649    Bang,
650    Colon,
651    ColonColon,
652    Dot,
653    Comma,
654    Equal,
655    Langle,
656    Lparen,
657    Lbrace,
658    Lbracket,
659    Minus,
660    Plus,
661    Semicolon,
662    SlashSlash,
663    Slash,
664    Star,
665    Rangle,
666    Rparen,
667    Rbrace,
668    Rbracket,
669    Rstab,
670    DocComment(DocumentationType),
671    HexValue(IntValue),
672    HexWord(WordValue),
673    BinValue(BinEncodedValue),
674    Int(u64),
675    Ident(&'input str),
676    ConstantIdent(&'input str),
677    QuotedIdent(&'input str),
678    QuotedString(&'input str),
679    Comment,
680    Eof,
681}
682
683impl fmt::Display for Token<'_> {
684    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
685        match self {
686            Token::Add => write!(f, "add"),
687            Token::Addrspace => write!(f, "addrspace"),
688            Token::Adv => write!(f, "adv"),
689            Token::AdvMap => write!(f, "adv_map"),
690            Token::AdvStack => write!(f, "adv_stack"),
691            Token::InsertHdword => write!(f, "insert_hdword"),
692            Token::InsertHdwordWithDomain => write!(f, "insert_hdword_d"),
693            Token::InsertHqword => write!(f, "insert_hqword"),
694            Token::InsertHperm => write!(f, "insert_hperm"),
695            Token::InsertMem => write!(f, "insert_mem"),
696            Token::AdvLoadw => write!(f, "adv_loadw"),
697            Token::AdvPipe => write!(f, "adv_pipe"),
698            Token::AdvPush => write!(f, "adv_push"),
699            Token::PushMapval => write!(f, "push_mapval"),
700            Token::PushMapvalCount => write!(f, "push_mapval_count"),
701            Token::PushMapvaln => write!(f, "push_mapvaln"),
702            Token::PushMtnode => write!(f, "push_mtnode"),
703            Token::And => write!(f, "and"),
704            Token::Assert => write!(f, "assert"),
705            Token::Assertz => write!(f, "assertz"),
706            Token::AssertEq => write!(f, "assert_eq"),
707            Token::AssertEqw => write!(f, "assert_eqw"),
708            Token::EvalCircuit => write!(f, "eval_circuit"),
709            Token::Begin => write!(f, "begin"),
710            Token::Byte => write!(f, "byte"),
711            Token::Caller => write!(f, "caller"),
712            Token::Call => write!(f, "call"),
713            Token::Cdrop => write!(f, "cdrop"),
714            Token::Cdropw => write!(f, "cdropw"),
715            Token::Clk => write!(f, "clk"),
716            Token::Const => write!(f, "const"),
717            Token::CryptoStream => write!(f, "crypto_stream"),
718            Token::Cswap => write!(f, "cswap"),
719            Token::Cswapw => write!(f, "cswapw"),
720            Token::Debug => write!(f, "debug"),
721            Token::Div => write!(f, "div"),
722            Token::Drop => write!(f, "drop"),
723            Token::Dropw => write!(f, "dropw"),
724            Token::Dup => write!(f, "dup"),
725            Token::Dupw => write!(f, "dupw"),
726            Token::Dynexec => write!(f, "dynexec"),
727            Token::Dyncall => write!(f, "dyncall"),
728            Token::Else => write!(f, "else"),
729            Token::Emit => write!(f, "emit"),
730            Token::End => write!(f, "end"),
731            Token::Enum => write!(f, "enum"),
732            Token::Eq => write!(f, "eq"),
733            Token::Eqw => write!(f, "eqw"),
734            Token::Ext2Add => write!(f, "ext2add"),
735            Token::Ext2Div => write!(f, "ext2div"),
736            Token::Ext2Inv => write!(f, "ext2inv"),
737            Token::Ext2Mul => write!(f, "ext2mul"),
738            Token::Ext2Neg => write!(f, "ext2neg"),
739            Token::Ext2Sub => write!(f, "ext2sub"),
740            Token::Err => write!(f, "err"),
741            Token::Exec => write!(f, "exec"),
742            Token::Exp => write!(f, "exp"),
743            Token::ExpU => write!(f, "exp.u"),
744            Token::Export => write!(f, "export"),
745            Token::False => write!(f, "false"),
746            Token::Felt => write!(f, "felt"),
747            Token::FriExt2Fold4 => write!(f, "fri_ext2fold4"),
748            Token::Gt => write!(f, "gt"),
749            Token::Gte => write!(f, "gte"),
750            Token::Hash => write!(f, "hash"),
751            Token::HasMapkey => write!(f, "has_mapkey"),
752            Token::Hperm => write!(f, "hperm"),
753            Token::Hmerge => write!(f, "hmerge"),
754            Token::I1 => write!(f, "i1"),
755            Token::I8 => write!(f, "i8"),
756            Token::I16 => write!(f, "i16"),
757            Token::I32 => write!(f, "i32"),
758            Token::I64 => write!(f, "i64"),
759            Token::I128 => write!(f, "i128"),
760            Token::If => write!(f, "if"),
761            Token::ILog2 => write!(f, "ilog2"),
762            Token::Inv => write!(f, "inv"),
763            Token::IsOdd => write!(f, "is_odd"),
764            Token::Local => write!(f, "local"),
765            Token::Locaddr => write!(f, "locaddr"),
766            Token::LocLoad => write!(f, "loc_load"),
767            Token::LocLoadw => write!(f, "loc_loadw"),
768            Token::LocLoadwBe => write!(f, "loc_loadw_be"),
769            Token::LocLoadwLe => write!(f, "loc_loadw_le"),
770            Token::LocStore => write!(f, "loc_store"),
771            Token::LocStorew => write!(f, "loc_storew"),
772            Token::LocStorewBe => write!(f, "loc_storew_be"),
773            Token::LocStorewLe => write!(f, "loc_storew_le"),
774            Token::Lt => write!(f, "lt"),
775            Token::Lte => write!(f, "lte"),
776            Token::Mem => write!(f, "mem"),
777            Token::MemLoad => write!(f, "mem_load"),
778            Token::MemLoadw => write!(f, "mem_loadw"),
779            Token::MemLoadwBe => write!(f, "mem_loadw_be"),
780            Token::MemLoadwLe => write!(f, "mem_loadw_le"),
781            Token::MemStore => write!(f, "mem_store"),
782            Token::MemStorew => write!(f, "mem_storew"),
783            Token::MemStorewBe => write!(f, "mem_storew_be"),
784            Token::MemStorewLe => write!(f, "mem_storew_le"),
785            Token::MemStream => write!(f, "mem_stream"),
786            Token::Movdn => write!(f, "movdn"),
787            Token::Movdnw => write!(f, "movdnw"),
788            Token::Movup => write!(f, "movup"),
789            Token::Movupw => write!(f, "movupw"),
790            Token::MtreeGet => write!(f, "mtree_get"),
791            Token::MtreeMerge => write!(f, "mtree_merge"),
792            Token::MtreeSet => write!(f, "mtree_set"),
793            Token::MtreeVerify => write!(f, "mtree_verify"),
794            Token::Mul => write!(f, "mul"),
795            Token::Neg => write!(f, "neg"),
796            Token::Neq => write!(f, "neq"),
797            Token::Not => write!(f, "not"),
798            Token::Nop => write!(f, "nop"),
799            Token::Or => write!(f, "or"),
800            Token::Padw => write!(f, "padw"),
801            Token::Pow2 => write!(f, "pow2"),
802            Token::Proc => write!(f, "proc"),
803            Token::Procref => write!(f, "procref"),
804            Token::Ptr => write!(f, "ptr"),
805            Token::Pub => write!(f, "pub"),
806            Token::Push => write!(f, "push"),
807            Token::HornerBase => write!(f, "horner_eval_base"),
808            Token::HornerExt => write!(f, "horner_eval_ext"),
809            Token::LogPrecompile => write!(f, "log_precompile"),
810            Token::Repeat => write!(f, "repeat"),
811            Token::Reversew => write!(f, "reversew"),
812            Token::Reversedw => write!(f, "reversedw"),
813            Token::Sdepth => write!(f, "sdepth"),
814            Token::Stack => write!(f, "stack"),
815            Token::Struct => write!(f, "struct"),
816            Token::Sub => write!(f, "sub"),
817            Token::Swap => write!(f, "swap"),
818            Token::Swapw => write!(f, "swapw"),
819            Token::Swapdw => write!(f, "swapdw"),
820            Token::Syscall => write!(f, "syscall"),
821            Token::Trace => write!(f, "trace"),
822            Token::True => write!(f, "true"),
823            Token::Type => write!(f, "type"),
824            Token::Use => write!(f, "use"),
825            Token::U8 => write!(f, "u8"),
826            Token::U16 => write!(f, "u16"),
827            Token::U32 => write!(f, "u32"),
828            Token::U32And => write!(f, "u32and"),
829            Token::U32Assert => write!(f, "u32assert"),
830            Token::U32Assert2 => write!(f, "u32assert2"),
831            Token::U32Assertw => write!(f, "u32assertw"),
832            Token::U32Cast => write!(f, "u32cast"),
833            Token::U32Div => write!(f, "u32div"),
834            Token::U32Divmod => write!(f, "u32divmod"),
835            Token::U32Gt => write!(f, "u32gt"),
836            Token::U32Gte => write!(f, "u32gte"),
837            Token::U32Lt => write!(f, "u32lt"),
838            Token::U32Lte => write!(f, "u32lte"),
839            Token::U32Max => write!(f, "u32max"),
840            Token::U32Min => write!(f, "u32min"),
841            Token::U32Mod => write!(f, "u32mod"),
842            Token::U32Not => write!(f, "u32not"),
843            Token::U32Or => write!(f, "u32or"),
844            Token::U32OverflowingAdd => write!(f, "u32overflowing_add"),
845            Token::U32OverflowingAdd3 => write!(f, "u32overflowing_add3"),
846            Token::U32WideningAdd => write!(f, "u32widening_add"),
847            Token::U32WideningAdd3 => write!(f, "u32widening_add3"),
848            Token::U32WideningMadd => write!(f, "u32widening_madd"),
849            Token::U32WideningMul => write!(f, "u32widening_mul"),
850            Token::U32OverflowingSub => write!(f, "u32overflowing_sub"),
851            Token::U32Popcnt => write!(f, "u32popcnt"),
852            Token::U32Clz => write!(f, "u32clz"),
853            Token::U32Ctz => write!(f, "u32ctz"),
854            Token::U32Clo => write!(f, "u32clo"),
855            Token::U32Cto => write!(f, "u32cto"),
856            Token::U32Rotl => write!(f, "u32rotl"),
857            Token::U32Rotr => write!(f, "u32rotr"),
858            Token::U32Shl => write!(f, "u32shl"),
859            Token::U32Shr => write!(f, "u32shr"),
860            Token::U32Split => write!(f, "u32split"),
861            Token::U32Test => write!(f, "u32test"),
862            Token::U32Testw => write!(f, "u32testw"),
863            Token::U32WrappingAdd => write!(f, "u32wrapping_add"),
864            Token::U32WrappingAdd3 => write!(f, "u32wrapping_add3"),
865            Token::U32WrappingMadd => write!(f, "u32wrapping_madd"),
866            Token::U32WrappingMul => write!(f, "u32wrapping_mul"),
867            Token::U32WrappingSub => write!(f, "u32wrapping_sub"),
868            Token::U32Xor => write!(f, "u32xor"),
869            Token::U64 => write!(f, "u64"),
870            Token::U128 => write!(f, "u128"),
871            Token::While => write!(f, "while"),
872            Token::Word => write!(f, "word"),
873            Token::Event => write!(f, "event"),
874            Token::Xor => write!(f, "xor"),
875            Token::At => write!(f, "@"),
876            Token::Bang => write!(f, "!"),
877            Token::Colon => write!(f, ":"),
878            Token::ColonColon => write!(f, "::"),
879            Token::Dot => write!(f, "."),
880            Token::Comma => write!(f, ","),
881            Token::Equal => write!(f, "="),
882            Token::Langle => write!(f, "<"),
883            Token::Lparen => write!(f, "("),
884            Token::Lbrace => write!(f, "{{"),
885            Token::Lbracket => write!(f, "["),
886            Token::Minus => write!(f, "-"),
887            Token::Plus => write!(f, "+"),
888            Token::Semicolon => write!(f, ";"),
889            Token::SlashSlash => write!(f, "//"),
890            Token::Slash => write!(f, "/"),
891            Token::Star => write!(f, "*"),
892            Token::Rangle => write!(f, ">"),
893            Token::Rparen => write!(f, ")"),
894            Token::Rbrace => write!(f, "}}"),
895            Token::Rbracket => write!(f, "]"),
896            Token::Rstab => write!(f, "->"),
897            Token::Range => write!(f, ".."),
898            Token::DocComment(DocumentationType::Module(_)) => f.write_str("module doc"),
899            Token::DocComment(DocumentationType::Form(_)) => f.write_str("doc comment"),
900            Token::HexValue(_) => f.write_str("hex-encoded value"),
901            Token::HexWord(_) => f.write_str("hex-encoded word"),
902            Token::BinValue(_) => f.write_str("bin-encoded value"),
903            Token::Int(_) => f.write_str("integer"),
904            Token::Ident(_) => f.write_str("identifier"),
905            Token::ConstantIdent(_) => f.write_str("constant identifier"),
906            Token::QuotedIdent(_) => f.write_str("quoted identifier"),
907            Token::QuotedString(_) => f.write_str("quoted string"),
908            Token::Comment => f.write_str("comment"),
909            Token::Eof => write!(f, "end of file"),
910        }
911    }
912}
913
914impl<'input> Token<'input> {
915    /// Returns true if this token represents the name of an instruction.
916    ///
917    /// This is used to simplify diagnostic output related to expected tokens so as not to
918    /// overwhelm the user with a ton of possible expected instruction variants.
919    pub fn is_instruction(&self) -> bool {
920        matches!(
921            self,
922            Token::Add
923                | Token::Adv
924                | Token::InsertHdword
925                | Token::InsertHdwordWithDomain
926                | Token::InsertHqword
927                | Token::InsertHperm
928                | Token::InsertMem
929                | Token::AdvLoadw
930                | Token::AdvPipe
931                | Token::AdvPush
932                | Token::AdvStack
933                | Token::PushMapval
934                | Token::PushMapvalCount
935                | Token::PushMapvaln
936                | Token::PushMtnode
937                | Token::And
938                | Token::Assert
939                | Token::Assertz
940                | Token::AssertEq
941                | Token::AssertEqw
942                | Token::EvalCircuit
943                | Token::Caller
944                | Token::Call
945                | Token::Cdrop
946                | Token::Cdropw
947                | Token::Clk
948                | Token::CryptoStream
949                | Token::Cswap
950                | Token::Cswapw
951                | Token::Debug
952                | Token::Div
953                | Token::Drop
954                | Token::Dropw
955                | Token::Dup
956                | Token::Dupw
957                | Token::Dynexec
958                | Token::Dyncall
959                | Token::Emit
960                | Token::Eq
961                | Token::Eqw
962                | Token::Ext2Add
963                | Token::Ext2Div
964                | Token::Ext2Inv
965                | Token::Ext2Mul
966                | Token::Ext2Neg
967                | Token::Ext2Sub
968                | Token::Exec
969                | Token::Exp
970                | Token::ExpU
971                | Token::FriExt2Fold4
972                | Token::Gt
973                | Token::Gte
974                | Token::Hash
975                | Token::Hperm
976                | Token::Hmerge
977                | Token::HornerBase
978                | Token::HornerExt
979                | Token::LogPrecompile
980                | Token::ILog2
981                | Token::Inv
982                | Token::IsOdd
983                | Token::Local
984                | Token::Locaddr
985                | Token::LocLoad
986                | Token::LocLoadw
987                | Token::LocLoadwBe
988                | Token::LocLoadwLe
989                | Token::LocStore
990                | Token::LocStorew
991                | Token::LocStorewBe
992                | Token::LocStorewLe
993                | Token::Lt
994                | Token::Lte
995                | Token::Mem
996                | Token::MemLoad
997                | Token::MemLoadw
998                | Token::MemLoadwBe
999                | Token::MemLoadwLe
1000                | Token::MemStore
1001                | Token::MemStorew
1002                | Token::MemStorewBe
1003                | Token::MemStorewLe
1004                | Token::MemStream
1005                | Token::Movdn
1006                | Token::Movdnw
1007                | Token::Movup
1008                | Token::Movupw
1009                | Token::MtreeGet
1010                | Token::MtreeMerge
1011                | Token::MtreeSet
1012                | Token::MtreeVerify
1013                | Token::Mul
1014                | Token::Neg
1015                | Token::Neq
1016                | Token::Not
1017                | Token::Nop
1018                | Token::Or
1019                | Token::Padw
1020                | Token::Pow2
1021                | Token::Procref
1022                | Token::Push
1023                | Token::Repeat
1024                | Token::Reversew
1025                | Token::Reversedw
1026                | Token::Sdepth
1027                | Token::Stack
1028                | Token::Sub
1029                | Token::Swap
1030                | Token::Swapw
1031                | Token::Swapdw
1032                | Token::Syscall
1033                | Token::Trace
1034                | Token::U32And
1035                | Token::U32Assert
1036                | Token::U32Assert2
1037                | Token::U32Assertw
1038                | Token::U32Cast
1039                | Token::U32Div
1040                | Token::U32Divmod
1041                | Token::U32Gt
1042                | Token::U32Gte
1043                | Token::U32Lt
1044                | Token::U32Lte
1045                | Token::U32Max
1046                | Token::U32Min
1047                | Token::U32Mod
1048                | Token::U32Not
1049                | Token::U32Or
1050                | Token::U32OverflowingAdd
1051                | Token::U32OverflowingAdd3
1052                | Token::U32WideningAdd
1053                | Token::U32WideningAdd3
1054                | Token::U32WideningMadd
1055                | Token::U32WideningMul
1056                | Token::U32OverflowingSub
1057                | Token::U32Popcnt
1058                | Token::U32Clz
1059                | Token::U32Ctz
1060                | Token::U32Clo
1061                | Token::U32Cto
1062                | Token::U32Rotl
1063                | Token::U32Rotr
1064                | Token::U32Shl
1065                | Token::U32Shr
1066                | Token::U32Split
1067                | Token::U32Test
1068                | Token::U32Testw
1069                | Token::U32WrappingAdd
1070                | Token::U32WrappingAdd3
1071                | Token::U32WrappingMadd
1072                | Token::U32WrappingMul
1073                | Token::U32WrappingSub
1074                | Token::U32Xor
1075                | Token::Xor
1076        )
1077    }
1078
1079    /// Returns true if this token represents the name of an type or a type-related keyword.
1080    ///
1081    /// This is used to simplify diagnostic output related to expected tokens so as not to
1082    /// overwhelm the user with a ton of possible expected tokens.
1083    pub fn is_type_keyword(&self) -> bool {
1084        matches!(
1085            self,
1086            Token::Addrspace
1087                | Token::Ptr
1088                | Token::I1
1089                | Token::I8
1090                | Token::I16
1091                | Token::I32
1092                | Token::I64
1093                | Token::I128
1094                | Token::U8
1095                | Token::U16
1096                | Token::U32
1097                | Token::U64
1098                | Token::U128
1099                | Token::Felt
1100                | Token::Word
1101                | Token::Struct
1102        )
1103    }
1104
1105    const KEYWORDS: &'static [(&'static str, Token<'static>)] = &[
1106        ("add", Token::Add),
1107        ("addrspace", Token::Addrspace),
1108        ("adv", Token::Adv),
1109        ("adv_map", Token::AdvMap),
1110        ("eval_circuit", Token::EvalCircuit),
1111        ("insert_hdword", Token::InsertHdword),
1112        ("insert_hdword_d", Token::InsertHdwordWithDomain),
1113        ("insert_hqword", Token::InsertHqword),
1114        ("insert_hperm", Token::InsertHperm),
1115        ("insert_mem", Token::InsertMem),
1116        ("adv_loadw", Token::AdvLoadw),
1117        ("adv_pipe", Token::AdvPipe),
1118        ("adv_push", Token::AdvPush),
1119        ("adv_stack", Token::AdvStack),
1120        ("push_mapval", Token::PushMapval),
1121        ("push_mapval_count", Token::PushMapvalCount),
1122        ("push_mapvaln", Token::PushMapvaln),
1123        ("push_mtnode", Token::PushMtnode),
1124        ("and", Token::And),
1125        ("assert", Token::Assert),
1126        ("assertz", Token::Assertz),
1127        ("assert_eq", Token::AssertEq),
1128        ("assert_eqw", Token::AssertEqw),
1129        ("begin", Token::Begin),
1130        ("byte", Token::Byte),
1131        ("caller", Token::Caller),
1132        ("call", Token::Call),
1133        ("cdrop", Token::Cdrop),
1134        ("cdropw", Token::Cdropw),
1135        ("clk", Token::Clk),
1136        ("const", Token::Const),
1137        ("crypto_stream", Token::CryptoStream),
1138        ("cswap", Token::Cswap),
1139        ("cswapw", Token::Cswapw),
1140        ("debug", Token::Debug),
1141        ("div", Token::Div),
1142        ("drop", Token::Drop),
1143        ("dropw", Token::Dropw),
1144        ("dup", Token::Dup),
1145        ("dupw", Token::Dupw),
1146        ("dynexec", Token::Dynexec),
1147        ("dyncall", Token::Dyncall),
1148        ("else", Token::Else),
1149        ("emit", Token::Emit),
1150        ("end", Token::End),
1151        ("enum", Token::Enum),
1152        ("eq", Token::Eq),
1153        ("eqw", Token::Eqw),
1154        ("ext2add", Token::Ext2Add),
1155        ("ext2div", Token::Ext2Div),
1156        ("ext2inv", Token::Ext2Inv),
1157        ("ext2mul", Token::Ext2Mul),
1158        ("ext2neg", Token::Ext2Neg),
1159        ("ext2sub", Token::Ext2Sub),
1160        ("err", Token::Err),
1161        ("exec", Token::Exec),
1162        ("exp", Token::Exp),
1163        ("exp.u", Token::ExpU),
1164        ("export", Token::Export),
1165        ("false", Token::False),
1166        ("felt", Token::Felt),
1167        ("fri_ext2fold4", Token::FriExt2Fold4),
1168        ("gt", Token::Gt),
1169        ("gte", Token::Gte),
1170        ("hash", Token::Hash),
1171        ("has_mapkey", Token::HasMapkey),
1172        ("hperm", Token::Hperm),
1173        ("hmerge", Token::Hmerge),
1174        ("i1", Token::I1),
1175        ("i8", Token::I8),
1176        ("i16", Token::I16),
1177        ("i32", Token::I32),
1178        ("i64", Token::I64),
1179        ("i128", Token::I128),
1180        ("if", Token::If),
1181        ("ilog2", Token::ILog2),
1182        ("inv", Token::Inv),
1183        ("is_odd", Token::IsOdd),
1184        ("local", Token::Local),
1185        ("locaddr", Token::Locaddr),
1186        ("loc_load", Token::LocLoad),
1187        ("loc_loadw", Token::LocLoadw),
1188        ("loc_loadw_be", Token::LocLoadwBe),
1189        ("loc_loadw_le", Token::LocLoadwLe),
1190        ("loc_store", Token::LocStore),
1191        ("loc_storew", Token::LocStorew),
1192        ("loc_storew_be", Token::LocStorewBe),
1193        ("loc_storew_le", Token::LocStorewLe),
1194        ("lt", Token::Lt),
1195        ("lte", Token::Lte),
1196        ("mem", Token::Mem),
1197        ("mem_load", Token::MemLoad),
1198        ("mem_loadw", Token::MemLoadw),
1199        ("mem_loadw_be", Token::MemLoadwBe),
1200        ("mem_loadw_le", Token::MemLoadwLe),
1201        ("mem_store", Token::MemStore),
1202        ("mem_storew", Token::MemStorew),
1203        ("mem_storew_be", Token::MemStorewBe),
1204        ("mem_storew_le", Token::MemStorewLe),
1205        ("mem_stream", Token::MemStream),
1206        ("movdn", Token::Movdn),
1207        ("movdnw", Token::Movdnw),
1208        ("movup", Token::Movup),
1209        ("movupw", Token::Movupw),
1210        ("mtree_get", Token::MtreeGet),
1211        ("mtree_merge", Token::MtreeMerge),
1212        ("mtree_set", Token::MtreeSet),
1213        ("mtree_verify", Token::MtreeVerify),
1214        ("mul", Token::Mul),
1215        ("neg", Token::Neg),
1216        ("neq", Token::Neq),
1217        ("not", Token::Not),
1218        ("nop", Token::Nop),
1219        ("or", Token::Or),
1220        ("padw", Token::Padw),
1221        ("pow2", Token::Pow2),
1222        ("proc", Token::Proc),
1223        ("procref", Token::Procref),
1224        ("ptr", Token::Ptr),
1225        ("push", Token::Push),
1226        ("pub", Token::Pub),
1227        ("horner_eval_base", Token::HornerBase),
1228        ("horner_eval_ext", Token::HornerExt),
1229        ("log_precompile", Token::LogPrecompile),
1230        ("repeat", Token::Repeat),
1231        ("reversew", Token::Reversew),
1232        ("reversedw", Token::Reversedw),
1233        ("sdepth", Token::Sdepth),
1234        ("stack", Token::Stack),
1235        ("struct", Token::Struct),
1236        ("sub", Token::Sub),
1237        ("swap", Token::Swap),
1238        ("swapw", Token::Swapw),
1239        ("swapdw", Token::Swapdw),
1240        ("syscall", Token::Syscall),
1241        ("trace", Token::Trace),
1242        ("true", Token::True),
1243        ("type", Token::Type),
1244        ("use", Token::Use),
1245        ("u8", Token::U8),
1246        ("u16", Token::U16),
1247        ("u32", Token::U32),
1248        ("u32and", Token::U32And),
1249        ("u32assert", Token::U32Assert),
1250        ("u32assert2", Token::U32Assert2),
1251        ("u32assertw", Token::U32Assertw),
1252        ("u32cast", Token::U32Cast),
1253        ("u32div", Token::U32Div),
1254        ("u32divmod", Token::U32Divmod),
1255        ("u32gt", Token::U32Gt),
1256        ("u32gte", Token::U32Gte),
1257        ("u32lt", Token::U32Lt),
1258        ("u32lte", Token::U32Lte),
1259        ("u32max", Token::U32Max),
1260        ("u32min", Token::U32Min),
1261        ("u32mod", Token::U32Mod),
1262        ("u32not", Token::U32Not),
1263        ("u32or", Token::U32Or),
1264        ("u32overflowing_add", Token::U32OverflowingAdd),
1265        ("u32overflowing_add3", Token::U32OverflowingAdd3),
1266        ("u32widening_add", Token::U32WideningAdd),
1267        ("u32widening_add3", Token::U32WideningAdd3),
1268        ("u32widening_madd", Token::U32WideningMadd),
1269        ("u32widening_mul", Token::U32WideningMul),
1270        ("u32overflowing_sub", Token::U32OverflowingSub),
1271        ("u32popcnt", Token::U32Popcnt),
1272        ("u32clz", Token::U32Clz),
1273        ("u32ctz", Token::U32Ctz),
1274        ("u32clo", Token::U32Clo),
1275        ("u32cto", Token::U32Cto),
1276        ("u32rotl", Token::U32Rotl),
1277        ("u32rotr", Token::U32Rotr),
1278        ("u32shl", Token::U32Shl),
1279        ("u32shr", Token::U32Shr),
1280        ("u32split", Token::U32Split),
1281        ("u32test", Token::U32Test),
1282        ("u32testw", Token::U32Testw),
1283        ("u32wrapping_add", Token::U32WrappingAdd),
1284        ("u32wrapping_add3", Token::U32WrappingAdd3),
1285        ("u32wrapping_madd", Token::U32WrappingMadd),
1286        ("u32wrapping_mul", Token::U32WrappingMul),
1287        ("u32wrapping_sub", Token::U32WrappingSub),
1288        ("u32xor", Token::U32Xor),
1289        ("u64", Token::U64),
1290        ("u128", Token::U128),
1291        ("while", Token::While),
1292        ("word", Token::Word),
1293        ("event", Token::Event),
1294        ("xor", Token::Xor),
1295    ];
1296
1297    /// Constructs a DFA capable of recognizing Miden Assembly keywords.
1298    ///
1299    /// Constructing the state machine is expensive, so it should not be done in hot code. Instead,
1300    /// prefer to construct it once and reuse it many times.
1301    ///
1302    /// Currently we construct an instance of this searcher in the lexer, which is then used to
1303    /// select a keyword token or construct an identifier token depending on whether a given string
1304    /// is a known keyword.
1305    pub fn keyword_searcher() -> aho_corasick::AhoCorasick {
1306        use aho_corasick::AhoCorasick;
1307
1308        // Execute a search for any of the keywords above, matching longest first, and requiring
1309        // the match to cover the entire input.
1310        AhoCorasick::builder()
1311            .match_kind(aho_corasick::MatchKind::LeftmostLongest)
1312            .start_kind(aho_corasick::StartKind::Anchored)
1313            .build(Self::KEYWORDS.iter().map(|(kw, _)| kw).copied())
1314            .expect("unable to build aho-corasick searcher for token")
1315    }
1316
1317    /// Returns an appropriate [Token] depending on whether the given string is a keyword or an
1318    /// identifier.
1319    ///
1320    /// NOTE: This constructs and throws away an expensive-to-construct Aho-Corasick state machine.
1321    /// You should not call this from any code on a hot path. Instead, construct the state machine
1322    /// once using [Token::keyword_searcher], and reuse it for all searches using
1323    /// [Token::from_keyword_or_ident_with_searcher].
1324    ///
1325    /// Currently, this function is only called along one code path, which is when we are
1326    /// constructing a parser error in which we wish to determine which, if any, of the expected
1327    /// tokens are instruction opcode keywords, so we can collapse them into a more user-friendly
1328    /// error message. This is not on a hot path, so we don't care if it is a bit slow.
1329    pub fn from_keyword_or_ident(s: &'input str) -> Self {
1330        let searcher = Self::keyword_searcher();
1331        Self::from_keyword_or_ident_with_searcher(s, &searcher)
1332    }
1333
1334    /// This is the primary function you should use when you wish to get an appropriate token for
1335    /// a given input string, depending on whether it is a keyword or an identifier.
1336    ///
1337    /// See [Token::keyword_searcher] for additional information on how this is meant to be used.
1338    pub fn from_keyword_or_ident_with_searcher(
1339        s: &'input str,
1340        searcher: &aho_corasick::AhoCorasick,
1341    ) -> Self {
1342        let input = aho_corasick::Input::new(s).anchored(aho_corasick::Anchored::Yes);
1343        match searcher.find(input) {
1344            // No match, it's an ident
1345            None => Token::Ident(s),
1346            // If the match is not exact, it's an ident
1347            Some(matched) if matched.len() != s.len() => Token::Ident(s),
1348            // Otherwise clone the Token corresponding to the keyword that was matched
1349            Some(matched) => Self::KEYWORDS[matched.pattern().as_usize()].1.clone(),
1350        }
1351    }
1352
1353    /// Parses a [Token] from a string corresponding to that token.
1354    ///
1355    /// This solely exists to aid in constructing more user-friendly error messages in certain
1356    /// scenarios, and is otherwise not used (nor should it be). It is quite expensive to call due
1357    /// to invoking [Token::keyword_searcher] under the covers. See the documentation for that
1358    /// function for more details.
1359    pub fn parse(s: &'input str) -> Option<Token<'input>> {
1360        match Token::from_keyword_or_ident(s) {
1361            Token::Ident(_) => {
1362                // Nope, try again
1363                match s {
1364                    "@" => Some(Token::At),
1365                    "!" => Some(Token::Bang),
1366                    ":" => Some(Token::Colon),
1367                    "::" => Some(Token::ColonColon),
1368                    "." => Some(Token::Dot),
1369                    "," => Some(Token::Comma),
1370                    "=" => Some(Token::Equal),
1371                    "<" => Some(Token::Langle),
1372                    "(" => Some(Token::Lparen),
1373                    "{" => Some(Token::Lbrace),
1374                    "[" => Some(Token::Lbracket),
1375                    "-" => Some(Token::Minus),
1376                    "+" => Some(Token::Plus),
1377                    ";" => Some(Token::Semicolon),
1378                    "//" => Some(Token::SlashSlash),
1379                    "/" => Some(Token::Slash),
1380                    "*" => Some(Token::Star),
1381                    ">" => Some(Token::Rangle),
1382                    ")" => Some(Token::Rparen),
1383                    "}" => Some(Token::Rbrace),
1384                    "]" => Some(Token::Rbracket),
1385                    "->" => Some(Token::Rstab),
1386                    ".." => Some(Token::Range),
1387                    "end of file" => Some(Token::Eof),
1388                    "module doc" => {
1389                        Some(Token::DocComment(DocumentationType::Module(String::new())))
1390                    },
1391                    "doc comment" => {
1392                        Some(Token::DocComment(DocumentationType::Form(String::new())))
1393                    },
1394                    "comment" => Some(Token::Comment),
1395                    "hex-encoded value" => Some(Token::HexValue(IntValue::U8(0))),
1396                    "hex-encoded word" => Some(Token::HexWord(WordValue([Felt::ZERO; 4]))),
1397                    "bin-encoded value" => Some(Token::BinValue(BinEncodedValue::U8(0))),
1398                    "integer" => Some(Token::Int(0)),
1399                    "identifier" => Some(Token::Ident("")),
1400                    "constant identifier" => Some(Token::ConstantIdent("")),
1401                    "quoted identifier" => Some(Token::QuotedIdent("")),
1402                    "quoted string" => Some(Token::QuotedString("")),
1403                    _ => None,
1404                }
1405            },
1406            // We matched a keyword
1407            token => Some(token),
1408        }
1409    }
1410}
miden_assembly_syntax/parser/token.rs

miden_assembly_syntax/parser/
token.rs
