Enum Token

pub enum Token<'input> {
    Add,
    Adv,
    AdvMap,
    InsertHdword,
    InsertHdwordWithDomain,
    InsertHperm,
    InsertMem,
    AdvLoadw,
    AdvPipe,
    AdvPush,
    AdvStack,
    PushMapval,
    PushMapvaln,
    PushMtnode,
    PushSmtpeek,
    PushSmtset,
    PushSmtget,
    PushU64Div,
    PushFalconDiv,
    And,
    Assert,
    Assertz,
    AssertEq,
    AssertEqw,
    ArithmeticCircuitEval,
    Begin,
    Breakpoint,
    Caller,
    Call,
    Cdrop,
    Cdropw,
    Clk,
    Const,
    Cswap,
    Cswapw,
    Debug,
    Div,
    Drop,
    Dropw,
    Dup,
    Dupw,
    Dynexec,
    Dyncall,
    Else,
    Emit,
    End,
    Eq,
    Eqw,
    Ext2Add,
    Ext2Div,
    Ext2Inv,
    Ext2Mul,
    Ext2Neg,
    Ext2Sub,
    Err,
    Exec,
    Export,
    Exp,
    ExpU,
    False,
    FriExt2Fold4,
    Gt,
    Gte,
    Hash,
    HasMapkey,
    HornerBase,
    HornerExt,
    Hperm,
    Hmerge,
    If,
    ILog2,
    Inv,
    IsOdd,
    Local,
    Locaddr,
    LocLoad,
    LocLoadw,
    LocStore,
    LocStorew,
    Lt,
    Lte,
    Mem,
    MemLoad,
    MemLoadw,
    MemStore,
    MemStorew,
    MemStream,
    Movdn,
    Movdnw,
    Movup,
    Movupw,
    MtreeGet,
    MtreeMerge,
    MtreeSet,
    MtreeVerify,
    Mul,
    Neg,
    Neq,
    Not,
    Nop,
    Or,
    Padw,
    Pow2,
    Proc,
    Procref,
    Push,
    Repeat,
    Sdepth,
    Stack,
    Sub,
    Swap,
    Swapw,
    Swapdw,
    Syscall,
    Trace,
    True,
    Use,
    U32And,
    U32Assert,
    U32Assert2,
    U32Assertw,
    U32Cast,
    U32Div,
    U32Divmod,
    U32Gt,
    U32Gte,
    U32Lt,
    U32Lte,
    U32Max,
    U32Min,
    U32Mod,
    U32Not,
    U32Or,
    U32OverflowingAdd,
    U32OverflowingAdd3,
    U32OverflowingMadd,
    U32OverflowingMul,
    U32OverflowingSub,
    U32Popcnt,
    U32Clz,
    U32Ctz,
    U32Clo,
    U32Cto,
    U32Rotl,
    U32Rotr,
    U32Shl,
    U32Shr,
    U32Split,
    U32Test,
    U32Testw,
    U32WrappingAdd,
    U32WrappingAdd3,
    U32WrappingMadd,
    U32WrappingMul,
    U32WrappingSub,
    U32Xor,
    While,
    Xor,
    At,
    Bang,
    ColonColon,
    Dot,
    Comma,
    Equal,
    Lparen,
    Lbracket,
    Minus,
    Plus,
    SlashSlash,
    Slash,
    Star,
    Rparen,
    Rbracket,
    Rstab,
    DocComment(DocumentationType),
    HexValue(IntValue),
    BinValue(BinEncodedValue),
    Int(u64),
    Ident(&'input str),
    ConstantIdent(&'input str),
    QuotedIdent(&'input str),
    QuotedString(&'input str),
    Comment,
    Eof,
}

The token type produced by crate::parser::Lexer, and consumed by the parser.

Variants

Add

Adv

AdvMap

InsertHdword

InsertHdwordWithDomain

InsertHperm

InsertMem

AdvLoadw

AdvPipe

AdvPush

AdvStack

PushMapval

PushMapvaln

PushMtnode

PushSmtpeek

PushSmtset

PushSmtget

PushU64Div

PushFalconDiv

And

Assert

Assertz

AssertEq

AssertEqw

ArithmeticCircuitEval

Begin

Breakpoint

Caller

Call

Cdrop

Cdropw

Clk

Const

Cswap

Cswapw

Debug

Div

Drop

Dropw

Dup

Dupw

Dynexec

Dyncall

Else

Emit

End

Eq

Eqw

Ext2Add

Ext2Div

Ext2Inv

Ext2Mul

Ext2Neg

Ext2Sub

Err

Exec

Export

Exp

ExpU

False

FriExt2Fold4

Gt

Gte

Hash

HasMapkey

HornerBase

HornerExt

Hperm

Hmerge

If

ILog2

Inv

IsOdd

Local

Locaddr

LocLoad

LocLoadw

LocStore

LocStorew

Lt

Lte

Mem

MemLoad

MemLoadw

MemStore

MemStorew

MemStream

Movdn

Movdnw

Movup

Movupw

MtreeGet

MtreeMerge

MtreeSet

MtreeVerify

Mul

Neg

Neq

Not

Nop

Or

Padw

Pow2

Proc

Procref

Push

Repeat

Sdepth

Stack

Sub

Swap

Swapw

Swapdw

Syscall

Trace

True

Use

U32And

U32Assert

U32Assert2

U32Assertw

U32Cast

U32Div

U32Divmod

U32Gt

U32Gte

U32Lt

U32Lte

U32Max

U32Min

U32Mod

U32Not

U32Or

U32OverflowingAdd

U32OverflowingAdd3

U32OverflowingMadd

U32OverflowingMul

U32OverflowingSub

U32Popcnt

U32Clz

U32Ctz

U32Clo

U32Cto

U32Rotl

U32Rotr

U32Shl

U32Shr

U32Split

U32Test

U32Testw

U32WrappingAdd

U32WrappingAdd3

U32WrappingMadd

U32WrappingMul

U32WrappingSub

U32Xor

While

Xor

At

Bang

ColonColon

Dot

Comma

Equal

Lparen

Lbracket

Minus

Plus

SlashSlash

Slash

Star

Rparen

Rbracket

Rstab

DocComment(DocumentationType)

HexValue(IntValue)

BinValue(BinEncodedValue)

Int(u64)

Ident(&'input str)

ConstantIdent(&'input str)

QuotedIdent(&'input str)

QuotedString(&'input str)

Comment

Eof

Implementations

impl<'input> Token<'input>

pub fn is_instruction(&self) -> bool

Returns true if this token represents the name of an instruction.

This is used to simplify diagnostic output related to expected tokens so as not to overwhelm the user with a ton of possible expected instruction variants.

pub fn keyword_searcher() -> AhoCorasick

Constructs a DFA capable of recognizing Miden Assembly keywords.

Constructing the state machine is expensive, so it should not be done in hot code. Instead, prefer to construct it once and reuse it many times.

Currently we construct an instance of this searcher in the lexer, which is then used to select a keyword token or construct an identifier token depending on whether a given string is a known keyword.

pub fn from_keyword_or_ident(s: &'input str) -> Self

Returns an appropriate Token depending on whether the given string is a keyword or an identifier.

NOTE: This constructs and throws away an expensive-to-construct Aho-Corasick state machine. You should not call this from any code on a hot path. Instead, construct the state machine once using Token::keyword_searcher, and reuse it for all searches using Token::from_keyword_or_ident_with_searcher.

Currently, this function is only called along one code path, which is when we are constructing a parser error in which we wish to determine which, if any, of the expected tokens are instruction opcode keywords, so we can collapse them into a more user-friendly error message. This is not on a hot path, so we don’t care if it is a bit slow.

pub fn from_keyword_or_ident_with_searcher( s: &'input str, searcher: &AhoCorasick, ) -> Self

This is the primary function you should use when you wish to get an appropriate token for a given input string, depending on whether it is a keyword or an identifier.

See Token::keyword_searcher for additional information on how this is meant to be used.

pub fn parse(s: &'input str) -> Option<Token<'input>>

Parses a Token from a string corresponding to that token.

This solely exists to aid in constructing more user-friendly error messages in certain scenarios, and is otherwise not used (nor should it be). It is quite expensive to call due to invoking Token::keyword_searcher under the covers. See the documentation for that function for more details.

Trait Implementations

impl<'input> Clone for Token<'input>

fn clone(&self) -> Token<'input>

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<'input> Debug for Token<'input>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Display for Token<'_>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.