Struct lib_ruby_parser::Lexer[−][src]

pub struct Lexer {
    pub lex_state: LexState,
    pub cond: StackState,
    pub cmdarg: StackState,
    pub static_env: StaticEnvironment,
    // some fields omitted
}

A struct responsible for converting a given input into a sequence of tokens

Fields

lex_state: LexState

Current state of the lexer, used internally for testing

cond: StackState

Internal field, used to differentiate kDO_COND vs kDO, exposed for internal testing

cmdarg: StackState

Internal field, used to differentiate kDO_BLOCK vs kDO, exposed for internal testing

static_env: StaticEnvironment

Stack of sets of variables in current scopes. Each stack item represents locals in the scope.

You can use it to pre-define some locals and parse your input as if these locals exist.

For example, you can parse the following code

a = b + c

as

Send(LocalVar(a), "+", LocalVar(b))

by declaring a and b as locals using

parser.lexer.static_env.declare("a")
parser.lexer.static_env.declare("b")
parser.parse()

Implementations

`impl Lexer`[src]

`pub fn new<Bytes, Name, Decoder>( bytes: Bytes, name: Name, decoder: Decoder ) -> Self where Bytes: Into<List<u8>>, Name: Into<StringPtr>, Decoder: Into<MaybePtr<CustomDecoder>>,` [src]

Constructs an instance of Lexer

`pub fn set_debug(&mut self, debug: Type)`[src]

Enables printing additional debugging during lexing

`pub fn tokenize_until_eof(&mut self) -> Vec<Token>`[src]

Tokenizes given input until EOF

Keep in mind that Lexer in Ruby is driven by Parser, and so this method on its own can return a wrong sequence of tokens. It’s used internally to test simple inputs.

If you need to get tokens better use ParserResult::tokens field

`impl Lexer`[src]

`pub const END_OF_INPUT: i32`[src]

Token "end-of-input", to be returned by the scanner.

`pub const YYerror: i32`[src]

Token error, to be returned by the scanner.

`pub const YYUNDEF: i32`[src]

Token "invalid token", to be returned by the scanner.

`pub const kCLASS: i32`[src]

Token "`class'", to be returned by the scanner.

`pub const kMODULE: i32`[src]

Token "`module'", to be returned by the scanner.

`pub const kDEF: i32`[src]

Token "`def'", to be returned by the scanner.

`pub const kUNDEF: i32`[src]

Token "`undef'", to be returned by the scanner.

`pub const kBEGIN: i32`[src]

Token "`begin'", to be returned by the scanner.

`pub const kRESCUE: i32`[src]

Token "`rescue'", to be returned by the scanner.

`pub const kENSURE: i32`[src]

Token "`ensure'", to be returned by the scanner.

`pub const kEND: i32`[src]

Token "`end'", to be returned by the scanner.

`pub const kIF: i32`[src]

Token "`if'", to be returned by the scanner.

`pub const kUNLESS: i32`[src]

Token "`unless'", to be returned by the scanner.

`pub const kTHEN: i32`[src]

Token "`then'", to be returned by the scanner.

`pub const kELSIF: i32`[src]

Token "`elsif'", to be returned by the scanner.

`pub const kELSE: i32`[src]

Token "`else'", to be returned by the scanner.

`pub const kCASE: i32`[src]

Token "`case'", to be returned by the scanner.

`pub const kWHEN: i32`[src]

Token "`when'", to be returned by the scanner.

`pub const kWHILE: i32`[src]

Token "`while'", to be returned by the scanner.

`pub const kUNTIL: i32`[src]

Token "`until'", to be returned by the scanner.

`pub const kFOR: i32`[src]

Token "`for'", to be returned by the scanner.

`pub const kBREAK: i32`[src]

Token "`break'", to be returned by the scanner.

`pub const kNEXT: i32`[src]

Token "`next'", to be returned by the scanner.

`pub const kREDO: i32`[src]

Token "`redo'", to be returned by the scanner.

`pub const kRETRY: i32`[src]

Token "`retry'", to be returned by the scanner.

`pub const kIN: i32`[src]

Token "`in'", to be returned by the scanner.

`pub const kDO: i32`[src]

Token "`do'", to be returned by the scanner.

`pub const kDO_COND: i32`[src]

Token "`do' for condition", to be returned by the scanner.

`pub const kDO_BLOCK: i32`[src]

Token "`do' for block", to be returned by the scanner.

`pub const kDO_LAMBDA: i32`[src]

Token "`do' for lambda", to be returned by the scanner.

`pub const kRETURN: i32`[src]

Token "`return'", to be returned by the scanner.

`pub const kYIELD: i32`[src]

Token "`yield'", to be returned by the scanner.

`pub const kSUPER: i32`[src]

Token "`super'", to be returned by the scanner.

`pub const kSELF: i32`[src]

Token "`self'", to be returned by the scanner.

`pub const kNIL: i32`[src]

Token "`nil'", to be returned by the scanner.

`pub const kTRUE: i32`[src]

Token "`true'", to be returned by the scanner.

`pub const kFALSE: i32`[src]

Token "`false'", to be returned by the scanner.

`pub const kAND: i32`[src]

Token "`and'", to be returned by the scanner.

`pub const kOR: i32`[src]

Token "`or'", to be returned by the scanner.

`pub const kNOT: i32`[src]

Token "`not'", to be returned by the scanner.

`pub const kIF_MOD: i32`[src]

Token "`if' modifier", to be returned by the scanner.

`pub const kUNLESS_MOD: i32`[src]

Token "`unless' modifier", to be returned by the scanner.

`pub const kWHILE_MOD: i32`[src]

Token "`while' modifier", to be returned by the scanner.

`pub const kUNTIL_MOD: i32`[src]

Token "`until' modifier", to be returned by the scanner.

`pub const kRESCUE_MOD: i32`[src]

Token "`rescue' modifier", to be returned by the scanner.

`pub const kALIAS: i32`[src]

Token "`alias'", to be returned by the scanner.

`pub const kDEFINED: i32`[src]

Token "`defined?'", to be returned by the scanner.

`pub const klBEGIN: i32`[src]

Token "`BEGIN'", to be returned by the scanner.

`pub const klEND: i32`[src]

Token "`END'", to be returned by the scanner.

`pub const kLINE: i32`[src]

Token "`__LINE__'", to be returned by the scanner.

`pub const kFILE: i32`[src]

Token "`__FILE__'", to be returned by the scanner.

`pub const kENCODING: i32`[src]

Token "`__ENCODING__'", to be returned by the scanner.

`pub const tIDENTIFIER: i32`[src]

Token "local variable or method", to be returned by the scanner.

`pub const tFID: i32`[src]

Token "method", to be returned by the scanner.

`pub const tGVAR: i32`[src]

Token "global variable", to be returned by the scanner.

`pub const tIVAR: i32`[src]

Token "instance variable", to be returned by the scanner.

`pub const tCONSTANT: i32`[src]

Token "constant", to be returned by the scanner.

`pub const tCVAR: i32`[src]

Token "class variable", to be returned by the scanner.

`pub const tLABEL: i32`[src]

Token "label", to be returned by the scanner.

`pub const tINTEGER: i32`[src]

Token "integer literal", to be returned by the scanner.

`pub const tFLOAT: i32`[src]

Token "float literal", to be returned by the scanner.

`pub const tRATIONAL: i32`[src]

Token "rational literal", to be returned by the scanner.

`pub const tIMAGINARY: i32`[src]

Token "imaginary literal", to be returned by the scanner.

`pub const tCHAR: i32`[src]

Token "char literal", to be returned by the scanner.

`pub const tNTH_REF: i32`[src]

Token "numbered reference", to be returned by the scanner.

`pub const tBACK_REF: i32`[src]

Token "back reference", to be returned by the scanner.

`pub const tSTRING_CONTENT: i32`[src]

Token "literal content", to be returned by the scanner.

`pub const tREGEXP_END: i32`[src]

Token tREGEXP_END, to be returned by the scanner.

`pub const tDOT: i32`[src]

Token tDOT, to be returned by the scanner.

`pub const tBACKSLASH: i32`[src]

Token "backslash", to be returned by the scanner.

`pub const tSP: i32`[src]

Token "escaped space", to be returned by the scanner.

`pub const tSLASH_T: i32`[src]

Token "escaped horizontal tab", to be returned by the scanner.

`pub const tSLASH_F: i32`[src]

Token "escaped form feed", to be returned by the scanner.

`pub const tSLASH_R: i32`[src]

Token "escaped carriage return", to be returned by the scanner.

`pub const tVTAB: i32`[src]

Token "escaped vertical tab", to be returned by the scanner.

`pub const tUPLUS: i32`[src]

Token "unary+", to be returned by the scanner.

`pub const tUMINUS: i32`[src]

Token "unary-", to be returned by the scanner.

`pub const tPOW: i32`[src]

Token "**", to be returned by the scanner.

`pub const tCMP: i32`[src]

Token "<=>", to be returned by the scanner.

`pub const tEQ: i32`[src]

Token "==", to be returned by the scanner.

`pub const tEQQ: i32`[src]

Token "===", to be returned by the scanner.

`pub const tNEQ: i32`[src]

Token "!=", to be returned by the scanner.

`pub const tGEQ: i32`[src]

Token ">=", to be returned by the scanner.

`pub const tLEQ: i32`[src]

Token "<=", to be returned by the scanner.

`pub const tANDOP: i32`[src]

Token "&&", to be returned by the scanner.

`pub const tOROP: i32`[src]

Token "||", to be returned by the scanner.

`pub const tMATCH: i32`[src]

Token "=~", to be returned by the scanner.

`pub const tNMATCH: i32`[src]

Token "!~", to be returned by the scanner.

`pub const tDOT2: i32`[src]

Token "..", to be returned by the scanner.

`pub const tDOT3: i32`[src]

Token "...", to be returned by the scanner.

`pub const tBDOT2: i32`[src]

Token "(..", to be returned by the scanner.

`pub const tBDOT3: i32`[src]

Token "(...", to be returned by the scanner.

`pub const tAREF: i32`[src]

Token "[]", to be returned by the scanner.

`pub const tASET: i32`[src]

Token "[]=", to be returned by the scanner.

`pub const tLSHFT: i32`[src]

Token "<<", to be returned by the scanner.

`pub const tRSHFT: i32`[src]

Token ">>", to be returned by the scanner.

`pub const tANDDOT: i32`[src]

Token "&.", to be returned by the scanner.

`pub const tCOLON2: i32`[src]

Token "::", to be returned by the scanner.

`pub const tCOLON3: i32`[src]

Token ":: at EXPR_BEG", to be returned by the scanner.

`pub const tOP_ASGN: i32`[src]

Token "operator-assignment", to be returned by the scanner.

`pub const tASSOC: i32`[src]

Token "=>", to be returned by the scanner.

`pub const tLPAREN: i32`[src]

Token "(", to be returned by the scanner.

`pub const tLPAREN_ARG: i32`[src]

Token "( arg", to be returned by the scanner.

`pub const tRPAREN: i32`[src]

Token ")", to be returned by the scanner.

`pub const tLBRACK: i32`[src]

Token "[", to be returned by the scanner.

`pub const tLBRACE: i32`[src]

Token "{", to be returned by the scanner.

`pub const tLBRACE_ARG: i32`[src]

Token "{ arg", to be returned by the scanner.

`pub const tSTAR: i32`[src]

Token "*", to be returned by the scanner.

`pub const tDSTAR: i32`[src]

Token "**arg", to be returned by the scanner.

`pub const tAMPER: i32`[src]

Token "&", to be returned by the scanner.

`pub const tLAMBDA: i32`[src]

Token "->", to be returned by the scanner.

`pub const tSYMBEG: i32`[src]

Token "symbol literal", to be returned by the scanner.

`pub const tSTRING_BEG: i32`[src]

Token "string begin", to be returned by the scanner.

`pub const tXSTRING_BEG: i32`[src]

Token "backtick literal", to be returned by the scanner.

`pub const tREGEXP_BEG: i32`[src]

Token "regexp literal", to be returned by the scanner.

`pub const tWORDS_BEG: i32`[src]

Token "word list", to be returned by the scanner.

`pub const tQWORDS_BEG: i32`[src]

Token "verbatim word list", to be returned by the scanner.

`pub const tSYMBOLS_BEG: i32`[src]

Token "symbol list", to be returned by the scanner.

`pub const tQSYMBOLS_BEG: i32`[src]

Token "verbatim symbol list", to be returned by the scanner.

`pub const tSTRING_END: i32`[src]

Token "string end", to be returned by the scanner.

`pub const tSTRING_DEND: i32`[src]

Token "tRCURLY", to be returned by the scanner.

`pub const tSTRING_DBEG: i32`[src]

Token tSTRING_DBEG, to be returned by the scanner.

`pub const tSTRING_DVAR: i32`[src]

Token tSTRING_DVAR, to be returned by the scanner.

`pub const tLAMBEG: i32`[src]

Token tLAMBEG, to be returned by the scanner.

`pub const tLABEL_END: i32`[src]

Token tLABEL_END, to be returned by the scanner.

`pub const tCOMMA: i32`[src]

Token ",", to be returned by the scanner.

`pub const tLCURLY: i32`[src]

Token "{ (tLCURLY)", to be returned by the scanner.

`pub const tRCURLY: i32`[src]

Token "}", to be returned by the scanner.

`pub const tLBRACK2: i32`[src]

Token "[ (tLBRACK2)", to be returned by the scanner.

`pub const tEQL: i32`[src]

Token "=", to be returned by the scanner.

`pub const tPIPE: i32`[src]

Token "|", to be returned by the scanner.

`pub const tAMPER2: i32`[src]

Token "& (tAMPER2)", to be returned by the scanner.

`pub const tGT: i32`[src]

Token ">", to be returned by the scanner.

`pub const tLT: i32`[src]

Token "<", to be returned by the scanner.

`pub const tBACK_REF2: i32`[src]

Token "`", to be returned by the scanner.

`pub const tCARET: i32`[src]

Token "^", to be returned by the scanner.

`pub const tLPAREN2: i32`[src]

Token "( (tLPAREN2)", to be returned by the scanner.

`pub const tRBRACK: i32`[src]

Token "]", to be returned by the scanner.

`pub const tSEMI: i32`[src]

Token ";", to be returned by the scanner.

`pub const tSPACE: i32`[src]

Token " ", to be returned by the scanner.

`pub const tNL: i32`[src]

Token "\n", to be returned by the scanner.

`pub const tPLUS: i32`[src]

Token "+", to be returned by the scanner.

`pub const tMINUS: i32`[src]

Token "-", to be returned by the scanner.

`pub const tSTAR2: i32`[src]

Token "* (tSTAR2)", to be returned by the scanner.

`pub const tDIVIDE: i32`[src]

Token "/", to be returned by the scanner.

`pub const tPERCENT: i32`[src]

Token "%", to be returned by the scanner.

`pub const tTILDE: i32`[src]

Token "~", to be returned by the scanner.

`pub const tBANG: i32`[src]

Token "!", to be returned by the scanner.

`pub const tLOWEST: i32`[src]

Token tLOWEST, to be returned by the scanner.

`pub const tEH: i32`[src]

Token tEH, to be returned by the scanner.

`pub const tCOLON: i32`[src]

Token tCOLON, to be returned by the scanner.

`pub const tUMINUS_NUM: i32`[src]

Token tUMINUS_NUM, to be returned by the scanner.

`pub const tLAST_TOKEN: i32`[src]

Token tLAST_TOKEN, to be returned by the scanner.

Trait Implementations

`impl Debug for Lexer`[src]

`fn fmt(&self, f: &mut Formatter<'_>) -> Result`[src]

`impl Default for Lexer`[src]

`fn default() -> Lexer`[src]

Auto Trait Implementations

`impl !RefUnwindSafe for Lexer`

`impl !Send for Lexer`

`impl !Sync for Lexer`

`impl Unpin for Lexer`

`impl !UnwindSafe for Lexer`

Blanket Implementations

`impl<T> Any for T where T: 'static + ?Sized,` [src]

`pub fn type_id(&self) -> TypeId`[src]

`impl<T> Borrow<T> for T where T: ?Sized,` [src]

`pub fn borrow(&self) -> &T`[src]

`impl<T> BorrowMut<T> for T where T: ?Sized,` [src]

`pub fn borrow_mut(&mut self) -> &mut T`[src]

`impl<T> From<T> for T`[src]

`pub fn from(t: T) -> T`[src]

`impl<T, U> Into for T where U: From<T>,` [src]

`pub fn into(self) -> U`[src]

`impl<T, U> TryFrom for T where U: Into<T>,` [src]

`type Error = Infallible`

The type returned in the event of a conversion error.

`pub fn try_from(value: U) -> Result<T, <T as TryFrom>::Error>`[src]

`impl<T, U> TryInto for T where U: TryFrom<T>,` [src]

`type Error = >::Error`

The type returned in the event of a conversion error.

Struct lib_ruby_parser::Lexer⎘[−][src]

Fields

Implementations

impl Lexer[src]

pub fn new<Bytes, Name, Decoder>( bytes: Bytes, name: Name, decoder: Decoder) -> Self where Bytes: Into<List<u8>>, Name: Into<StringPtr>, Decoder: Into<MaybePtr<CustomDecoder>>, [src]

pub fn set_debug(&mut self, debug: Type)[src]

pub fn tokenize_until_eof(&mut self) -> Vec<Token>[src]

impl Lexer[src]

pub const END_OF_INPUT: i32[src]

pub const YYerror: i32[src]

pub const YYUNDEF: i32[src]

pub const kCLASS: i32[src]

pub const kMODULE: i32[src]

pub const kDEF: i32[src]

pub const kUNDEF: i32[src]

pub const kBEGIN: i32[src]

pub const kRESCUE: i32[src]

pub const kENSURE: i32[src]

pub const kEND: i32[src]

pub const kIF: i32[src]

pub const kUNLESS: i32[src]

pub const kTHEN: i32[src]

pub const kELSIF: i32[src]

pub const kELSE: i32[src]

pub const kCASE: i32[src]

pub const kWHEN: i32[src]

pub const kWHILE: i32[src]

pub const kUNTIL: i32[src]

pub const kFOR: i32[src]

pub const kBREAK: i32[src]

pub const kNEXT: i32[src]

pub const kREDO: i32[src]

pub const kRETRY: i32[src]

pub const kIN: i32[src]

pub const kDO: i32[src]

pub const kDO_COND: i32[src]

pub const kDO_BLOCK: i32[src]

pub const kDO_LAMBDA: i32[src]

pub const kRETURN: i32[src]

pub const kYIELD: i32[src]

pub const kSUPER: i32[src]

pub const kSELF: i32[src]

pub const kNIL: i32[src]

pub const kTRUE: i32[src]

pub const kFALSE: i32[src]

pub const kAND: i32[src]

pub const kOR: i32[src]

pub const kNOT: i32[src]

pub const kIF_MOD: i32[src]

pub const kUNLESS_MOD: i32[src]

pub const kWHILE_MOD: i32[src]

pub const kUNTIL_MOD: i32[src]

pub const kRESCUE_MOD: i32[src]

pub const kALIAS: i32[src]

pub const kDEFINED: i32[src]

pub const klBEGIN: i32[src]

pub const klEND: i32[src]

pub const k__LINE__: i32[src]

pub const k__FILE__: i32[src]

pub const k__ENCODING__: i32[src]

pub const tIDENTIFIER: i32[src]

pub const tFID: i32[src]

pub const tGVAR: i32[src]

pub const tIVAR: i32[src]

pub const tCONSTANT: i32[src]

pub const tCVAR: i32[src]

pub const tLABEL: i32[src]

pub const tINTEGER: i32[src]

pub const tFLOAT: i32[src]

pub const tRATIONAL: i32[src]

pub const tIMAGINARY: i32[src]

pub const tCHAR: i32[src]

pub const tNTH_REF: i32[src]

pub const tBACK_REF: i32[src]

pub const tSTRING_CONTENT: i32[src]

pub const tREGEXP_END: i32[src]

pub const tDOT: i32[src]

pub const tBACKSLASH: i32[src]

pub const tSP: i32[src]

pub const tSLASH_T: i32[src]

Struct lib_ruby_parser::Lexer[−][src]

`impl Lexer`[src]

`pub fn new<Bytes, Name, Decoder>( bytes: Bytes, name: Name, decoder: Decoder ) -> Self where Bytes: Into<List<u8>>, Name: Into<StringPtr>, Decoder: Into<MaybePtr<CustomDecoder>>,` [src]

`pub fn set_debug(&mut self, debug: Type)`[src]

`pub fn tokenize_until_eof(&mut self) -> Vec<Token>`[src]

`impl Lexer`[src]

`pub const END_OF_INPUT: i32`[src]

`pub const YYerror: i32`[src]

`pub const YYUNDEF: i32`[src]

`pub const kCLASS: i32`[src]

`pub const kMODULE: i32`[src]

`pub const kDEF: i32`[src]

`pub const kUNDEF: i32`[src]

`pub const kBEGIN: i32`[src]

`pub const kRESCUE: i32`[src]

`pub const kENSURE: i32`[src]

`pub const kEND: i32`[src]

`pub const kIF: i32`[src]

`pub const kUNLESS: i32`[src]

`pub const kTHEN: i32`[src]

`pub const kELSIF: i32`[src]

`pub const kELSE: i32`[src]

`pub const kCASE: i32`[src]

`pub const kWHEN: i32`[src]

`pub const kWHILE: i32`[src]

`pub const kUNTIL: i32`[src]

`pub const kFOR: i32`[src]

`pub const kBREAK: i32`[src]

`pub const kNEXT: i32`[src]

`pub const kREDO: i32`[src]

`pub const kRETRY: i32`[src]

`pub const kIN: i32`[src]

`pub const kDO: i32`[src]

`pub const kDO_COND: i32`[src]

`pub const kDO_BLOCK: i32`[src]

`pub const kDO_LAMBDA: i32`[src]

`pub const kRETURN: i32`[src]

`pub const kYIELD: i32`[src]

`pub const kSUPER: i32`[src]

`pub const kSELF: i32`[src]

`pub const kNIL: i32`[src]

`pub const kTRUE: i32`[src]

`pub const kFALSE: i32`[src]

`pub const kAND: i32`[src]

`pub const kOR: i32`[src]

`pub const kNOT: i32`[src]

`pub const kIF_MOD: i32`[src]

`pub const kUNLESS_MOD: i32`[src]

`pub const kWHILE_MOD: i32`[src]

`pub const kUNTIL_MOD: i32`[src]

`pub const kRESCUE_MOD: i32`[src]

`pub const kALIAS: i32`[src]

`pub const kDEFINED: i32`[src]

`pub const klBEGIN: i32`[src]

`pub const klEND: i32`[src]

`pub const kLINE: i32`[src]

`pub const kFILE: i32`[src]

`pub const kENCODING: i32`[src]

`pub const tIDENTIFIER: i32`[src]

`pub const tFID: i32`[src]

`pub const tGVAR: i32`[src]

`pub const tIVAR: i32`[src]

`pub const tCONSTANT: i32`[src]

`pub const tCVAR: i32`[src]

`pub const tLABEL: i32`[src]

`pub const tINTEGER: i32`[src]

`pub const tFLOAT: i32`[src]

`pub const tRATIONAL: i32`[src]

`pub const tIMAGINARY: i32`[src]

`pub const tCHAR: i32`[src]

`pub const tNTH_REF: i32`[src]

`pub const tBACK_REF: i32`[src]

`pub const tSTRING_CONTENT: i32`[src]

`pub const tREGEXP_END: i32`[src]

`pub const tDOT: i32`[src]

`pub const tBACKSLASH: i32`[src]

`pub const tSP: i32`[src]

`pub const tSLASH_T: i32`[src]

`pub const tSLASH_F: i32`[src]

`pub const tSLASH_R: i32`[src]