1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45
pub mod token; pub mod iterator; pub mod lexlets; use crate::tracking::SourceReference; use token::{ Token, TokenStream }; use iterator::LexerIterator; use lexlets::{ LEXLETS, LexerResult }; pub fn lex (source_ref: SourceReference) -> TokenStream { let mut it = LexerIterator::new(source_ref); let mut tokens: Vec<Token> = Vec::new(); 'main_loop: while it.valid() { 'lexlet_loop: for lexlet in LEXLETS { match lexlet(&mut it) { LexerResult::None => continue 'lexlet_loop, LexerResult::Ignore => continue 'main_loop, LexerResult::Some(token) => { tokens.push(token); continue 'main_loop } } } it.simple_error( format!( "Failed to find lexical match for codepoint '{}'", it.curr.escape_default() ) ); it.advance(); } TokenStream { tokens, source_ref: it.source_ref } }