pub mod token;
pub mod iterator;
pub mod lexlets;


use crate::tracking::SourceReference;

use token::{ Token, TokenStream };
use iterator::LexerIterator;
use lexlets::{ LEXLETS, LexerResult };



pub fn lex (source_ref: SourceReference) -> TokenStream {
  let mut it = LexerIterator::new(source_ref);

  let mut tokens: Vec<Token> = Vec::new();

  'main_loop: while it.valid() {
    'lexlet_loop: for lexlet in LEXLETS {
      match lexlet(&mut it) {
        LexerResult::None => continue 'lexlet_loop,
        LexerResult::Ignore => continue 'main_loop,
        LexerResult::Some(token) => {
          tokens.push(token);
          continue 'main_loop
        }
      }
    }

    it.simple_error(
      format!(
        "Failed to find lexical match for codepoint '{}'",
        it.curr.escape_default()
      )
    );

    it.advance();
  }

  TokenStream {
    tokens,
    source_ref: it.source_ref
  }
}