marktwin 0.5.0

use crate::syntax::SyntaxKind;
use smol_str::SmolStr;
use std::str::Chars;

#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct LexerToken {
  pub kind: SyntaxKind,
  pub text: SmolStr,
}

#[derive(Debug, Clone)]
pub struct Lexer<'text> {
  text: &'text str,
}

impl Lexer<'_> {
  pub fn new(text: &str) -> Lexer {
    Lexer { text }
  }
}

impl<'text> Iterator for Lexer<'text> {
  type Item = LexerToken;

  fn next(&mut self) -> Option<Self::Item> {
    let token = next_token(self.text);
    if let Some(ref token) = &token {
      self.text = &self.text[token.text.len()..];
    }
    token
  }
}

pub fn lex(text: &str) -> Vec<LexerToken> {
  Lexer::new(text).collect()
}

/// Lexes the token at the start of the input
///
/// This is the core function of the lexer.
/// It does not require any contextual information. Contextual tokens are
/// handled by the parser by remapping the raw tokens.
fn next_token(input: &str) -> Option<LexerToken> {
  let mut chars = input.chars();
  let old_chars = chars.clone();
  if let Some(token) = maybe_punctuator(&mut chars) {
    return Some(token);
  }
  while !chars.as_str().is_empty() {
    let old_chars = chars.clone();
    if maybe_punctuator(&mut chars).is_some() {
      chars = old_chars;
      break;
    }
  }
  let token_len = old_chars.as_str().len() - chars.as_str().len();
  if token_len == 0 {
    None
  } else {
    let text: SmolStr = input[..token_len].into();
    Some(LexerToken {
      kind: SyntaxKind::TokenText,
      text,
    })
  }
}

fn maybe_punctuator(chars: &mut Chars) -> Option<LexerToken> {
  let old_chars = chars.clone();
  let first = match chars.next() {
    None => return None,
    Some(c) => c,
  };
  let kind: SyntaxKind = match first {
    ' ' => {
      loop {
        let saved = chars.clone();
        if chars.next() != Some(' ') {
          *chars = saved;
          break;
        }
      }
      SyntaxKind::TokenSpace
    }
    '\n' => SyntaxKind::TokenNewline,
    '\r' => match chars.next() {
      Some('\n') => SyntaxKind::TokenNewline,
      _ => return None,
    },
    ':' => SyntaxKind::TokenColon,
    '{' => SyntaxKind::TokenOpenBrace,
    '}' => SyntaxKind::TokenCloseBrace,
    '[' => SyntaxKind::TokenOpenBracket,
    ']' => SyntaxKind::TokenCloseBracket,
    '(' => SyntaxKind::TokenOpenParen,
    ')' => SyntaxKind::TokenCloseParen,
    '_' => SyntaxKind::TokenUnderscore,
    '/' => SyntaxKind::TokenSlash,
    '\\' => SyntaxKind::TokenBackslash,
    '*' => SyntaxKind::TokenStar,
    '~' => SyntaxKind::TokenTilde,
    _ => return None,
  };
  let token_len = old_chars.as_str().len() - chars.as_str().len();
  let text: SmolStr = old_chars.as_str()[..token_len].into();
  Some(LexerToken { kind, text })
}

#[cfg(test)]
mod lexer_tests {
  use crate::lexer::{lex, LexerToken};
  use ::test_generator::test_resources;
  use std::path::{Path, PathBuf};
  use std::{fs, io};

  #[test_resources("./test-resources/[!.]*/")]
  fn test_lex_mkt(path: &str) {
    let path: PathBuf = Path::join(Path::new(".."), path);
    let _name = path
      .components()
      .last()
      .unwrap()
      .as_os_str()
      .to_str()
      .expect("Failed to retrieve sample name");

    let mkt_path = path.join("main.mkt");
    let mkt_text: String = ::std::fs::read_to_string(mkt_path).expect("Failed to read input");

    let actual_tokens = lex(&mkt_text);

    let actual_lex_text = {
      let mut actual_lex_text: Vec<u8> = Vec::new();
      dump_tokens(&mut actual_lex_text, &actual_tokens).unwrap();
      String::from_utf8(actual_lex_text).unwrap()
    };

    fs::write(path.join("local-main.lex.txt"), &actual_lex_text).unwrap();

    let expected_lex_text: String = fs::read_to_string(path.join("main.lex.txt")).expect("Failed to read Lex");

    assert_eq!(&actual_lex_text, &expected_lex_text);
  }

  fn dump_tokens<W: io::Write>(writer: &mut W, tokens: &[LexerToken]) -> Result<(), io::Error> {
    for token in tokens {
      writeln!(writer, "{:?} {:?}", token.kind, token.text,)?;
    }
    Ok(())
  }
}