skullbrain-lexer 0.1.0

#![doc = include_str!("../README.md")]

mod lexer;
mod token;

#[cfg(test)]
mod tests;

use std::iter::FusedIterator;

pub(crate) use grammar::*;
pub(crate) use lexer::*;
pub use token::*;

/// Provides constants of language tokens, particularly keywords and special symbols.
pub mod grammar {
    #![allow(missing_docs)]
    include!(concat!(env!("OUT_DIR"), "/grammar.rs"));
}

/// Traverses a character stream and produces tokens.
pub struct Lexer<I>
where
    I: Iterator<Item = char> + FusedIterator,
{
    cursor: LexerCursor<I>,
    buffered_token: Option<Token>,
}

impl<I> Lexer<I>
where
    I: Iterator<Item = char> + FusedIterator,
{
    /// Creates a new Lexer that will process the given character iterator.
    pub fn new(iter: I) -> Self {
        Self {
            cursor: LexerCursor::new(iter),
            buffered_token: None,
        }
    }
}

impl<I> Iterator for Lexer<I>
where
    I: Iterator<Item = char> + FusedIterator,
{
    type Item = Token;

    /// Returns the next token in the stream.
    ///
    /// This method properly handles the cursor's tuple return type by:
    /// 1. First returning any buffered token from previous iterations
    /// 2. When processing new tokens from the cursor:
    ///    - If there's only one token, return it directly
    ///    - If there's a pair (token, unknown_token), buffer the token and return the unknown_token first
    fn next(&mut self) -> Option<Self::Item> {
        match self.buffered_token.take() {
            Some(tok) => Some(tok),
            None => match self.cursor.next()? {
                (tok, unknown_token) if unknown_token.is_none() => Some(tok),
                (tok, unknown_token) => {
                    debug_assert!(self.buffered_token.is_none());
                    self.buffered_token = Some(tok);
                    unknown_token
                }
            },
        }
    }
}