skullbrain-lexer 0.1.0

use itertools::{peek_nth, PeekNth};
/// A cursor that traverses a character stream and produces tokens.
///
/// The first character in the iterator is considered to be at position `0`.
/// This lexer processes the entire iterator, emitting tokens along with their spans.
pub(crate) struct LexerCursor<I>
where
    I: Iterator<Item = char>,
{
    pub(super) iter: CursorIter<I>,
}

pub(super) struct CursorIter<I>
where
    I: Iterator<Item = char>,
{
    iter: PeekNth<I>,
    current_byte_len: u16,
}

impl<I> LexerCursor<I>
where
    I: Iterator<Item = char>,
{
    pub(super) const fn current_token_offset(&self) -> u16 {
        self.iter.current_byte_len
    }

    pub(super) fn reset_current_token_offset(&mut self) {
        self.iter.current_byte_len = 0;
    }

    pub(super) fn take_while(&mut self, mut accept: impl FnMut(&char) -> bool) {
        while self.iter.peeking_next(|c| accept(c)).is_some() {}
    }
}

impl<I> LexerCursor<I>
where
    I: Iterator<Item = char>,
{
    /// Creates a new `LexerCursor` from an iterator of characters.
    ///
    /// # Arguments
    ///
    /// * `iter` - An iterator over characters.
    ///
    /// # Returns
    ///
    pub fn new(iter: I) -> Self {
        Self {
            iter: CursorIter {
                current_byte_len: 0,
                iter: peek_nth(iter),
            },
        }
    }
}

impl<I> CursorIter<I>
where
    I: Iterator<Item = char>,
{
    pub(super) fn peek_nth(&mut self, n: usize) -> Option<&<CursorIter<I> as Iterator>::Item> {
        self.iter.peek_nth(n)
    }

    pub(super) fn peeking_next<F>(&mut self, accept: F) -> Option<<Self as Iterator>::Item>
    where
        Self: Sized,
        F: FnOnce(&<Self as Iterator>::Item) -> bool,
    {
        // Delegate to Self::next
        self.iter.peek().filter(|item| accept(item))?;
        self.next()
    }
}
impl<I> Iterator for CursorIter<I>
where
    I: Iterator<Item = char>,
{
    type Item = char;

    fn next(&mut self) -> Option<Self::Item> {
        let ch = self.iter.next()?;
        self.current_byte_len += ch.len_utf8() as u16;
        Some(ch)
    }
}