lexxor 0.9.2

// =============================
// Lexxor Input Abstractions
// =============================
//
// This module provides input sources for the Lexxor lexer, supporting both in-memory strings and buffered streaming input (e.g., files).
//
// The core abstraction is the `LexxorInput` trait, which yields one character at a time as `Result<Option<char>, LexxorInputError>`, allowing for end-of-input and error reporting.
//
// - `InputString`: Efficient, fixed-size buffer for string input.
// - `InputReader`: Buffered, streaming input from any type implementing `Read`, with UTF-8 and buffer boundary handling.
//
// All types are designed for robust, panic-free operation with clear error propagation.
//
use crate::LexxError;
use std::error::Error;
use std::fmt;
use std::fmt::Debug;
use std::io::Read;
use std::str::{from_utf8, from_utf8_unchecked};

/// Maximum size for input buffers in [`InputString`] and [`InputReader`].
/// Strings or streams longer than this are truncated or paged in chunks.
pub const BUFFER_SIZE: usize = 1024;

/// Error type for input sources used by Lexx.
/// Used to signal I/O or decoding errors during input processing.
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum LexxorInputError {
    /// An error occurred in the input source or during decoding.
    Error(String),
}

impl fmt::Display for LexxorInputError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match *self {
            LexxorInputError::Error(ref s) => {
                write!(f, "an error occurred: {:?}", s)
            }
        }
    }
}

impl Error for LexxorInputError {
    #[allow(deprecated)]
    fn description(&self) -> &str {
        match *self {
            LexxorInputError::Error(..) => "an error occurred",
        }
    }
}

impl From<LexxorInputError> for LexxError {
    fn from(lie: LexxorInputError) -> LexxError {
        match lie {
            LexxorInputError::Error(e) => LexxError::Error(e),
        }
    }
}

/// Trait for providing character input to the Lexxorlexer.
///
/// Implementors yield one character at a time, or an error, until end-of-input.
///
/// # Example
/// ```rust
/// use lexxor::input::{LexxorInput, InputString};
/// let mut input = InputString::new("The\n".to_string());
/// assert!(matches!(input.next(), Ok(Some('T'))));
/// assert!(matches!(input.next(), Ok(Some('h'))));
/// assert!(matches!(input.next(), Ok(Some('e'))));
/// assert!(matches!(input.next(), Ok(Some('\n'))));
/// assert!(matches!(input.next(), Ok(None)));
/// ```
pub trait LexxorInput: Debug {
    /// Returns the next character, or `Ok(None)` at EOF, or an error.
    fn next(&mut self) -> Result<Option<char>, LexxorInputError>;
}

/// In-memory input source for Lexx, using a fixed-size buffer of chars.
///
/// Suitable for small or moderate strings. Strings longer than `BUFFER_SIZE` are truncated.
///
/// # Example
/// ```rust
/// use lexxor::input::{InputString, LexxorInput};
/// let mut input = InputString::new("abc".to_string());
/// assert_eq!(input.next().unwrap(), Some('a'));
/// assert_eq!(input.next().unwrap(), Some('b'));
/// assert_eq!(input.next().unwrap(), Some('c'));
/// assert_eq!(input.next().unwrap(), None);
/// ```
#[derive(Debug)]
pub struct InputString {
    /// Current index into the buffer.
    index: usize,
    /// Number of valid chars in the buffer.
    size: usize,
    /// Fixed-size buffer of chars.
    chars: Box<[char; BUFFER_SIZE]>,
}

impl InputString {
    /// Creates a new `InputString` from the given string.
    ///
    /// If the input string is longer than `BUFFER_SIZE`, it is truncated.
    ///
    /// * `text` - The string to use as input.
    pub fn new(text: String) -> Self {
        let mut chars = Box::new(['x'; BUFFER_SIZE]);
        let mut size: usize = 0;
        let cs = text.chars();
        for c in cs {
            chars[size] = c;
            size += 1;
            if size == BUFFER_SIZE {
                break;
            }
        }
        InputString {
            index: 0,
            size,
            chars,
        }
    }
}

impl LexxorInput for InputString {
    /// Returns each character in the string one at a time until EOF, then returns `Ok(None)`.
    fn next(&mut self) -> Result<Option<char>, LexxorInputError> {
        if self.index < self.size {
            let c = self.chars[self.index];
            self.index += 1;
            return Ok(Some(c));
        }
        Ok(None)
    }
}

/// Buffered, streaming input source for Lexx, reading from any `Read` implementor (e.g., files).
///
/// Handles UTF-8 and buffer rollover for multi-byte chars split across buffer boundaries.
/// Suitable for large files and streaming input.
///
/// # Example
/// ```rust
/// use std::io::Cursor;
/// use lexxor::input::{InputReader, LexxorInput};
/// let data = b"xyz";
/// let mut reader = InputReader::new(Cursor::new(&data[..]));
/// assert_eq!(reader.next().unwrap(), Some('x'));
/// assert_eq!(reader.next().unwrap(), Some('y'));
/// assert_eq!(reader.next().unwrap(), Some('z'));
/// assert_eq!(reader.next().unwrap(), None);
/// ```
#[derive(Debug)]
pub struct InputReader<R>
where
    R: Read + Debug,
{
    /// Current index into the char buffer.
    index: usize,
    /// Number of valid chars in the buffer.
    size: usize,
    /// Start of rollover bytes for incomplete UTF-8 at buffer end.
    rollover_start: usize,
    /// End of rollover bytes.
    rollover_end: usize,
    /// The underlying reader.
    reader: R,
    /// Buffer for raw bytes from the reader.
    buffer: Box<[u8; BUFFER_SIZE]>,
    /// Buffer for decoded chars.
    text: Box<[char; BUFFER_SIZE]>,
}

impl<R> InputReader<R>
where
    R: Read + Debug,
{
    /// Creates a new `InputReader` from any type implementing `Read`.
    ///
    /// The input is buffered and decoded as UTF-8, with rollover handling for multi-byte chars that span buffer boundaries.
    pub fn new(input: R) -> Self {
        let buffer = Box::new([0; BUFFER_SIZE]);
        let text = Box::new(['x'; BUFFER_SIZE]);
        InputReader {
            index: 1,
            size: 0,
            rollover_start: 0,
            rollover_end: 0,
            reader: input,
            buffer,
            text,
        }
    }
}

impl<R> LexxorInput for InputReader<R>
where
    R: Read + Debug,
{
    /// Returns the next character from the stream, handling buffer refills and UTF-8 boundaries.
    /// Returns `Ok(None)` at EOF.
    fn next(&mut self) -> Result<Option<char>, LexxorInputError> {
        if self.index < self.size {
            let c = self.text[self.index];
            self.index += 1;
            return Ok(Some(c));
        }
        // Read new bytes into buffer
        let n: usize = if self.rollover_start == 0 {
            self.reader.read(self.buffer.as_mut()).unwrap()
        } else {
            // Move rollover bytes to front
            let rollover_len = self.rollover_end - self.rollover_start;
            self.buffer
                .copy_within(self.rollover_start..self.rollover_end, 0);
            let read_bytes = self.reader.read(&mut self.buffer[rollover_len..]).unwrap();
            let n = read_bytes + rollover_len;
            self.rollover_start = 0;
            n
        };
        if n == 0 {
            return Ok(None);
        }
        self.index = 0;
        // Handle incomplete UTF-8 at buffer end efficiently
        let valid_up_to = match from_utf8(&self.buffer[..n]) {
            Ok(_) => n,
            Err(e) => {
                let end = e.valid_up_to();
                if end != n {
                    self.rollover_start = end;
                    self.rollover_end = n;
                }
                end
            }
        };
        let se = unsafe { from_utf8_unchecked(&self.buffer[..valid_up_to]) };
        self.size = 0;
        for c in se.chars() {
            self.text[self.size] = c;
            self.size += 1;
        }
        if self.size == 0 {
            return Ok(None);
        }
        self.index += 1;
        Ok(Some(self.text[self.index - 1]))
    }
}

// =============================
// End of LexxorInput Abstractions
// =============================

#[cfg(test)]
mod tests {
    use std::collections::HashMap;
    use std::fs::File;
    use std::time::Instant;

    use crate::input::InputReader;
    use crate::matcher::float::FloatMatcher;
    use crate::matcher::integer::IntegerMatcher;
    use crate::matcher::symbol::SymbolMatcher;
    use crate::matcher::whitespace::WhitespaceMatcher;
    use crate::matcher::word::WordMatcher;
    use crate::token::{
        TOKEN_TYPE_FLOAT, TOKEN_TYPE_INTEGER, TOKEN_TYPE_SYMBOL, TOKEN_TYPE_WHITESPACE,
        TOKEN_TYPE_WORD, Token,
    };
    use crate::{LexxError, Lexxer, Lexxor};

    #[test]
    fn lexxor_parse_large_file() {
        let mut integers = 0;
        let mut floats = 0;
        let mut whitespace = 0;
        let mut unique_words = HashMap::new();
        let mut words = 0;
        let mut symbols = 0;
        let mut total = 0;
        let mut lines = 0;

        let start = Instant::now();

        let file = File::open("./test_data/Varney-the-Vampire.txt").unwrap();

        let input_file = InputReader::new(file);

        let mut lexxor = make_test_lexxor(input_file);

        loop {
            match lexxor.next_token() {
                Ok(Some(token)) => {
                    total += 1;
                    lines = token.line;
                    match token.token_type {
                        TOKEN_TYPE_INTEGER => {
                            integers += 1;
                        }
                        TOKEN_TYPE_FLOAT => {
                            floats += 1;
                        }
                        TOKEN_TYPE_WHITESPACE => {
                            whitespace += 1;
                        }
                        TOKEN_TYPE_SYMBOL => {
                            //println!("'{}'", token.value);
                            symbols += 1;
                        }
                        TOKEN_TYPE_WORD => {
                            words += 1;
                            let count = unique_words.entry(token.value).or_insert(0);
                            *count += 1;
                        }
                        _ => {
                            unreachable!("Don't know what this is!")
                        }
                    }
                }
                Err(e) => match e {
                    LexxError::TokenNotFound(_) => {
                        unreachable!("Should not have failed finding a token file");
                    }
                    LexxError::Error(_) => {
                        unreachable!("Should not have failed parsing file");
                    }
                },
                Ok(None) => break,
            }
        }

        let duration = start.elapsed();
        println!("Time elapsed is: {:?}", duration);
        assert_eq!(743524, total);
        assert_eq!(124, integers);
        assert_eq!(1, floats);
        assert_eq!(332189, whitespace);
        assert_eq!(72301, symbols);
        assert_eq!(338909, words);
        assert_eq!(13267, unique_words.len());
        assert_eq!(43680, lines);
    }

    #[test]
    fn lexxor_parse_utf_file() {
        let file = File::open("./test_data/utf-8-sampler.txt").unwrap();

        let input_file = InputReader::new(file);

        let lexxor = make_test_lexxor(input_file);

        let mut final_token: Token = Token {
            value: "".to_string(),
            token_type: 0,
            len: 0,
            line: 0,
            column: 0,
            precedence: 0,
        };

        for token in lexxor {
            if token.token_type != TOKEN_TYPE_WHITESPACE {
                final_token = token;
            }
        }

        assert_eq!(TOKEN_TYPE_SYMBOL, final_token.token_type);
        assert_eq!(72, final_token.column);
        assert_eq!(204, final_token.line);
        assert_eq!(String::from("▁▂▃▄▅▆▇█"), final_token.value);
    }

    fn make_test_lexxor(input_file: InputReader<File>) -> Box<Lexxor<512>> {
        Box::new(Lexxor::<512>::new(
            Box::new(input_file),
            vec![
                Box::new(IntegerMatcher {
                    index: 0,
                    precedence: 0,
                    running: true,
                }),
                Box::new(FloatMatcher {
                    index: 0,
                    precedence: 0,
                    dot: false,
                    float: false,
                    running: true,
                }),
                Box::new(WhitespaceMatcher {
                    index: 0,
                    column: 0,
                    line: 0,
                    precedence: 0,
                    running: true,
                }),
                Box::new(WordMatcher {
                    index: 0,
                    precedence: 0,
                    running: true,
                }),
                Box::new(SymbolMatcher {
                    index: 0,
                    precedence: 0,
                    running: true,
                }),
            ],
        ))
    }
}