neotoma 0.1.1

//! UTF-8 decoding utilities for character-level parsing.
//!
//! This module provides low-level UTF-8 decoding functionality used by
//! character-aware parsers like [`Utf8Class`](crate::utf8class::Utf8Class) and [`Utf8Until`](crate::until::Utf8Until). It includes
//! manual UTF-8 validation and decoding with optimizations for ASCII characters.
//!
//! The utilities handle proper UTF-8 validation, preventing overlong encodings
//! and ensuring continuation bytes are correctly formatted. ASCII characters
//! are handled through a fast path for optimal performance.

use crate::{
    parser::{Parsable, Source},
    result::Error,
};

/// Reads a single UTF-8 character from the source, returning the character.
///
/// This function handles UTF-8 decoding with proper validation and error handling.
/// It includes optimizations for ASCII characters and manual decoding for multi-byte
/// UTF-8 sequences.
///
/// # Examples
///
/// ```rust
/// use neotoma::utf8util::read_utf8_char;
/// use neotoma::parser::Source;
/// use std::io::Cursor;
///
/// let mut input = Cursor::new("Hello 世界".as_bytes());
/// let mut source = Source::new(&mut input);
///
/// let ch1 = read_utf8_char(&mut source).unwrap();
/// assert_eq!(ch1, 'H');
///
/// // Skip "ello " (5 characters)
/// for _ in 0..5 { read_utf8_char(&mut source).unwrap(); }
/// let ch2 = read_utf8_char(&mut source).unwrap();
/// assert_eq!(ch2, '世');
/// ```
pub fn read_utf8_char<S>(source: &mut Source<S>) -> Result<char, Error>
where
    S: Parsable,
{
    // Read the first byte to determine character length
    let first_byte = source.peek1()?;

    // Fast path for ASCII characters - handle immediately
    if first_byte & 0x80 == 0 {
        source.advance(1);
        return Ok(first_byte as char);
    }

    // Determine multi-byte character length
    let char_len = if first_byte & 0xE0 == 0xC0 {
        // 2-byte character (110xxxxx)
        2
    } else if first_byte & 0xF0 == 0xE0 {
        // 3-byte character (1110xxxx)
        3
    } else if first_byte & 0xF8 == 0xF0 {
        // 4-byte character (11110xxx)
        4
    } else {
        // Invalid UTF-8 start byte
        return Err(Error::NoMatch);
    };

    // Read the full character bytes for multi-byte sequences
    let bytes = source.read(char_len)?;

    // Validate continuation bytes and decode manually for better performance
    let mut code_point = (first_byte & (0xFF >> (char_len + 1))) as u32;

    for byte in bytes.iter().take(char_len).skip(1) {
        if byte & 0xC0 != 0x80 {
            // Invalid continuation byte
            return Err(Error::NoMatch);
        }
        code_point = (code_point << 6) | ((byte & 0x3F) as u32);
    }

    // Check for overlong encodings
    let min_code_point = match char_len {
        2 => 0x80,
        3 => 0x800,
        4 => 0x10000,
        _ => 0,
    };

    if code_point < min_code_point {
        // This is an overlong encoding
        return Err(Error::NoMatch);
    }

    // Convert to char (this validates the code point)
    char::from_u32(code_point).ok_or(Error::NoMatch)
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::Cursor;

    #[test]
    fn test_read_ascii_char() {
        let mut input = Cursor::new(b"Hello");
        let mut source = crate::parser::Source::new(&mut input);

        let ch = read_utf8_char(&mut source).unwrap();
        assert_eq!(ch, 'H');
    }

    #[test]
    fn test_read_two_byte_utf8() {
        let mut input = Cursor::new("café".as_bytes());
        let mut source = crate::parser::Source::new(&mut input);

        // Skip 'c', 'a'
        read_utf8_char(&mut source).unwrap();
        read_utf8_char(&mut source).unwrap();

        let ch = read_utf8_char(&mut source).unwrap();
        assert_eq!(ch, 'f');

        let ch = read_utf8_char(&mut source).unwrap();
        assert_eq!(ch, 'é'); // 2-byte UTF-8
    }

    #[test]
    fn test_read_three_byte_utf8() {
        let mut input = Cursor::new("世界".as_bytes());
        let mut source = crate::parser::Source::new(&mut input);

        let ch1 = read_utf8_char(&mut source).unwrap();
        assert_eq!(ch1, '世'); // 3-byte UTF-8

        let ch2 = read_utf8_char(&mut source).unwrap();
        assert_eq!(ch2, '界'); // 3-byte UTF-8
    }

    #[test]
    fn test_read_four_byte_utf8() {
        let mut input = Cursor::new("👋🌍".as_bytes());
        let mut source = crate::parser::Source::new(&mut input);

        let ch1 = read_utf8_char(&mut source).unwrap();
        assert_eq!(ch1, '👋'); // 4-byte UTF-8 (emoji)

        let ch2 = read_utf8_char(&mut source).unwrap();
        assert_eq!(ch2, '🌍'); // 4-byte UTF-8 (emoji)
    }

    #[test]
    fn test_read_mixed_utf8() {
        let mut input = Cursor::new("A世👋".as_bytes());
        let mut source = crate::parser::Source::new(&mut input);

        let ch1 = read_utf8_char(&mut source).unwrap();
        assert_eq!(ch1, 'A'); // ASCII

        let ch2 = read_utf8_char(&mut source).unwrap();
        assert_eq!(ch2, '世'); // 3-byte UTF-8

        let ch3 = read_utf8_char(&mut source).unwrap();
        assert_eq!(ch3, '👋'); // 4-byte UTF-8
    }

    #[test]
    fn test_invalid_utf8_start_byte() {
        let mut input = Cursor::new(b"\xFF\xFE");
        let mut source = crate::parser::Source::new(&mut input);

        let result = read_utf8_char(&mut source);
        assert!(result.is_err());
    }

    #[test]
    fn test_invalid_continuation_byte() {
        // 0xC2 expects a continuation byte, but 0xFF is not valid
        let mut input = Cursor::new(b"\xC2\xFF");
        let mut source = crate::parser::Source::new(&mut input);

        let result = read_utf8_char(&mut source);
        assert!(result.is_err());
    }

    #[test]
    fn test_truncated_utf8() {
        // 0xC2 expects a continuation byte, but input ends
        let mut input = Cursor::new(b"\xC2");
        let mut source = crate::parser::Source::new(&mut input);

        let result = read_utf8_char(&mut source);
        assert!(result.is_err());
    }

    #[test]
    fn test_overlong_encoding() {
        // Overlong encoding of 'A' (should be 0x41, not 0xC1 0x81)
        let mut input = Cursor::new(b"\xC1\x81");
        let mut source = crate::parser::Source::new(&mut input);

        let result = read_utf8_char(&mut source);
        assert!(result.is_err());
    }

    #[test]
    fn test_empty_input() {
        let mut input = Cursor::new(b"");
        let mut source = crate::parser::Source::new(&mut input);

        let result = read_utf8_char(&mut source);
        assert!(result.is_err());
    }
}