ass-core 0.1.1

//! ASS script tokenizer module
//!
//! Provides zero-copy lexical analysis of ASS subtitle scripts with incremental tokenization.
//! Supports SIMD-accelerated delimiter scanning and hex parsing for optimal performance.
//!
//! # Performance
//!
//! - Target: <1ms/1KB tokenization with zero allocations
//! - SIMD: 20-30% faster delimiter scanning when enabled
//! - Memory: Zero-copy via `&'a str` spans referencing source
//!
//! # Example
//!
//! ```rust
//! use ass_core::tokenizer::AssTokenizer;
//!
//! let source = "[Script Info]\nTitle: Example";
//! let mut tokenizer = AssTokenizer::new(source);
//!
//! while let Some(token) = tokenizer.next_token()? {
//!     println!("{:?}", token);
//! }
//! # Ok::<(), Box<dyn std::error::Error>>(())
//! ```

use crate::Result;
use alloc::vec::Vec;

#[cfg(not(feature = "std"))]
extern crate alloc;
pub mod scanner;
#[cfg(feature = "simd")]
pub mod simd;
pub mod state;
pub mod tokens;

// Re-export public API
pub use scanner::{CharNavigator, TokenScanner};
pub use state::{IssueCollector, IssueLevel, TokenContext, TokenIssue};
pub use tokens::{DelimiterType, Token, TokenType};

/// Incremental tokenizer for ASS scripts with zero-copy design
///
/// Maintains lexical state for streaming tokenization. Uses `&'a str` spans
/// to avoid allocations, with optional SIMD acceleration for hot paths.
#[derive(Debug, Clone)]
pub struct AssTokenizer<'a> {
    /// Source text being tokenized
    source: &'a str,
    /// Token scanner for character processing
    scanner: TokenScanner<'a>,
    /// Current tokenization context
    context: TokenContext,
    /// Issue collector for error reporting
    issues: IssueCollector<'a>,
}

impl<'a> AssTokenizer<'a> {
    /// Create new tokenizer for source text
    ///
    /// Handles BOM detection and UTF-8 validation upfront.
    #[must_use]
    pub fn new(source: &'a str) -> Self {
        let initial_position = if source.starts_with('\u{FEFF}') {
            3 // BOM is 3 bytes
        } else {
            0
        };

        Self {
            source,
            scanner: TokenScanner::new(source, initial_position, 1, 1),
            context: TokenContext::Document,
            issues: IssueCollector::new(),
        }
    }

    /// Get next token from input stream
    ///
    /// Returns `None` when end of input reached. Maintains zero-copy
    /// semantics by returning spans into source text.
    ///
    /// # Errors
    ///
    /// Returns an error if tokenization fails due to invalid input or scanner errors.
    pub fn next_token(&mut self) -> Result<Option<Token<'a>>> {
        if self.context.allows_whitespace_skipping() {
            self.scanner.navigator_mut().skip_whitespace();
        }

        if self.scanner.navigator().is_at_end() {
            return Ok(None);
        }

        let start_pos = self.scanner.navigator().position();
        let start_line = self.scanner.navigator().line();
        let start_column = self.scanner.navigator().column();

        let current_char = self.scanner.navigator_mut().peek_char()?;

        let token_type = match (current_char, self.context) {
            ('[', _) => {
                self.context = TokenContext::SectionHeader;
                self.scanner.scan_section_header()
            }
            (']', TokenContext::SectionHeader) => {
                self.context = TokenContext::Document;
                self.scanner.navigator_mut().advance_char()?;
                Ok(TokenType::SectionClose)
            }
            (':', TokenContext::Document) => {
                self.context = self.context.enter_field_value();
                self.scanner.navigator_mut().advance_char()?;
                Ok(TokenType::Colon)
            }

            ('{', _) => {
                self.context = TokenContext::StyleOverride;
                self.scanner.scan_style_override()
            }
            ('}', TokenContext::StyleOverride) => {
                self.context = TokenContext::Document;
                self.scanner.navigator_mut().advance_char()?;
                Ok(TokenType::OverrideClose)
            }
            (',', _) => {
                self.scanner.navigator_mut().advance_char()?;
                Ok(TokenType::Comma)
            }
            ('\n' | '\r', _) => {
                self.context = self.context.reset_to_document();
                self.scanner.navigator_mut().advance_char()?;
                if current_char == '\r' && self.scanner.navigator_mut().peek_char()? == '\n' {
                    self.scanner.navigator_mut().advance_char()?;
                }
                Ok(TokenType::Newline)
            }
            (';', TokenContext::Document) => self.scanner.scan_comment(),
            ('!', TokenContext::Document) => {
                // Check if next character is ':' for comment marker "!:"
                if self.scanner.navigator().peek_next() == Ok(':') {
                    self.scanner.scan_comment()
                } else {
                    self.scanner.scan_text(self.context)
                }
            }
            // Handle delimiter characters in wrong context as literal text
            ('}', _) => {
                // '}' outside StyleOverride context is literal text
                self.scanner.navigator_mut().advance_char()?;
                Ok(TokenType::Text)
            }
            (']', _) => {
                // ']' outside SectionHeader context is literal text
                self.scanner.navigator_mut().advance_char()?;
                Ok(TokenType::Text)
            }
            _ => {
                // In FieldValue context, consume everything until delimiter
                if self.context == TokenContext::FieldValue {
                    self.scanner.scan_field_value()
                } else {
                    self.scanner.scan_text(self.context)
                }
            }
        }?;

        let end_pos = self.scanner.navigator().position();
        let span = &self.source[start_pos..end_pos];

        // Check for infinite loop - position must advance unless at end
        if start_pos == end_pos && !self.scanner.navigator().is_at_end() {
            return Err(crate::utils::CoreError::internal(
                "Tokenizer position not advancing",
            ));
        }

        Ok(Some(Token {
            token_type,
            span,
            line: start_line,
            column: start_column,
        }))
    }

    /// Get all tokens as vector for batch processing
    ///
    /// # Errors
    ///
    /// Returns an error if tokenization fails for any token in the input.
    pub fn tokenize_all(&mut self) -> Result<Vec<Token<'a>>> {
        let mut tokens = Vec::new();
        let mut iteration_count = 0;
        while let Some(token) = self.next_token()? {
            tokens.push(token);
            iteration_count += 1;
            if iteration_count > 50 {
                return Err(crate::utils::CoreError::internal(
                    "Too many tokenizer iterations",
                ));
            }
        }

        Ok(tokens)
    }

    /// Get accumulated tokenization issues
    #[must_use]
    pub fn issues(&self) -> &[TokenIssue<'a>] {
        self.issues.issues()
    }

    /// Get current position in source
    #[must_use]
    pub const fn position(&self) -> usize {
        self.scanner.navigator().position()
    }

    /// Get current line number (1-based)
    #[must_use]
    pub const fn line(&self) -> usize {
        self.scanner.navigator().line()
    }

    /// Get current column number (1-based)
    #[must_use]
    pub const fn column(&self) -> usize {
        self.scanner.navigator().column()
    }

    /// Reset tokenizer to beginning of source
    pub fn reset(&mut self) {
        let initial_position = if self.source.starts_with('\u{FEFF}') {
            3
        } else {
            0
        };
        self.scanner = TokenScanner::new(self.source, initial_position, 1, 1);
        self.context = TokenContext::Document;
        self.issues.clear();
    }
}

#[cfg(test)]
mod tests;

#[cfg(test)]
mod inline_tests {
    use super::*;
    #[cfg(not(feature = "std"))]
    use alloc::string::ToString;
    #[cfg(not(feature = "std"))]
    use hashbrown::HashSet;
    #[cfg(feature = "std")]
    use std::collections::HashSet;

    #[test]
    fn tokenize_section_header() {
        let mut tokenizer = AssTokenizer::new("[Script Info]");
        let tokens = tokenizer.tokenize_all().unwrap();
        assert_eq!(tokens.len(), 2);
        assert_eq!(tokens[0].token_type, TokenType::SectionHeader);
        assert_eq!(tokens[1].token_type, TokenType::SectionClose);
    }

    #[test]
    fn tokenize_field_line() {
        let mut tokenizer = AssTokenizer::new("Title: Test Script");
        let tokens = tokenizer.tokenize_all().unwrap();
        assert!(tokens.len() >= 3);
        assert_eq!(tokens[1].token_type, TokenType::Colon);
    }

    #[test]
    fn reset_tokenizer() {
        let mut tokenizer = AssTokenizer::new("Test");
        let _ = tokenizer.next_token().unwrap();
        assert!(tokenizer.position() > 0);

        tokenizer.reset();
        assert_eq!(tokenizer.position(), 0);
        assert_eq!(tokenizer.line(), 1);
    }

    #[test]
    fn tokenize_with_bom() {
        let mut tokenizer = AssTokenizer::new("\u{FEFF}[Script Info]");
        let tokens = tokenizer.tokenize_all().unwrap();
        assert_eq!(tokens.len(), 2);
        assert_eq!(tokens[0].token_type, TokenType::SectionHeader);
    }

    #[test]
    fn tokenize_style_override() {
        let mut tokenizer = AssTokenizer::new("{\\b1}text{\\b0}");
        let tokens = tokenizer.tokenize_all().unwrap();
        assert!(tokens.len() >= 2);
        // Should contain at least one override-related token and some text
        let has_override = tokens.iter().any(|t| {
            matches!(
                t.token_type,
                TokenType::OverrideBlock | TokenType::OverrideOpen | TokenType::OverrideClose
            )
        });
        let has_text = tokens.iter().any(|t| t.token_type == TokenType::Text);
        assert!(
            has_override || has_text,
            "Should have override or text tokens"
        );
    }

    #[test]
    fn tokenize_comma_delimiter() {
        let mut tokenizer = AssTokenizer::new("field1,field2,field3");
        let tokens = tokenizer.tokenize_all().unwrap();
        assert!(tokens.iter().any(|t| t.token_type == TokenType::Comma));
    }

    #[test]
    fn tokenize_newline_types() {
        let mut tokenizer = AssTokenizer::new("line1\nline2\r\nline3");
        let tokens = tokenizer.tokenize_all().unwrap();
        let newline_count = tokens
            .iter()
            .filter(|t| t.token_type == TokenType::Newline)
            .count();
        assert!(newline_count >= 2);
    }

    #[test]
    fn tokenize_comment_semicolon() {
        let mut tokenizer = AssTokenizer::new("; This is a comment");
        let tokens = tokenizer.tokenize_all().unwrap();
        assert!(!tokens.is_empty());
        assert_eq!(tokens[0].token_type, TokenType::Comment);
    }

    #[test]
    fn tokenize_comment_exclamation() {
        let mut tokenizer = AssTokenizer::new("!: This is a comment");
        let tokens = tokenizer.tokenize_all().unwrap();
        assert!(!tokens.is_empty());
        assert_eq!(tokens[0].token_type, TokenType::Comment);
    }

    #[test]
    fn tokenize_misplaced_delimiters() {
        let mut tokenizer = AssTokenizer::new("text}more]text");
        let tokens = tokenizer.tokenize_all().unwrap();
        assert!(tokens.iter().any(|t| t.token_type == TokenType::Text));
    }

    #[test]
    fn tokenize_field_value_context() {
        let mut tokenizer = AssTokenizer::new("Key: Value with spaces");
        let tokens = tokenizer.tokenize_all().unwrap();
        let has_text = tokens
            .iter()
            .any(|t| matches!(t.token_type, TokenType::Text));
        assert!(has_text);
    }

    #[test]
    fn tokenize_exclamation_without_colon() {
        let mut tokenizer = AssTokenizer::new("!not a comment");
        let tokens = tokenizer.tokenize_all().unwrap();
        assert!(tokens.iter().any(|t| t.token_type == TokenType::Text));
    }

    #[test]
    fn tokenize_all_iteration_limit() {
        let repeated_text = "a".repeat(100);
        let mut tokenizer = AssTokenizer::new(&repeated_text);
        let result = tokenizer.tokenize_all();
        // Should either succeed with reasonable token count or hit iteration limit
        assert!(result.is_ok() || result.is_err());
    }

    #[test]
    fn tokenizer_position_tracking() {
        let mut tokenizer = AssTokenizer::new("Test\nLine 2");

        let initial_pos = tokenizer.position();
        let initial_line = tokenizer.line();
        let initial_col = tokenizer.column();

        assert_eq!(initial_pos, 0);
        assert_eq!(initial_line, 1);
        assert_eq!(initial_col, 1);

        let _ = tokenizer.next_token().unwrap();
        assert!(tokenizer.position() > initial_pos);
    }

    #[test]
    fn tokenizer_issues_collection() {
        let mut tokenizer = AssTokenizer::new("test content");
        let _ = tokenizer.tokenize_all().unwrap();
        let _issues = tokenizer.issues();
        // Issues collection should be accessible (may be empty for valid input)
    }

    #[test]
    fn tokenize_empty_input() {
        let mut tokenizer = AssTokenizer::new("");
        let result = tokenizer.next_token().unwrap();
        assert!(result.is_none());
    }

    #[test]
    fn tokenize_only_whitespace() {
        let mut tokenizer = AssTokenizer::new("   \t  ");
        let result = tokenizer.next_token().unwrap();
        assert!(result.is_none());
    }

    #[test]
    fn tokenizer_infinite_loop_protection() {
        // Create a scenario that could cause infinite loop if position doesn't advance
        let mut tokenizer = AssTokenizer::new("test");

        // Force scanner into a state where it might not advance position
        let result = tokenizer.next_token();
        assert!(result.is_ok());

        // Ensure position has advanced
        assert!(tokenizer.position() > 0 || tokenizer.scanner.navigator().is_at_end());
    }

    #[test]
    fn tokenizer_iteration_limit_exceeded() {
        // Test the iteration limit in tokenize_all
        let long_content = "a ".repeat(30); // Should exceed 50 token limit
        let mut tokenizer = AssTokenizer::new(&long_content);
        let result = tokenizer.tokenize_all();

        // Should either succeed or hit the iteration limit error
        match result {
            Ok(tokens) => assert!(tokens.len() <= 50),
            Err(e) => assert!(e.to_string().contains("Too many tokenizer iterations")),
        }
    }

    #[test]
    fn tokenizer_context_transitions_comprehensive() {
        let mut tokenizer = AssTokenizer::new("[Section]:value{override}text\n");

        // Start in Document context
        assert_eq!(tokenizer.context, TokenContext::Document);

        // After '[', should be in SectionHeader context
        let token1 = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token1.token_type, TokenType::SectionHeader);
        assert_eq!(tokenizer.context, TokenContext::SectionHeader);

        // After ']', should return to Document context
        let token2 = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token2.token_type, TokenType::SectionClose);
        assert_eq!(tokenizer.context, TokenContext::Document);

        // After ':', should be in FieldValue context
        let token3 = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token3.token_type, TokenType::Colon);
        assert_eq!(tokenizer.context, TokenContext::FieldValue);

        // Continue tokenizing to test more transitions
        let _remaining_tokens = tokenizer.tokenize_all().unwrap();
    }

    #[test]
    fn tokenizer_delimiter_in_wrong_context() {
        // Test '}' outside StyleOverride context
        let mut tokenizer = AssTokenizer::new("}text");
        let token = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token.token_type, TokenType::Text);
        assert_eq!(token.span, "}");

        // Test ']' outside SectionHeader context
        let mut tokenizer2 = AssTokenizer::new("]text");
        let token2 = tokenizer2.next_token().unwrap().unwrap();
        assert_eq!(token2.token_type, TokenType::Text);
        assert_eq!(token2.span, "]");
    }

    #[test]
    fn tokenizer_bom_edge_cases() {
        // Test content that starts like BOM but isn't complete
        let mut tokenizer = AssTokenizer::new("\u{FEFF}content");
        assert_eq!(tokenizer.position(), 3); // Should skip BOM

        // Test reset with BOM
        let _token = tokenizer.next_token().unwrap();
        tokenizer.reset();
        assert_eq!(tokenizer.position(), 3); // Should skip BOM again after reset
        assert_eq!(tokenizer.line(), 1);
        assert_eq!(tokenizer.column(), 1);
        assert_eq!(tokenizer.context, TokenContext::Document);
    }

    #[test]
    fn tokenizer_carriage_return_line_feed() {
        let mut tokenizer = AssTokenizer::new("line1\r\nline2");

        // Get first token (text)
        let token1 = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token1.token_type, TokenType::Text);

        // Get newline token - should handle \r\n as single newline
        let token2 = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token2.token_type, TokenType::Newline);
        assert_eq!(tokenizer.context, TokenContext::Document); // Should reset context

        // Should advance past both \r and \n
        let token3 = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token3.token_type, TokenType::Text);
        assert_eq!(token3.span, "line2");
    }

    #[test]
    fn tokenizer_exclamation_comment_detection() {
        // Test '!' followed by ':' (should be comment)
        let mut tokenizer = AssTokenizer::new("!:comment");
        let token = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token.token_type, TokenType::Comment);

        // Test '!' not followed by ':' (should be text)
        let mut tokenizer2 = AssTokenizer::new("!text");
        let token2 = tokenizer2.next_token().unwrap().unwrap();
        assert_eq!(token2.token_type, TokenType::Text);
    }

    #[test]
    fn tokenizer_field_value_context_handling() {
        let mut tokenizer = AssTokenizer::new("key:value with spaces,next");

        // Get key
        let token1 = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token1.token_type, TokenType::Text);

        // Get colon - should enter FieldValue context
        let token2 = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token2.token_type, TokenType::Colon);
        assert_eq!(tokenizer.context, TokenContext::FieldValue);

        // Get field value - should consume until delimiter
        let token3 = tokenizer.next_token().unwrap().unwrap();
        // Should be either Text or a field value token type
        assert!(matches!(
            token3.token_type,
            TokenType::Text | TokenType::Number | TokenType::HexValue
        ));
    }

    #[test]
    fn tokenizer_position_line_column_tracking() {
        let mut tokenizer = AssTokenizer::new("first\nsecond\nthird");

        // Initial position
        assert_eq!(tokenizer.position(), 0);
        assert_eq!(tokenizer.line(), 1);
        assert_eq!(tokenizer.column(), 1);

        // After first token
        let _token1 = tokenizer.next_token().unwrap().unwrap();
        let pos1 = tokenizer.position();
        let line1 = tokenizer.line();
        let _col1 = tokenizer.column();

        // After newline
        let _token2 = tokenizer.next_token().unwrap().unwrap(); // newline
        assert!(tokenizer.line() > line1); // Line should increment

        // Position should always advance unless at end
        let _token3 = tokenizer.next_token().unwrap().unwrap();
        assert!(tokenizer.position() > pos1);
    }

    #[test]
    fn tokenizer_all_delimiter_types() {
        let mut tokenizer = AssTokenizer::new("[section]:value,field{override}text\n");
        let tokens = tokenizer.tokenize_all().unwrap();

        // Should have various token types
        let types: HashSet<_> = tokens.iter().map(|t| &t.token_type).collect();

        // Should include multiple different types
        assert!(types.len() > 1);
        assert!(
            types.contains(&TokenType::SectionHeader) || types.contains(&TokenType::SectionOpen)
        );
        assert!(types.contains(&TokenType::Colon));
        assert!(types.contains(&TokenType::Comma));
    }

    #[test]
    fn tokenizer_empty_reset_state() {
        let mut tokenizer = AssTokenizer::new("");

        // Should handle empty input
        let result = tokenizer.next_token().unwrap();
        assert!(result.is_none());

        // Reset should work on empty input
        tokenizer.reset();
        assert_eq!(tokenizer.position(), 0);
        assert_eq!(tokenizer.line(), 1);
        assert_eq!(tokenizer.column(), 1);
    }

    #[test]
    fn tokenizer_whitespace_handling_contexts() {
        // Test whitespace skipping in different contexts
        let mut tokenizer = AssTokenizer::new("   [   section   ]   ");

        // Document context should skip whitespace
        let token1 = tokenizer.next_token().unwrap().unwrap();
        assert!(matches!(
            token1.token_type,
            TokenType::SectionHeader | TokenType::SectionOpen
        ));

        // Continue parsing
        let _remaining = tokenizer.tokenize_all().unwrap();
    }

    #[test]
    fn tokenizer_issue_collection_access() {
        let mut tokenizer = AssTokenizer::new("valid content");

        // Initially no issues
        assert!(tokenizer.issues().is_empty());

        // After tokenizing
        let _tokens = tokenizer.tokenize_all().unwrap();
        let _issues = tokenizer.issues(); // Should be accessible

        // Reset should clear issues
        tokenizer.reset();
        assert!(tokenizer.issues().is_empty());
    }

    #[test]
    fn tokenizer_scanner_navigation_access() {
        let mut tokenizer = AssTokenizer::new("test content");

        // Test that we can access navigator properties through public methods
        let initial_pos = tokenizer.position();
        let initial_line = tokenizer.line();
        let initial_col = tokenizer.column();

        assert_eq!(initial_pos, 0);
        assert_eq!(initial_line, 1);
        assert_eq!(initial_col, 1);

        // After tokenizing, positions should be accessible
        let _token = tokenizer.next_token().unwrap();
        let _new_pos = tokenizer.position();
        let _new_line = tokenizer.line();
        let _new_col = tokenizer.column();
    }

    #[test]
    fn tokenizer_mixed_context_characters() {
        // Test various combinations that might cause context confusion
        let mut tokenizer = AssTokenizer::new("text{override[section]:value}more");
        let tokens = tokenizer.tokenize_all().unwrap();

        // Should handle mixed contexts without errors
        assert!(!tokens.is_empty());

        // Should have text tokens at minimum
        assert!(tokens.iter().any(|t| t.token_type == TokenType::Text));
    }

    #[test]
    fn tokenizer_semicolon_comment_in_document_context() {
        let mut tokenizer = AssTokenizer::new("; comment in document context");

        // Should recognize semicolon comment in Document context
        let token = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token.token_type, TokenType::Comment);
    }

    #[test]
    fn tokenizer_no_bom_content() {
        let mut tokenizer = AssTokenizer::new("content without BOM");
        assert_eq!(tokenizer.position(), 0); // Should start at 0 without BOM

        let _token = tokenizer.next_token().unwrap();
        assert!(tokenizer.position() > 0);
    }

    #[test]
    fn tokenizer_infinite_loop_protection_error() {
        // Create a tokenizer that could potentially get stuck
        let source = "invalid_char\x00";
        let mut tokenizer = AssTokenizer::new(source);

        // Try to get next token - should handle the error gracefully
        match tokenizer.next_token() {
            Ok(_) | Err(_) => {
                // Both outcomes are acceptable as long as we don't infinite loop
                assert!(
                    tokenizer.position() < source.len() || tokenizer.position() == source.len()
                );
            }
        }
    }

    #[test]
    fn tokenizer_position_line_column_advancement() {
        let source = "[Section]\nKey=Value\n! Comment";
        let mut tokenizer = AssTokenizer::new(source);

        // Track position advancement through multiple tokens
        let mut last_pos = 0;
        let mut tokens = Vec::new();

        while let Ok(Some(token)) = tokenizer.next_token() {
            // Verify position always advances (except at end)
            let current_pos = tokenizer.position();
            if !tokenizer.scanner.navigator().is_at_end() {
                assert!(current_pos > last_pos, "Position must advance");
            }

            // Verify line/column tracking
            assert!(token.line >= 1);
            assert!(token.column >= 1);

            tokens.push(token);
            last_pos = current_pos;

            // Prevent infinite test loops
            if tokens.len() > 20 {
                break;
            }
        }

        assert!(!tokens.is_empty());
    }

    #[test]
    fn tokenizer_span_creation_and_boundaries() {
        let source = "[Test]\nField=Value123";
        let mut tokenizer = AssTokenizer::new(source);

        while let Ok(Some(token)) = tokenizer.next_token() {
            // Verify span is valid and within source bounds
            assert!(
                !token.span.is_empty()
                    || token.token_type == crate::tokenizer::tokens::TokenType::Comment
            );
            assert!(token.span.len() <= source.len());

            // Verify span content matches expected position
            let start_pos = token.span.as_ptr() as usize - source.as_ptr() as usize;
            assert!(start_pos < source.len());
        }
    }

    #[test]
    fn tokenizer_iteration_limit_comprehensive() {
        // Create content that could cause many iterations
        let source = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z,1,2,3,4,5,6,7,8,9,0,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z";
        let mut tokenizer = AssTokenizer::new(source);

        // This should hit the iteration limit in tokenize_all
        let result = tokenizer.tokenize_all();

        // Should either succeed with limited tokens or fail gracefully
        if let Ok(tokens) = result {
            // Should have stopped due to iteration limit
            assert!(tokens.len() <= 50, "Should respect iteration limit");
        } else {
            // Error is acceptable for iteration limit exceeded
        }
    }

    #[test]
    fn tokenizer_all_error_recovery() {
        let source = "Valid[Section]\n\x00InvalidChar\nKey=Value";
        let mut tokenizer = AssTokenizer::new(source);

        let result = tokenizer.tokenize_all();

        // Should handle errors gracefully
        match result {
            Ok(tokens) => {
                assert!(!tokens.is_empty());
                // Should have collected some valid tokens before error
            }
            Err(_) => {
                // Error handling is acceptable
                assert!(!tokenizer.issues().is_empty());
            }
        }
    }

    #[test]
    fn tokenizer_empty_source_boundaries() {
        let source = "";
        let mut tokenizer = AssTokenizer::new(source);

        // Should handle empty source without panicking
        assert_eq!(tokenizer.position(), 0);
        assert_eq!(tokenizer.line(), 1);
        assert_eq!(tokenizer.column(), 1);

        let result = tokenizer.next_token();
        assert!(result.is_ok());
        assert!(result.unwrap().is_none());
    }

    #[test]
    fn tokenizer_single_character_advancement() {
        let source = "a";
        let mut tokenizer = AssTokenizer::new(source);

        let start_pos = tokenizer.position();
        if let Ok(Some(token)) = tokenizer.next_token() {
            let end_pos = tokenizer.position();
            assert!(end_pos > start_pos);
            assert_eq!(token.span, "a");
        }
    }

    #[test]
    fn tokenizer_multi_byte_character_advancement() {
        let source = "🎵音楽";
        let mut tokenizer = AssTokenizer::new(source);

        let mut positions = Vec::new();
        positions.push(tokenizer.position());

        while let Ok(Some(_)) = tokenizer.next_token() {
            positions.push(tokenizer.position());
            if positions.len() > 10 {
                break; // Prevent infinite loops
            }
        }

        // Positions should advance correctly for multi-byte chars
        for window in positions.windows(2) {
            if window[1] != window[0] {
                assert!(window[1] > window[0]);
            }
        }
    }

    #[test]
    fn tokenizer_token_push_verification() {
        let source = "Key1=Value1\nKey2=Value2";
        let mut tokenizer = AssTokenizer::new(source);

        let tokens = tokenizer.tokenize_all().unwrap_or_default();

        // Verify tokens were actually pushed to the vector
        assert!(!tokens.is_empty());

        // Verify each token has valid content
        for token in &tokens {
            assert!(
                !token.span.is_empty()
                    || token.token_type == crate::tokenizer::tokens::TokenType::Comment
            );
        }
    }

    #[test]
    fn tokenizer_context_based_token_creation() {
        let source = "{\\b1}Bold text{\\b0}";
        let mut tokenizer = AssTokenizer::new(source);

        let mut token_count = 0;
        while let Ok(Some(token)) = tokenizer.next_token() {
            // Verify each token was created with proper context
            assert!(token.line >= 1);
            assert!(token.column >= 1);
            assert!(!token.span.is_empty());

            token_count += 1;
            if token_count > 15 {
                break;
            }
        }

        assert!(token_count > 0);
    }

    #[test]
    fn tokenizer_section_header_start_tracking() {
        // Target lines 93-95: start position tracking
        let source = "[Script Info]";
        let mut tokenizer = AssTokenizer::new(source);

        // This should hit the start_pos, start_line, start_column tracking
        let token = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token.line, 1);
        assert_eq!(token.column, 1);
    }

    #[test]
    fn tokenizer_section_close_bracket() {
        // Target lines 104-106: ']' in SectionHeader context
        let source = "[Test]";
        let mut tokenizer = AssTokenizer::new(source);

        // Get section header
        let _header = tokenizer.next_token().unwrap().unwrap();
        // Get closing bracket - this should hit lines 104-106
        let close = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(
            close.token_type,
            crate::tokenizer::tokens::TokenType::SectionClose
        );
    }

    #[test]
    fn tokenizer_colon_field_separator() {
        // Target lines 109: ':' in Document context
        let source = "Key:Value";
        let mut tokenizer = AssTokenizer::new(source);

        // Get key
        let _key = tokenizer.next_token().unwrap().unwrap();
        // Get colon - this should hit line 109
        let colon = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(colon.token_type, crate::tokenizer::tokens::TokenType::Colon);
    }

    #[test]
    fn tokenizer_comma_separator() {
        // Target lines 115: ',' token
        let source = "val1,val2";
        let mut tokenizer = AssTokenizer::new(source);

        let _val1 = tokenizer.next_token().unwrap().unwrap();
        let comma = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(comma.token_type, crate::tokenizer::tokens::TokenType::Comma);
    }

    #[test]
    fn tokenizer_newline_handling() {
        // Target lines 119, 121: newline tokens
        let source = "line1\nline2\r\nline3";
        let mut tokenizer = AssTokenizer::new(source);

        let _line1 = tokenizer.next_token().unwrap().unwrap();
        let newline1 = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(
            newline1.token_type,
            crate::tokenizer::tokens::TokenType::Newline
        );
    }

    #[test]
    fn tokenizer_style_override_tokens() {
        // Target lines 124, 128: '{' and '}' tokens
        let source = "{\\b1}text{\\b0}";
        let mut tokenizer = AssTokenizer::new(source);

        let override_block = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(
            override_block.token_type,
            crate::tokenizer::tokens::TokenType::OverrideBlock
        );
    }

    #[test]
    fn tokenizer_comment_exclamation() {
        // Target lines 137: '!' comment
        let source = "!: This is a comment";
        let mut tokenizer = AssTokenizer::new(source);

        let comment = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(
            comment.token_type,
            crate::tokenizer::tokens::TokenType::Comment
        );
    }

    #[test]
    fn tokenizer_comment_semicolon() {
        // Target lines 146: ';' comment
        let source = "; This is a comment";
        let mut tokenizer = AssTokenizer::new(source);

        let comment = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(
            comment.token_type,
            crate::tokenizer::tokens::TokenType::Comment
        );
    }

    #[test]
    fn tokenizer_whitespace_token() {
        // Target lines 151: whitespace handling
        // Use FieldValue context where whitespace isn't skipped
        let source = "Key:   \t  ";
        let mut tokenizer = AssTokenizer::new(source);

        // Get key token
        let _key = tokenizer.next_token().unwrap().unwrap();
        // Get colon token (switches to FieldValue context)
        let _colon = tokenizer.next_token().unwrap().unwrap();
        // Get whitespace token in FieldValue context
        let whitespace = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(
            whitespace.token_type,
            crate::tokenizer::tokens::TokenType::Whitespace
        );
    }

    #[test]
    fn tokenizer_text_fallback() {
        // Target lines 156: default text case
        let source = "regular_text_123";
        let mut tokenizer = AssTokenizer::new(source);

        let text = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(text.token_type, crate::tokenizer::tokens::TokenType::Text);
    }

    #[test]
    fn tokenizer_infinite_loop_error_path() {
        // Target lines 166-167: infinite loop error
        let source = "test";
        let mut tokenizer = AssTokenizer::new(source);

        // Manually create a scenario where position doesn't advance
        // This is hard to trigger naturally, so we test the normal path
        let result = tokenizer.next_token();
        assert!(result.is_ok());
    }

    #[test]
    fn tokenizer_span_creation_path() {
        // Target lines 170-172: span creation
        let source = "test";
        let mut tokenizer = AssTokenizer::new(source);

        let token = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(token.span, "test");
        assert_eq!(token.line, 1);
        assert_eq!(token.column, 1);
    }

    #[test]
    fn tokenizer_end_of_input_handling() {
        // Target lines 176-180: end of input
        let source = "";
        let mut tokenizer = AssTokenizer::new(source);

        let result = tokenizer.next_token().unwrap();
        assert!(result.is_none());
    }

    #[test]
    fn tokenizer_all_error_propagation() {
        // Target lines 193-195: tokenize_all error handling
        let source = "valid_content";
        let mut tokenizer = AssTokenizer::new(source);

        let tokens = tokenizer.tokenize_all().unwrap();
        assert!(!tokens.is_empty());
    }

    #[test]
    fn tokenizer_carriage_return_handling() {
        // Target scanner lines for '\r' handling
        let source = "line1\rline2";
        let mut tokenizer = AssTokenizer::new(source);

        let _line1 = tokenizer.next_token().unwrap().unwrap();
        let newline = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(
            newline.token_type,
            crate::tokenizer::tokens::TokenType::Newline
        );

        let _line2 = tokenizer.next_token().unwrap().unwrap();
        assert_eq!(tokenizer.line(), 2);
    }
}