ass_core/tokenizer/
next_token.rs

1//! Core per-token stepping logic for [`AssTokenizer`].
2//!
3//! Implements `next_token`, the context-sensitive scanner step that drives
4//! incremental tokenization while preserving zero-copy `&'a str` spans.
5
6use super::{AssTokenizer, Token, TokenContext, TokenType};
7use crate::Result;
8
9impl<'a> AssTokenizer<'a> {
10    /// Get next token from input stream
11    ///
12    /// Returns `None` when end of input reached. Maintains zero-copy
13    /// semantics by returning spans into source text.
14    ///
15    /// # Errors
16    ///
17    /// Returns an error if tokenization fails due to invalid input or scanner errors.
18    pub fn next_token(&mut self) -> Result<Option<Token<'a>>> {
19        if self.context.allows_whitespace_skipping() {
20            self.scanner.navigator_mut().skip_whitespace();
21        }
22
23        if self.scanner.navigator().is_at_end() {
24            return Ok(None);
25        }
26
27        let start_pos = self.scanner.navigator().position();
28        let start_line = self.scanner.navigator().line();
29        let start_column = self.scanner.navigator().column();
30
31        let current_char = self.scanner.navigator_mut().peek_char()?;
32
33        let token_type = match (current_char, self.context) {
34            ('[', _) => {
35                self.context = TokenContext::SectionHeader;
36                self.scanner.scan_section_header()
37            }
38            (']', TokenContext::SectionHeader) => {
39                self.context = TokenContext::Document;
40                self.scanner.navigator_mut().advance_char()?;
41                Ok(TokenType::SectionClose)
42            }
43            (':', TokenContext::Document) => {
44                self.context = self.context.enter_field_value();
45                self.scanner.navigator_mut().advance_char()?;
46                Ok(TokenType::Colon)
47            }
48
49            ('{', _) => {
50                self.context = TokenContext::StyleOverride;
51                self.scanner.scan_style_override()
52            }
53            ('}', TokenContext::StyleOverride) => {
54                self.context = TokenContext::Document;
55                self.scanner.navigator_mut().advance_char()?;
56                Ok(TokenType::OverrideClose)
57            }
58            (',', _) => {
59                self.scanner.navigator_mut().advance_char()?;
60                Ok(TokenType::Comma)
61            }
62            ('\n' | '\r', _) => {
63                self.context = self.context.reset_to_document();
64                self.scanner.navigator_mut().advance_char()?;
65                if current_char == '\r' && self.scanner.navigator_mut().peek_char()? == '\n' {
66                    self.scanner.navigator_mut().advance_char()?;
67                }
68                Ok(TokenType::Newline)
69            }
70            (';', TokenContext::Document) => self.scanner.scan_comment(),
71            ('!', TokenContext::Document) => {
72                // Check if next character is ':' for comment marker "!:"
73                if self.scanner.navigator().peek_next() == Ok(':') {
74                    self.scanner.scan_comment()
75                } else {
76                    self.scanner.scan_text(self.context)
77                }
78            }
79            // Handle delimiter characters in wrong context as literal text
80            ('}', _) => {
81                // '}' outside StyleOverride context is literal text
82                self.scanner.navigator_mut().advance_char()?;
83                Ok(TokenType::Text)
84            }
85            (']', _) => {
86                // ']' outside SectionHeader context is literal text
87                self.scanner.navigator_mut().advance_char()?;
88                Ok(TokenType::Text)
89            }
90            _ => {
91                // In FieldValue context, consume everything until delimiter
92                if self.context == TokenContext::FieldValue {
93                    self.scanner.scan_field_value()
94                } else {
95                    self.scanner.scan_text(self.context)
96                }
97            }
98        }?;
99
100        let end_pos = self.scanner.navigator().position();
101        let span = &self.source[start_pos..end_pos];
102
103        // Check for infinite loop - position must advance unless at end
104        if start_pos == end_pos && !self.scanner.navigator().is_at_end() {
105            return Err(crate::utils::CoreError::internal(
106                "Tokenizer position not advancing",
107            ));
108        }
109
110        Ok(Some(Token {
111            token_type,
112            span,
113            line: start_line,
114            column: start_column,
115        }))
116    }
117}
ass_core/tokenizer/next_token.rs

ass_core/tokenizer/
next_token.rs
