1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
//! Text and field-value scanning routines for the token scanner.
//!
//! Implements general text scanning, field-value scanning, and hex-value
//! detection, including the optional SIMD-accelerated fast paths.
use super::token_scanner::TokenScanner;
use crate::tokenizer::{state::TokenContext, tokens::TokenType};
use crate::Result;
#[cfg(feature = "simd")]
use crate::tokenizer::simd;
impl TokenScanner<'_> {
/// Scan general text content
///
/// # Errors
///
/// Returns an error if character navigation fails.
pub fn scan_text(&mut self, context: TokenContext) -> Result<TokenType> {
let start = self.navigator.position();
// Use SIMD delimiter scanning when available and context doesn't affect delimiters
#[cfg(feature = "simd")]
{
// Only use SIMD when context doesn't change delimiter behavior
let use_simd = !matches!(context, TokenContext::FieldValue);
if use_simd {
if let Some(delimiter_pos) = self.scan_delimiters_simd(start) {
self.navigator.position = delimiter_pos;
} else {
self.navigator.position = self.source.len();
}
self.navigator.chars = self.source[self.navigator.position..].chars();
self.navigator.peek_char = None;
}
}
// Fallback to scalar scanning (or when SIMD can't be used due to context)
#[cfg(not(feature = "simd"))]
let use_scalar = true;
#[cfg(feature = "simd")]
let use_scalar = matches!(context, TokenContext::FieldValue);
if use_scalar {
while !self.navigator.is_at_end() {
let ch = self.navigator.peek_char()?;
// Check for delimiters based on context
let is_delimiter = match context {
TokenContext::FieldValue => {
// In field values, don't treat colon as delimiter (for time formats)
matches!(ch, ',' | '{' | '}' | '[' | ']' | '\n' | '\r')
}
_ => {
// In other contexts, treat colon as delimiter
matches!(ch, ',' | ':' | '{' | '}' | '[' | ']' | '\n' | '\r')
|| (ch == ';' && context == TokenContext::Document)
}
};
if is_delimiter {
break;
}
self.navigator.advance_char()?;
}
}
let span = &self.source[start..self.navigator.position()];
if context == TokenContext::SectionHeader {
Ok(TokenType::SectionName)
} else if Self::is_hex_value(span) {
Ok(TokenType::HexValue)
} else if !span.is_empty()
&& span
.chars()
.all(|c| c.is_ascii_digit() || c == '.' || c == '-')
{
Ok(TokenType::Number)
} else if !span.is_empty() && span.chars().all(char::is_whitespace) {
Ok(TokenType::Whitespace)
} else {
Ok(TokenType::Text)
}
}
/// Check if a span represents a hex value
pub(super) fn is_hex_value(span: &str) -> bool {
// Check &H format first (standard ASS hex format)
if let Some(after_prefix) = span.strip_prefix("&H") {
let hex_part = after_prefix
.strip_suffix('&')
.map_or(after_prefix, |stripped| stripped);
if !hex_part.is_empty()
&& hex_part.len() % 2 == 0
&& hex_part.len() <= 8
&& hex_part.chars().all(|c| c.is_ascii_hexdigit())
{
#[cfg(feature = "simd")]
{
return TokenScanner::parse_hex_simd(hex_part).is_some();
}
#[cfg(not(feature = "simd"))]
{
return true;
}
}
}
// Raw hex without &H prefix is very rare in ASS files - don't detect it
// to avoid conflicts with numbers and text
false
}
/// Fast delimiter scanning using SIMD when available
#[cfg(feature = "simd")]
fn scan_delimiters_simd(&self, start: usize) -> Option<usize> {
simd::scan_delimiters(&self.source[start..]).map(|offset| start + offset)
}
/// Fast hex parsing using SIMD when available
#[cfg(feature = "simd")]
fn parse_hex_simd(hex_str: &str) -> Option<u32> {
simd::parse_hex_u32(hex_str)
}
/// Scan field value content in field value context
///
/// In field value context, colons are not delimiters (for time formats)
/// and we consume until comma, newline, or end of input.
///
/// # Errors
///
/// Returns an error if character navigation fails.
pub fn scan_field_value(&mut self) -> Result<TokenType> {
let start = self.navigator.position();
while !self.navigator.is_at_end() {
let ch = self.navigator.peek_char()?;
// Stop at delimiters that end field values
if ch == ',' || ch == '\n' || ch == '\r' || ch == '{' || ch == '[' {
break;
}
self.navigator.advance_char()?;
}
let span = &self.source[start..self.navigator.position()];
if !span.is_empty()
&& span
.chars()
.all(|c| c.is_ascii_digit() || c == '.' || c == '-' || c == ':')
{
Ok(TokenType::Number)
} else if !span.is_empty() && span.chars().all(char::is_whitespace) {
Ok(TokenType::Whitespace)
} else {
Ok(TokenType::Text)
}
}
}