ass_core/tokenizer/mod.rs
1//! ASS script tokenizer module
2//!
3//! Provides zero-copy lexical analysis of ASS subtitle scripts with incremental tokenization.
4//! Supports SIMD-accelerated delimiter scanning and hex parsing for optimal performance.
5//!
6//! # Performance
7//!
8//! - Target: <1ms/1KB tokenization with zero allocations
9//! - SIMD: 20-30% faster delimiter scanning when enabled
10//! - Memory: Zero-copy via `&'a str` spans referencing source
11//!
12//! # Example
13//!
14//! ```rust
15//! use ass_core::tokenizer::AssTokenizer;
16//!
17//! let source = "[Script Info]\nTitle: Example";
18//! let mut tokenizer = AssTokenizer::new(source);
19//!
20//! while let Some(token) = tokenizer.next_token()? {
21//! println!("{:?}", token);
22//! }
23//! # Ok::<(), Box<dyn std::error::Error>>(())
24//! ```
25
26mod ass_tokenizer;
27mod next_token;
28pub mod scanner;
29#[cfg(feature = "simd")]
30pub mod simd;
31pub mod state;
32pub mod tokens;
33
34// Re-export public API
35pub use ass_tokenizer::AssTokenizer;
36pub use scanner::{CharNavigator, TokenScanner};
37pub use state::{IssueCollector, IssueLevel, TokenContext, TokenIssue};
38pub use tokens::{DelimiterType, Token, TokenType};
39
40#[cfg(test)]
41mod tests;
42
43#[cfg(test)]
44mod inline1_tests;
45#[cfg(test)]
46mod inline2_tests;
47#[cfg(test)]
48mod inline3_tests;
49#[cfg(test)]
50mod inline4_tests;
51#[cfg(test)]
52mod inline5_tests;
53#[cfg(test)]
54mod inline6_tests;