ass-core 0.1.2

High-performance ASS subtitle format parser and analyzer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131

//! Context-state, scanner, and SIMD component-level tokenizer tests.

#[cfg(not(feature = "std"))]
extern crate alloc;
#[cfg(not(feature = "std"))]
use alloc::{string::ToString, vec};

#[test]
fn tokenizer_uuencoded_data_context() {
    use crate::tokenizer::state::TokenContext;

    // Test UuEncodedData context behavior
    let context = TokenContext::UuEncodedData;
    assert!(!context.allows_whitespace_skipping());
    assert!(!context.is_delimited_block());
    assert_eq!(context.closing_delimiter(), None);
    assert_eq!(context.enter_field_value(), TokenContext::UuEncodedData);
    assert_eq!(context.reset_to_document(), TokenContext::Document);
}

#[test]
fn tokenizer_drawing_commands_context() {
    use crate::tokenizer::state::TokenContext;

    // Test DrawingCommands context behavior
    let context = TokenContext::DrawingCommands;
    assert!(context.allows_whitespace_skipping());
    assert!(!context.is_delimited_block());
    assert_eq!(context.closing_delimiter(), None);
    assert_eq!(context.enter_field_value(), TokenContext::DrawingCommands);
    assert_eq!(context.reset_to_document(), TokenContext::Document);
}

#[test]
fn tokenizer_critical_issues() {
    use crate::tokenizer::state::{IssueCollector, IssueLevel, TokenIssue};

    let mut collector = IssueCollector::new();

    // Test critical issue handling
    let critical_issue = TokenIssue::critical("Critical error".to_string(), "test", 1, 1);
    collector.add_issue(critical_issue);

    assert!(collector.has_issues());
    assert!(collector.has_errors());
    assert_eq!(collector.issue_count(), 1);

    // Test issue level properties
    assert!(IssueLevel::Critical.is_error());
    assert!(IssueLevel::Critical.should_abort());
    assert_eq!(IssueLevel::Critical.as_str(), "critical");
}

#[test]
fn tokenizer_simd_edge_cases() {
    #[cfg(feature = "simd")]
    {
        use crate::tokenizer::simd;

        // Test SIMD with exactly 16 bytes (chunk boundary)
        let text_16 = "0123456789abcdef";
        assert_eq!(text_16.len(), 16);
        let result = simd::scan_delimiters(text_16);
        assert!(result.is_none());

        // Test SIMD with delimiter at chunk boundary
        let text_with_delimiter = "0123456789abcde:";
        let result = simd::scan_delimiters(text_with_delimiter);
        assert_eq!(result, Some(15));

        // Test hex parsing edge cases
        let invalid_hex = simd::parse_hex_u32("GGGG");
        assert!(invalid_hex.is_none());

        // Test UTF-8 validation with mixed content
        let mixed_utf8 = "Hello, δΈ–η•Œ! πŸ¦€";
        let result = simd::validate_utf8_batch(mixed_utf8.as_bytes());
        assert!(result.is_ok());
    }
}

#[test]
fn tokenizer_scanner_edge_cases() {
    use crate::tokenizer::scanner::TokenScanner;

    // Test scanner with empty input
    let scanner = TokenScanner::new("", 0, 1, 1);
    assert!(scanner.navigator().is_at_end());

    // Test scanner navigation methods
    let mut scanner = TokenScanner::new("test", 0, 1, 1);
    let nav = scanner.navigator_mut();

    // Test character advancement and position tracking
    let ch = nav.advance_char();
    assert!(ch.is_ok());
    assert_eq!(ch.unwrap(), 't');
    assert_eq!(nav.position(), 1);
    assert_eq!(nav.column(), 2);
}

#[test]
fn tokenizer_hex_value_detection() {
    use crate::tokenizer::scanner::TokenScanner;

    let mut scanner = TokenScanner::new("&H123ABC", 0, 1, 1);

    // Test hex value scanning
    let result = scanner.scan_text(crate::tokenizer::state::TokenContext::Document);
    assert!(result.is_ok());
}

#[test]
fn tokenizer_field_value_comprehensive() {
    use crate::tokenizer::scanner::TokenScanner;

    // Test field value scanning with various content types
    let test_cases = vec![
        "simple_value",
        "123.45",
        "&H123ABC",
        "value,with,commas",
        "value:with:colons",
    ];

    for test_input in test_cases {
        let mut scanner = TokenScanner::new(test_input, 0, 1, 1);
        let result = scanner.scan_field_value();
        assert!(result.is_ok(), "Failed to scan field value: {test_input}");
    }
}