ass-core 0.1.2

High-performance ASS subtitle format parser and analyzer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142

//! Position tracking, reset, and context-transition tests for [`AssTokenizer`].

use crate::tokenizer::{AssTokenizer, TokenType};

#[test]
fn position_tracking() {
    let mut tokenizer = AssTokenizer::new("Hello\nWorld");

    assert_eq!(tokenizer.position(), 0);
    assert_eq!(tokenizer.line(), 1);
    assert_eq!(tokenizer.column(), 1);

    let _token1 = tokenizer.next_token().unwrap().unwrap();
    assert!(tokenizer.position() > 0);

    let _token2 = tokenizer.next_token().unwrap().unwrap(); // newline
    assert_eq!(tokenizer.line(), 2);

    let _token3 = tokenizer.next_token().unwrap().unwrap();
    assert_eq!(tokenizer.line(), 2);
}

#[test]
fn reset_functionality() {
    let mut tokenizer = AssTokenizer::new("Test content");

    // Consume some tokens
    let _token = tokenizer.next_token().unwrap().unwrap();
    assert!(tokenizer.position() > 0);

    // Reset and verify
    tokenizer.reset();
    assert_eq!(tokenizer.position(), 0);
    assert_eq!(tokenizer.line(), 1);
    assert_eq!(tokenizer.column(), 1);

    // Should be able to tokenize again
    let token = tokenizer.next_token().unwrap().unwrap();
    assert_eq!(token.span, "Test content");
}

#[test]
fn reset_with_bom() {
    let mut tokenizer = AssTokenizer::new("\u{FEFF}Test");

    // Should start after BOM
    assert_eq!(tokenizer.position(), 3);

    let _token = tokenizer.next_token().unwrap().unwrap();

    // Reset should go back to after BOM
    tokenizer.reset();
    assert_eq!(tokenizer.position(), 3);
}

#[test]
fn issues_collection() {
    let mut tokenizer = AssTokenizer::new("Valid content");
    let _tokens = tokenizer.tokenize_all().unwrap();

    // Initially no issues
    assert!(tokenizer.issues().is_empty());
}

#[test]
fn context_transitions() {
    let mut tokenizer = AssTokenizer::new("[Section]\nField: Value\n{\\override}text");

    // Should handle all context transitions properly
    let tokens = tokenizer.tokenize_all().unwrap();
    assert!(!tokens.is_empty());

    // Verify we can tokenize various contexts without panicking
    for token in &tokens {
        assert!(
            !token.span.is_empty()
                || matches!(
                    token.token_type,
                    TokenType::Newline
                        | TokenType::Colon
                        | TokenType::Comma
                        | TokenType::SectionClose
                        | TokenType::OverrideClose
                )
        );
    }
}

#[test]
fn empty_section_header() {
    let mut tokenizer = AssTokenizer::new("[]");

    let token1 = tokenizer.next_token().unwrap().unwrap();
    assert_eq!(token1.token_type, TokenType::SectionHeader);
    assert_eq!(token1.span, "[");

    let token2 = tokenizer.next_token().unwrap().unwrap();
    assert_eq!(token2.token_type, TokenType::SectionClose);
}

#[test]
fn empty_style_override() {
    let mut tokenizer = AssTokenizer::new("{}");

    let token1 = tokenizer.next_token().unwrap().unwrap();
    assert_eq!(token1.token_type, TokenType::OverrideBlock);
    assert_eq!(token1.span, "{");

    let token2 = tokenizer.next_token().unwrap().unwrap();
    assert_eq!(token2.token_type, TokenType::OverrideClose);
}

#[test]
fn line_column_tracking() {
    let mut tokenizer = AssTokenizer::new("abc\ndef\nghi");

    let token1 = tokenizer.next_token().unwrap().unwrap();
    assert_eq!(token1.line, 1);
    assert_eq!(token1.column, 1);

    let _newline1 = tokenizer.next_token().unwrap().unwrap();

    let token2 = tokenizer.next_token().unwrap().unwrap();
    assert_eq!(token2.line, 2);
    assert_eq!(token2.column, 1);
}

#[test]
fn tokenize_all_empty() {
    let mut tokenizer = AssTokenizer::new("");
    let tokens = tokenizer.tokenize_all().unwrap();
    assert!(tokens.is_empty());
}

#[test]
fn tokenize_all_whitespace() {
    let mut tokenizer = AssTokenizer::new("   \t\n  ");
    let tokens = tokenizer.tokenize_all().unwrap();
    // Should only have the newline token
    assert_eq!(tokens.len(), 1);
    assert_eq!(tokens[0].token_type, TokenType::Newline);
}