ass-core 0.1.2

High-performance ASS subtitle format parser and analyzer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

//! Tests for `TokenScanner::scan_text` content scanning and delimiters.

use crate::tokenizer::scanner::TokenScanner;
use crate::tokenizer::state::TokenContext;
use crate::tokenizer::tokens::TokenType;

#[test]
fn token_scanner_text_simple() {
    let source = "Hello World";
    let mut scanner = TokenScanner::new(source, 0, 1, 1);

    let result = scanner.scan_text(TokenContext::Document).unwrap();
    assert_eq!(result, TokenType::Text);

    assert!(scanner.navigator().is_at_end());
}

#[test]
fn token_scanner_text_with_delimiters() {
    let source = "Hello:World";
    let mut scanner = TokenScanner::new(source, 0, 1, 1);

    let result = scanner.scan_text(TokenContext::Document).unwrap();
    assert_eq!(result, TokenType::Text);

    // Should stop at colon
    assert_eq!(scanner.navigator_mut().peek_char().unwrap(), ':');
}

#[test]
fn token_scanner_text_field_value() {
    let source = "Test Script Title";
    let mut scanner = TokenScanner::new(source, 0, 1, 1);

    let result = scanner.scan_text(TokenContext::FieldValue).unwrap();
    assert_eq!(result, TokenType::Text);

    assert!(scanner.navigator().is_at_end());
}

#[test]
fn token_scanner_text_empty() {
    let source = "";
    let mut scanner = TokenScanner::new(source, 0, 1, 1);

    let result = scanner.scan_text(TokenContext::Document).unwrap();
    assert_eq!(result, TokenType::Text);

    assert!(scanner.navigator().is_at_end());
}

#[test]
fn token_scanner_text_whitespace_only() {
    let source = "   \t  ";
    let mut scanner = TokenScanner::new(source, 0, 1, 1);

    let result = scanner.scan_text(TokenContext::Document).unwrap();
    assert_eq!(result, TokenType::Text);

    assert!(scanner.navigator().is_at_end());
}

#[test]
fn token_scanner_text_unicode() {
    let source = "こんにちは世界";
    let mut scanner = TokenScanner::new(source, 0, 1, 1);

    let result = scanner.scan_text(TokenContext::Document).unwrap();
    assert_eq!(result, TokenType::Text);

    assert!(scanner.navigator().is_at_end());
}

#[test]
fn token_scanner_text_with_special_chars() {
    let source = "Test with émojis 🎬 and symbols ©®™";
    let mut scanner = TokenScanner::new(source, 0, 1, 1);

    let result = scanner.scan_text(TokenContext::Document).unwrap();
    assert_eq!(result, TokenType::Text);

    assert!(scanner.navigator().is_at_end());
}

#[test]
fn token_scanner_text_stops_at_brace() {
    let source = "Text{override}";
    let mut scanner = TokenScanner::new(source, 0, 1, 1);

    let result = scanner.scan_text(TokenContext::Document).unwrap();
    assert_eq!(result, TokenType::Text);

    assert_eq!(scanner.navigator_mut().peek_char().unwrap(), '{');
}

#[test]
fn token_scanner_text_stops_at_comma() {
    let source = "Value1,Value2";
    let mut scanner = TokenScanner::new(source, 0, 1, 1);

    let result = scanner.scan_text(TokenContext::Document).unwrap();
    assert_eq!(result, TokenType::Text);

    assert_eq!(scanner.navigator_mut().peek_char().unwrap(), ',');
}

#[test]
fn token_scanner_text_stops_at_newline() {
    let source = "Line1\nLine2";
    let mut scanner = TokenScanner::new(source, 0, 1, 1);

    let result = scanner.scan_text(TokenContext::Document).unwrap();
    assert_eq!(result, TokenType::Text);

    assert_eq!(scanner.navigator_mut().peek_char().unwrap(), '\n');
}