texform-core 0.1.0

Parser, document tree, and serializer for TeXForm (internal; use the texform crate)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

use logos::Logos;
use texform_core::lexer::Token;

#[test]
fn test_command_sequences() {
    let mut lex = Token::lexer(r"\alpha \frac \$ ~");
    assert_eq!(lex.next(), Some(Ok(Token::ControlSeq("alpha".to_string()))));
    assert_eq!(lex.next(), Some(Ok(Token::Whitespaces)));
    assert_eq!(lex.next(), Some(Ok(Token::ControlSeq("frac".to_string()))));
    assert_eq!(lex.next(), Some(Ok(Token::Whitespaces)));
    assert_eq!(lex.next(), Some(Ok(Token::ControlSeq("$".to_string()))));
    assert_eq!(lex.next(), Some(Ok(Token::Whitespaces)));
    assert_eq!(lex.next(), Some(Ok(Token::ActiveChar)));
}

#[test]
fn test_structural_tokens() {
    let mut lex = Token::lexer(r"{a^b_c}$&");
    assert_eq!(lex.next(), Some(Ok(Token::LBrace)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('a'))));
    assert_eq!(lex.next(), Some(Ok(Token::Superscript)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('b'))));
    assert_eq!(lex.next(), Some(Ok(Token::Subscript)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('c'))));
    assert_eq!(lex.next(), Some(Ok(Token::RBrace)));
    assert_eq!(lex.next(), Some(Ok(Token::MathShift)));
    assert_eq!(lex.next(), Some(Ok(Token::Alignment)));
}

#[test]
fn test_whitespace_and_comments() {
    let mut lex = Token::lexer("a  \t\n  b % comment\nc");
    assert_eq!(lex.next(), Some(Ok(Token::Char('a'))));
    assert_eq!(lex.next(), Some(Ok(Token::Whitespaces)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('b'))));
    assert_eq!(lex.next(), Some(Ok(Token::Whitespaces)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('c'))));
    assert_eq!(lex.next(), None);
}

#[test]
fn test_brackets() {
    let mut lex = Token::lexer(r"\frac[1]{2}");
    assert_eq!(lex.next(), Some(Ok(Token::ControlSeq("frac".to_string()))));
    assert_eq!(lex.next(), Some(Ok(Token::LBracket)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('1'))));
    assert_eq!(lex.next(), Some(Ok(Token::RBracket)));
    assert_eq!(lex.next(), Some(Ok(Token::LBrace)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('2'))));
    assert_eq!(lex.next(), Some(Ok(Token::RBrace)));
}

#[test]
fn test_star_token() {
    let mut lex = Token::lexer(r"\section*{Title}");
    assert_eq!(
        lex.next(),
        Some(Ok(Token::ControlSeq("section".to_string())))
    );
    assert_eq!(lex.next(), Some(Ok(Token::Star)));
    assert_eq!(lex.next(), Some(Ok(Token::LBrace)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('T'))));
    assert_eq!(lex.next(), Some(Ok(Token::Char('i'))));
    assert_eq!(lex.next(), Some(Ok(Token::Char('t'))));
    assert_eq!(lex.next(), Some(Ok(Token::Char('l'))));
    assert_eq!(lex.next(), Some(Ok(Token::Char('e'))));
    assert_eq!(lex.next(), Some(Ok(Token::RBrace)));

    let mut lex = Token::lexer(r"a*b");
    assert_eq!(lex.next(), Some(Ok(Token::Char('a'))));
    assert_eq!(lex.next(), Some(Ok(Token::Star)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('b'))));
}

#[test]
fn test_parameter_marker() {
    let mut lex = Token::lexer(r"#1 #2");
    assert_eq!(lex.next(), Some(Ok(Token::Parameter)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('1'))));
    assert_eq!(lex.next(), Some(Ok(Token::Whitespaces)));
    assert_eq!(lex.next(), Some(Ok(Token::Parameter)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('2'))));
}

#[test]
fn test_invalid_characters() {
    let mut lex = Token::lexer("a\x00b");
    assert_eq!(lex.next(), Some(Ok(Token::Char('a'))));
    assert_eq!(lex.next(), Some(Err(())));
    assert_eq!(lex.next(), Some(Ok(Token::Char('b'))));

    let mut lex = Token::lexer("x\x7Fy");
    assert_eq!(lex.next(), Some(Ok(Token::Char('x'))));
    assert_eq!(lex.next(), Some(Err(())));
    assert_eq!(lex.next(), Some(Ok(Token::Char('y'))));

    let mut lex = Token::lexer("m\x01n");
    assert_eq!(lex.next(), Some(Ok(Token::Char('m'))));
    assert_eq!(lex.next(), Some(Err(())));
    assert_eq!(lex.next(), Some(Ok(Token::Char('n'))));
}

#[test]
fn test_active_char() {
    let mut lex = Token::lexer("a~b");
    assert_eq!(lex.next(), Some(Ok(Token::Char('a'))));
    assert_eq!(lex.next(), Some(Ok(Token::ActiveChar)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('b'))));
}

#[test]
fn test_prime_token() {
    let mut lex = Token::lexer("f'");
    assert_eq!(lex.next(), Some(Ok(Token::Char('f'))));
    assert_eq!(lex.next(), Some(Ok(Token::Prime(1))));

    let mut lex = Token::lexer("f''");
    assert_eq!(lex.next(), Some(Ok(Token::Char('f'))));
    assert_eq!(lex.next(), Some(Ok(Token::Prime(2))));

    let mut lex = Token::lexer("f'");
    assert_eq!(lex.next(), Some(Ok(Token::Char('f'))));
    assert_eq!(lex.next(), Some(Ok(Token::Prime(1))));
}

#[test]
fn test_nbsp_whitespace() {
    let mut lex = Token::lexer("a\u{00A0}b");
    assert_eq!(lex.next(), Some(Ok(Token::Char('a'))));
    assert_eq!(lex.next(), Some(Ok(Token::Whitespaces)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('b'))));

    let mut lex = Token::lexer("a \u{00A0}\t b");
    assert_eq!(lex.next(), Some(Ok(Token::Char('a'))));
    assert_eq!(lex.next(), Some(Ok(Token::Whitespaces)));
    assert_eq!(lex.next(), Some(Ok(Token::Char('b'))));
}