rlsp_yaml_parser/
token.rs

1// SPDX-License-Identifier: MIT
2
3use crate::pos::Pos;
4
5/// Token codes corresponding to the YAML 1.2 token vocabulary.
6#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7pub enum Code {
8    BeginMapping,
9    EndMapping,
10    BeginSequence,
11    EndSequence,
12    BeginScalar,
13    EndScalar,
14    BeginComment,
15    EndComment,
16    BeginAnchor,
17    EndAnchor,
18    BeginAlias,
19    EndAlias,
20    BeginTag,
21    EndTag,
22    BeginDocument,
23    EndDocument,
24    BeginNode,
25    EndNode,
26    BeginPair,
27    EndPair,
28    DirectivesEnd,
29    DocumentEnd,
30    Text,
31    Indicator,
32    Meta,
33    LineFeed,
34    LineFold,
35    White,
36    Indent,
37    Break,
38    Error,
39}
40
41/// A single token emitted by the parser.
42#[derive(Debug, Clone, Copy, PartialEq, Eq)]
43pub struct Token<'input> {
44    pub code: Code,
45    pub pos: Pos,
46    pub text: &'input str,
47}
48
49#[cfg(test)]
50#[allow(clippy::indexing_slicing, clippy::expect_used, clippy::unwrap_used)]
51mod tests {
52    use super::*;
53
54    #[test]
55    fn token_carries_code_and_pos() {
56        let pos = Pos {
57            byte_offset: 5,
58            char_offset: 5,
59            line: 2,
60            column: 3,
61        };
62        let token = Token {
63            code: Code::BeginMapping,
64            pos,
65            text: "{",
66        };
67
68        assert_eq!(token.code, Code::BeginMapping);
69        assert_eq!(token.pos, pos);
70    }
71
72    #[test]
73    fn code_variants_are_distinct() {
74        assert_ne!(Code::BeginMapping, Code::EndMapping);
75        assert_ne!(Code::Text, Code::Indicator);
76        assert_ne!(Code::BeginScalar, Code::EndScalar);
77        assert_ne!(Code::BeginDocument, Code::EndDocument);
78    }
79
80    #[test]
81    fn code_enum_is_exhaustively_matchable() {
82        // Compile-time check: if a variant is added without updating this
83        // match, the test fails to compile.
84        let code = Code::BeginMapping;
85        let _ = match code {
86            Code::BeginMapping => 0,
87            Code::EndMapping => 1,
88            Code::BeginSequence => 2,
89            Code::EndSequence => 3,
90            Code::BeginScalar => 4,
91            Code::EndScalar => 5,
92            Code::BeginComment => 6,
93            Code::EndComment => 7,
94            Code::BeginAnchor => 8,
95            Code::EndAnchor => 9,
96            Code::BeginAlias => 10,
97            Code::EndAlias => 11,
98            Code::BeginTag => 12,
99            Code::EndTag => 13,
100            Code::BeginDocument => 14,
101            Code::EndDocument => 15,
102            Code::BeginNode => 16,
103            Code::EndNode => 17,
104            Code::BeginPair => 18,
105            Code::EndPair => 19,
106            Code::DirectivesEnd => 20,
107            Code::DocumentEnd => 21,
108            Code::Text => 22,
109            Code::Indicator => 23,
110            Code::Meta => 24,
111            Code::LineFeed => 25,
112            Code::LineFold => 26,
113            Code::White => 27,
114            Code::Indent => 28,
115            Code::Break => 29,
116            Code::Error => 30,
117        };
118    }
119
120    #[test]
121    fn token_is_debug_formattable() {
122        let token = Token {
123            code: Code::Text,
124            pos: Pos::ORIGIN,
125            text: "hello",
126        };
127        let s = format!("{token:?}");
128        assert!(!s.is_empty());
129    }
130}
rlsp_yaml_parser/token.rs

rlsp_yaml_parser/
token.rs
