rust_yaml/scanner/
tokens.rs

1//! YAML token definitions
2
3use crate::Position;
4use std::fmt;
5
6/// Quote style for string scalars
7#[derive(Debug, Clone, PartialEq, Eq)]
8pub enum QuoteStyle {
9    /// No quotes (plain scalar)
10    Plain,
11    /// Single quotes ('string')
12    Single,
13    /// Double quotes ("string")
14    Double,
15}
16
17/// Represents a YAML token with position information
18#[derive(Debug, Clone, PartialEq, Eq)]
19pub struct Token {
20    /// The type of token
21    pub token_type: TokenType,
22    /// Start position of the token
23    pub start_position: Position,
24    /// End position of the token
25    pub end_position: Position,
26}
27
28/// Types of YAML tokens
29#[derive(Debug, Clone, PartialEq, Eq)]
30pub enum TokenType {
31    // Stream tokens
32    /// Start of stream
33    StreamStart,
34    /// End of stream
35    StreamEnd,
36
37    // Document tokens
38    /// Document start marker (---)
39    DocumentStart,
40    /// Document end marker (...)
41    DocumentEnd,
42
43    // Directive tokens
44    /// YAML version directive (%YAML)
45    YamlDirective(u8, u8), // major, minor version
46    /// Tag directive (%TAG)
47    TagDirective(String, String), // handle, prefix
48
49    // Block structure tokens
50    /// Block sequence start
51    BlockSequenceStart,
52    /// Block mapping start
53    BlockMappingStart,
54    /// Block end
55    BlockEnd,
56
57    // Flow structure tokens
58    /// Flow sequence start ([)
59    FlowSequenceStart,
60    /// Flow sequence end (])
61    FlowSequenceEnd,
62    /// Flow mapping start ({)
63    FlowMappingStart,
64    /// Flow mapping end (})
65    FlowMappingEnd,
66
67    // Entry tokens
68    /// Block entry (-)
69    BlockEntry,
70    /// Flow entry (,)
71    FlowEntry,
72
73    // Key-value tokens
74    /// Key indicator (? in complex keys)
75    Key,
76    /// Value indicator (:)
77    Value,
78
79    // Scalar tokens
80    /// Scalar value with quote style
81    Scalar(String, QuoteStyle),
82    /// Literal block scalar (|)
83    BlockScalarLiteral(String),
84    /// Folded block scalar (>)
85    BlockScalarFolded(String),
86
87    // Reference tokens
88    /// Alias (*name)
89    Alias(String),
90    /// Anchor (&name)
91    Anchor(String),
92
93    // Tag token
94    /// Tag (!tag or !!tag)
95    Tag(String),
96
97    // Comment token (for round-trip support)
98    /// Comment (# text)
99    Comment(String),
100}
101
102impl Token {
103    /// Create a new token
104    pub const fn new(
105        token_type: TokenType,
106        start_position: Position,
107        end_position: Position,
108    ) -> Self {
109        Self {
110            token_type,
111            start_position,
112            end_position,
113        }
114    }
115
116    /// Create a simple token at a single position
117    pub const fn simple(token_type: TokenType, position: Position) -> Self {
118        Self::new(token_type, position, position)
119    }
120
121    /// Get the token type
122    pub const fn token_type(&self) -> &TokenType {
123        &self.token_type
124    }
125
126    /// Get the start position
127    pub const fn start_position(&self) -> Position {
128        self.start_position
129    }
130
131    /// Get the end position
132    pub const fn end_position(&self) -> Position {
133        self.end_position
134    }
135
136    /// Check if this is a scalar token
137    pub const fn is_scalar(&self) -> bool {
138        matches!(
139            self.token_type,
140            TokenType::Scalar(_, _)
141                | TokenType::BlockScalarLiteral(_)
142                | TokenType::BlockScalarFolded(_)
143        )
144    }
145
146    /// Get scalar value if this is a scalar token
147    pub fn as_scalar(&self) -> Option<&str> {
148        match &self.token_type {
149            TokenType::Scalar(s, _)
150            | TokenType::BlockScalarLiteral(s)
151            | TokenType::BlockScalarFolded(s) => Some(s),
152            _ => None,
153        }
154    }
155
156    /// Get scalar value and quote style if this is a scalar token
157    pub fn as_scalar_with_style(&self) -> Option<(&str, QuoteStyle)> {
158        match &self.token_type {
159            TokenType::Scalar(s, style) => Some((s, style.clone())),
160            TokenType::BlockScalarLiteral(s) => Some((s, QuoteStyle::Plain)), // Block scalars are considered plain
161            TokenType::BlockScalarFolded(s) => Some((s, QuoteStyle::Plain)), // Block scalars are considered plain
162            _ => None,
163        }
164    }
165
166    /// Check if this is a flow collection start token
167    pub const fn is_flow_collection_start(&self) -> bool {
168        matches!(
169            self.token_type,
170            TokenType::FlowSequenceStart | TokenType::FlowMappingStart
171        )
172    }
173
174    /// Check if this is a flow collection end token
175    pub const fn is_flow_collection_end(&self) -> bool {
176        matches!(
177            self.token_type,
178            TokenType::FlowSequenceEnd | TokenType::FlowMappingEnd
179        )
180    }
181
182    /// Check if this is a block collection start token
183    pub const fn is_block_collection_start(&self) -> bool {
184        matches!(
185            self.token_type,
186            TokenType::BlockSequenceStart | TokenType::BlockMappingStart
187        )
188    }
189
190    /// Check if this is a document boundary token
191    pub const fn is_document_boundary(&self) -> bool {
192        matches!(
193            self.token_type,
194            TokenType::DocumentStart | TokenType::DocumentEnd
195        )
196    }
197}
198
199impl fmt::Display for Token {
200    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
201        match &self.token_type {
202            TokenType::StreamStart => write!(f, "STREAM-START"),
203            TokenType::StreamEnd => write!(f, "STREAM-END"),
204            TokenType::DocumentStart => write!(f, "DOCUMENT-START"),
205            TokenType::DocumentEnd => write!(f, "DOCUMENT-END"),
206            TokenType::BlockSequenceStart => write!(f, "BLOCK-SEQUENCE-START"),
207            TokenType::BlockMappingStart => write!(f, "BLOCK-MAPPING-START"),
208            TokenType::BlockEnd => write!(f, "BLOCK-END"),
209            TokenType::FlowSequenceStart => write!(f, "FLOW-SEQUENCE-START"),
210            TokenType::FlowSequenceEnd => write!(f, "FLOW-SEQUENCE-END"),
211            TokenType::FlowMappingStart => write!(f, "FLOW-MAPPING-START"),
212            TokenType::FlowMappingEnd => write!(f, "FLOW-MAPPING-END"),
213            TokenType::BlockEntry => write!(f, "BLOCK-ENTRY"),
214            TokenType::FlowEntry => write!(f, "FLOW-ENTRY"),
215            TokenType::Key => write!(f, "KEY"),
216            TokenType::Value => write!(f, "VALUE"),
217            TokenType::Scalar(s, style) => write!(f, "SCALAR({}, {:?})", s, style),
218            TokenType::BlockScalarLiteral(s) => write!(f, "LITERAL({})", s),
219            TokenType::BlockScalarFolded(s) => write!(f, "FOLDED({})", s),
220            TokenType::Alias(name) => write!(f, "ALIAS({})", name),
221            TokenType::Anchor(name) => write!(f, "ANCHOR({})", name),
222            TokenType::Tag(tag) => write!(f, "TAG({})", tag),
223            TokenType::Comment(text) => write!(f, "COMMENT({})", text),
224            TokenType::YamlDirective(major, minor) => {
225                write!(f, "YAML-DIRECTIVE({}.{})", major, minor)
226            }
227            TokenType::TagDirective(handle, prefix) => {
228                write!(f, "TAG-DIRECTIVE({}, {})", handle, prefix)
229            }
230        }
231    }
232}
233
234impl fmt::Display for TokenType {
235    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
236        match self {
237            Self::StreamStart => write!(f, "StreamStart"),
238            Self::StreamEnd => write!(f, "StreamEnd"),
239            Self::DocumentStart => write!(f, "DocumentStart"),
240            Self::DocumentEnd => write!(f, "DocumentEnd"),
241            Self::BlockSequenceStart => write!(f, "BlockSequenceStart"),
242            Self::BlockMappingStart => write!(f, "BlockMappingStart"),
243            Self::BlockEnd => write!(f, "BlockEnd"),
244            Self::FlowSequenceStart => write!(f, "FlowSequenceStart"),
245            Self::FlowSequenceEnd => write!(f, "FlowSequenceEnd"),
246            Self::FlowMappingStart => write!(f, "FlowMappingStart"),
247            Self::FlowMappingEnd => write!(f, "FlowMappingEnd"),
248            Self::BlockEntry => write!(f, "BlockEntry"),
249            Self::FlowEntry => write!(f, "FlowEntry"),
250            Self::Key => write!(f, "Key"),
251            Self::Value => write!(f, "Value"),
252            Self::Scalar(s, style) => write!(f, "Scalar({}, {:?})", s, style),
253            Self::BlockScalarLiteral(s) => write!(f, "BlockScalarLiteral({})", s),
254            Self::BlockScalarFolded(s) => write!(f, "BlockScalarFolded({})", s),
255            Self::Alias(name) => write!(f, "Alias({})", name),
256            Self::Anchor(name) => write!(f, "Anchor({})", name),
257            Self::Tag(tag) => write!(f, "Tag({})", tag),
258            Self::Comment(text) => write!(f, "Comment({})", text),
259            Self::YamlDirective(major, minor) => write!(f, "YamlDirective({}.{})", major, minor),
260            Self::TagDirective(handle, prefix) => write!(f, "TagDirective({}, {})", handle, prefix),
261        }
262    }
263}
264
265#[cfg(test)]
266mod tests {
267    use super::*;
268
269    #[test]
270    fn test_token_creation() {
271        let pos1 = Position::at(1, 1, 0);
272        let pos2 = Position::at(1, 5, 4);
273
274        let token = Token::new(
275            TokenType::Scalar("hello".to_string(), QuoteStyle::Plain),
276            pos1,
277            pos2,
278        );
279
280        assert_eq!(token.start_position(), pos1);
281        assert_eq!(token.end_position(), pos2);
282        assert!(token.is_scalar());
283        assert_eq!(token.as_scalar(), Some("hello"));
284    }
285
286    #[test]
287    fn test_token_type_checks() {
288        let scalar_token = Token::simple(
289            TokenType::Scalar("test".to_string(), QuoteStyle::Plain),
290            Position::start(),
291        );
292        let flow_start = Token::simple(TokenType::FlowSequenceStart, Position::start());
293        let doc_start = Token::simple(TokenType::DocumentStart, Position::start());
294
295        assert!(scalar_token.is_scalar());
296        assert!(!scalar_token.is_flow_collection_start());
297
298        assert!(flow_start.is_flow_collection_start());
299        assert!(!flow_start.is_scalar());
300
301        assert!(doc_start.is_document_boundary());
302        assert!(!doc_start.is_scalar());
303    }
304
305    #[test]
306    fn test_token_display() {
307        let scalar = Token::simple(
308            TokenType::Scalar("hello".to_string(), QuoteStyle::Plain),
309            Position::start(),
310        );
311        assert_eq!(format!("{}", scalar), "SCALAR(hello, Plain)");
312
313        let flow_start = Token::simple(TokenType::FlowSequenceStart, Position::start());
314        assert_eq!(format!("{}", flow_start), "FLOW-SEQUENCE-START");
315    }
316
317    #[test]
318    fn test_token_type_display() {
319        assert_eq!(format!("{}", TokenType::StreamStart), "StreamStart");
320        assert_eq!(
321            format!(
322                "{}",
323                TokenType::Scalar("test".to_string(), QuoteStyle::Plain)
324            ),
325            "Scalar(test, Plain)"
326        );
327        assert_eq!(
328            format!("{}", TokenType::FlowSequenceStart),
329            "FlowSequenceStart"
330        );
331    }
332}
rust_yaml/scanner/tokens.rs

rust_yaml/scanner/
tokens.rs
