vexy_json_core/ast/
token.rs

1// this_file: src/ast/token.rs
2#![allow(missing_docs)]
3
4//! Token types and definitions for the vexy_json lexer.
5//!
6//! This module defines all the token types that can be produced during
7//! lexical analysis of vexy_json input. The tokens support both standard JSON
8//! syntax and vexy_json's forgiving extensions like comments and unquoted strings.
9
10/// Represents a token in the vexy_json language.
11///
12/// This enum is used by the lexer to break down the input string into meaningful units.
13use logos::Logos;
14
15#[derive(Debug, Clone, Copy, PartialEq, Logos)]
16pub enum Token {
17    /// Opening curly brace '{' for objects.
18    #[token("{")]
19    LeftBrace,
20    /// Closing curly brace '}' for objects.
21    #[token("}")]
22    RightBrace,
23    /// Opening square bracket '[' for arrays.
24    #[token("[")]
25    LeftBracket,
26    /// Closing square bracket ']' for arrays.
27    #[token("]")]
28    RightBracket,
29    /// Comma ',' separator.
30    #[token(",")]
31    Comma,
32    /// Colon ':' separator between keys and values.
33    #[token(":")]
34    Colon,
35    /// JSON null literal.
36    #[token("null")]
37    Null,
38    /// JSON true literal.
39    #[token("true")]
40    True,
41    /// JSON false literal.
42    #[token("false")]
43    False,
44
45    /// Newline character '\n' or '\r' (used for newline-as-comma feature).
46    #[token("\n")]
47    #[token("\r")]
48    Newline,
49
50    /// Whitespace (spaces, tabs) - skipped during parsing.
51    #[regex(r"[ \t]+", logos::skip)]
52
53    /// String literal (double or single quoted).
54    #[regex(r#""(?:[^"\\]|\\.)*""#)]
55    #[regex(r#"'(?:[^'\\]|\\.)*'"#)]
56    String,
57
58    /// Unquoted string (used for object keys in forgiving mode). (Basic, will be refined)
59    #[regex(r"[a-zA-Z_$][a-zA-Z0-9_$-]*")]
60    UnquotedString,
61
62    /// Numeric literal. (Basic, will be refined)
63    #[regex(r"(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?")]
64    #[regex(r"-(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?")]
65    Number,
66
67    /// Single-line comment starting with '//' or '#'.
68    #[regex(r"//[^\r\n]*")]
69    #[regex(r"#[^\r\n]*")]
70    SingleLineComment,
71
72    /// Multi-line comment enclosed in '/* */'.
73    #[regex(r"/\*([^*]|\*[^/])*\*/")]
74    MultiLineComment,
75
76    /// End of file/input. (Logos usually handles this implicitly)
77    Eof,
78
79    /// Represents a lexical error.
80    Error, // Catch-all for lexing errors. Logos requires this.
81}
vexy_json_core/ast/token.rs

vexy_json_core/ast/
token.rs
