rill_json/
token.rs

1//! Defines the `Token` and `TokenType` enums.
2//!
3//! These are used as an intermediate representation between the
4//! `Tokenizer` (lexer) and the `StreamingParser` (parser).
5//! This module is part of the library's internal API.
6
7// --- 2. Token Structs ---
8
9/// The specific type of a `Token`.
10///
11/// This represents the smallest meaningful units of JSON grammar.
12#[derive(Debug, PartialEq, Clone)]
13pub enum TokenType {
14    /// `{`
15    LeftBrace,
16    /// `}`
17    RightBrace,
18    /// `[`
19    LeftBracket,
20    /// `]`
21    RightBracket,
22    /// `:`
23    Colon,
24    /// `,`
25    Comma,
26    /// A string, e.g., `"hello"`
27    String(String),
28    /// A number, e.g., `123.4`
29    Number(f64),
30    /// A boolean, `true` or `false`
31    Boolean(bool),
32    /// The `null` literal
33    Null,
34}
35
36/// A single token produced by the `Tokenizer`.
37///
38/// It contains the `TokenType` and its location (line and column)
39/// in the source string, which is crucial for error reporting.
40#[derive(Debug, PartialEq, Clone)]
41pub struct Token {
42    /// The type of the token.
43    pub(crate) kind: TokenType,
44    /// The 1-indexed line number where the token starts.
45    pub(crate) line: usize,
46    /// The 1-indexed column number where the token starts.
47    pub(crate) column: usize,
48}
rill_json/token.rs

rill_json/
token.rs
