rill_json/
token.rs

1//! Defines the `Token` and `TokenType` enums.
2//!
3//! These are used as an intermediate representation between the
4//! `Tokenizer` (lexer) and the `StreamingParser` (parser).
5//! This module is part of the library's internal API.
6
7use crate::value::JsonNumber;
8use std::borrow::Cow;
9
10/// The specific type of a `Token`.
11///
12/// This represents the smallest meaningful units of JSON grammar.
13#[derive(Debug, PartialEq, Clone)]
14pub enum TokenType<'a> {
15    /// `{`
16    LeftBrace,
17    /// `}`
18    RightBrace,
19    /// `[`
20    LeftBracket,
21    /// `]`
22    RightBracket,
23    /// `:`
24    Colon,
25    /// `,`
26    Comma,
27    /// A string, e.g., `"hello"`
28    String(Cow<'a, str>),
29    /// A number, e.g., `123.4`
30    Number(JsonNumber),
31    /// A boolean, `true` or `false`
32    Boolean(bool),
33    /// The `null` literal
34    Null,
35}
36
37/// A single token produced by the `Tokenizer`.
38///
39/// It contains the `TokenType` and its location (line and column)
40/// in the source string, which is crucial for error reporting.
41#[derive(Debug, PartialEq, Clone)]
42pub struct Token<'a> {
43    /// The type of the token.
44    pub(crate) kind: TokenType<'a>,
45    /// The 1-indexed line number where the token starts.
46    pub(crate) line: usize,
47    /// The 1-indexed column number where the token starts.
48    pub(crate) column: usize,
49}
rill_json/token.rs

rill_json/
token.rs
