sim_codec_python/
types.rs

1//! Public, runtime-independent syntax data.
2
3use std::fmt;
4
5/// Half-open UTF-8 byte range in the original source.
6#[derive(Clone, Copy, Debug, Eq, PartialEq)]
7pub struct Span {
8    /// First byte.
9    pub start: usize,
10    /// Byte after the range.
11    pub end: usize,
12}
13
14/// Kind of a lossless lexical token.
15#[derive(Clone, Debug, Eq, PartialEq)]
16pub enum TokenKind {
17    /// Identifier, including contextual soft-keyword spellings.
18    Name,
19    /// Reserved keyword.
20    Keyword,
21    /// Numeric literal with its original spelling retained.
22    Number,
23    /// Ordinary string or bytes literal.
24    String,
25    /// Formatted string literal.
26    FString,
27    /// Python 3.14 template string literal.
28    TemplateString,
29    /// Operator or delimiter.
30    Operator,
31    /// Physical line ending.
32    Newline,
33    /// Significant indentation increase.
34    Indent,
35    /// Significant indentation decrease (zero-width).
36    Dedent,
37    /// Spaces, tabs, form feeds, comments, or escaped newlines.
38    Trivia,
39    /// End marker (zero-width).
40    End,
41}
42
43/// A token borrowing no source storage; text is recovered through [`Token::span`].
44#[derive(Clone, Debug, Eq, PartialEq)]
45pub struct Token {
46    /// Token category.
47    pub kind: TokenKind,
48    /// Original byte range.
49    pub span: Span,
50    /// One-based line.
51    pub line: usize,
52    /// Zero-based Unicode-scalar column.
53    pub column: usize,
54}
55
56/// Concrete source-tree node category.
57#[derive(Clone, Debug, Eq, PartialEq)]
58pub enum NodeKind {
59    /// Complete file input.
60    Module,
61    /// Logical statement.
62    Statement,
63    /// Indented suite.
64    Suite,
65    /// Delimited expression/grouping region.
66    Group,
67    /// Expression region whose precedence was admitted through the Pratt table.
68    Expression,
69}
70
71/// A concrete node covering a contiguous token range.
72#[derive(Clone, Debug, Eq, PartialEq)]
73pub struct Node {
74    /// Node category.
75    pub kind: NodeKind,
76    /// Half-open token-index range.
77    pub tokens: std::ops::Range<usize>,
78    /// Nested structural nodes.
79    pub children: Vec<Node>,
80}
81
82/// Complete lossless Python source tree.
83#[derive(Clone, Debug, Eq, PartialEq)]
84pub struct SyntaxTree {
85    source: String,
86    /// Lossless token stream, including trivia and layout markers.
87    pub tokens: Vec<Token>,
88    /// Structural root.
89    pub root: Node,
90}
91
92impl SyntaxTree {
93    /// Returns the exact input bytes as UTF-8 text.
94    #[must_use]
95    pub fn source(&self) -> &str {
96        &self.source
97    }
98
99    /// Re-emits source byte-for-byte. Zero-width layout markers contribute no bytes.
100    #[must_use]
101    pub fn preserve_source(&self) -> String {
102        self.source.clone()
103    }
104
105    pub(crate) fn new(source: &str, tokens: Vec<Token>, root: Node) -> Self {
106        Self {
107            source: source.to_owned(),
108            tokens,
109            root,
110        }
111    }
112}
113
114/// Resource limits shared by lexing and parsing.
115#[derive(Clone, Copy, Debug, Eq, PartialEq)]
116pub struct Limits {
117    /// Maximum source bytes.
118    pub max_bytes: usize,
119    /// Maximum emitted tokens (including trivia/layout).
120    pub max_tokens: usize,
121    /// Maximum bracket, f-string, and indentation nesting.
122    pub max_nesting: usize,
123    /// Maximum physical lines.
124    pub max_lines: usize,
125}
126
127impl Default for Limits {
128    fn default() -> Self {
129        Self {
130            max_bytes: 4 * 1024 * 1024,
131            max_tokens: 1_000_000,
132            max_nesting: 256,
133            max_lines: 250_000,
134        }
135    }
136}
137
138/// Stable diagnostic category.
139#[derive(Clone, Copy, Debug, Eq, PartialEq)]
140pub enum DiagnosticCode {
141    /// A configured resource bound was crossed.
142    ResourceLimit,
143    /// Indentation did not match an earlier level.
144    InvalidIndentation,
145    /// A tab/space combination has an ambiguous visual column.
146    AmbiguousIndentation,
147    /// A literal was not terminated.
148    UnterminatedLiteral,
149    /// A character cannot begin a Python token.
150    InvalidCharacter,
151    /// Delimiters are unmatched or crossed.
152    UnmatchedDelimiter,
153    /// Statement structure is invalid.
154    InvalidSyntax,
155}
156
157/// Deterministic, located syntax failure.
158#[derive(Clone, Debug, Eq, PartialEq)]
159pub struct Diagnostic {
160    /// Stable category.
161    pub code: DiagnosticCode,
162    /// Offending source range.
163    pub span: Span,
164    /// One-based source line.
165    pub line: usize,
166    /// Zero-based Unicode-scalar column.
167    pub column: usize,
168    /// Stable human-readable detail.
169    pub message: String,
170}
171
172impl fmt::Display for Diagnostic {
173    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
174        write!(f, "{}:{}: {}", self.line, self.column, self.message)
175    }
176}
177
178impl std::error::Error for Diagnostic {}
sim_codec_python/types.rs

sim_codec_python/
types.rs
