presk_syntax/
token.rs

1//! Tokens of the Presk surface syntax.
2
3use std::fmt;
4
5use crate::source::Span;
6
7/// Reserved words. `model`, `run`, `serve`, `stream`, `checkpoint`, `every`,
8/// `steps` and `to` are *contextual*: they lex as identifiers and the parser
9/// recognizes them by their text, because `model` is also an axis name and a
10/// configuration field.
11///
12/// ASCII spellings (both forms lex to the same token; the Unicode form is
13/// canonical and is what diagnostics print): `ℝ` = `Real`, `λ` = `\`,
14/// `·` = `@`, `⊗` = `><`, `∘` = `<<`.
15#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
16pub enum Kw {
17    Module,
18    Param,
19    Params,
20    Use,
21    Fn,
22    Const,
23    Dim,
24    Type,
25    Import,
26    As,
27    Static,
28    Assert,
29    Optimizer,
30    Training,
31    Inference,
32    Effect,
33    Let,
34    Uses,
35    Init,
36    Axis,
37    Idx,
38    Ein,
39    /// `ℝ`
40    Real,
41    /// `λ`
42    Lambda,
43}
44
45impl Kw {
46    pub fn from_ident(s: &str) -> Option<Kw> {
47        Some(match s {
48            "module" => Kw::Module,
49            "param" => Kw::Param,
50            "params" => Kw::Params,
51            "use" => Kw::Use,
52            "fn" => Kw::Fn,
53            "const" => Kw::Const,
54            "dim" => Kw::Dim,
55            "type" => Kw::Type,
56            "import" => Kw::Import,
57            "as" => Kw::As,
58            "static" => Kw::Static,
59            "assert" => Kw::Assert,
60            "optimizer" => Kw::Optimizer,
61            "training" => Kw::Training,
62            "inference" => Kw::Inference,
63            "effect" => Kw::Effect,
64            "let" => Kw::Let,
65            "uses" => Kw::Uses,
66            "init" => Kw::Init,
67            "axis" => Kw::Axis,
68            "idx" => Kw::Idx,
69            "ein" => Kw::Ein,
70            "ℝ" | "Real" => Kw::Real,
71            "λ" => Kw::Lambda,
72            _ => return None,
73        })
74    }
75
76    pub fn text(self) -> &'static str {
77        match self {
78            Kw::Module => "module",
79            Kw::Param => "param",
80            Kw::Params => "params",
81            Kw::Use => "use",
82            Kw::Fn => "fn",
83            Kw::Const => "const",
84            Kw::Dim => "dim",
85            Kw::Type => "type",
86            Kw::Import => "import",
87            Kw::As => "as",
88            Kw::Static => "static",
89            Kw::Assert => "assert",
90            Kw::Optimizer => "optimizer",
91            Kw::Training => "training",
92            Kw::Inference => "inference",
93            Kw::Effect => "effect",
94            Kw::Let => "let",
95            Kw::Uses => "uses",
96            Kw::Init => "init",
97            Kw::Axis => "axis",
98            Kw::Idx => "idx",
99            Kw::Ein => "ein",
100            Kw::Real => "ℝ",
101            Kw::Lambda => "λ",
102        }
103    }
104}
105
106#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
107pub enum Op {
108    /// `·` last-axis contraction
109    Dot,
110    Star,
111    Slash,
112    Percent,
113    Plus,
114    Minus,
115    Caret,
116    EqEq,
117    Le,
118    Ge,
119    Lt,
120    Gt,
121    Eq,
122    Arrow,
123    /// `|>`
124    Pipe,
125    /// `|`
126    Bar,
127    /// `⊗`
128    Tensor,
129    /// `∘`
130    Compose,
131    Period,
132    Comma,
133    Colon,
134    LParen,
135    RParen,
136    LBracket,
137    RBracket,
138    LBrace,
139    RBrace,
140    /// `...`
141    Ellipsis,
142    /// `..`, between the bounds of a slice
143    DotDot,
144}
145
146impl Op {
147    pub fn text(self) -> &'static str {
148        match self {
149            Op::Dot => "·",
150            Op::Star => "*",
151            Op::Slash => "/",
152            Op::Percent => "%",
153            Op::Plus => "+",
154            Op::Minus => "-",
155            Op::Caret => "^",
156            Op::EqEq => "==",
157            Op::Le => "<=",
158            Op::Ge => ">=",
159            Op::Lt => "<",
160            Op::Gt => ">",
161            Op::Eq => "=",
162            Op::Arrow => "->",
163            Op::Pipe => "|>",
164            Op::Bar => "|",
165            Op::Tensor => "⊗",
166            Op::Compose => "∘",
167            Op::Period => ".",
168            Op::Comma => ",",
169            Op::Colon => ":",
170            Op::LParen => "(",
171            Op::RParen => ")",
172            Op::LBracket => "[",
173            Op::RBracket => "]",
174            Op::LBrace => "{",
175            Op::RBrace => "}",
176            Op::Ellipsis => "...",
177            Op::DotDot => "..",
178        }
179    }
180}
181
182#[derive(Clone, Debug, PartialEq, Eq)]
183pub enum TokKind {
184    /// NFC-normalized identifier.
185    Ident(String),
186    Kw(Kw),
187    /// Digits only; the text is kept (arbitrary size).
188    Nat(String),
189    /// Decimal with a fraction and/or an exponent; the text is kept.
190    Float(String),
191    /// A number immediately followed by a unit, e.g. `5ms`, `204GB/s`.
192    Quantity { num: String, unit: String },
193    Str(String),
194    Op(Op),
195    /// One or more line breaks.
196    Newline,
197    Eof,
198}
199
200impl fmt::Display for TokKind {
201    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
202        match self {
203            TokKind::Ident(s) => write!(f, "identifier `{s}`"),
204            TokKind::Kw(k) => write!(f, "`{}`", k.text()),
205            TokKind::Nat(s) | TokKind::Float(s) => write!(f, "number `{s}`"),
206            TokKind::Quantity { num, unit } => write!(f, "quantity `{num}{unit}`"),
207            TokKind::Str(s) => write!(f, "string {s:?}"),
208            TokKind::Op(o) => write!(f, "`{}`", o.text()),
209            TokKind::Newline => write!(f, "end of line"),
210            TokKind::Eof => write!(f, "end of file"),
211        }
212    }
213}
214
215#[derive(Clone, Debug, PartialEq, Eq)]
216pub struct Token {
217    pub kind: TokKind,
218    pub span: Span,
219}
presk_syntax/token.rs

presk_syntax/
token.rs
