typr_core/components/syntax/
mod.rs

1//! The TypR **syntax manifest** — the single source of truth for every
2//! lexeme the language colors.
3//!
4//! ## Why this module exists
5//!
6//! Before it, the same syntax was written down six times by hand (the parser
7//! here, VSCode's `typr.tmLanguage.json`, the Playground's Monarch rules,
8//! the Docusaurus Prism rules, `editors/vim/syntax/typr.vim`, and the LSP's
9//! semantic tokens) and the copies had already drifted apart: the editor
10//! grammars colored `impl`/`trait`/`struct`/`enum`/`where`/`mut`/`Option`/
11//! `Result` — Rust keywords copy-pasted in that TypR has never had — while
12//! `opaque`, `module`, `record`, `object`, `interface`, `typeconstructor`,
13//! `recursive`, `embed`, `@export` and the kind sigils were colored nowhere.
14//!
15//! TextMate (VSCode/Positron/Shiki/Monaco) and tree-sitter (Neovim/Helix/Zed)
16//! do not convert into one another, so the source of truth has to sit one
17//! level *above* any grammar format: a plain data description of the lexemes,
18//! from which each target grammar is **generated**. That is this manifest.
19//! `typr syntax` (typr-cli) serializes it and renders the targets.
20//!
21//! ## Consumers
22//!
23//! `editors/vscode/syntaxes/typr.tmLanguage.json` is the only artifact rendered
24//! today, and it feeds four targets: the VSCode/Positron extension, the
25//! playground (Shiki → Monaco, `src/lib/monaco-typr.ts`), the Docusaurus
26//! documentation (`@shikijs/rehype`, `src/syntax/shiki.ts`) and — the day TypR
27//! clears Linguist's 2000-file bar — GitHub. Both sites keep a copy under
28//! `syntaxes/`, pushed there by the release's `grammar` job; it is not
29//! hand-edited, and their CI warns when it drifts. Both read the grammar with
30//! Shiki's JavaScript regex engine rather than Oniguruma, which is why
31//! `RuleBody::Match` sticks to a dialect both engines accept.
32//!
33//! ## Invariant
34//!
35//! Every word-like literal the parser matches with `tag("…")` must appear in
36//! this manifest. `tests::every_parser_tag_is_in_the_manifest` reads the
37//! parser sources with `include_str!` and fails the build when one is
38//! missing, so adding a keyword to the grammar without teaching the editors
39//! about it is a test failure rather than a silent divergence.
40
41use serde::{Deserialize, Serialize};
42
43/// Bumped whenever the shape of the serialized manifest changes in a way a
44/// consumer (a generator, a downstream editor plugin) has to notice.
45pub const SCHEMA_VERSION: u32 = 1;
46
47/// Identity of the language, for the target grammars' preamble.
48#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
49pub struct Language {
50    /// Editor language id (VSCode `contributes.languages[].id`, Vim filetype).
51    pub id: String,
52    /// Human-readable display name.
53    pub name: String,
54    /// TextMate scope name.
55    pub scope_name: String,
56    pub extensions: Vec<String>,
57    pub line_comment: String,
58}
59
60/// A kind sigil (RFC `ai_context/sigils.md` §3.2.1), prefixed to a
61/// single-uppercase-letter generic to fix its kind: `#N`, `%R`, `@T`, `^S`,
62/// `?B`, `$L`.
63#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
64pub struct Sigil {
65    pub sigil: String,
66    /// The kind it denotes (`Number`, `Record`, …).
67    pub kind: String,
68    /// Suffix appended to `entity.name.type.parameter` in target scopes.
69    pub scope_suffix: String,
70    /// Reserved by the RFC but not parsed yet. Generators MUST skip these:
71    /// `~`, `&` and `!` are live *operators* today, so coloring them as
72    /// sigils would mis-highlight real code.
73    pub reserved: bool,
74    pub note: Option<String>,
75}
76
77/// What a rule matches. Ordering of `SyntaxManifest::rules` is significant:
78/// earlier rules win, in every target format.
79#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
80#[serde(tag = "kind", rename_all = "snake_case")]
81pub enum RuleBody {
82    /// Whole-word alternation, guarded by identifier boundaries.
83    Words {
84        lexemes: Vec<String>,
85        /// When set, the word only counts glued to what this regex matches —
86        /// the delimiter the parser consumes along with it (`c(`, `seq[`,
87        /// `R {`). Without it a variable named `c` would be colored as the
88        /// vector constructor.
89        #[serde(default, skip_serializing_if = "Option::is_none")]
90        followed_by: Option<String>,
91    },
92    /// Symbol alternation, emitted longest-first so `<=` beats `<`.
93    Symbols { lexemes: Vec<String> },
94    /// `@`-prefixed annotations — a word rule whose `\b` prefix guard has to
95    /// be dropped, since `@` is not a word character.
96    Annotations { lexemes: Vec<String> },
97    /// The kind sigils, expanded by each generator from `SyntaxManifest::sigils`.
98    Sigils,
99    /// A raw regex, in the ECMAScript-ish dialect TextMate/Oniguruma and
100    /// Shiki's JS engine both accept.
101    Match {
102        pattern: String,
103        /// The same rule in Vim's `\v` (very-magic) dialect. Vim's regex
104        /// engine diverges from TextMate/Oniguruma too far for one pattern to
105        /// serve both: lookaround is postfix (`atom@=`/`atom@!` instead of
106        /// `(?=atom)`/`(?!atom)`), and in `\v` mode bare `<`/`>` mean
107        /// word-boundary while a *literal* `<`/`>` character needs escaping
108        /// — the opposite of `\b`. Kept side by side with `pattern` so both
109        /// targets are generated from the same rule instead of one being
110        /// hand-transcribed later.
111        vim: String,
112    },
113    /// A begin/end span (strings).
114    Span {
115        begin: String,
116        end: String,
117        /// Whether `\\.` escape sequences are highlighted inside.
118        escapes: bool,
119    },
120}
121
122/// One highlighting rule: a named bucket of lexemes plus the scope the
123/// targets should paint them with.
124#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
125pub struct Rule {
126    /// Repository key in TextMate, group name elsewhere.
127    pub name: String,
128    /// TextMate scope name. Other targets map it to their own convention.
129    pub scope: String,
130    #[serde(flatten)]
131    pub body: RuleBody,
132    /// Why this rule exists / what it deliberately excludes.
133    pub note: Option<String>,
134}
135
136impl Rule {
137    fn new(name: &str, scope: &str, body: RuleBody) -> Self {
138        Rule {
139            name: name.to_string(),
140            scope: scope.to_string(),
141            body,
142            note: None,
143        }
144    }
145
146    fn with_note(mut self, note: &str) -> Self {
147        self.note = Some(note.to_string());
148        self
149    }
150
151    /// The lexemes this rule contributes to the manifest's coverage set.
152    pub fn lexemes(&self) -> &[String] {
153        match &self.body {
154            RuleBody::Words { lexemes, .. } | RuleBody::Symbols { lexemes } | RuleBody::Annotations { lexemes } => {
155                lexemes
156            }
157            _ => &[],
158        }
159    }
160}
161
162#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
163pub struct SyntaxManifest {
164    pub schema_version: u32,
165    pub language: Language,
166    pub sigils: Vec<Sigil>,
167    /// Ordered; earlier rules take precedence in the generated grammars.
168    pub rules: Vec<Rule>,
169}
170
171fn words(v: &[&str]) -> RuleBody {
172    RuleBody::Words {
173        lexemes: v.iter().map(|s| s.to_string()).collect(),
174        followed_by: None,
175    }
176}
177
178/// A word the parser only ever matches glued to a delimiter.
179fn words_before(v: &[&str], followed_by: &str) -> RuleBody {
180    RuleBody::Words {
181        lexemes: v.iter().map(|s| s.to_string()).collect(),
182        followed_by: Some(followed_by.to_string()),
183    }
184}
185
186fn symbols(v: &[&str]) -> RuleBody {
187    RuleBody::Symbols {
188        lexemes: v.iter().map(|s| s.to_string()).collect(),
189    }
190}
191
192fn annotations(v: &[&str]) -> RuleBody {
193    RuleBody::Annotations {
194        lexemes: v.iter().map(|s| s.to_string()).collect(),
195    }
196}
197
198/// A raw regex rule, authored once in both dialects it has to serve.
199fn matches(pattern: &str, vim: &str) -> RuleBody {
200    RuleBody::Match {
201        pattern: pattern.to_string(),
202        vim: vim.to_string(),
203    }
204}
205
206impl Default for SyntaxManifest {
207    fn default() -> Self {
208        Self::new()
209    }
210}
211
212impl SyntaxManifest {
213    /// The manifest itself. Every entry below is a lexeme the parser in
214    /// `processes/parsing/` (or the tokenizer in `components/language/
215    /// operators.rs`) actually matches — nothing aspirational, nothing
216    /// inherited from another language's grammar.
217    pub fn new() -> Self {
218        SyntaxManifest {
219            schema_version: SCHEMA_VERSION,
220            language: Language {
221                id: "typr".to_string(),
222                name: "typR".to_string(),
223                scope_name: "source.typr".to_string(),
224                extensions: vec![".ty".to_string()],
225                line_comment: "#".to_string(),
226            },
227            sigils: Self::sigils(),
228            rules: Self::rules(),
229        }
230    }
231
232    /// RFC `ai_context/sigils.md` §3.2.1. Six are parsed today
233    /// (`parsing/types.rs::index_generic`, `label_generic`, `kinded_generic`),
234    /// three are reserved and must not reach a grammar.
235    fn sigils() -> Vec<Sigil> {
236        let live = |sigil: &str, kind: &str, suffix: &str| Sigil {
237            sigil: sigil.to_string(),
238            kind: kind.to_string(),
239            scope_suffix: suffix.to_string(),
240            reserved: false,
241            note: None,
242        };
243        let reserved = |sigil: &str| Sigil {
244            sigil: sigil.to_string(),
245            kind: "Reserved".to_string(),
246            scope_suffix: "reserved".to_string(),
247            reserved: true,
248            note: Some(
249                "Reserved by sigils.md §3.2.1 for a future kind, not parsed yet. \
250                 Already live as an operator, so generators must not emit it."
251                    .to_string(),
252            ),
253        };
254        vec![
255            live("#", "Number", "number"),
256            live("%", "Record", "record"),
257            live("@", "Interface", "interface"),
258            live("^", "String", "string"),
259            live("?", "Boolean", "boolean"),
260            live("$", "Label", "label"),
261            reserved("~"),
262            reserved("&"),
263            reserved("!"),
264        ]
265    }
266
267    fn rules() -> Vec<Rule> {
268        vec![
269            // Ordering is load-bearing throughout: every target grammar tries
270            // these in sequence at each position, so a shorter lexeme placed
271            // ahead of a longer one silently eats its prefix. This is exactly
272            // what a hand-written grammar cannot keep straight (the Vim plugin
273            // matched single-character operators before the kind sigils and
274            // swallowed them) and what generating from one ordered list fixes
275            // for every target at once.
276            Rule::new(
277                "strings.raw-r",
278                "string.quoted.other.raw.typr",
279                RuleBody::Span {
280                    begin: "r#\"".to_string(),
281                    end: "\"#".to_string(),
282                    escapes: false,
283                },
284            )
285            .with_note(
286                "`extern (...) -> T r#\"...\"#` raw R body — verbatim, no escape processing. \
287                 Ahead of `comments` so its inner `#` never opens one.",
288            ),
289            Rule::new(
290                "comments",
291                "comment.line.number-sign.typr",
292                matches(
293                    "#(?!(?:Self|[A-Z])(?![A-Za-z0-9_])).*$",
294                    "#%(%(Self|[A-Z])[A-Za-z0-9_]@!)@!.*$",
295                ),
296            )
297            .with_note(
298                "TypR comments are `#` only — never `//`, which the parser rejects outright \
299                 (`wrong_comment` in parsing/mod.rs). The lookahead is the one place a \
300                 regex grammar has to guess where the parser uses context: `#N` and `#Self` \
301                 are the Number kind sigil, so they are left to `types.sigil-generic`; \
302                 anything else after `#` opens a comment.",
303            ),
304            Rule::new(
305                "strings.double",
306                "string.quoted.double.typr",
307                RuleBody::Span {
308                    begin: "\"".to_string(),
309                    end: "\"".to_string(),
310                    escapes: true,
311                },
312            ),
313            Rule::new(
314                "strings.single",
315                "string.quoted.single.typr",
316                RuleBody::Span {
317                    begin: "'".to_string(),
318                    end: "'".to_string(),
319                    escapes: true,
320                },
321            ),
322            Rule::new(
323                "strings.backtick",
324                "string.quoted.other.backtick.typr",
325                RuleBody::Span {
326                    begin: "`".to_string(),
327                    end: "`".to_string(),
328                    escapes: false,
329                },
330            )
331            .with_note("R-style non-syntactic name (`parsing/elements.rs::quoted_variable`)."),
332            Rule::new(
333                "numbers.float",
334                "constant.numeric.float.typr",
335                matches("\\b[0-9]+\\.[0-9]+\\b", "<[0-9]+\\.[0-9]+>"),
336            )
337            .with_note("`parsing/elements.rs::number` — digits, a dot, digits. No exponent form exists."),
338            Rule::new(
339                "numbers.integer",
340                "constant.numeric.integer.typr",
341                matches("\\b[0-9]+\\b", "<[0-9]+>"),
342            ),
343            Rule::new(
344                "annotations",
345                "storage.modifier.annotation.typr",
346                annotations(&["@export", "@pub", "@testable", "@extern", "@importFrom"]),
347            )
348            .with_note(
349                "Visibility/interop annotations (RFC-TR-032). Ahead of the `@T` interface \
350                 sigil and of any bare-`@` rule, both of which would eat their `@`.",
351            ),
352            Rule::new(
353                "keywords.control",
354                "keyword.control.typr",
355                words(&["if", "else", "match", "for", "while", "loop", "break", "next", "return"]),
356            )
357            .with_note(
358                "No `continue`: the loop-skip keyword is `next;` (R spelling), \
359                 `parsing/elements.rs::next_exp`.",
360            ),
361            Rule::new(
362                "keywords.declaration",
363                "keyword.declaration.typr",
364                words(&[
365                    "let",
366                    "fn",
367                    "function",
368                    "type",
369                    "opaque",
370                    "typeconstructor",
371                    "recursive",
372                    "interface",
373                    "record",
374                    "object",
375                    "module",
376                    "mod",
377                    "import",
378                    "use",
379                    "extern",
380                    "embed",
381                ]),
382            )
383            .with_note(
384                "No `impl`/`trait`/`struct`/`enum`/`where`/`mut`/`pub`: none has ever existed \
385                 in TypR — they were Rust leftovers in the hand-written grammars. Visibility \
386                 is spelled with the `@pub`/`@export` annotations.",
387            ),
388            Rule::new("keywords.cast", "keyword.operator.cast.typr", words(&["as!", "as"]))
389                .with_note("`as!` first, or the `as` alternative eats its prefix."),
390            Rule::new(
391                "keywords.operator-word",
392                "keyword.operator.word.typr",
393                words(&["and", "or", "in"]),
394            )
395            .with_note("Word-spelled operators (`components/language/operators.rs::bool_op`/`op`)."),
396            Rule::new(
397                "constants",
398                "constant.language.typr",
399                words(&["true", "TRUE", "false", "FALSE", "null", "NULL", "na", "NA"]),
400            )
401            .with_note(
402                "Both spellings are real: `parsing/elements.rs` accepts the R form and the \
403                 lowercase TypR form. No `NaN`/`Inf` — the parser has a tag for neither.",
404            ),
405            Rule::new(
406                "types.primitive",
407                "support.type.primitive.typr",
408                words(&["int", "num", "char", "bool", "logic", "Any", "Empty", "Self"]),
409            )
410            .with_note("`logic` is the accepted alias of `bool` (`parsing/types.rs::boolean_type`)."),
411            Rule::new(
412                "types.builtin",
413                "support.type.builtin.typr",
414                words(&[
415                    "Vec",
416                    "Array",
417                    "Tuple",
418                    "Record",
419                    "UnknownFunction",
420                    "dataframe",
421                    "data.frame",
422                    "data__frame",
423                    "list",
424                    "tuple",
425                ]),
426            )
427            .with_note(
428                "No `Option`/`Result`/`List`/`Matrix`: TypR has none of them. These are the \
429                 names usable bare in a type position; the ones that only exist in front of \
430                 a delimiter are in `types.constructor` and `keywords.block`.",
431            ),
432            Rule::new(
433                "types.builtin-indexed",
434                "support.type.builtin.typr",
435                words_before(&["df"], "\\s*\\["),
436            )
437            .with_note(
438                "`df` is a type name the parser accepts bare too (`parsing/types.rs::dataframe_type`), \
439                 but `df` is also the single most common data-frame *variable* name in R. \
440                 Coloring it only in `df[...]` under-colors a bare `x: df` annotation; the \
441                 alternative over-colors every `df` in every program, which is worse. \
442                 `dataframe`, the unambiguous spelling, stays in `types.builtin`.",
443            ),
444            Rule::new(
445                "types.constructor",
446                "support.function.builtin.typr",
447                words_before(&["c", "seq", "Class", "library"], "\\s*[\\[(]"),
448            )
449            .with_note(
450                "The parser only ever matches these glued to their opening delimiter \
451                 (`c(`, `seq[`, `Class(`, `library(`), so the delimiter is part of the rule. \
452                 Without it, every variable named `c` would be colored as a builtin.",
453            ),
454            Rule::new(
455                "keywords.block",
456                "keyword.other.block.typr",
457                words_before(&["R", "JS", "Test"], "\\s*[\\[{]"),
458            )
459            .with_note(
460                "Escape-hatch and test block heads: `R { ... }`, `JS { ... }`, `Test { ... }`, \
461                 `Test[...]`. Delimiter-guarded for the same reason as `types.constructor` — \
462                 and `R` in particular is a single uppercase letter, i.e. also a valid \
463                 generic name, so an unguarded rule would fight `types.generic`.",
464            ),
465            Rule::new(
466                "types.variant",
467                "entity.name.type.variant.typr",
468                matches("\\.[A-Z][A-Za-z0-9_]*\\b", "\\.[A-Z][A-Za-z0-9_]*>"),
469            )
470            .with_note("Union-variant tag, `.Variant` (`parsing/elements.rs::tag_exp`)."),
471            Rule::new(
472                "types.sigil-generic",
473                "entity.name.type.parameter.typr",
474                RuleBody::Sigils,
475            )
476            .with_note(
477                "Sigil + generic. A generic name is a *single* uppercase letter or `Self` \
478                     (`parsing/types.rs::upper_case_generic`), so the rule stops there instead \
479                     of running on into a PascalCase alias.",
480            ),
481            Rule::new(
482                "types.generic",
483                "entity.name.type.parameter.typr",
484                matches("\\b[A-Z](?![A-Za-z0-9_])", "<[A-Z][A-Za-z0-9_]@!"),
485            ),
486            Rule::new(
487                "types.alias",
488                "entity.name.type.typr",
489                matches("\\b[A-Z][A-Za-z0-9_]+\\b", "<[A-Z][A-Za-z0-9_]+>"),
490            )
491            .with_note("PascalCase alias / type-constructor name (`parsing/types.rs::pascal_case_no_space`)."),
492            Rule::new("operators.arrow", "keyword.operator.arrow.typr", symbols(&["->", "=>"]))
493                .with_note("Ahead of comparison and arithmetic, which own `-`, `=` and `>`."),
494            Rule::new("operators.bind", "keyword.operator.assignment.typr", symbols(&["<-"]))
495                .with_note("Ahead of comparison, whose `<` would otherwise split `<-`."),
496            Rule::new(
497                "operators.comparison",
498                "keyword.operator.comparison.typr",
499                symbols(&["==", "!=", "<=", ">=", "<", ">"]),
500            ),
501            Rule::new(
502                "operators.logical",
503                "keyword.operator.logical.typr",
504                symbols(&["&&", "||", "&", "!"]),
505            )
506            .with_note(
507                "After comparison, so `!=` is not read as `!` then `=`. `|` belongs to \
508                 `operators.type-union`, which it doubles as.",
509            ),
510            Rule::new("operators.pipe", "keyword.operator.pipe.typr", symbols(&["|>"])).with_note(
511                "No `|>>`: it has no type-checking or transpiling arm and no stdlib \
512                     signature — see the tokenizer purge note on `operators.rs::op`.",
513            ),
514            Rule::new(
515                "operators.type-union",
516                "keyword.operator.type-union.typr",
517                matches("\\|(?!>)", "\\|\\>@!"),
518            ),
519            Rule::new(
520                "operators.custom",
521                "keyword.operator.custom.typr",
522                matches("%[^%\\s]*%", "\\%[^%\\s]*\\%"),
523            )
524            .with_note("R-style custom infix (`operators.rs::custom_op`). Ahead of the bare `%` modulo."),
525            Rule::new(
526                "operators.arithmetic",
527                "keyword.operator.arithmetic.typr",
528                symbols(&["+", "-", "*", "/", "%"]),
529            )
530            .with_note(
531                "No `^`/`++`/`--`/`**`/`//`: `op()` recognizes none of them. `^` is the String \
532                 kind sigil and nothing else — the hand-written grammars listed it as \
533                 exponentiation, which TypR does not have.",
534            ),
535            Rule::new(
536                "operators.vectorial-block",
537                "keyword.operator.vectorial.typr",
538                symbols(&["@{", "}@"]),
539            )
540            .with_note("`@{ ... }@` vectorized block (`parsing/elements.rs::vectorial_bloc`)."),
541            Rule::new(
542                "operators.spread",
543                "keyword.operator.spread.typr",
544                symbols(&["...", ".."]),
545            )
546            .with_note(
547                "`...` runtime spread / variadic, `..` nominal spread (`Point:{ ..source }`). \
548                     Longest-first inside the rule, and ahead of `operators.access`'s `.`.",
549            ),
550            Rule::new(
551                "operators.access",
552                "keyword.operator.access.typr",
553                symbols(&["::", "$", "."]),
554            ),
555            Rule::new("operators.assign", "keyword.operator.assignment.typr", symbols(&["="])).with_note(
556                "Bare `=` is never an infix operator in TypR — only a binder, a named-field \
557                     separator and a default-value separator. Last of the `=`-shaped rules so \
558                     `==`, `=>` and `!=` are already claimed.",
559            ),
560            Rule::new(
561                "operators.lambda",
562                "keyword.operator.lambda.typr",
563                matches("\\\\(?=[({:])", "\\\\[({:]@="),
564            )
565            .with_note("`\\(x) ...` lambda shorthand (`parsing/elements.rs::lambda`)."),
566            Rule::new(
567                "functions.call",
568                "entity.name.function.typr",
569                matches("\\b([a-z_][A-Za-z0-9_]*)\\s*(?=\\()", "<[a-z_][A-Za-z0-9_]*\\s*\\(@="),
570            )
571            .with_note("After every keyword rule, so `if (`/`while (` stay keywords."),
572            Rule::new(
573                "variables.parameter",
574                "variable.parameter.typr",
575                matches(
576                    "\\b([a-z_][A-Za-z0-9_]*)\\s*(?=:(?!:))",
577                    "<[a-z_][A-Za-z0-9_]*\\s*%(:%(:@!))@=",
578                ),
579            )
580            .with_note("`(?!:)` keeps the `x` of `x::y` out — that is namespace access, not an annotation."),
581            Rule::new(
582                "variables.other",
583                "variable.other.typr",
584                matches("\\b[a-z_][A-Za-z0-9_]*\\b", "<[a-z_][A-Za-z0-9_]*>"),
585            )
586            .with_note(
587                "An identifier starts lowercase-or-underscore and continues in `[A-Za-z0-9_]` — \
588                 no dots, unlike R (`parsing/elements.rs::starting_char`/`body_char`). \
589                 `data.frame` is a single dedicated tag, not an identifier.",
590            ),
591            Rule::new("punctuation.terminator", "punctuation.terminator.typr", symbols(&[";"])),
592            Rule::new(
593                "punctuation.separator",
594                "punctuation.separator.typr",
595                symbols(&[",", ":"]),
596            ),
597            Rule::new(
598                "punctuation.brackets",
599                "punctuation.section.brackets.typr",
600                matches("[\\[\\](){}]", "[][(){}]"),
601            ),
602        ]
603    }
604
605    /// Every lexeme the manifest claims, flattened — what the parser-coverage
606    /// test checks against.
607    pub fn all_lexemes(&self) -> Vec<&str> {
608        self.rules
609            .iter()
610            .flat_map(|r| r.lexemes())
611            .map(|s| s.as_str())
612            .collect()
613    }
614
615    pub fn to_json(&self) -> String {
616        serde_json::to_string_pretty(self).expect("the syntax manifest is plain data; it cannot fail to serialize")
617    }
618}
619
620#[cfg(test)]
621mod tests {
622    use super::*;
623    use std::collections::HashSet;
624
625    /// The parser sources scanned for `tag("…")` literals. Everything that
626    /// introduces a *word* of TypR syntax lives in one of these.
627    const PARSER_SOURCES: &[(&str, &str)] = &[
628        (
629            "parsing/elements.rs",
630            include_str!("../../processes/parsing/elements.rs"),
631        ),
632        ("parsing/mod.rs", include_str!("../../processes/parsing/mod.rs")),
633        ("parsing/types.rs", include_str!("../../processes/parsing/types.rs")),
634        (
635            "parsing/indexation.rs",
636            include_str!("../../processes/parsing/indexation.rs"),
637        ),
638        ("language/operators.rs", include_str!("../language/operators.rs")),
639    ];
640
641    /// Word-like `tag("…")` literals that deliberately have no manifest
642    /// entry. Every exemption needs a reason; the list is meant to stay tiny.
643    const NOT_A_LEXEME: &[(&str, &str)] = &[
644        (
645            "@",
646            "Bare `@` heads a module signature declaration (`@name: T;`) and prefixes \
647             the annotations. Painting a lone `@` would break `@export` and the `@T` \
648             interface sigil, both of which the manifest already covers.",
649        ),
650        (
651            "logical",
652            "Not TypR syntax: `RIndex::Logical`'s parser in parsing/indexation.rs \
653             matches R's `TRUE`/`FALSE`, and this literal is only in a rustdoc line.",
654        ),
655    ];
656
657    /// Pulls every `tag("…")` string literal out of a Rust source, handling
658    /// the `\"` escapes in literals like `tag("r#\"")`.
659    fn scan_tag_literals(source: &str) -> Vec<String> {
660        let mut out = Vec::new();
661        let bytes: Vec<char> = source.chars().collect();
662        let needle: Vec<char> = "tag(\"".chars().collect();
663        let mut i = 0;
664        while i + needle.len() <= bytes.len() {
665            if bytes[i..i + needle.len()] != needle[..] {
666                i += 1;
667                continue;
668            }
669            // `tag_no_case(` / `stag(` etc. would end in the same suffix; require
670            // the character before `tag` to not be an identifier character.
671            let preceded_by_ident = i > 0 && (bytes[i - 1].is_alphanumeric() || bytes[i - 1] == '_');
672            if preceded_by_ident {
673                i += 1;
674                continue;
675            }
676            let mut j = i + needle.len();
677            let mut literal = String::new();
678            while j < bytes.len() {
679                match bytes[j] {
680                    '\\' if j + 1 < bytes.len() => {
681                        literal.push(bytes[j + 1]);
682                        j += 2;
683                    }
684                    '"' => break,
685                    c => {
686                        literal.push(c);
687                        j += 1;
688                    }
689                }
690            }
691            out.push(literal);
692            i = j.max(i + 1);
693        }
694        out
695    }
696
697    /// A parser tag carries its delimiter along (`Array[`, `library(`,
698    /// `break;`, `return `) because that is what the combinator consumes.
699    /// The manifest stores the bare word.
700    fn normalize(literal: &str) -> String {
701        literal.trim_end_matches(['(', '[', ';', ' ']).to_string()
702    }
703
704    /// Word-like: the shape the manifest is responsible for. Symbols
705    /// (`->`, `{`, `..`, `\`) are covered by hand-written symbol rules and
706    /// are not part of this check.
707    fn is_word_like(lexeme: &str) -> bool {
708        let mut chars = lexeme.chars();
709        match chars.next() {
710            Some(c) if c.is_ascii_alphabetic() || c == '@' => {}
711            _ => return false,
712        }
713        let rest: Vec<char> = chars.collect();
714        let (body, _) = match rest.split_last() {
715            Some((&'!', body)) => (body, true),
716            _ => (&rest[..], false),
717        };
718        body.iter().all(|c| c.is_ascii_alphanumeric() || *c == '_' || *c == '.')
719    }
720
721    /// The invariant this whole module exists to enforce: adding a keyword to
722    /// the parser without adding it to the manifest fails here, instead of
723    /// silently leaving four editor grammars behind.
724    #[test]
725    fn every_parser_tag_is_in_the_manifest() {
726        let manifest = SyntaxManifest::new();
727        let known: HashSet<&str> = manifest.all_lexemes().into_iter().collect();
728        let exempt: HashSet<&str> = NOT_A_LEXEME.iter().map(|(l, _)| *l).collect();
729
730        let mut missing: Vec<(String, &str)> = Vec::new();
731        for (file, source) in PARSER_SOURCES {
732            for literal in scan_tag_literals(source) {
733                let lexeme = normalize(&literal);
734                if lexeme.is_empty() || !is_word_like(&lexeme) {
735                    continue;
736                }
737                if known.contains(lexeme.as_str()) || exempt.contains(lexeme.as_str()) {
738                    continue;
739                }
740                missing.push((lexeme, file));
741            }
742        }
743        missing.sort();
744        missing.dedup();
745
746        assert!(
747            missing.is_empty(),
748            "these lexemes are parsed by TypR but absent from the syntax manifest, so no \
749             editor would color them:\n{}\n\nAdd each to a rule in \
750             components/syntax/mod.rs (or to NOT_A_LEXEME with a reason), then regenerate \
751             the grammars with `typr syntax --write`.",
752            missing
753                .iter()
754                .map(|(lexeme, file)| format!("  - `{lexeme}` (from {file})"))
755                .collect::<Vec<_>>()
756                .join("\n")
757        );
758    }
759
760    /// The other direction: the manifest must not invent syntax. This is the
761    /// bug that started the project — `impl`, `trait`, `struct`, `enum`,
762    /// `where`, `mut`, `pub`, `continue`, `Option`, `Result`, `List`,
763    /// `Matrix`, `Some`, `None`, `Ok`, `Err` were colored by VSCode and the
764    /// Playground while existing nowhere in TypR.
765    #[test]
766    fn manifest_claims_no_word_the_parser_does_not_know() {
767        let mut parsed: HashSet<String> = HashSet::new();
768        for (_, source) in PARSER_SOURCES {
769            for literal in scan_tag_literals(source) {
770                parsed.insert(normalize(&literal));
771            }
772        }
773        // Words the grammar reaches through a dedicated combinator rather
774        // than a `tag()`, so the scan above cannot see them.
775        for extra in ["function", "and", "or"] {
776            parsed.insert(extra.to_string());
777        }
778
779        let manifest = SyntaxManifest::new();
780        let invented: Vec<&str> = manifest
781            .all_lexemes()
782            .into_iter()
783            .filter(|l| is_word_like(l))
784            .filter(|l| !parsed.contains(*l))
785            .collect();
786
787        assert!(
788            invented.is_empty(),
789            "the manifest colors words TypR does not have: {invented:?}. \
790             Remove them — this is exactly the Rust-keyword copy-paste the manifest replaced."
791        );
792    }
793
794    #[test]
795    fn reserved_sigils_are_flagged() {
796        let manifest = SyntaxManifest::new();
797        let reserved: Vec<&str> = manifest
798            .sigils
799            .iter()
800            .filter(|s| s.reserved)
801            .map(|s| s.sigil.as_str())
802            .collect();
803        // sigils.md §3.2.1 reserves these three; all three are live operators
804        // today, so a generator emitting them would mis-color real code.
805        assert_eq!(reserved, vec!["~", "&", "!"]);
806        assert_eq!(manifest.sigils.iter().filter(|s| !s.reserved).count(), 6);
807    }
808
809    #[test]
810    fn rule_names_are_unique() {
811        let manifest = SyntaxManifest::new();
812        let mut names: Vec<&str> = manifest.rules.iter().map(|r| r.name.as_str()).collect();
813        let count = names.len();
814        names.sort();
815        names.dedup();
816        assert_eq!(names.len(), count, "rule names double as TextMate repository keys");
817    }
818}
typr_core/components/syntax/mod.rs

typr_core/components/syntax/
mod.rs
