rustyfi_syntax/

leaf.rs

//! Leaf token types and the delimiter `Group` aliases.
//!
//! The unit and single-field leaves (`KwLet`, `VarTok`, `IntTok`, ...) are
//! generated by `token_leaves!` in [`crate::token`] and re-exported here (so
//! `use crate::leaf::*` still names them). This module keeps only what that
//! macro cannot express — the multi-field qualified-name leaves,
//! `LengthTok`/`LiteralTok`, the multi-variant `BinOpTok`, and the
//! `AnyHorzCmdTok`/`AnyVertCmdTok` alternations — plus the delimiter `Group`
//! aliases, which are the core atom-generic `syan::nested::group::Group`.

use crate::span::Span;
// Re-export the generated leaves (and `Token`/`Atom`) so downstream
// `use crate::leaf::*` resolves every leaf type from one place.
pub use crate::token::*;
use syan::error::ParseError;
use syan::parse::unparse::Emitter;
use syan::parse::{Parse, ParseStream, Unparse};
use syan::span::Spanned;

/// Multi-field qualified-name leaves: a `(Vec<String>, String)` payload
/// (module path + bare name). `token_leaves!` supports only unit and
/// single-field variants, so these stay hand-written — same peek → match →
/// push-back-on-mismatch shape as the generated leaves.
macro_rules! qualified_name_tokens {
    ($($(#[$doc:meta])* $name:ident => $variant:ident, $desc:literal;)*) => {
        $(
            $(#[$doc])*
            #[derive(Clone, Debug, PartialEq)]
            pub struct $name {
                pub mods: Vec<String>,
                pub name: String,
                pub span: Span,
            }

            impl Parse<Atom> for $name {
                type Error = ParseError<Span>;

                fn parse_stream<S: ParseStream<Atom = Atom>>(
                        stream: &mut S,
                    ) -> Result<Self, Self::Error> {
                    match stream.next() {
                        Some(Atom { slot: Token::$variant(mods, name), span }) => {
                            Ok($name { mods, name, span })
                        }
                        Some(atom) => {
                            let span = atom.span;
                            stream.push(atom);
                            Err(ParseError::expected(span, $desc))
                        }
                        None => Err(ParseError::eof(Span::default())),
                    }
                }
            }

            impl Unparse<Atom> for $name {
                fn unparse<S: Emitter<Atom>>(&self, sink: &mut S) -> Result<(), S::Error> {
                    sink.write_one(Atom {
                        slot: Token::$variant(self.mods.clone(), self.name.clone()),
                        span: self.span,
                    })
                }
            }

            impl Spanned for $name {
                type Span = Span;
                fn span(&self) -> Span {
                    self.span
                }
            }
        )*
    };
}

qualified_name_tokens! {
    /// `#var` (or `#Mod.var`) in inline text, before the mode switches to active.
    VarInHorzTok => VarInHorz, "a variable reference in inline text";
    /// `#var` (or `#Mod.var`) in block text, before the mode switches to active.
    VarInVertTok => VarInVert, "a variable reference in block text";
    /// `#var` (or `#Mod.var`) in math mode.
    VarInMathTok => VarInMath, "a variable reference in math";
    /// A module-qualified variable, e.g. `Mod.x`.
    VarWithModTok => VarWithMod, "a qualified variable name";
    /// A dotted module path ending in an UPPER segment, e.g. `A.B.C`
    /// (upstream `LONG_UPPER`) — module-expression paths, functor
    /// application operands, and signature paths (`mod_chain`,
    /// `sigexpr_bot`). V0_1-only (a lex error under V0_0).
    LongUpperTok => LongUpper, "a qualified module path";
    /// A module-qualified inline command, e.g. `\Mod.cmd`.
    HorzCmdWithModTok => HorzCmdWithMod, "a qualified inline command";
    /// A module-qualified block command, e.g. `+Mod.cmd`.
    VertCmdWithModTok => VertCmdWithMod, "a qualified block command";
    /// A module-qualified math command, e.g. `\Mod.cmd` in math mode.
    MathCmdWithModTok => MathCmdWithMod, "a qualified math command";
}

/// Either sigil-only (`\cmd`) or module-qualified (`\Mod.cmd`) inline command
/// name — `hcmd` in `parser.mly`. Not itself recursive, so (unlike
/// [`crate::cst::ast::InlineElem`]) it can be a plain top-level derive.
#[derive(Parse, Unparse, Debug, Clone, PartialEq)]
pub enum AnyHorzCmdTok {
    Plain(HorzCmdTok),
    Mod(HorzCmdWithModTok),
}

/// Either sigil-only (`+cmd`) or module-qualified (`+Mod.cmd`) block command
/// name — `vcmd` in `parser.mly`.
#[derive(Parse, Unparse, Debug, Clone, PartialEq)]
pub enum AnyVertCmdTok {
    Plain(VertCmdTok),
    Mod(VertCmdWithModTok),
}

/// Either sigil-only (`\cmd`) or module-qualified (`\Mod.cmd`) math
/// command name in math mode — the math analogue of `AnyHorzCmdTok`.
#[derive(Parse, Unparse, Debug, Clone, PartialEq)]
pub enum AnyMathCmdTok {
    Plain(MathCmdTok),
    Mod(MathCmdWithModTok),
}

/// A length constant such as `12pt` (two payload fields, hand-written).
#[derive(Clone, Debug, PartialEq)]
pub struct LengthTok {
    pub value: f64,
    pub unit: String,
    pub span: Span,
}

impl Parse<Atom> for LengthTok {
    type Error = ParseError<Span>;

    fn parse_stream<S: ParseStream<Atom = Atom>>(
            stream: &mut S,
        ) -> Result<Self, Self::Error> {
        match stream.next() {
            Some(Atom {
                slot: Token::LengthConst(value, unit),
                span,
            }) => Ok(LengthTok { value, unit, span }),
            Some(atom) => {
                let span = atom.span;
                stream.push(atom);
                Err(ParseError::expected(span, "a length constant"))
            }
            None => Err(ParseError::eof(Span::default())),
        }
    }
}

impl Unparse<Atom> for LengthTok {
    fn unparse<S: Emitter<Atom>>(&self, sink: &mut S) -> Result<(), S::Error> {
        sink.write_one(Atom {
            slot: Token::LengthConst(self.value, self.unit.clone()),
            span: self.span,
        })
    }
}

impl Spanned for LengthTok {
    type Span = Span;
    fn span(&self) -> Span {
        self.span
    }
}

/// A backtick string literal with its space-trimming flags.
#[derive(Clone, Debug, PartialEq)]
pub struct LiteralTok {
    pub body: String,
    pub omit_pre: bool,
    pub omit_post: bool,
    pub span: Span,
}

impl Parse<Atom> for LiteralTok {
    type Error = ParseError<Span>;

    fn parse_stream<S: ParseStream<Atom = Atom>>(
            stream: &mut S,
        ) -> Result<Self, Self::Error> {
        match stream.next() {
            Some(Atom {
                slot:
                    Token::Literal {
                        body,
                        omit_pre,
                        omit_post,
                    },
                span,
            }) => Ok(LiteralTok {
                body,
                omit_pre,
                omit_post,
                span,
            }),
            Some(atom) => {
                let span = atom.span;
                stream.push(atom);
                Err(ParseError::expected(span, "a string literal"))
            }
            None => Err(ParseError::eof(Span::default())),
        }
    }
}

impl Unparse<Atom> for LiteralTok {
    fn unparse<S: Emitter<Atom>>(&self, sink: &mut S) -> Result<(), S::Error> {
        sink.write_one(Atom {
            slot: Token::Literal {
                body: self.body.clone(),
                omit_pre: self.omit_pre,
                omit_post: self.omit_post,
            },
            span: self.span,
        })
    }
}

impl Spanned for LiteralTok {
    type Span = Span;
    fn span(&self) -> Span {
        self.span
    }
}

/// A standalone `*` (`EXACT_TIMES` in `parser.mly`) in *type*-expression
/// position — the product-type separator (`type point = length * length`,
/// `cst.rs`'s `TypeProd`). Hand-written (not `#[leaf(...)]`-generated on
/// `Token::ExactTimes` in `token.rs`) because `Token::ExactTimes` already
/// doubles as one of `BinOpTok`'s matched variants for the *expression*-level
/// `*`; this leaf lets `TypeProd` consume the same token without pulling in
/// the rest of the binop set, mirroring `LengthTok`'s boilerplate.
#[derive(Clone, Debug, PartialEq)]
pub struct ExactTimesTok {
    pub span: Span,
}

impl Parse<Atom> for ExactTimesTok {
    type Error = ParseError<Span>;

    fn parse_stream<S: ParseStream<Atom = Atom>>(
            stream: &mut S,
        ) -> Result<Self, Self::Error> {
        match stream.next() {
            Some(Atom {
                slot: Token::ExactTimes,
                span,
            }) => Ok(ExactTimesTok { span }),
            Some(atom) => {
                let span = atom.span;
                stream.push(atom);
                Err(ParseError::expected(span, "'*'"))
            }
            None => Err(ParseError::eof(Span::default())),
        }
    }
}

impl Unparse<Atom> for ExactTimesTok {
    fn unparse<S: Emitter<Atom>>(&self, sink: &mut S) -> Result<(), S::Error> {
        sink.write_one(Atom {
            slot: Token::ExactTimes,
            span: self.span,
        })
    }
}

impl Spanned for ExactTimesTok {
    type Span = Span;
    fn span(&self) -> Span {
        self.span
    }
}

/// A binary-operator token: the `binop` family of `nxlor`..`nxrtimes` plus the
/// `mod`/`::` operators that also act as binops (`parser.mly`'s `binop`
/// nonterminal, minus `UNOP_EXCLAM`/`BEFORE`/`LNOT`, which are not simple
/// infix operators in this grammar's flattened operator-chain shape).
/// Deliberately excludes `Token::Bar` (match-arm separator) and
/// `Token::ExactAmp` (the `&`-prefixed "next" unary operator) — see
/// `cst.rs`'s note on `Bar` handling.
#[derive(Clone, Debug, PartialEq)]
pub struct BinOpTok {
    pub tok: Token,
    pub span: Span,
}

impl Parse<Atom> for BinOpTok {
    type Error = ParseError<Span>;

    fn parse_stream<S: ParseStream<Atom = Atom>>(
            stream: &mut S,
        ) -> Result<Self, Self::Error> {
        match stream.next() {
            Some(Atom { slot, span })
                if matches!(
                    slot,
                    Token::BinopPlus(_)
                        | Token::BinopMinus(_)
                        | Token::BinopTimes(_)
                        | Token::BinopDivides(_)
                        | Token::BinopEq(_)
                        | Token::BinopLt(_)
                        | Token::BinopGt(_)
                        | Token::BinopAmp(_)
                        | Token::BinopBar(_)
                        | Token::BinopHat(_)
                        | Token::ExactMinus
                        | Token::ExactTimes
                        | Token::Mod
                        | Token::Cons
                ) =>
            {
                Ok(BinOpTok { tok: slot, span })
            }
            Some(atom) => {
                let span = atom.span;
                stream.push(atom);
                Err(ParseError::expected(span, "a binary operator"))
            }
            None => Err(ParseError::eof(Span::default())),
        }
    }
}

impl Unparse<Atom> for BinOpTok {
    fn unparse<S: Emitter<Atom>>(&self, sink: &mut S) -> Result<(), S::Error> {
        sink.write_one(Atom {
            slot: self.tok.clone(),
            span: self.span,
        })
    }
}

impl Spanned for BinOpTok {
    type Span = Span;
    fn span(&self) -> Span {
        self.span
    }
}

impl BinOpTok {
    /// The operator's source text, e.g. `"+"`, `"mod"`, `"::"`.
    pub fn op_text(&self) -> String {
        match &self.tok {
            Token::BinopPlus(s)
            | Token::BinopMinus(s)
            | Token::BinopTimes(s)
            | Token::BinopDivides(s)
            | Token::BinopEq(s)
            | Token::BinopLt(s)
            | Token::BinopGt(s)
            | Token::BinopAmp(s)
            | Token::BinopBar(s)
            | Token::BinopHat(s) => s.clone(),
            Token::ExactMinus => "-".to_string(),
            Token::ExactTimes => "*".to_string(),
            Token::Mod => "mod".to_string(),
            Token::Cons => "::".to_string(),
            _ => unreachable!("BinOpTok only ever holds one of the matched variants"),
        }
    }
}

/// The operator token accepted inside a `( ‹op› )` NAMING form (see
/// [`OpNameTok`]) — [`BinOpTok`]'s whole alternative set, PLUS
/// `!`/`before`. Upstream `parser.mly`'s `binop` nonterminal is used in
/// exactly two productions, both naming forms (`VAL LPAREN binop RPAREN`
/// and `LPAREN binop RPAREN` as a bare atomic-expression reference) —
/// *never* for infix chaining — and it accepts `UNOP_EXCLAM`/`BEFORE`/
/// `LNOT` there, unlike this grammar's flattened infix operator chain
/// (`cst.rs`'s `OpRhs`, which uses [`BinOpTok`] directly and correctly
/// excludes them: `!`/`before` are not simple infix operators here).
/// `LNOT` (`not`) has no reserved keyword token in this port at all (see the
/// `not` note beside [`crate::token::Token::Mod`]): it lexes as an ordinary
/// `VarTok`, so `let not x = ..`/`val not : ty` (unparenthesized) already
/// parse via the plain-name arm of `cst.rs`'s `BindName`/`ast::Atomic::
/// OpRef`'s sibling `Var` arm, with no token to add here. Kept as its own
/// struct (mirroring
/// [`BinOpTok`]'s shape) rather than widening `BinOpTok` itself, so the
/// infix-chain grammar's exclusion stays intact by construction — nothing
/// downstream of [`BinOpTok::parse`] can ever see `!`/`before`.
#[derive(Clone, Debug, PartialEq)]
pub struct NamingOpTok {
    pub tok: Token,
    pub span: Span,
}

impl Parse<Atom> for NamingOpTok {
    type Error = ParseError<Span>;

    fn parse_stream<S: ParseStream<Atom = Atom>>(
            stream: &mut S,
        ) -> Result<Self, Self::Error> {
        match stream.next() {
            Some(Atom { slot, span })
                if matches!(
                    slot,
                    Token::BinopPlus(_)
                        | Token::BinopMinus(_)
                        | Token::BinopTimes(_)
                        | Token::BinopDivides(_)
                        | Token::BinopEq(_)
                        | Token::BinopLt(_)
                        | Token::BinopGt(_)
                        | Token::BinopAmp(_)
                        | Token::BinopBar(_)
                        | Token::BinopHat(_)
                        | Token::ExactMinus
                        | Token::ExactTimes
                        | Token::Mod
                        | Token::Cons
                        | Token::UnopExclam(_)
                        | Token::Before
                ) =>
            {
                Ok(NamingOpTok { tok: slot, span })
            }
            Some(atom) => {
                let span = atom.span;
                stream.push(atom);
                Err(ParseError::expected(
                    span,
                    "a binary operator, '!', or 'before'",
                ))
            }
            None => Err(ParseError::eof(Span::default())),
        }
    }
}

impl Unparse<Atom> for NamingOpTok {
    fn unparse<S: Emitter<Atom>>(&self, sink: &mut S) -> Result<(), S::Error> {
        sink.write_one(Atom {
            slot: self.tok.clone(),
            span: self.span,
        })
    }
}

impl Spanned for NamingOpTok {
    type Span = Span;
    fn span(&self) -> Span {
        self.span
    }
}

impl NamingOpTok {
    /// The operator's source text, e.g. `"+"`, `"mod"`, `"::"`, `"!"`,
    /// `"before"` — [`BinOpTok::op_text`] plus the two naming-only
    /// additions.
    pub fn op_text(&self) -> String {
        match &self.tok {
            Token::BinopPlus(s)
            | Token::BinopMinus(s)
            | Token::BinopTimes(s)
            | Token::BinopDivides(s)
            | Token::BinopEq(s)
            | Token::BinopLt(s)
            | Token::BinopGt(s)
            | Token::BinopAmp(s)
            | Token::BinopBar(s)
            | Token::BinopHat(s)
            | Token::UnopExclam(s) => s.clone(),
            Token::ExactMinus => "-".to_string(),
            Token::ExactTimes => "*".to_string(),
            Token::Mod => "mod".to_string(),
            Token::Cons => "::".to_string(),
            Token::Before => "before".to_string(),
            _ => unreachable!("NamingOpTok only ever holds one of the matched variants"),
        }
    }
}

/// `( ‹op› )` — a parenthesized (possibly user-defined) operator name.
/// Two surface uses share this leaf: a binding-position NAME (`cst.rs`'s
/// `BindName`, e.g. `let (+++>) = ..` / `val (-->) : ty` — the gap blocking
/// `itemize.satyh`/`progsynt.satyh`) and a bare atomic-expression reference
/// to an operator as a first-class value (`cst.rs`'s `ast::Atomic::OpRef`,
/// e.g. `(+++)`). `.name` is the operator's text (`NamingOpTok::op_text`);
/// `.span` covers the whole `( .. )`, delimiters included. Both are
/// precomputed here (rather than left to the `Spanned` trait) so a
/// downstream crate with no direct `syan` dependency (`rustyfi-lang`) can
/// read them as plain fields, exactly like every other leaf's `.name`/
/// `.span`. Hand-written (not `#[leaf(...)]`-generated) for the same reason
/// as [`BinOpTok`]: it spans three atoms, not one. The inner operator is a
/// [`NamingOpTok`] (not a bare [`BinOpTok`]) so `(!)`/`(before)` — valid
/// only in this naming position, per upstream's `binop` nonterminal — also
/// parse; see [`NamingOpTok`]'s doc comment.
#[derive(Clone, Debug, PartialEq)]
pub struct OpNameTok {
    pub name: String,
    pub span: Span,
    pub lparen: LParenTok,
    pub op: NamingOpTok,
    pub rparen: RParenTok,
}

impl Parse<Atom> for OpNameTok {
    type Error = ParseError<Span>;

    fn parse_stream<S: ParseStream<Atom = Atom>>(
            stream: &mut S,
        ) -> Result<Self, Self::Error> {
        let lparen = LParenTok::parse_stream(&mut *stream)?;
        let op = NamingOpTok::parse_stream(&mut *stream)?;
        let rparen = RParenTok::parse_stream(&mut *stream)?;
        let name = op.op_text();
        let span = lparen.span().unite(rparen.span());
        Ok(OpNameTok {
            name,
            span,
            lparen,
            op,
            rparen,
        })
    }
}

impl Unparse<Atom> for OpNameTok {
    fn unparse<S: Emitter<Atom>>(&self, sink: &mut S) -> Result<(), S::Error> {
        self.lparen.unparse(sink)?;
        self.op.unparse(sink)?;
        self.rparen.unparse(sink)
    }
}

impl Spanned for OpNameTok {
    type Span = Span;
    fn span(&self) -> Span {
        self.span
    }
}

/// `@stage: persistent` / `@stage: 0` / `@stage: 1` header token
/// (`cst.rs`'s `Header::Stage`). Hand-written like [`BinOpTok`] above: the
/// three spellings are separate unit `Token` variants with no shared
/// payload for a `#[leaf(...)]` derive to key off, so this matches any of
/// them and keeps whichever one matched (needed for a lossless round-trip —
/// `Unparse` replays exactly the token that was read).
#[derive(Clone, Debug, PartialEq)]
pub struct HeaderStageTok {
    pub tok: Token,
    pub span: Span,
}

impl Parse<Atom> for HeaderStageTok {
    type Error = ParseError<Span>;

    fn parse_stream<S: ParseStream<Atom = Atom>>(
            stream: &mut S,
        ) -> Result<Self, Self::Error> {
        match stream.next() {
            Some(Atom { slot, span })
                if matches!(
                    slot,
                    Token::HeaderStage0 | Token::HeaderStage1 | Token::HeaderPersistent0
                ) =>
            {
                Ok(HeaderStageTok { tok: slot, span })
            }
            Some(atom) => {
                let span = atom.span;
                stream.push(atom);
                Err(ParseError::expected(span, "'@stage:'"))
            }
            None => Err(ParseError::eof(Span::default())),
        }
    }
}

impl Unparse<Atom> for HeaderStageTok {
    fn unparse<S: Emitter<Atom>>(&self, sink: &mut S) -> Result<(), S::Error> {
        sink.write_one(Atom {
            slot: self.tok.clone(),
            span: self.span,
        })
    }
}

impl Spanned for HeaderStageTok {
    type Span = Span;
    fn span(&self) -> Span {
        self.span
    }
}

// ---- delimiter groups ---------------------------------------------------------

// Atom-generic groups: `syan::nested::group::Group<T, Open, Close>` supplies
// `Parse`/`Unparse` (flat-family blankets over the leaf open/close atoms),
// `Spanned` (delimiter-only, so empty groups still have a span), `Deref` to the
// slot, and `EmptyGroup for Group<(), _, _>`. The field names the grammar reads
// are `.open`/`.slot`/`.close`.

/// `( … )` in program mode.
pub type ParenGroup<T> = syan::nested::group::Group<T, LParenTok, RParenTok>;
/// `(| … |)` record.
pub type RecordGroup<T> = syan::nested::group::Group<T, BRecordTok, ERecordTok>;
/// `[ … ]` list.
pub type ListGroup<T> = syan::nested::group::Group<T, BListTok, EListTok>;
/// `{ … }` inline text.
pub type InlineGroup<T> = syan::nested::group::Group<T, BHorzGrpTok, EHorzGrpTok>;
/// `'< … >` / `< … >` block text.
pub type BlockGroup<T> = syan::nested::group::Group<T, BVertGrpTok, EVertGrpTok>;
/// `${ … }` / `{ … }` math (the latter when already inside math mode).
pub type MathGroup<T> = syan::nested::group::Group<T, BMathGrpTok, EMathGrpTok>;
/// `Mod.( … )` — the open-module-scope expression (`cst.rs`'s
/// `Atomic::OpenModule`). The open delimiter (`OpenModuleTok`, carrying the
/// module name) is `Mod.(`; the close is a plain `)` (`RParenTok`, exactly
/// like [`ParenGroup`]'s).
pub type OpenModuleGroup<T> = syan::nested::group::Group<T, OpenModuleTok, RParenTok>;

/// `Unparse` for an EMPTY group used as an ordinary field rather than as the
/// target of a `#[group(self.x)]` — `PatBot::Unit { paren: ParenGroup<()> }`
/// and its three siblings, which spell `()` with nothing inside.
///
/// syan core has a generic `Parse` for `Group<T, O, C>` but no generic
/// `Unparse`: the only `Unparse for Group<..>` impls there are the
/// `proc_macro2` ones, where a delimited group is a single `TokenTree` rather
/// than three atoms. A `#[group(..)]` holder never needs it (it is emitted
/// through `GroupUnparse::unparse_group`), so the gap only shows up for a
/// holder standing alone as a field, where `#[derive(Unparse)]` synthesizes an
/// ordinary `FieldTy: Unparse<Atom>` predicate.
///
/// Hence a local node rather than an `impl Unparse<Atom> for ParenGroup<()>`:
/// `Atom` is `syan::span::WithSpan<Token, Span>`, an alias for a FOREIGN type,
/// so nothing in that impl would be local and the orphan rule rejects it
/// (E0117). The fields keep `Group`'s `open`/`close` names, the derived
/// `Parse` is the same two-token sequence `Group`'s generic `Parse` produces,
/// and the delimiters emit in source order, so `parse . unparse` still
/// round-trips.
#[derive(Parse, Unparse, Debug, Clone, PartialEq)]
pub struct UnitParen {
    pub open: LParenTok,
    pub close: RParenTok,
}

impl Spanned for UnitParen {
    type Span = Span;
    fn span(&self) -> Span {
        self.open.span().unite(self.close.span())
    }
}

// ---- PartialEq bridge for the generated leaves --------------------------------

// `token_leaves!` emits only `Clone` + `Debug` per leaf, but the surface
// grammar (`cst.rs`) derives `PartialEq` on the nodes that embed these leaves
// (and the `Group` aliases need their delimiter leaves `PartialEq` too), so
// every generated leaf must also be `PartialEq` — span included.
macro_rules! leaf_eq {
    (
        span_only: $($unit:ident),* $(,)?;
        with_fields: $($payload:ident { $($field:ident),* }),* $(,)?
    ) => {
        $(
            impl PartialEq for $unit {
                fn eq(&self, other: &Self) -> bool {
                    self.0 == other.0
                }
            }
        )*
        $(
            impl PartialEq for $payload {
                fn eq(&self, other: &Self) -> bool {
                    $(self.$field == other.$field &&)* self.span == other.span
                }
            }
        )*
    };
}

leaf_eq! {
    span_only:
        KwLet, KwLetRec, KwLetHorz, KwLetVert, KwLetMath, KwAnd, KwIn, KwFun,
        KwIf, KwThen, KwElse, KwTrue, KwFalse, ArrowTok, DefEqTok, ListPunctTok,
        CommaTok, LParenTok, RParenTok, BRecordTok, ERecordTok, BListTok, EListTok,
        BHorzGrpTok, EHorzGrpTok, BVertGrpTok, EVertGrpTok, SpaceTok, BreakTok,
        EndActiveTok, EoiTok, KwMatch, KwWith, KwWhen, KwAs, KwType, KwOf, BarTok,
        WildcardTok, ConsTok, ColonTok, ExactMinusTok, KwLetMutable, KwWhile, KwDo,
        KwBefore, OverwriteEqTok, AccessTok, KwModule, KwStruct, KwSig, KwEnd,
        KwOpen, KwVal, KwDirect, OptionalTok, OmissionTok, SuperscriptTok,
        SubscriptTok, SepTok, BMathGrpTok, EMathGrpTok, HorzCmdTypeTok,
        VertCmdTypeTok, MathCmdTypeTok, OptionalTypeTok, OptionalArrowTok,
        ConstraintTok, CommandTok, KwRec, KwInline, KwBlock, KwMutable,
        CoerceTok, KwSignature, KwInclude, KwUse, KwPackage, KwMath,
        KwPersistent, ExactAmpTok, ExactTildeTok;
    with_fields:
        VarTok { name }, CtorTok { name }, IntTok { value }, FloatTok { value },
        HorzCmdTok { name }, VertCmdTok { name }, CharTok { text }, ItemTok { depth },
        CodeTextTok { text },
        HeaderRequireTok { content }, HeaderImportTok { content }, TypeVarTok { name },
        UnopExclamTok { text }, MathCharTok { text }, MathCmdTok { name },
        PrimesTok { count }, OpenModuleTok { name }, RowVarTok { name }
}