lang_forge/
kind.rs

1//! [`Kind`]: the one kind type every forged language uses for its tokens and
2//! nodes.
3
4use core::fmt;
5
6use syntax_lang::TokenKind;
7
8/// The kind of a token or node in a forged language's syntax tree.
9///
10/// A schematic declares its vocabulary as text — rule names, keywords,
11/// punctuation — so a forged language cannot have a Rust `enum` of its own.
12/// `Kind` stands in for one: a small `Copy` value, assigned when the language is
13/// forged, that compares as cheaply as an enum discriminant. Tokens and nodes
14/// share the type, the model [`syntax_lang`] is built on.
15///
16/// Kinds are looked up by name with [`Language::kind`](crate::Language::kind)
17/// and named with [`Language::kind_name`](crate::Language::kind_name). Look a
18/// kind up once, keep it, and compare against it while walking trees: the
19/// comparison is a single integer compare.
20///
21/// | Name | Kind of |
22/// |---|---|
23/// | a rule name, such as `let_stmt` | the node the rule builds |
24/// | a literal's text, such as `let` or `+=` | the keyword or symbol token |
25/// | a Pratt level's `node`, or `binary`, `prefix`, `postfix` | an operator node |
26/// | `IDENT`, `NUMBER`, `STRING`, `NEWLINE` | the built-in token classes |
27/// | `WHITESPACE`, `COMMENT` | trivia tokens |
28/// | `UNKNOWN` | a character the lexer did not recognize (trivia) |
29/// | `ERROR` | a node wrapping tokens the parser skipped |
30///
31/// A kind belongs to the language that produced it. Comparing kinds from two
32/// different languages is meaningless, and naming one language's kind with
33/// another language returns the wrong name or `"<unknown>"`.
34///
35/// # Trivia
36///
37/// [`TokenKind::is_trivia`] answers without the language: whitespace, comments,
38/// unrecognized characters, and — unless the schematic sets `newlines = true` —
39/// line breaks are trivia. Trivia is kept in the tree, so it stays lossless,
40/// but the parser never sees it.
41///
42/// # Examples
43///
44/// ```
45/// use lang_forge::Language;
46/// use lang_forge::syntax_lang::TokenKind;
47///
48/// let lang = Language::from_lsf(
49///     r#"
50///     [language]
51///     name = "sum"
52///
53///     [rules]
54///     sum = "NUMBER ('+' NUMBER)*"
55///     "#,
56/// )?;
57///
58/// let plus = lang.kind("+").expect("the grammar uses '+'");
59/// let space = lang.kind("WHITESPACE").expect("built in");
60/// assert_eq!(lang.kind_name(plus), "+");
61/// assert!(space.is_trivia());
62/// assert!(!plus.is_trivia());
63///
64/// let parse = lang.parse("1 + 2");
65/// let pluses = parse.tree().tokens().filter(|t| *t.kind() == plus).count();
66/// assert_eq!(pluses, 1);
67/// # Ok::<(), lang_forge::Error>(())
68/// ```
69#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
70pub struct Kind(u16);
71
72/// The bit that marks a kind as trivia. Kept inside the value so that
73/// [`TokenKind::is_trivia`] needs no access to the language.
74const TRIVIA: u16 = 0x8000;
75
76/// The largest number of kinds a language can have: indexes use the 15 bits
77/// below the trivia flag.
78pub(crate) const MAX_KINDS: usize = TRIVIA as usize;
79
80impl Kind {
81    /// A kind with the given index, flagged as trivia or not.
82    #[inline]
83    pub(crate) const fn new(index: u16, trivia: bool) -> Self {
84        Self(if trivia { index | TRIVIA } else { index })
85    }
86
87    /// The kind's position in its language's kind table.
88    #[inline]
89    pub(crate) const fn index(self) -> usize {
90        (self.0 & !TRIVIA) as usize
91    }
92}
93
94impl TokenKind for Kind {
95    #[inline]
96    fn is_trivia(&self) -> bool {
97        self.0 & TRIVIA != 0
98    }
99}
100
101impl fmt::Debug for Kind {
102    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
103        write!(f, "Kind({})", self.index())
104    }
105}
106
107#[cfg(test)]
108mod tests {
109    use super::*;
110
111    #[test]
112    fn test_kind_trivia_flag_is_independent_of_index() {
113        let plain = Kind::new(7, false);
114        let trivia = Kind::new(7, true);
115        assert_eq!(plain.index(), 7);
116        assert_eq!(trivia.index(), 7);
117        assert!(!plain.is_trivia());
118        assert!(trivia.is_trivia());
119        assert_ne!(plain, trivia);
120    }
121
122    #[test]
123    fn test_kind_debug_shows_index_only() {
124        assert_eq!(alloc::format!("{:?}", Kind::new(3, true)), "Kind(3)");
125    }
126
127    #[test]
128    fn test_kind_max_index_fits_below_flag() {
129        let last = Kind::new((MAX_KINDS - 1) as u16, false);
130        assert_eq!(last.index(), MAX_KINDS - 1);
131        assert!(!last.is_trivia());
132    }
133}
lang_forge/kind.rs

lang_forge/
kind.rs
