weavatrix_parse/
syntax.rs

1//! Lexical shape of each supported language.
2//!
3//! Languages differ in a small number of lexical decisions - how a comment
4//! starts, which quotes open a string, whether a backslash escapes, whether
5//! indentation is significant - and agree on everything else. Describing those
6//! differences as data keeps one tokenizer correct for all of them instead of
7//! one hand-written scanner per language.
8
9/// A language this crate can tokenize.
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
11#[non_exhaustive]
12pub enum Language {
13    JavaScript,
14    TypeScript,
15    Graphql,
16    Protobuf,
17    Rust,
18    Python,
19    Go,
20    Java,
21    CSharp,
22    C,
23    Cpp,
24    Sql,
25    Solidity,
26    Swift,
27    Terraform,
28    Html,
29    Xml,
30    Markdown,
31    /// Markdown with JavaScript imports and components.
32    Mdx,
33    ReStructuredText,
34    AsciiDoc,
35    Css,
36    /// SCSS, Sass and Less, which differ from CSS by allowing `//` comments
37    /// and nesting selectors.
38    Scss,
39    Bash,
40    Yaml,
41}
42
43impl Language {
44    /// The language a file extension selects, if this crate handles it.
45    #[must_use]
46    pub fn from_extension(extension: &str) -> Option<Self> {
47        Some(match extension.to_ascii_lowercase().as_str() {
48            "js" | "jsx" | "mjs" | "cjs" => Self::JavaScript,
49            "ts" | "tsx" | "mts" | "cts" => Self::TypeScript,
50            "graphql" | "gql" => Self::Graphql,
51            "proto" => Self::Protobuf,
52            "rs" => Self::Rust,
53            "py" | "pyi" => Self::Python,
54            "go" => Self::Go,
55            "java" => Self::Java,
56            "cs" => Self::CSharp,
57            "c" | "h" => Self::C,
58            "cc" | "cpp" | "cxx" | "hh" | "hpp" | "hxx" => Self::Cpp,
59            "sql" | "psql" => Self::Sql,
60            "sol" => Self::Solidity,
61            "swift" => Self::Swift,
62            "tf" | "tfvars" | "hcl" => Self::Terraform,
63            "html" | "htm" | "xhtml" | "vue" | "svelte" => Self::Html,
64            "xml" | "xsd" | "xsl" | "xslt" | "pom" | "csproj" | "vbproj" | "fsproj" | "props"
65            | "targets" | "plist" | "storyboard" | "xib" | "resx" | "nuspec" => Self::Xml,
66            "md" | "markdown" | "mdown" | "mkd" | "mkdn" => Self::Markdown,
67            "mdx" => Self::Mdx,
68            "rst" => Self::ReStructuredText,
69            "adoc" | "asciidoc" | "asc" => Self::AsciiDoc,
70            "css" => Self::Css,
71            "scss" | "sass" | "less" => Self::Scss,
72            "sh" | "bash" | "zsh" => Self::Bash,
73            "yaml" | "yml" => Self::Yaml,
74            _ => return None,
75        })
76    }
77
78    #[must_use]
79    pub const fn as_str(self) -> &'static str {
80        match self {
81            Self::JavaScript => "javascript",
82            Self::TypeScript => "typescript",
83            Self::Graphql => "graphql",
84            Self::Protobuf => "protobuf",
85            Self::Rust => "rust",
86            Self::Python => "python",
87            Self::Go => "go",
88            Self::Java => "java",
89            Self::CSharp => "csharp",
90            Self::C => "c",
91            Self::Cpp => "cpp",
92            Self::Sql => "sql",
93            Self::Solidity => "solidity",
94            Self::Swift => "swift",
95            Self::Terraform => "terraform",
96            Self::Xml => "xml",
97            Self::Markdown => "markdown",
98            Self::Mdx => "mdx",
99            Self::ReStructuredText => "rst",
100            Self::AsciiDoc => "asciidoc",
101            Self::Html => "html",
102            Self::Css => "css",
103            Self::Scss => "scss",
104            Self::Bash => "bash",
105            Self::Yaml => "yaml",
106        }
107    }
108
109    /// The lexical rules this language follows.
110    // This is a table, not an algorithm: one arm per language, each a literal.
111    // Splitting it to satisfy a line count would scatter the table across
112    // functions and make the languages harder to compare against each other,
113    // which is the whole point of writing them as data.
114    #[allow(clippy::too_many_lines)]
115    #[must_use]
116    pub const fn syntax(self) -> Syntax {
117        match self {
118            Self::JavaScript | Self::TypeScript => Syntax {
119                line_comments: &["//"],
120                block_comment: Some(("/*", "*/")),
121                nested_block_comments: false,
122                quotes: &['"', '\'', '`'],
123                interpolated_quote: Some('`'),
124                escapes: true,
125                regex_literals: true,
126                raw_strings: false,
127                triple_quotes: false,
128                char_literals: false,
129                significant_indentation: false,
130                identifier_extra: &['$', '_'],
131            },
132            Self::Graphql => Syntax {
133                line_comments: &["#"],
134                block_comment: None,
135                nested_block_comments: false,
136                quotes: &['"'],
137                interpolated_quote: None,
138                escapes: true,
139                regex_literals: false,
140                raw_strings: false,
141                // GraphQL descriptions use block strings.
142                triple_quotes: true,
143                char_literals: false,
144                significant_indentation: false,
145                identifier_extra: &['_'],
146            },
147            Self::Protobuf => Syntax {
148                line_comments: &["//"],
149                block_comment: Some(("/*", "*/")),
150                nested_block_comments: false,
151                quotes: &['"', '\''],
152                interpolated_quote: None,
153                escapes: true,
154                regex_literals: false,
155                raw_strings: false,
156                triple_quotes: false,
157                char_literals: false,
158                significant_indentation: false,
159                identifier_extra: &['_'],
160            },
161            Self::Rust => Syntax {
162                line_comments: &["//"],
163                block_comment: Some(("/*", "*/")),
164                // Rust block comments nest, so a naive scanner ends one early.
165                nested_block_comments: true,
166                quotes: &['"'],
167                interpolated_quote: None,
168                escapes: true,
169                regex_literals: false,
170                raw_strings: true,
171                triple_quotes: false,
172                char_literals: true,
173                significant_indentation: false,
174                identifier_extra: &['_'],
175            },
176            Self::Python => Syntax {
177                line_comments: &["#"],
178                block_comment: None,
179                nested_block_comments: false,
180                quotes: &['"', '\''],
181                interpolated_quote: None,
182                escapes: true,
183                regex_literals: false,
184                raw_strings: false,
185                triple_quotes: true,
186                char_literals: false,
187                significant_indentation: true,
188                identifier_extra: &['_'],
189            },
190            // Solidity is lexically a C-family language; only its keywords
191            // differ, and those live with the structural rules rather than here.
192            Self::Go | Self::Java | Self::CSharp | Self::C | Self::Cpp | Self::Solidity => Syntax {
193                line_comments: &["//"],
194                block_comment: Some(("/*", "*/")),
195                nested_block_comments: false,
196                quotes: &['"', '\'', '`'],
197                interpolated_quote: None,
198                escapes: true,
199                regex_literals: false,
200                raw_strings: false,
201                triple_quotes: false,
202                char_literals: false,
203                significant_indentation: false,
204                identifier_extra: &['_'],
205            },
206            Self::Sql => Syntax {
207                line_comments: &["--"],
208                block_comment: Some(("/*", "*/")),
209                nested_block_comments: false,
210                quotes: &['\'', '"'],
211                interpolated_quote: None,
212                // SQL doubles a quote to escape it rather than using a
213                // backslash, which the tokenizer handles explicitly.
214                escapes: false,
215                regex_literals: false,
216                raw_strings: false,
217                triple_quotes: false,
218                char_literals: false,
219                significant_indentation: false,
220                identifier_extra: &['_', '$'],
221            },
222            Self::Swift => Syntax {
223                line_comments: &["//"],
224                block_comment: Some(("/*", "*/")),
225                // Swift block comments nest, as Rust's do.
226                nested_block_comments: true,
227                quotes: &['"'],
228                interpolated_quote: None,
229                escapes: true,
230                raw_strings: false,
231                regex_literals: false,
232                triple_quotes: true,
233                char_literals: false,
234                significant_indentation: false,
235                // `$0` names a closure argument, and `_` a wildcard.
236                identifier_extra: &['_', '$'],
237            },
238            Self::Terraform => Syntax {
239                line_comments: &["#", "//"],
240                block_comment: Some(("/*", "*/")),
241                nested_block_comments: false,
242                quotes: &['"'],
243                interpolated_quote: Some('"'),
244                escapes: true,
245                regex_literals: false,
246                raw_strings: false,
247                triple_quotes: false,
248                char_literals: false,
249                significant_indentation: false,
250                identifier_extra: &['_', '-'],
251            },
252            // A tag is punctuation and identifiers around a quoted value, so
253            // the ordinary token model fits once `-` and `:` are allowed in a
254            // name: `data-count`, `aria-label`, `xlink:href`, `v-on:click`.
255            Self::Html => Syntax {
256                line_comments: &[],
257                block_comment: Some(("<!--", "-->")),
258                nested_block_comments: false,
259                quotes: &['"', '\''],
260                interpolated_quote: None,
261                // HTML escapes with entities rather than backslashes, so a
262                // backslash before a quote does not extend the value.
263                escapes: false,
264                regex_literals: false,
265                raw_strings: false,
266                triple_quotes: false,
267                char_literals: false,
268                significant_indentation: false,
269                identifier_extra: &['_', '-', ':', '.', '@'],
270            },
271            // CSS has no line comment: `//` is invalid there, and treating it
272            // as one would swallow the rest of a line in a valid stylesheet.
273            Self::Css => Syntax {
274                line_comments: &[],
275                block_comment: Some(("/*", "*/")),
276                nested_block_comments: false,
277                quotes: &['"', '\''],
278                interpolated_quote: None,
279                escapes: true,
280                regex_literals: false,
281                raw_strings: false,
282                triple_quotes: false,
283                char_literals: false,
284                significant_indentation: false,
285                identifier_extra: &['_', '-'],
286            },
287            Self::Scss => Syntax {
288                line_comments: &["//"],
289                block_comment: Some(("/*", "*/")),
290                nested_block_comments: false,
291                quotes: &['"', '\''],
292                interpolated_quote: None,
293                escapes: true,
294                regex_literals: false,
295                raw_strings: false,
296                triple_quotes: false,
297                char_literals: false,
298                significant_indentation: false,
299                identifier_extra: &['_', '-', '$', '@'],
300            },
301            // XML shares HTML's shape; what differs is which attributes name a
302            // file, and that belongs with the extractor rather than here.
303            Self::Xml => Syntax {
304                line_comments: &[],
305                block_comment: Some(("<!--", "-->")),
306                nested_block_comments: false,
307                quotes: &['"', '\''],
308                interpolated_quote: None,
309                escapes: false,
310                regex_literals: false,
311                raw_strings: false,
312                triple_quotes: false,
313                char_literals: false,
314                significant_indentation: false,
315                identifier_extra: &['_', '-', ':', '.'],
316            },
317            // Prose has no token structure worth the name: a `"` is a quotation
318            // mark, not a literal, and `//` is part of a URL. These rules exist
319            // so the tokenizer stays total over every language; the document
320            // extractors read lines directly, which is the honest model.
321            Self::Markdown | Self::Mdx | Self::ReStructuredText | Self::AsciiDoc => Syntax {
322                line_comments: &[],
323                block_comment: Some(("<!--", "-->")),
324                nested_block_comments: false,
325                quotes: &[],
326                interpolated_quote: None,
327                escapes: false,
328                regex_literals: false,
329                raw_strings: false,
330                triple_quotes: false,
331                char_literals: false,
332                significant_indentation: true,
333                identifier_extra: &['_', '-', '.'],
334            },
335            Self::Bash => Syntax {
336                line_comments: &["#"],
337                block_comment: None,
338                nested_block_comments: false,
339                quotes: &['"', '\''],
340                interpolated_quote: Some('"'),
341                escapes: true,
342                regex_literals: false,
343                raw_strings: false,
344                triple_quotes: false,
345                char_literals: false,
346                significant_indentation: false,
347                identifier_extra: &['_'],
348            },
349            Self::Yaml => Syntax {
350                line_comments: &["#"],
351                block_comment: None,
352                nested_block_comments: false,
353                quotes: &['"', '\''],
354                interpolated_quote: None,
355                escapes: true,
356                regex_literals: false,
357                raw_strings: false,
358                triple_quotes: false,
359                char_literals: false,
360                significant_indentation: true,
361                identifier_extra: &['_', '-', '.'],
362            },
363        }
364    }
365}
366
367/// The lexical rules of one language.
368///
369/// The flags are independent lexical facts rather than a state machine, so
370/// they are listed plainly instead of being packed into an option type that
371/// would obscure which language has which behaviour.
372#[derive(Debug, Clone, Copy)]
373#[non_exhaustive]
374#[allow(clippy::struct_excessive_bools)]
375pub struct Syntax {
376    pub line_comments: &'static [&'static str],
377    pub block_comment: Option<(&'static str, &'static str)>,
378    pub nested_block_comments: bool,
379    pub quotes: &'static [char],
380    /// Quote that opens a string containing `${...}` expressions.
381    pub interpolated_quote: Option<char>,
382    pub escapes: bool,
383    /// Whether `/` can open a regular-expression literal.
384    pub regex_literals: bool,
385    /// Whether `r"..."` and `r#"..."#` forms exist.
386    pub raw_strings: bool,
387    /// Whether `"""..."""` spans lines.
388    pub triple_quotes: bool,
389    /// Whether `'` opens a character literal that a lifetime is also written
390    /// with. Rust needs this: `'a` is a lifetime and `'"'` is a quote
391    /// character, and treating `'` as an ordinary quote or as ordinary
392    /// punctuation gets one of the two wrong.
393    pub char_literals: bool,
394    pub significant_indentation: bool,
395    pub identifier_extra: &'static [char],
396}
weavatrix_parse/syntax.rs

weavatrix_parse/
syntax.rs
