textmate_lang/grammar.rs
1//! The grammar: a scope name, top-level patterns, a repository of named
2//! rules, and injections.
3
4use alloc::{string::String, vec::Vec};
5
6use crate::{Error, Rule, Style, check, json, rule::Text};
7
8/// A TextMate grammar, built in Rust and emitted as `tmLanguage.json`.
9///
10/// A grammar is a scope name (`source.demo`) and a list of top-level
11/// patterns — the [`Rule`]s the tokenizer tries at each position — plus a
12/// repository of named rules that patterns include by name, and optionally
13/// injections into other grammars. Build one by chaining: every method takes
14/// the grammar by value and returns it, and nothing is checked until it is
15/// emitted.
16///
17/// [`to_json`](Grammar::to_json) validates the whole grammar — every regular
18/// expression against the [supported Oniguruma subset](crate::validate_regex),
19/// every scope name, every include, every capture — and returns the JSON, or
20/// the first problem as an [`Error`] that says where it is. Output is
21/// deterministic: the same grammar always produces the same bytes, whatever
22/// order its repository entries, injections, and captures were added in.
23///
24/// # Examples
25///
26/// ```
27/// use textmate_lang::{BeginEnd, Grammar, Match, Rule};
28///
29/// let json = Grammar::new("source.demo")
30/// .name("Demo")
31/// .file_type("demo")
32/// .pattern(Rule::include("comments"))
33/// .pattern(Rule::include("strings"))
34/// .pattern(Match::new(r"\b(if|else|while|return)\b").scope("keyword.control.demo"))
35/// .pattern(Match::new(r"\b\d+(\.\d+)?\b").scope("constant.numeric.demo"))
36/// .repository("comments", Match::new("#.*$").scope("comment.line.number-sign.demo"))
37/// .repository(
38/// "strings",
39/// BeginEnd::new("\"", "\"")
40/// .scope("string.quoted.double.demo")
41/// .pattern(Match::new(r"\\.").scope("constant.character.escape.demo")),
42/// )
43/// .to_json()?;
44///
45/// assert!(json.starts_with("{\n \"name\": \"Demo\",\n \"scopeName\": \"source.demo\","));
46/// assert!(json.contains("\"match\": \"\\\\b(if|else|while|return)\\\\b\""));
47/// # Ok::<(), textmate_lang::Error>(())
48/// ```
49#[derive(Clone, Debug, PartialEq, Eq)]
50pub struct Grammar {
51 pub(crate) scope_name: Text,
52 pub(crate) name: Option<Text>,
53 pub(crate) file_types: Vec<Text>,
54 pub(crate) first_line_match: Option<Text>,
55 pub(crate) injection_selector: Option<Text>,
56 pub(crate) patterns: Vec<Rule>,
57 pub(crate) repository: Vec<(Text, Rule)>,
58 pub(crate) injections: Vec<(Text, Rule)>,
59}
60
61impl Grammar {
62 /// Starts an empty grammar whose root scope is `scope_name`
63 /// (`"scopeName"`).
64 ///
65 /// The scope name is how editors and other grammars refer to this one:
66 /// by convention `source.<language>` for programming languages and
67 /// `text.<format>` for markup. It is a single scope without spaces or
68 /// substitutions ([`Error::InvalidScope`]), checked when the grammar is
69 /// emitted.
70 ///
71 /// | Parameter | Meaning |
72 /// |---|---|
73 /// | `scope_name` | The root scope, such as `source.rust`: a `&'static str` (not copied) or a `String`. |
74 ///
75 /// # Examples
76 ///
77 /// ```
78 /// use textmate_lang::Grammar;
79 ///
80 /// let json = Grammar::new("source.toml").to_json_compact()?;
81 /// assert_eq!(json, r#"{"scopeName":"source.toml","patterns":[]}"#);
82 /// # Ok::<(), textmate_lang::Error>(())
83 /// ```
84 #[must_use]
85 pub fn new(scope_name: impl Into<Text>) -> Grammar {
86 Grammar {
87 scope_name: scope_name.into(),
88 name: None,
89 file_types: Vec::new(),
90 first_line_match: None,
91 injection_selector: None,
92 patterns: Vec::new(),
93 repository: Vec::new(),
94 injections: Vec::new(),
95 }
96 }
97
98 /// The grammar's root scope, as given to [`new`](Grammar::new).
99 ///
100 /// # Examples
101 ///
102 /// ```
103 /// use textmate_lang::Grammar;
104 ///
105 /// assert_eq!(Grammar::new("source.demo").scope_name(), "source.demo");
106 /// ```
107 #[must_use]
108 pub fn scope_name(&self) -> &str {
109 &self.scope_name
110 }
111
112 /// Sets the display name (`"name"`), shown in editors' language
113 /// pickers. Any text; it is escaped on output.
114 ///
115 /// # Examples
116 ///
117 /// ```
118 /// use textmate_lang::Grammar;
119 ///
120 /// let json = Grammar::new("source.demo").name("Demo \"Lang\"").to_json_compact()?;
121 /// assert!(json.starts_with(r#"{"name":"Demo \"Lang\"","#));
122 /// # Ok::<(), textmate_lang::Error>(())
123 /// ```
124 #[must_use]
125 pub fn name(mut self, name: impl Into<Text>) -> Grammar {
126 self.name = Some(name.into());
127 self
128 }
129
130 /// Adds a file extension the grammar applies to (`"fileTypes"`), such as
131 /// `rs` or `tmLanguage.json`. Must not be empty ([`Error::InvalidName`]).
132 ///
133 /// # Examples
134 ///
135 /// ```
136 /// use textmate_lang::Grammar;
137 ///
138 /// let json = Grammar::new("source.demo").file_type("dm").file_type("demo").to_json_compact()?;
139 /// assert!(json.contains(r#""fileTypes":["dm","demo"]"#));
140 /// # Ok::<(), textmate_lang::Error>(())
141 /// ```
142 #[must_use]
143 pub fn file_type(mut self, extension: impl Into<Text>) -> Grammar {
144 self.file_types.push(extension.into());
145 self
146 }
147
148 /// Sets a pattern matched against a file's first line to recognize the
149 /// language when the extension does not (`"firstLineMatch"`), such as a
150 /// shebang.
151 ///
152 /// # Examples
153 ///
154 /// ```
155 /// use textmate_lang::Grammar;
156 ///
157 /// let json = Grammar::new("source.demo").first_line_match(r"^#!.*\bdemo\b").to_json()?;
158 /// assert!(json.contains(r#""firstLineMatch": "^#!.*\\bdemo\\b""#));
159 /// # Ok::<(), textmate_lang::Error>(())
160 /// ```
161 #[must_use]
162 pub fn first_line_match(mut self, regex: impl Into<Text>) -> Grammar {
163 self.first_line_match = Some(regex.into());
164 self
165 }
166
167 /// Makes this an injection grammar (`"injectionSelector"`): its
168 /// top-level patterns are injected into the scopes the selector matches
169 /// in other grammars, as VS Code does for grammars that contribute
170 /// `injectTo`.
171 ///
172 /// | Parameter | Meaning |
173 /// |---|---|
174 /// | `selector` | A scope selector, such as `L:source.js -comment -string`. |
175 ///
176 /// # Examples
177 ///
178 /// ```
179 /// use textmate_lang::{Grammar, Match};
180 ///
181 /// let json = Grammar::new("todo.injection")
182 /// .injection_selector("L:comment")
183 /// .pattern(Match::new(r"\bTODO\b").scope("keyword.todo"))
184 /// .to_json()?;
185 /// assert!(json.contains(r#""injectionSelector": "L:comment""#));
186 /// # Ok::<(), textmate_lang::Error>(())
187 /// ```
188 #[must_use]
189 pub fn injection_selector(mut self, selector: impl Into<Text>) -> Grammar {
190 self.injection_selector = Some(selector.into());
191 self
192 }
193
194 /// Adds a top-level pattern (`"patterns"`). Patterns are tried in the
195 /// order they are added: at each position the earliest match wins, and
196 /// the first pattern wins a tie.
197 ///
198 /// # Examples
199 ///
200 /// ```
201 /// use textmate_lang::{Grammar, Match};
202 ///
203 /// let json = Grammar::new("source.demo")
204 /// .pattern(Match::new("==|!=").scope("keyword.operator.comparison.demo"))
205 /// .pattern(Match::new("=").scope("keyword.operator.assignment.demo"))
206 /// .to_json()?;
207 /// let comparison = json.find("comparison").unwrap();
208 /// let assignment = json.find("assignment").unwrap();
209 /// assert!(comparison < assignment);
210 /// # Ok::<(), textmate_lang::Error>(())
211 /// ```
212 #[must_use]
213 pub fn pattern(mut self, rule: impl Into<Rule>) -> Grammar {
214 self.patterns.push(rule.into());
215 self
216 }
217
218 /// Adds a repository entry called `name` (`"repository"`), which rules
219 /// include with [`Rule::include`]. Names are ASCII letters, digits, `_`,
220 /// `-`, and `.` ([`Error::InvalidName`]) and unique
221 /// ([`Error::DuplicateRepository`]). Entries are emitted sorted by name.
222 ///
223 /// # Examples
224 ///
225 /// ```
226 /// use textmate_lang::{Grammar, Match, Rule};
227 ///
228 /// let mut grammar = Grammar::new("source.demo");
229 /// for (name, regex) in [("numbers", r"\d+"), ("booleans", r"\b(true|false)\b")] {
230 /// grammar = grammar
231 /// .pattern(Rule::include(name))
232 /// .repository(name, Match::new(regex).scope(format!("constant.{name}.demo")));
233 /// }
234 /// let json = grammar.to_json()?;
235 /// assert!(json.find("\"booleans\": {").unwrap() < json.find("\"numbers\": {").unwrap());
236 /// # Ok::<(), textmate_lang::Error>(())
237 /// ```
238 #[must_use]
239 pub fn repository(mut self, name: impl Into<Text>, rule: impl Into<Rule>) -> Grammar {
240 self.repository.push((name.into(), rule.into()));
241 self
242 }
243
244 /// Adds an injection (`"injections"`): `rule` is tried, inside this
245 /// grammar, wherever the scope stack matches `selector`. Selectors are
246 /// unique ([`Error::DuplicateInjection`]); entries are emitted sorted.
247 ///
248 /// # Examples
249 ///
250 /// ```
251 /// use textmate_lang::{Grammar, Match, Rule};
252 ///
253 /// let json = Grammar::new("source.demo")
254 /// .injection(
255 /// "L:string.quoted.demo",
256 /// Rule::patterns([Match::new(r"\{\w+\}").scope("variable.interpolation.demo").into()]),
257 /// )
258 /// .to_json()?;
259 /// assert!(json.contains(r#""L:string.quoted.demo": {"#));
260 /// # Ok::<(), textmate_lang::Error>(())
261 /// ```
262 #[must_use]
263 pub fn injection(mut self, selector: impl Into<Text>, rule: impl Into<Rule>) -> Grammar {
264 self.injections.push((selector.into(), rule.into()));
265 self
266 }
267
268 /// Checks the whole grammar without emitting it.
269 ///
270 /// # Errors
271 ///
272 /// The first problem found, as an [`Error`]; see its documentation for
273 /// the order of the checks.
274 ///
275 /// # Examples
276 ///
277 /// ```
278 /// use textmate_lang::{Error, Grammar, Match};
279 ///
280 /// let grammar = Grammar::new("source.demo").pattern(Match::new("(a)").capture(2, "x"));
281 /// assert_eq!(
282 /// grammar.validate(),
283 /// Err(Error::CaptureOutOfRange { path: "/patterns/0/captures/2".into(), group: 2, groups: 1 }),
284 /// );
285 /// ```
286 pub fn validate(&self) -> Result<(), Error> {
287 check::check(self).map(|_| ())
288 }
289
290 /// Validates the grammar and returns it as pretty-printed
291 /// `tmLanguage.json`: two-space indentation, a final newline. The
292 /// Tier-1 call: build a grammar, call this, write the file.
293 ///
294 /// # Errors
295 ///
296 /// The first problem found, as an [`Error`].
297 ///
298 /// # Examples
299 ///
300 /// ```
301 /// use textmate_lang::{Grammar, Match};
302 ///
303 /// let json = Grammar::new("source.demo")
304 /// .pattern(Match::new(r"\bdemo\b").scope("keyword.demo"))
305 /// .to_json()?;
306 /// assert_eq!(json, r#"{
307 /// "scopeName": "source.demo",
308 /// "patterns": [
309 /// {
310 /// "name": "keyword.demo",
311 /// "match": "\\bdemo\\b"
312 /// }
313 /// ]
314 /// }
315 /// "#);
316 /// // std::fs::write("syntaxes/demo.tmLanguage.json", json)?;
317 /// # Ok::<(), textmate_lang::Error>(())
318 /// ```
319 pub fn to_json(&self) -> Result<String, Error> {
320 let mut out = String::new();
321 self.write_json(&mut out, Style::Pretty)?;
322 Ok(out)
323 }
324
325 /// Validates the grammar and returns it as compact JSON, with no
326 /// whitespace.
327 ///
328 /// # Errors
329 ///
330 /// The first problem found, as an [`Error`].
331 ///
332 /// # Examples
333 ///
334 /// ```
335 /// use textmate_lang::{Grammar, Match};
336 ///
337 /// let json = Grammar::new("source.demo")
338 /// .pattern(Match::new(r"\bdemo\b").scope("keyword.demo"))
339 /// .to_json_compact()?;
340 /// assert_eq!(json, r#"{"scopeName":"source.demo","patterns":[{"name":"keyword.demo","match":"\\bdemo\\b"}]}"#);
341 /// # Ok::<(), textmate_lang::Error>(())
342 /// ```
343 pub fn to_json_compact(&self) -> Result<String, Error> {
344 let mut out = String::new();
345 self.write_json(&mut out, Style::Compact)?;
346 Ok(out)
347 }
348
349 /// Validates the grammar and appends it to `out` in `style`, reusing the
350 /// buffer's capacity. On an error nothing is appended.
351 ///
352 /// # Errors
353 ///
354 /// The first problem found, as an [`Error`].
355 ///
356 /// # Examples
357 ///
358 /// ```
359 /// use textmate_lang::{Grammar, Style};
360 ///
361 /// let mut out = String::with_capacity(4096);
362 /// for scope in ["source.a", "source.b"] {
363 /// out.clear();
364 /// Grammar::new(scope).write_json(&mut out, Style::Compact)?;
365 /// assert!(out.contains(scope));
366 /// }
367 /// assert!(Grammar::new("bad scope").write_json(&mut out, Style::Compact).is_err());
368 /// assert!(out.contains("source.b"));
369 /// # Ok::<(), textmate_lang::Error>(())
370 /// ```
371 pub fn write_json(&self, out: &mut String, style: Style) -> Result<(), Error> {
372 let checked = check::check(self)?;
373 json::write(self, &checked, out, style == Style::Pretty);
374 Ok(())
375 }
376}