textmate_lang/
lib.rs

1//! # textmate_lang
2//!
3//! Generate [TextMate grammars](https://macromates.com/manual/en/language_grammars)
4//! (`tmLanguage.json`) from Rust: describe a language's tokens with
5//! [`Grammar`] and [`Rule`]s, and emit the JSON that VS Code, Sublime Text,
6//! TextMate, Shiki, and other editors and highlighters load to highlight it,
7//! no tree-sitter required.
8//!
9//! Emitting validates everything a grammar file cannot check for itself.
10//! Every regular expression is held to a documented subset of Oniguruma
11//! syntax, and refused with a byte offset and a reason when it strays
12//! outside. Scope names, scope selectors, captures, and includes are checked;
13//! includes that would send a tokenizer into endless recursion are found.
14//! The JSON writer is the crate's own and escapes every string it writes, so
15//! no name, pattern, or scope can change the structure of the output.
16//!
17//! ## The pieces
18//!
19//! - [`Grammar`] — the scope name, top-level patterns, the repository of
20//!   named rules, and injections. [`Grammar::to_json`] validates it and
21//!   returns the JSON.
22//! - [`Rule`] — an entry of a pattern list: a [`Match`], a [`BeginEnd`] or
23//!   [`BeginWhile`] region, an include ([`Rule::include`] and friends), or a
24//!   list ([`Rule::patterns`]). [`Capture`] says what to do with a capture
25//!   group.
26//! - [`Error`] — what validation found, and where, as a JSON Pointer into
27//!   the output.
28//! - [`validate_regex`] and [`escape_regex`] — the regular-expression check
29//!   on its own, and literal text made safe to match.
30//!
31//! ## Example
32//!
33//! ```
34//! use textmate_lang::{BeginEnd, Grammar, Match, Rule, escape_regex};
35//!
36//! let operators = ["+", "-", "*", "/", "=="].map(escape_regex).join("|");
37//! let json = Grammar::new("source.calc")
38//!     .name("Calc")
39//!     .file_type("calc")
40//!     .pattern(Rule::include("comment"))
41//!     .pattern(Match::new(r"\b\d+(\.\d+)?\b").scope("constant.numeric.calc"))
42//!     .pattern(Match::new(operators).scope("keyword.operator.calc"))
43//!     .pattern(
44//!         Match::new(r"\b(let)\s+([a-z_]\w*)")
45//!             .capture(1, "storage.type.calc")
46//!             .capture(2, "variable.other.calc"),
47//!     )
48//!     .repository(
49//!         "comment",
50//!         BeginEnd::new("#", "$")
51//!             .scope("comment.line.number-sign.calc")
52//!             .begin_capture(0, "punctuation.definition.comment.calc"),
53//!     )
54//!     .to_json()?;
55//!
56//! assert!(json.contains(r#""match": "\\+|\\-|\\*|\\/|\\=\\=""#));
57//! // std::fs::write("syntaxes/calc.tmLanguage.json", json)?;
58//! # Ok::<(), textmate_lang::Error>(())
59//! ```
60//!
61//! ## What is checked
62//!
63//! - **Regular expressions** (`match`, `begin`, `end`, `while`,
64//!   `firstLineMatch`) are non-empty and in the subset `docs/API.md`
65//!   documents: balanced groups and classes, known escapes, quantifiers with
66//!   one meaning in every engine, fixed-length look-behind, back-references
67//!   to groups that exist (in `end` and `while`, to groups of `begin`), and
68//!   no mixing of named and numbered groups.
69//! - **Scope names** are dot-separated segments of ASCII letters, digits,
70//!   `_`, `+`, and `-`; rule names may list several scopes and substitute
71//!   capture text with `$n`, which must name an existing group.
72//! - **Captures** name groups the pattern has, each at most once.
73//! - **Includes** of repository entries resolve; includes that form a cycle
74//!   of pattern lists, which tokenizers expand forever, are refused, while
75//!   recursion through a region (how nested constructs are written) is
76//!   accepted.
77//! - **Selectors** (`injectionSelector`, injection keys) parse.
78//! - Rules nest at most [`MAX_RULE_DEPTH`] deep and patterns at most
79//!   [`MAX_REGEX_DEPTH`]; validation and emission are iterative.
80//!
81//! What cannot be checked here: rules in other grammars that external
82//! includes name, and whether a pattern matches what its author intended.
83//!
84//! ## Features
85//!
86//! - `std` (default) — the standard library. Without it the crate is `no_std`
87//!   and needs only `alloc`. There are no dependencies.
88
89#![cfg_attr(not(feature = "std"), no_std)]
90#![cfg_attr(docsrs, feature(doc_cfg))]
91#![forbid(unsafe_code)]
92#![deny(
93    warnings,
94    missing_docs,
95    unsafe_op_in_unsafe_fn,
96    unused_must_use,
97    unused_results,
98    clippy::unwrap_used,
99    clippy::expect_used,
100    clippy::panic,
101    clippy::todo,
102    clippy::unimplemented,
103    clippy::unreachable,
104    clippy::dbg_macro,
105    clippy::print_stdout,
106    clippy::print_stderr,
107    clippy::undocumented_unsafe_blocks
108)]
109
110extern crate alloc;
111
112mod check;
113mod error;
114mod grammar;
115mod json;
116mod regex;
117mod rule;
118mod scope;
119
120pub use check::MAX_RULE_DEPTH;
121pub use error::Error;
122pub use grammar::Grammar;
123pub use json::Style;
124pub use regex::{MAX_REGEX_DEPTH, RegexError, RegexErrorKind, escape_regex, validate_regex};
125pub use rule::{BeginEnd, BeginWhile, Capture, Match, Rule};
126
127/// Compiles and runs the `rust` code blocks in `README.md` and `docs/API.md` as
128/// part of `cargo test`, so the published examples cannot drift from the API.
129///
130/// Present only while collecting doctests (`#[cfg(doctest)]`); it is not part of
131/// the public surface and does not appear in the built library or its docs.
132#[cfg(doctest)]
133#[doc = include_str!("../README.md")]
134#[doc = include_str!("../docs/API.md")]
135pub struct MarkdownDocTests;
textmate_lang/lib.rs

textmate_lang/
lib.rs
