lang_forge/lib.rs
1//! # lang_forge
2//!
3//! LexerSketch: forge a working language front end — lexer, parser, and
4//! lossless syntax tree — from a `.lsf` schematic.
5//!
6//! A schematic is a short NOML document that describes a language: its name,
7//! how its tokens look, its grammar, and the capabilities (passes) it
8//! includes. [`Language::from_lsf`] reads it, checks it, and compiles it into
9//! tables; [`Language::parse`] then turns source text into a
10//! [`syntax_lang::Node`] tree with [`diag_lang::Diagnostic`]s for anything
11//! malformed. There is no code generation step and nothing to build: the
12//! language is ready the moment the schematic is forged.
13//!
14//! lang-forge is the capstone of the `-lang` language-construction family. Its
15//! trees are `syntax-lang` trees, the family's lossless CST, which the
16//! formatter, incremental reparser, language server, and tree-sitter crates
17//! are designed to consume; the adapters that connect a forged language to
18//! those crates are not part of lang-forge and arrive with LexerSketch. Its
19//! diagnostics render with `diag-lang`, and its capabilities run on
20//! `pass-lang`.
21//!
22//! ## A first language
23//!
24//! ```
25//! use lang_forge::Language;
26//!
27//! let calc = Language::from_lsf(
28//! r##"
29//! [language]
30//! name = "calc"
31//!
32//! [lexer]
33//! line_comments = ["#"]
34//!
35//! [rules]
36//! program = "stmt*"
37//! stmt = "'let' IDENT '=' expr ';' | expr ';'"
38//! group = "'(' expr ')'"
39//!
40//! [rules.expr]
41//! operand = "NUMBER | IDENT | group"
42//! levels = [
43//! { left = ["+", "-"] },
44//! { left = ["*", "/"] },
45//! { prefix = ["-"] },
46//! { right = ["^"] },
47//! ]
48//! "##,
49//! )?;
50//!
51//! let parse = calc.parse("let area = 3 * r ^ 2; # circle-ish\n");
52//! assert!(!parse.has_errors());
53//!
54//! // `^` binds tighter than `*`, so the product's right operand is `r ^ 2`.
55//! let binary = calc.kind("binary").expect("the default operator node");
56//! let product = parse.tree().descendants().find(|n| *n.kind() == binary).expect("3 * r ^ 2");
57//! assert_eq!(product.text(parse.source()), Some("3 * r ^ 2"));
58//! assert_eq!(product.child_nodes().last().and_then(|n| n.text(parse.source())), Some("r ^ 2"));
59//! # Ok::<(), lang_forge::Error>(())
60//! ```
61//!
62//! ## The rule language
63//!
64//! Each entry of `[rules]` is a rule. A string rule is a sequence of elements:
65//!
66//! | Element | Matches |
67//! |---|---|
68//! | `'text'` or `"text"` | a keyword (if it looks like an identifier) or a symbol |
69//! | `IDENT`, `NUMBER`, `STRING`, `NEWLINE` | a token of that built-in class |
70//! | `name` | the rule `name`, as a child node |
71//! | `a b` | `a` then `b` |
72//! | `a \| b` | `a`, or else `b` — the first that matches wins |
73//! | `a*`, `a+`, `a?` | zero or more, one or more, zero or one |
74//! | `( ... )` | grouping |
75//!
76//! A rule builds a node named after itself, unless its name starts with `_`,
77//! in which case its children are placed directly in the parent. The first
78//! rule is the start rule unless `[language] start` names another; its node
79//! is the root of every tree.
80//!
81//! A table rule, `[rules.name]`, is an *expression rule*: an `operand` and
82//! operator `levels`, lowest precedence first. Each level is `left`,
83//! `right`, or `none` (binary, by associativity), `prefix`, or `postfix`,
84//! with an optional `then` (more grammar after the operator, for calls,
85//! indexing, or `?:`) and an optional `node` name.
86//!
87//! ## Format 2
88//!
89//! A sketch that says `[sketch] format = 2` uses LSF2's syntax: custom token
90//! classes (regular expressions), lexer modes with a mode stack, string
91//! classes with interpolation, heredocs, and raw delimiters, contextual and
92//! case-insensitive keywords, indentation layout, labelled fields on tree
93//! edges, predicates (`&e`, `!e`), text back-references, `EOF` /
94//! `LINE_START` / `NL_BEFORE`, injections, `[ast]` supertypes, and a check
95//! that rejects greedy repetitions that would silently reject valid input.
96//! A sketch with no `[sketch]` table, or `format = 1`, forges exactly as
97//! lang-forge 1.x did.
98//!
99//! ```
100//! use lang_forge::Language;
101//!
102//! let lang = Language::from_lsf(
103//! r##"
104//! [sketch]
105//! format = 2
106//!
107//! [language]
108//! name = "tmpl"
109//! version = "1.0.0"
110//!
111//! [lexer]
112//! initial_mode = "text"
113//!
114//! [lexer.tokens]
115//! OPEN = { literal = "{{", modes = ["text"], action = "push main" }
116//! CLOSE = { literal = "}}", action = "pop" }
117//! VAR = { regex = '\$[a-z]+' }
118//!
119//! [lexer.modes.text]
120//! tokens = ["OPEN"]
121//! text = "TEXT"
122//!
123//! [rules]
124//! page = "(TEXT | hole)*"
125//! hole = "OPEN value:VAR filters:('|' IDENT)* CLOSE"
126//! "##,
127//! )?;
128//!
129//! let parse = lang.parse("Hello, {{ $name | upper }}!");
130//! assert!(!parse.has_errors());
131//! let hole = parse.tree().child_nodes().next().expect("a hole");
132//! let fields: Vec<&str> = (0..hole.len())
133//! .filter_map(|i| lang.field_label(hole, i).and_then(|l| lang.label_name(l)))
134//! .collect();
135//! assert_eq!(fields, ["value", "filters", "filters"]);
136//! # Ok::<(), lang_forge::Error>(())
137//! ```
138//!
139//! A forged language can be saved as an image ([`Language::to_image`]) and
140//! loaded without forging ([`Language::from_image`]); a sketch can span
141//! several files ([`Sketch`]); and every diagnostic carries a stable code
142//! (`LSF` for sketches, `LF0xxx` lexical and `LF1xxx` parse errors in
143//! source).
144//!
145//! ## Errors and recovery
146//!
147//! Forging reports every problem in a schematic at once, each with a span
148//! into the schematic ([`Error`]). Parsing never fails: a missing token is
149//! reported and assumed, an unexpected token is reported and wrapped in an
150//! `ERROR` node, and the tree always covers the whole source.
151//!
152//! ## Features
153//!
154//! - `std` (default) — the standard library. Without it the crate is
155//! `no_std` and needs only `alloc`.
156//!
157//! ## Re-exports
158//!
159//! [`syntax_lang`], [`diag_lang`], and [`pass_lang`] are re-exported whole, so
160//! code that walks trees, renders diagnostics, or writes capability passes
161//! names the same versions lang-forge was built against.
162//!
163//! ## Stability
164//!
165//! This is `2.0.0-alpha.1`, a pre-release of the next major version. The
166//! format-1 surface and behaviour of 1.x are kept (format-1 sketches forge
167//! and parse exactly as before); the breaking changes are listed in the
168//! CHANGELOG with a migration guide. The format-2 surface may still change
169//! before `2.0.0`. The 1.x promise and what 2.0 changes are set out in
170//! [`docs/API.md`](https://github.com/jamesgober/lang-forge/blob/main/docs/API.md#stability).
171
172#![cfg_attr(not(feature = "std"), no_std)]
173#![cfg_attr(docsrs, feature(doc_cfg))]
174#![forbid(unsafe_code)]
175#![deny(missing_docs)]
176#![deny(unsafe_op_in_unsafe_fn)]
177#![deny(unused_must_use)]
178#![deny(unused_results)]
179#![deny(clippy::unwrap_used)]
180#![deny(clippy::expect_used)]
181#![deny(clippy::todo)]
182#![deny(clippy::unimplemented)]
183#![deny(clippy::print_stdout)]
184#![deny(clippy::print_stderr)]
185#![deny(clippy::dbg_macro)]
186#![deny(clippy::unreachable)]
187#![deny(clippy::undocumented_unsafe_blocks)]
188
189extern crate alloc;
190#[cfg(test)]
191extern crate std;
192
193mod codes;
194mod error;
195mod fields;
196mod grammar;
197mod grammar2;
198mod image;
199mod image_impls;
200mod inject;
201mod kind;
202mod language;
203mod layout;
204mod lexer;
205mod noml;
206mod overlap;
207mod parse;
208mod parser;
209mod regex;
210mod rule;
211mod scan;
212mod schematic;
213mod set;
214mod sketch;
215mod spec2;
216mod suggest;
217mod tree;
218
219pub use error::Error;
220pub use fields::{Cardinality, Field};
221pub use image::{IMAGE_FORMAT, ImageError};
222pub use kind::Kind;
223pub use language::{Capability, Language};
224pub use parse::{Injection, Parse};
225pub use sketch::Sketch;
226
227// Re-exported whole: trees are `syntax_lang` trees, problems are `diag_lang`
228// diagnostics, and capabilities are `pass_lang` passes.
229pub use diag_lang;
230pub use pass_lang;
231pub use syntax_lang;
232
233/// Compiles and runs the `rust` code blocks in `README.md` and `docs/API.md` as
234/// part of `cargo test`, so the published examples cannot drift from the API.
235///
236/// Present only while collecting doctests (`#[cfg(doctest)]`); it is not part of
237/// the public surface and does not appear in the built library or its docs.
238#[cfg(doctest)]
239#[doc = include_str!("../README.md")]
240#[doc = include_str!("../docs/API.md")]
241pub struct MarkdownDocTests;