ashlar_syntax/
lib.rs

1//! Headless syntax highlighting for ashlar: small, forward-only, linear-time lexers for common
2//! languages, and a line-state API so a view can re-highlight only the lines that changed.
3//!
4//! No dependencies, no UI; builds for `wasm32-unknown-unknown`. Colours live in the UI layer
5//! (`ashlar_ui::code_view::SyntaxTheme`); this crate only says which byte range is which [`Kind`].
6//!
7//! ```
8//! use ashlar_syntax::{highlight, lang_for_path, Kind};
9//!
10//! let lang = lang_for_path("src/main.rs");
11//! assert_eq!(lang, Some("rust"));
12//! let spans = highlight(lang, "fn main() { let x = 42; }");
13//! assert_eq!(spans.first(), Some(&(0..2, Kind::Keyword)));
14//! // ordered, non-overlapping and covering the whole input
15//! assert_eq!(spans.last().map(|(r, _)| r.end), Some(25));
16//! ```
17//!
18//! Incremental use: keep the [`State`] each line starts in. After an edit, re-highlight from the
19//! edited line and stop as soon as a line ends in the same state it ended in before.
20//!
21//! ```
22//! use ashlar_syntax::{highlight_line, Lang, State};
23//!
24//! let mut spans = Vec::new();
25//! let after_first = highlight_line(Lang::Rust, "let s = /* open", State::START, &mut spans);
26//! assert_ne!(after_first, State::START); // inside a block comment
27//! spans.clear();
28//! let after_second = highlight_line(Lang::Rust, "still a comment */ x", after_first, &mut spans);
29//! assert_eq!(after_second, State::START);
30//! ```
31
32mod lang;
33mod scan;
34
35use std::ops::Range;
36
37/// What a span of source text is, for colouring.
38#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
39#[non_exhaustive]
40pub enum Kind {
41    /// Anything without a colour of its own: identifiers, whitespace, text.
42    Plain,
43    /// Language keywords (`fn`, `if`, `return`, `SELECT`).
44    Keyword,
45    /// Type names (`u32`, `String`, a class name, a TOML table header).
46    Type,
47    /// Function and macro names at a call or a definition.
48    Function,
49    /// String and character literals, heredocs, YAML plain values, inline code.
50    String,
51    /// Comments.
52    Comment,
53    /// Numeric literals (and CSS colours and dimensions).
54    Number,
55    /// Named constants: `true`, `null`, `None`, `ALL_CAPS` names, HTML entities.
56    Constant,
57    /// Brackets, separators and similar.
58    Punctuation,
59    /// Operators (`+`, `&&`, `=`, `|`).
60    Operator,
61    /// Attributes, annotations, decorators, preprocessor lines, HTML attribute names.
62    Attribute,
63    /// Keys and properties: JSON/TOML/YAML keys, CSS properties.
64    Property,
65    /// Variables with a sigil or a role: `$HOME`, lifetimes, YAML anchors, SQL parameters.
66    Variable,
67    /// Markup tag names (`div`, a CSS element selector).
68    Tag,
69    /// Markdown headings.
70    Heading,
71    /// Markdown emphasis and strong emphasis.
72    Emphasis,
73    /// Markdown links and autolinks.
74    Link,
75    /// Added lines in a diff.
76    Inserted,
77    /// Removed lines in a diff.
78    Deleted,
79    /// Meta lines: diff headers and hunk markers, front matter fences, doctype.
80    Meta,
81}
82
83impl Kind {
84    /// Every kind, in declaration order.
85    pub const ALL: [Kind; 20] = [
86        Kind::Plain,
87        Kind::Keyword,
88        Kind::Type,
89        Kind::Function,
90        Kind::String,
91        Kind::Comment,
92        Kind::Number,
93        Kind::Constant,
94        Kind::Punctuation,
95        Kind::Operator,
96        Kind::Attribute,
97        Kind::Property,
98        Kind::Variable,
99        Kind::Tag,
100        Kind::Heading,
101        Kind::Emphasis,
102        Kind::Link,
103        Kind::Inserted,
104        Kind::Deleted,
105        Kind::Meta,
106    ];
107
108    /// A stable lowercase name (`"keyword"`), for themes and debugging.
109    pub fn name(self) -> &'static str {
110        match self {
111            Kind::Plain => "plain",
112            Kind::Keyword => "keyword",
113            Kind::Type => "type",
114            Kind::Function => "function",
115            Kind::String => "string",
116            Kind::Comment => "comment",
117            Kind::Number => "number",
118            Kind::Constant => "constant",
119            Kind::Punctuation => "punctuation",
120            Kind::Operator => "operator",
121            Kind::Attribute => "attribute",
122            Kind::Property => "property",
123            Kind::Variable => "variable",
124            Kind::Tag => "tag",
125            Kind::Heading => "heading",
126            Kind::Emphasis => "emphasis",
127            Kind::Link => "link",
128            Kind::Inserted => "inserted",
129            Kind::Deleted => "deleted",
130            Kind::Meta => "meta",
131        }
132    }
133}
134
135/// A highlighted byte range. Ranges from this crate are ordered, non-overlapping, non-empty, on
136/// `char` boundaries, cover the input, and neighbours never share a kind.
137pub type Span = (Range<usize>, Kind);
138
139/// The lexer state at the start of a line: whether it begins inside a block comment, a
140/// multi-line string, a heredoc… Opaque, `Copy` and cheap to compare; [`State::START`] is the
141/// state at the top of a file.
142#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
143pub struct State {
144    pub(crate) mode: u8,
145    pub(crate) n: u16,
146    pub(crate) h: u32,
147}
148
149impl State {
150    /// The state at the start of a file.
151    pub const START: State = State { mode: 0, n: 0, h: 0 };
152
153    /// Packs the state into a `u64`, for storage or for another [`Highlighter`] to carry its own.
154    pub fn to_bits(self) -> u64 {
155        u64::from(self.mode) | (u64::from(self.n) << 8) | (u64::from(self.h) << 32)
156    }
157
158    /// The inverse of [`State::to_bits`].
159    pub fn from_bits(bits: u64) -> State {
160        State { mode: (bits & 0xff) as u8, n: ((bits >> 8) & 0xffff) as u16, h: (bits >> 32) as u32 }
161    }
162
163    /// The state without its 32-bit payload, packed into 24 bits (used to nest a language inside
164    /// another one, such as script in HTML or a fenced block in Markdown).
165    pub(crate) fn pack24(self) -> u32 {
166        u32::from(self.mode) | (u32::from(self.n) << 8)
167    }
168
169    pub(crate) fn unpack24(v: u32) -> State {
170        State { mode: (v & 0xff) as u8, n: ((v >> 8) & 0xffff) as u16, h: 0 }
171    }
172}
173
174/// A language the built-in lexers know.
175#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
176#[non_exhaustive]
177pub enum Lang {
178    Rust,
179    TypeScript,
180    JavaScript,
181    Go,
182    C,
183    Cpp,
184    Java,
185    CSharp,
186    Python,
187    Shell,
188    Sql,
189    Json,
190    Toml,
191    Yaml,
192    Html,
193    Css,
194    Markdown,
195    Diff,
196}
197
198/// Names and aliases accepted by [`Lang::from_name`] (ASCII case-insensitive).
199const ALIASES: &[(&str, Lang)] = &[
200    ("rust", Lang::Rust),
201    ("rs", Lang::Rust),
202    ("typescript", Lang::TypeScript),
203    ("ts", Lang::TypeScript),
204    ("tsx", Lang::TypeScript),
205    ("mts", Lang::TypeScript),
206    ("cts", Lang::TypeScript),
207    ("javascript", Lang::JavaScript),
208    ("js", Lang::JavaScript),
209    ("jsx", Lang::JavaScript),
210    ("mjs", Lang::JavaScript),
211    ("cjs", Lang::JavaScript),
212    ("go", Lang::Go),
213    ("golang", Lang::Go),
214    ("c", Lang::C),
215    ("h", Lang::C),
216    ("cpp", Lang::Cpp),
217    ("c++", Lang::Cpp),
218    ("cc", Lang::Cpp),
219    ("cxx", Lang::Cpp),
220    ("hpp", Lang::Cpp),
221    ("hh", Lang::Cpp),
222    ("hxx", Lang::Cpp),
223    ("java", Lang::Java),
224    ("csharp", Lang::CSharp),
225    ("c#", Lang::CSharp),
226    ("cs", Lang::CSharp),
227    ("python", Lang::Python),
228    ("py", Lang::Python),
229    ("pyi", Lang::Python),
230    ("shell", Lang::Shell),
231    ("sh", Lang::Shell),
232    ("bash", Lang::Shell),
233    ("zsh", Lang::Shell),
234    ("console", Lang::Shell),
235    ("sql", Lang::Sql),
236    ("json", Lang::Json),
237    ("jsonc", Lang::Json),
238    ("json5", Lang::Json),
239    ("toml", Lang::Toml),
240    ("yaml", Lang::Yaml),
241    ("yml", Lang::Yaml),
242    ("html", Lang::Html),
243    ("htm", Lang::Html),
244    ("xml", Lang::Html),
245    ("svg", Lang::Html),
246    ("xhtml", Lang::Html),
247    ("vue", Lang::Html),
248    ("css", Lang::Css),
249    ("scss", Lang::Css),
250    ("less", Lang::Css),
251    ("markdown", Lang::Markdown),
252    ("md", Lang::Markdown),
253    ("mdx", Lang::Markdown),
254    ("diff", Lang::Diff),
255    ("patch", Lang::Diff),
256];
257
258/// Whole file names (no extension rule applies) and the language they hold.
259const FILE_NAMES: &[(&str, Lang)] = &[
260    (".bashrc", Lang::Shell),
261    (".zshrc", Lang::Shell),
262    (".profile", Lang::Shell),
263    (".bash_profile", Lang::Shell),
264    ("Cargo.lock", Lang::Toml),
265    ("PKGBUILD", Lang::Shell),
266    (".gitconfig", Lang::Toml),
267];
268
269impl Lang {
270    /// Every built-in language.
271    pub const ALL: [Lang; 18] = [
272        Lang::Rust,
273        Lang::TypeScript,
274        Lang::JavaScript,
275        Lang::Go,
276        Lang::C,
277        Lang::Cpp,
278        Lang::Java,
279        Lang::CSharp,
280        Lang::Python,
281        Lang::Shell,
282        Lang::Sql,
283        Lang::Json,
284        Lang::Toml,
285        Lang::Yaml,
286        Lang::Html,
287        Lang::Css,
288        Lang::Markdown,
289        Lang::Diff,
290    ];
291
292    /// The canonical lowercase name (`"rust"`, `"typescript"`).
293    pub fn name(self) -> &'static str {
294        match self {
295            Lang::Rust => "rust",
296            Lang::TypeScript => "typescript",
297            Lang::JavaScript => "javascript",
298            Lang::Go => "go",
299            Lang::C => "c",
300            Lang::Cpp => "cpp",
301            Lang::Java => "java",
302            Lang::CSharp => "csharp",
303            Lang::Python => "python",
304            Lang::Shell => "shell",
305            Lang::Sql => "sql",
306            Lang::Json => "json",
307            Lang::Toml => "toml",
308            Lang::Yaml => "yaml",
309            Lang::Html => "html",
310            Lang::Css => "css",
311            Lang::Markdown => "markdown",
312            Lang::Diff => "diff",
313        }
314    }
315
316    /// The language for a name, alias or file extension (`"rs"`, `"Rust"`, `"tsx"`, `"yml"`),
317    /// ignoring ASCII case and surrounding whitespace. Unknown names give `None`.
318    pub fn from_name(name: &str) -> Option<Lang> {
319        let name = name.trim();
320        ALIASES.iter().find(|(alias, _)| alias.eq_ignore_ascii_case(name)).map(|(_, lang)| *lang)
321    }
322
323    /// The language for a file path, from its file name or extension (`"src/lib.rs"`,
324    /// `"C:\\notes\\todo.md"`, `".bashrc"`).
325    pub fn for_path(path: &str) -> Option<Lang> {
326        let file = path.rsplit(['/', '\\']).next().unwrap_or(path);
327        if let Some((_, lang)) = FILE_NAMES.iter().find(|(name, _)| *name == file) {
328            return Some(*lang);
329        }
330        let (stem, ext) = file.rsplit_once('.')?;
331        if stem.is_empty() {
332            return None;
333        }
334        Lang::from_name(ext)
335    }
336}
337
338/// The canonical language name for a file path (`"rust"` for `"src/main.rs"`), or `None`.
339pub fn lang_for_path(path: &str) -> Option<&'static str> {
340    Lang::for_path(path).map(Lang::name)
341}
342
343/// Highlights `text` as `lang` (a name, alias or extension; see [`Lang::from_name`]).
344///
345/// The spans are ordered, non-overlapping, non-empty, on `char` boundaries and cover the whole
346/// input; neighbours never share a kind. An unknown or missing language gives one
347/// [`Kind::Plain`] span. Runs in time linear in the input and never panics.
348pub fn highlight(lang: Option<&str>, text: &str) -> Vec<Span> {
349    let mut out = Vec::new();
350    highlight_into(lang.and_then(Lang::from_name), text, &mut out);
351    out
352}
353
354/// [`highlight`] with a known language, appending to `out` (reuse it to save allocations).
355/// Offsets are relative to `text`; a span is merged into the last one of `out` when they touch
356/// and share a kind, so clear `out` first unless you mean to continue it.
357pub fn highlight_into(lang: Option<Lang>, text: &str, out: &mut Vec<Span>) {
358    if text.is_empty() {
359        return;
360    }
361    let Some(lang) = lang else {
362        out.push((0..text.len(), Kind::Plain));
363        return;
364    };
365    let mut state = State::START;
366    let mut base = 0usize;
367    for line in text.split_inclusive('\n') {
368        state = line_into(lang, line, state, base, out);
369        base = base.saturating_add(line.len());
370    }
371}
372
373/// Highlights one line (with or without its line ending), starting in `state`, appending spans
374/// relative to the line's start to `out`. Returns the state the next line starts in.
375pub fn highlight_line(lang: Lang, line: &str, state: State, out: &mut Vec<Span>) -> State {
376    line_into(lang, line, state, 0, out)
377}
378
379fn line_into(lang: Lang, line: &str, state: State, base: usize, out: &mut Vec<Span>) -> State {
380    let body = line.strip_suffix('\n').unwrap_or(line);
381    let body = body.strip_suffix('\r').unwrap_or(body);
382    let mut o = scan::Out::new(out, line, base);
383    let next = lang::lex(lang, body, state, &mut o);
384    o.finish();
385    next
386}
387
388/// A syntax highlighting engine. [`Builtin`] is the lexers of this crate; a bigger engine (a
389/// grammar-based one, say) can implement this and be used by the same views.
390pub trait Highlighter {
391    /// Whether this engine highlights `lang` (a name, alias or extension).
392    fn supports(&self, lang: &str) -> bool;
393
394    /// Highlights one line starting in `state`, appending line-relative spans to `out`, and
395    /// returns the state the next line starts in. Spans must follow the rules of [`Span`]; an
396    /// engine without line states may return `state` unchanged.
397    fn highlight_line(&self, lang: &str, line: &str, state: State, out: &mut Vec<Span>) -> State;
398
399    /// Highlights a whole text, appending spans relative to `text` to `out`.
400    fn highlight(&self, lang: &str, text: &str, out: &mut Vec<Span>) {
401        let mut state = State::START;
402        let mut base = 0usize;
403        let mut line_spans = Vec::new();
404        for line in text.split_inclusive('\n') {
405            line_spans.clear();
406            state = self.highlight_line(lang, line, state, &mut line_spans);
407            for (range, kind) in line_spans.drain(..) {
408                let range = range.start.saturating_add(base)..range.end.saturating_add(base);
409                match out.last_mut() {
410                    Some((last, k)) if *k == kind && last.end == range.start => last.end = range.end,
411                    _ => out.push((range, kind)),
412                }
413            }
414            base = base.saturating_add(line.len());
415        }
416    }
417}
418
419/// The built-in lexers as a [`Highlighter`].
420#[derive(Clone, Copy, Debug, Default)]
421pub struct Builtin;
422
423impl Highlighter for Builtin {
424    fn supports(&self, lang: &str) -> bool {
425        Lang::from_name(lang).is_some()
426    }
427
428    fn highlight_line(&self, lang: &str, line: &str, state: State, out: &mut Vec<Span>) -> State {
429        match Lang::from_name(lang) {
430            Some(l) => highlight_line(l, line, state, out),
431            None => {
432                if !line.is_empty() {
433                    out.push((0..line.len(), Kind::Plain));
434                }
435                state
436            }
437        }
438    }
439
440    fn highlight(&self, lang: &str, text: &str, out: &mut Vec<Span>) {
441        highlight_into(Lang::from_name(lang), text, out);
442    }
443}
ashlar_syntax/lib.rs

ashlar_syntax/
lib.rs
