syntaxmate/
lib.rs

1//! Rust-native TextMate syntax highlighting with bundled grammars and themes.
2//!
3//! [`Highlighter`] is the default batteries-included entry point, with
4//! structured spans plus safe HTML and ANSI convenience output.
5//! [`GrammarRegistry`] and [`Tokenizer`] provide the custom-grammar API.
6//!
7//! ```
8//! use syntaxmate::Highlighter;
9//!
10//! let highlighter = Highlighter::bundled()?;
11//! let document = highlighter.highlight("rust", "fn main() {}", "github-dark")?;
12//! assert!(document.status().is_complete());
13//! # Ok::<(), syntaxmate::Error>(())
14//! ```
15
16#![forbid(unsafe_code)]
17#![warn(missing_docs)]
18#![cfg_attr(docsrs, feature(doc_cfg))]
19#![cfg_attr(docsrs, doc(auto_cfg))]
20
21// Compile the actual guide snippets as doctests instead of maintaining copies.
22#[cfg(all(
23    doc,
24    feature = "bundled-grammars",
25    feature = "bundled-themes",
26    feature = "html"
27))]
28#[doc = include_str!("../README.md")]
29mod readme_examples {}
30
31#[cfg(all(
32    doc,
33    feature = "bundled-grammars",
34    feature = "bundled-themes",
35    feature = "html",
36    feature = "ansi"
37))]
38#[doc = include_str!("../docs/rendering.md")]
39mod rendering_examples {}
40
41mod catalog;
42#[allow(dead_code, unused_imports)]
43mod engine;
44mod error;
45#[allow(dead_code)]
46mod grammars;
47mod highlighter;
48mod render;
49mod theme;
50mod tokenizer;
51#[allow(dead_code)]
52mod types;
53
54pub use catalog::{AssetLicense, Catalog, CatalogSummary, LanguageInfo};
55pub use error::{
56    BundleError, BundleErrorKind, DiagnosticError, DiagnosticErrorKind, Error, GrammarError,
57    GrammarErrorKind, GrammarResource, JsonError, LimitExceeded, MissingInclude, RegexError,
58    RenderError, RenderErrorKind, Result, ThemeError, ThemeErrorKind,
59};
60pub use highlighter::{HighlightSession, Highlighter, HighlighterOptions};
61pub use highlighter::{
62    HighlightedDocument, HighlightedLine, HighlightedToken, Theme, style_document,
63};
64pub use render::RenderedOutput;
65#[cfg(feature = "ansi")]
66pub use render::{AnsiOptions, render_ansi, render_ansi_to};
67#[cfg(feature = "html")]
68pub use render::{HtmlOptions, html_stylesheet, render_html, render_html_to};
69pub use theme::{FontModifiers, RgbColor, Style};
70#[cfg(feature = "diagnostics")]
71pub use theme::{ResolvedThemeStyle, ThemeMatch, ThemeSelectorScore};
72pub use tokenizer::{
73    CheckpointTable, GrammarId, GrammarLimits, GrammarRegistry, HighlightStatus, PreparedLanguage,
74    PreparedLanguageStats, Scopes, Token, TokenizedDocument, TokenizedLine, Tokenizer,
75    TokenizerState,
76};
77pub(crate) use types::{HighlightScopeTable, ScopeAtomId};
78pub use types::{ScopeStackId, ThemeRule, TokenizerOptions};
79
80// Internal engine modules use these compact output types directly. They are
81// deliberately not part of the top-level documented facade.
82pub(crate) use types::{
83    HighlightedLine as EngineHighlightedLine, HighlightedText, LineTextFingerprint, SyntaxClass,
84    SyntaxSegment,
85};
86#[cfg(test)]
87#[path = "../tests/engine_capture_quality.rs"]
88mod engine_capture_quality;
89#[cfg(test)]
90#[path = "../tests/engine_regressions.rs"]
91mod engine_regressions;
92#[cfg(all(test, feature = "bundled-grammars", feature = "bundled-themes"))]
93mod public_api_tests;
94#[cfg(test)]
95#[path = "../tests/textmate_golden.rs"]
96mod textmate_golden;
97#[cfg(test)]
98#[path = "../tests/theme_golden.rs"]
99mod theme_golden;
100
101/// Feature-gated engine inspection APIs, outside the stable compatibility contract.
102#[cfg(feature = "diagnostics")]
103pub mod diagnostics {
104    use std::ops::Range;
105
106    pub use crate::engine::counters::{EngineCounters, PatternCompileCount, PatternHotspot};
107    use crate::engine::regex::{
108        AnchorContext, AutomataMatcher, FallbackMatcher, Matcher, RegexMatcher, parse, translate,
109    };
110    use crate::{Error, Result};
111
112    /// Matcher selection for diagnostic regex execution.
113    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
114    pub enum RegexEngine {
115        /// Selects a matcher using the normal engine routing rules.
116        Auto,
117        /// Requires the DFA matcher.
118        Dfa,
119        /// Uses the budgeted fallback matcher.
120        Fallback,
121    }
122
123    /// Anchor context for one diagnostic regex search.
124    #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
125    pub struct RegexAnchorContext {
126        /// Whether the start-of-file anchor may match.
127        pub allow_start_of_file: bool,
128        /// Byte position allowed to match the continuation anchor, if any.
129        pub continuation_position: Option<usize>,
130    }
131
132    /// Human-readable regex parsing and routing information.
133    #[derive(Debug, Clone, PartialEq, Eq)]
134    pub struct RegexInspection {
135        /// Parsed expression rendered as diagnostic text.
136        pub parsed: String,
137        /// Pattern after compatibility translation.
138        pub translated_pattern: String,
139        /// Diagnostic anchor-handling strategy.
140        pub anchor_strategy: String,
141        /// Selected matcher route.
142        pub route: String,
143    }
144
145    /// Result and execution metadata from a diagnostic regex search.
146    #[derive(Debug, Clone, PartialEq, Eq)]
147    pub struct RegexMatchReport {
148        /// Name of the matcher used.
149        pub engine: &'static str,
150        /// Matched byte range, or `None` when no match was found.
151        pub matched: Option<Range<usize>>,
152        /// Capture byte ranges, including the full match at index zero.
153        pub captures: Vec<Option<Range<usize>>>,
154        /// Fallback execution steps, when the matcher reports them.
155        pub steps: Option<usize>,
156    }
157
158    /// Inspects regex parsing, translation, and matcher routing.
159    pub fn inspect_regex(pattern: &str) -> RegexInspection {
160        let parsed = parse(pattern);
161        let translation = translate(pattern);
162        RegexInspection {
163            parsed: parsed.to_string(),
164            translated_pattern: translation.pattern,
165            anchor_strategy: format!("{:?}", translation.anchor_strategy),
166            route: format!("{:?}", translation.route),
167        }
168    }
169
170    /// Searches from a byte offset with explicit anchors, matcher choice, and fallback step budget.
171    pub fn match_regex(
172        pattern: &str,
173        line: &str,
174        from: usize,
175        anchors: RegexAnchorContext,
176        engine: RegexEngine,
177        fallback_budget: usize,
178    ) -> Result<RegexMatchReport> {
179        let context = AnchorContext {
180            allow_a: anchors.allow_start_of_file,
181            allow_g: anchors.continuation_position.is_some(),
182            g_pos: anchors.continuation_position.unwrap_or(0),
183        };
184        let (engine_name, result, steps) = match engine {
185            RegexEngine::Auto => {
186                let matcher = RegexMatcher::new(pattern);
187                let (result, steps) =
188                    matcher.find_report(line, from, context).map_err(|error| {
189                        Error::Diagnostic(crate::DiagnosticError::fallback(pattern, error))
190                    })?;
191                (matcher.engine_name(), result, steps)
192            }
193            RegexEngine::Dfa => {
194                let matcher = AutomataMatcher::new(pattern).map_err(|error| {
195                    Error::Diagnostic(crate::DiagnosticError::build(pattern, error.to_string()))
196                })?;
197                ("dfa", matcher.find(line, from, context), None)
198            }
199            RegexEngine::Fallback => {
200                let matcher = FallbackMatcher::with_budget(pattern, fallback_budget);
201                let report = matcher.try_find(line, from, context).map_err(|error| {
202                    Error::Diagnostic(crate::DiagnosticError::fallback(pattern, error))
203                })?;
204                ("fallback", report.result, Some(report.steps))
205            }
206        };
207        Ok(RegexMatchReport {
208            engine: engine_name,
209            matched: result.as_ref().map(|matched| matched.start..matched.end),
210            captures: result.map_or_else(Vec::new, |matched| matched.captures),
211            steps,
212        })
213    }
214}
syntaxmate/lib.rs

syntaxmate/
lib.rs
