syntect_tmlanguage/

translate.rs

//! Translates a parsed [`RawGrammar`] into `syntect`'s own
//! [`SyntaxDefinition`].
//!
//! ## Why this goes through `.sublime-syntax` YAML text
//!
//! The obvious approach — build `syntect::parsing::syntax_definition::{
//! Context, MatchPattern, ContextReference, ...}` values directly — turned
//! out not to be possible from outside the `syntect` crate: `syntax_definition.rs`
//! says "Everything here is public", but `ContextReference` (and each of its
//! variants) is `#[non_exhaustive]` with **no public constructor function**,
//! so external code cannot construct a `ContextReference::Named(..)` or
//! `::ByScope { .. }` value at all (confirmed against `syntect` 5.3.0; see
//! `TODO.canvas.md`, "Correction: `ContextReference` isn't externally
//! constructible").
//!
//! `syntect`'s own actually-public, stable construction path is
//! [`syntect::parsing::SyntaxDefinition::load_from_str`] — the same entry
//! point `.sublime-syntax` files go through. So this crate instead emits
//! `.sublime-syntax`-shaped YAML (via `yaml-rust`, the exact YAML crate
//! `syntect` itself parses with) and hands that text to `syntect`'s own
//! loader, which builds the real tree for us. This is arguably a better fit
//! for the project's Goal item 3 anyway ("works offline: convert once, ship
//! a generated `.sublime-syntax`") — see [`to_sublime_syntax_yaml`], which
//! exposes that intermediate text directly, valid to save as a
//! `.sublime-syntax` file as-is.
//!
//! ## Mapping from TextMate's model to sublime-syntax's
//!
//! - Top-level `patterns` become the `main` context (the conventional
//!   entry point `syntect` looks up by name).
//! - Each `repository` entry becomes its own named context, addressable via
//!   `include: "#name"`.
//! - `include: "$self"` / `"$base"` become `include: main`; any other
//!   `include` string is treated as an external scope name
//!   (`include: "scope:<name>"`).
//! - A `begin`/`end` rule becomes a `push` match on `begin` into a freshly
//!   named body context, whose own `pop: true` match on `end` is ordered
//!   relative to the body's own `patterns` per `applyEndPatternLast`
//!   (`end` first when absent/`false` — TextMate's default — last when
//!   `true`). `name`/`contentName` become that body context's
//!   `meta_scope`/`meta_content_scope`.
//!
//! `with_prototype` is **not** part of the `.tmLanguage.json` spec at all
//! (it's a `sublime-syntax`-only concept for prototype injection without
//! nesting under `patterns`) — nothing to translate.
//!
//! `begin`/`while` rules **are** real — VS Code's own `markdown.tmLanguage.json`
//! uses them extensively (blockquotes, most fenced-code-block variants) —
//! but `syntect` has no engine-level equivalent whatsoever: no YAML
//! `while:` key, no `MatchOperation` variant for it, checked by grepping
//! its source. A `begin`/`while` rule fails translation with
//! [`Error::WhileNotSupported`] rather than being silently mistranslated;
//! this means real-world grammars built around `while` (markdown chief
//! among them) can't be fully translated today. See `TODO.canvas.md` for
//! the analysis (this was originally — wrongly — listed as "not part of
//! the spec"; VS Code's markdown grammar disproved that).
//!
//! Injections (the grammar's `injections` key, VS Code's
//! `injectionSelector`) genuinely aren't handled either: structural
//! embedding (a fixed-boundary `begin`/`end` region whose body `include`s
//! another grammar's scope) already works through the mapping above and is
//! covered by a real end-to-end test, but true selector-based injection
//! isn't something `syntect` itself supports at all (checked by grepping
//! its source for "injection": zero matches) — see `TODO.canvas.md` for
//! the full analysis.
//!
//! `\G` anchors are rejected outright ([`Error::GAnchorNotSupported`]),
//! checked on every `match`/`begin`/`end` regex: confirmed empirically
//! (`examples/g_anchor_never_matches.rs`) that `\G` never matches at all
//! through `syntect`'s `onig` backend, in any position, because `syntect`
//! runs each pattern as an independent search rather than the iterative
//! same-string session real `\G` semantics depend on. This isn't a rare
//! corner case — VS Code's real `shell-unix-bash.tmLanguage.json` and
//! `html.tmLanguage.json` both hit it in commonly-exercised paths (adjacent
//! quoted command names; `<script>`/`<style>` content detection), so both
//! fail translation entirely today. Silently letting `\G` through would be
//! worse than rejecting it: the affected context just never closes,
//! corrupting everything parsed afterward, with no error at all.
//!
//! Capture indexes that `syntect`'s own YAML loader can't line up with a
//! "consuming" group in the regex (via its `get_consuming_capture_indexes`,
//! backed by the `regex-syntax` crate rather than true Oniguruma semantics)
//! are silently dropped by `syntect` itself, not by this crate.

use std::collections::HashMap;
use std::fmt::Write as _;
use std::path::{Path, PathBuf};

use syntect::parsing::{ParseSyntaxError, SyntaxDefinition};
use yaml_rust::yaml::Hash as YamlHash;
use yaml_rust::{Yaml, YamlEmitter};

use crate::raw::{RawCapture, RawGrammar, RawRule};

#[derive(Debug, thiserror::Error)]
pub enum Error {
    #[error("invalid JSON: {0}")]
    Json(#[from] serde_json::Error),
    #[error("failed to read grammar file {path:?}: {source}")]
    Io {
        path: PathBuf,
        #[source]
        source: std::io::Error,
    },
    #[error("a `begin` rule needs a matching `end` pattern")]
    BeginWithoutEnd,
    #[error(
        "`begin`/`while` rules (scope {scope:?}, while pattern {while_pattern:?}) aren't supported: \
         `syntect` has no engine-level equivalent of TextMate's `while` at all, so there's nothing \
         to translate this into (not just an unimplemented case in this crate)"
    )]
    WhileNotSupported {
        scope: Option<String>,
        while_pattern: String,
    },
    #[error(
        "regex {pattern:?} uses a `\\G` anchor, which never matches through `syntect`'s `onig` \
         backend (confirmed empirically: `syntect` runs each pattern as an independent search, \
         not the iterative same-string session `\\G` needs to mean anything) — translating it \
         would silently produce wrong parses (a context that never closes) rather than an error, \
         so this is rejected instead"
    )]
    GAnchorNotSupported { pattern: String },
    #[error("rule has none of `match`, `begin`/`end`, `include`, or nested `patterns`")]
    EmptyRule,
    #[error("non-numeric capture group index: {0:?}")]
    BadCaptureIndex(String),
    #[error("failed to emit intermediate sublime-syntax YAML: {0}")]
    YamlEmit(#[from] std::fmt::Error),
    #[error("failed to emit intermediate sublime-syntax YAML: {0}")]
    YamlDump(#[from] yaml_rust::EmitError),
    #[error("syntect rejected the generated sublime-syntax YAML: {0}")]
    Syntax(#[from] ParseSyntaxError),
}

/// Parses and translates a `.tmLanguage.json` grammar into a `syntect`
/// [`SyntaxDefinition`] in one step.
pub fn load(source: &str) -> Result<SyntaxDefinition, Error> {
    let grammar = RawGrammar::from_json(source)?;
    translate(&grammar)
}

/// Reads and translates a `.tmLanguage.json` grammar from an arbitrary
/// local file at runtime — the "load a custom grammar the user dropped in,
/// no bundling required" case from the project's Goal.
///
/// The result is a plain [`SyntaxDefinition`]; add it to a
/// [`syntect::parsing::SyntaxSetBuilder`] (alongside any other syntaxes)
/// and call `.build()` the same way you would for a hand-written
/// `.sublime-syntax` file — this crate doesn't wrap that step, since
/// `syntect`'s own builder already does it well.
///
/// For the build-time-bundling case (Goal item 3's "convert once, ship a
/// generated `.sublime-syntax`"), either save [`to_sublime_syntax_yaml`]'s
/// output as a `.sublime-syntax` file directly, or build a [`syntect::parsing::SyntaxSet`]
/// once from this function's output and cache it with
/// [`syntect::dumps::dump_to_file`]/[`syntect::dumps::from_dump_file`] — both
/// already handle any `Serialize`/`DeserializeOwned` type, `SyntaxSet`
/// included, so no wrapper is needed here either.
pub fn load_file(path: impl AsRef<Path>) -> Result<SyntaxDefinition, Error> {
    let path = path.as_ref();
    let source = std::fs::read_to_string(path).map_err(|source| Error::Io {
        path: path.to_path_buf(),
        source,
    })?;
    load(&source)
}

pub fn translate(grammar: &RawGrammar) -> Result<SyntaxDefinition, Error> {
    let yaml_text = to_sublime_syntax_yaml(grammar)?;
    SyntaxDefinition::load_from_str(&yaml_text, false, grammar.name.as_deref())
        .map_err(Error::Syntax)
}

/// Renders `grammar` as `.sublime-syntax`-shaped YAML text — the same
/// intermediate [`translate`] feeds to `syntect`'s own loader, exposed
/// directly because it's also valid to write out as a `.sublime-syntax`
/// file for offline/build-time use.
pub fn to_sublime_syntax_yaml(grammar: &RawGrammar) -> Result<String, Error> {
    let mut contexts = YamlHash::new();
    let mut namer = Namer::default();

    let main_items = translate_patterns(&grammar.patterns, &mut contexts, &mut namer)?;
    contexts.insert(Yaml::String("main".to_string()), Yaml::Array(main_items));

    let mut repo_names: Vec<&String> = grammar.repository.keys().collect();
    repo_names.sort();
    for name in repo_names {
        let rule = &grammar.repository[name];
        let items = repository_entry_to_context(rule, &mut contexts, &mut namer)?;
        contexts.insert(Yaml::String(name.clone()), Yaml::Array(items));
    }

    let mut doc = YamlHash::new();
    doc.insert(
        Yaml::String("name".to_string()),
        Yaml::String(
            grammar
                .name
                .clone()
                .unwrap_or_else(|| grammar.scope_name.clone()),
        ),
    );
    doc.insert(
        Yaml::String("scope".to_string()),
        Yaml::String(grammar.scope_name.clone()),
    );
    if !grammar.file_types.is_empty() {
        doc.insert(
            Yaml::String("file_extensions".to_string()),
            Yaml::Array(
                grammar
                    .file_types
                    .iter()
                    .cloned()
                    .map(Yaml::String)
                    .collect(),
            ),
        );
    }
    if let Some(flm) = &grammar.first_line_match {
        doc.insert(
            Yaml::String("first_line_match".to_string()),
            Yaml::String(flm.clone()),
        );
    }
    doc.insert(Yaml::String("contexts".to_string()), Yaml::Hash(contexts));

    let mut out = String::new();
    // `syntect::parsing::yaml_load` expects a `%YAML 1.2 ---` header, same
    // as real `.sublime-syntax` files; `YamlEmitter` only emits the `---`.
    writeln!(out, "%YAML 1.2")?;
    YamlEmitter::new(&mut out).dump(&Yaml::Hash(doc))?;
    Ok(out)
}

fn repository_entry_to_context(
    rule: &RawRule,
    contexts: &mut YamlHash,
    namer: &mut Namer,
) -> Result<Vec<Yaml>, Error> {
    if rule.match_.is_none() && rule.begin.is_none() && rule.include.is_none() {
        // A bare group entry (only `patterns`) — its patterns list *is* the context.
        return translate_patterns(&rule.patterns, contexts, namer);
    }
    let item = rule_to_item(rule, contexts, namer)?;
    Ok(vec![item])
}

fn translate_patterns(
    rules: &[RawRule],
    contexts: &mut YamlHash,
    namer: &mut Namer,
) -> Result<Vec<Yaml>, Error> {
    rules
        .iter()
        .map(|r| rule_to_item(r, contexts, namer))
        .collect()
}

fn rule_to_item(rule: &RawRule, contexts: &mut YamlHash, namer: &mut Namer) -> Result<Yaml, Error> {
    if let Some(include) = &rule.include {
        let mut item = YamlHash::new();
        item.insert(
            Yaml::String("include".to_string()),
            Yaml::String(resolve_include(include)),
        );
        return Ok(Yaml::Hash(item));
    }

    if let Some(begin) = &rule.begin {
        if let Some(while_pattern) = &rule.while_ {
            return Err(Error::WhileNotSupported {
                scope: rule.name.clone(),
                while_pattern: while_pattern.clone(),
            });
        }
        let end = rule.end.as_ref().ok_or(Error::BeginWithoutEnd)?;
        reject_g_anchor(begin)?;
        reject_g_anchor(end)?;

        let body_name = namer.next("begin-end-body");
        let mut body_items = Vec::new();
        if let Some(name) = &rule.name {
            let mut meta = YamlHash::new();
            meta.insert(
                Yaml::String("meta_scope".to_string()),
                Yaml::String(name.clone()),
            );
            body_items.push(Yaml::Hash(meta));
        }
        if let Some(content_name) = &rule.content_name {
            let mut meta = YamlHash::new();
            meta.insert(
                Yaml::String("meta_content_scope".to_string()),
                Yaml::String(content_name.clone()),
            );
            body_items.push(Yaml::Hash(meta));
        }

        let mut end_item = YamlHash::new();
        end_item.insert(Yaml::String("match".to_string()), Yaml::String(end.clone()));
        if let Some(captures) = captures_yaml(&rule.end_captures)? {
            end_item.insert(Yaml::String("captures".to_string()), captures);
        }
        end_item.insert(Yaml::String("pop".to_string()), Yaml::Boolean(true));
        let end_item = Yaml::Hash(end_item);

        let body_patterns = translate_patterns(&rule.patterns, contexts, namer)?;
        // `applyEndPatternLast` (default `false`) controls whether `end` is
        // tried before or after the body's own patterns at each position.
        if rule.apply_end_pattern_last {
            body_items.extend(body_patterns);
            body_items.push(end_item);
        } else {
            body_items.push(end_item);
            body_items.extend(body_patterns);
        }
        contexts.insert(
            Yaml::String(body_name.clone()),
            Yaml::Array(body_items),
        );

        let mut begin_item = YamlHash::new();
        begin_item.insert(
            Yaml::String("match".to_string()),
            Yaml::String(begin.clone()),
        );
        if let Some(captures) = captures_yaml(&rule.begin_captures)? {
            begin_item.insert(Yaml::String("captures".to_string()), captures);
        }
        begin_item.insert(Yaml::String("push".to_string()), Yaml::String(body_name));
        return Ok(Yaml::Hash(begin_item));
    }

    if let Some(m) = &rule.match_ {
        reject_g_anchor(m)?;
        let mut item = YamlHash::new();
        item.insert(Yaml::String("match".to_string()), Yaml::String(m.clone()));
        if let Some(name) = &rule.name {
            item.insert(
                Yaml::String("scope".to_string()),
                Yaml::String(name.clone()),
            );
        }
        if let Some(captures) = captures_yaml(&rule.captures)? {
            item.insert(Yaml::String("captures".to_string()), captures);
        }
        return Ok(Yaml::Hash(item));
    }

    if !rule.patterns.is_empty() {
        // An anonymous group node appearing directly in a `patterns` list.
        let name = namer.next("group");
        let items = translate_patterns(&rule.patterns, contexts, namer)?;
        contexts.insert(Yaml::String(name.clone()), Yaml::Array(items));
        let mut item = YamlHash::new();
        item.insert(Yaml::String("include".to_string()), Yaml::String(name));
        return Ok(Yaml::Hash(item));
    }

    Err(Error::EmptyRule)
}

/// Rejects a regex using an unescaped `\G` anchor — see
/// [`Error::GAnchorNotSupported`] for why this can't just be passed
/// through. Uses the same escape-parity scan as backreference detection
/// (an odd run of backslashes right before `G` means it's live).
fn reject_g_anchor(pattern: &str) -> Result<(), Error> {
    let mut backslash_run = 0usize;
    for c in pattern.chars() {
        if c == '\\' {
            backslash_run += 1;
            continue;
        }
        if c == 'G' && backslash_run % 2 == 1 {
            return Err(Error::GAnchorNotSupported {
                pattern: pattern.to_string(),
            });
        }
        backslash_run = 0;
    }
    Ok(())
}

fn resolve_include(include: &str) -> String {
    if include == "$self" || include == "$base" {
        return "main".to_string();
    }
    if let Some(name) = include.strip_prefix('#') {
        return name.to_string();
    }
    format!("scope:{include}")
}

fn captures_yaml(
    captures: &Option<HashMap<String, RawCapture>>,
) -> Result<Option<Yaml>, Error> {
    let Some(map) = captures else {
        return Ok(None);
    };
    let mut hash = YamlHash::new();
    for (key, capture) in map {
        let index: i64 = key
            .parse()
            .map_err(|_| Error::BadCaptureIndex(key.clone()))?;
        let Some(name) = &capture.name else {
            continue;
        };
        hash.insert(Yaml::Integer(index), Yaml::String(name.clone()));
    }
    if hash.is_empty() {
        return Ok(None);
    }
    Ok(Some(Yaml::Hash(hash)))
}

#[derive(Default)]
struct Namer {
    counter: usize,
}

impl Namer {
    fn next(&mut self, prefix: &str) -> String {
        self.counter += 1;
        format!("{prefix}-{}", self.counter)
    }
}