ferriki 1.1.1

use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;

use ferriki_textmate::{
    BacktrackingWarning, FontStyle, GrammarConfiguration, RawGrammar, StandardTokenType,
    SyncRegistry, parse_raw_grammar,
};
use serde_json::Value;

use crate::asset_catalog::StandardAssetCatalogs;
use crate::render::{RenderOptions, render_html_lines};
use crate::theme_data::{ThemeData, parse_theme_data};
use crate::tokens::{
    HighlightThemeMetadata, HighlightThemeToken, HighlightThemeTokenStyle, HighlightToken,
    HighlightTokensResult, HighlightTokensWithThemesResult, TokenizeOptions,
    convert_token_offsets_to_utf8, split_lines, token_from_metadata, utf16_to_byte_map,
};
use crate::{Error, ErrorKind, Result};

/// Compatibility bridge for the N-API host. Its token offsets are UTF-16.
/// Rust consumers should use [`Highlighter`] for UTF-8 byte offsets.
#[doc(hidden)]
pub struct HighlighterCore {
    standard_assets: Option<StandardAssetCatalogs>,
    registry: SyncRegistry,
    aliases: BTreeMap<String, String>,
    loaded_language_ids: BTreeSet<String>,
    loaded_custom_language_ids: BTreeSet<String>,
    loaded_language_order: Vec<String>,
    language_aliases: BTreeMap<String, Vec<String>>,
    injections: BTreeMap<String, Vec<String>>,
    themes: BTreeMap<String, ThemeData>,
    active_theme: Option<String>,
    regex_prefilter: bool,
}

/// Reusable, synchronous highlighter for Rust consumers.
///
/// A highlighter lazily loads grammars and themes from its asset catalogs.
/// Its mutable API makes cache updates explicit; instances are not shared
/// across threads. Create one instance and reuse it across many documents.
///
/// ```no_run
/// use std::path::Path;
/// use ferriki::{Highlighter, RenderOptions, StandardAssetCatalogs};
///
/// fn main() -> Result<(), Box<dyn std::error::Error>> {
///     let assets = StandardAssetCatalogs::load_from_root(Path::new("assets/standard"))?;
///     let mut highlighter = Highlighter::builder().with_assets(assets).build()?;
///     let highlighted = highlighter.highlight_html_lines(
///         "fn main() {}", "rust", "nord", &RenderOptions::default(),
///     )?;
///     assert_eq!(highlighted.lines.len(), 1);
///     Ok(())
/// }
/// ```
pub struct Highlighter {
    core: HighlighterCore,
}

impl std::fmt::Debug for Highlighter {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("Highlighter")
            .field("loaded_languages", &self.core.loaded_languages())
            .finish_non_exhaustive()
    }
}

/// Escaped, balanced inner HTML and the theme colors for one code block.
///
/// Each entry in `lines` corresponds to one source line. A Markdown renderer
/// owns the outer markup and should apply `foreground` and `background` to its
/// code-block wrapper.
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub struct HighlightedLines {
    pub lines: Vec<String>,
    pub foreground: String,
    pub background: String,
    pub theme_name: String,
}

/// A typed custom TextMate grammar registration.
#[derive(Clone, Debug, PartialEq)]
#[non_exhaustive]
pub struct LanguageRegistration {
    /// Language identifier used by `highlight`.
    pub id: String,
    /// Parsed TextMate grammar with a non-empty scope name.
    pub grammar: RawGrammar,
    /// Additional language identifiers.
    pub aliases: Vec<String>,
    /// Scopes into which this grammar injects.
    pub inject_to: Vec<String>,
}

/// Configuration for a reusable highlighter.
#[derive(Debug, Default)]
pub struct HighlighterBuilder {
    assets: Option<StandardAssetCatalogs>,
    languages: Vec<String>,
    themes: Vec<String>,
    regex_prefilter: Option<bool>,
}

impl Highlighter {
    /// Starts a highlighter builder. Assets are optional for custom-only use.
    pub fn builder() -> HighlighterBuilder {
        HighlighterBuilder::default()
    }

    /// Highlights source into per-line themed tokens.
    pub fn highlight(
        &mut self,
        code: &str,
        language: &str,
        theme: &str,
    ) -> Result<HighlightTokensResult> {
        self.highlight_with_options(code, language, theme, &TokenizeOptions::default())
    }

    /// Highlights source with explicit tokenizer limits and metadata options.
    pub fn highlight_with_options(
        &mut self,
        code: &str,
        language: &str,
        theme: &str,
        options: &TokenizeOptions,
    ) -> Result<HighlightTokensResult> {
        let mut result = self.core.tokenize(code, language, theme, options)?;
        convert_token_offsets_to_utf8(
            result
                .tokens
                .iter_mut()
                .flatten()
                .map(|token| &mut token.offset),
            code,
        )?;
        Ok(result)
    }

    /// Highlights source into aligned tokens with named theme variants.
    ///
    /// Each pair is `(variant_name, registered_theme_name)`. Variant names
    /// must be non-empty and unique, and at least one theme is required.
    /// Token boundaries are the union of each theme's boundaries; offsets
    /// are UTF-8 byte positions in the full source. Metadata follows request
    /// order, while token styles are keyed by variant name.
    pub fn highlight_with_themes(
        &mut self,
        code: &str,
        language: &str,
        themes: &[(&str, &str)],
    ) -> Result<HighlightTokensWithThemesResult> {
        self.highlight_with_themes_and_options(code, language, themes, &TokenizeOptions::default())
    }

    /// Highlights named theme variants with explicit tokenizer options.
    ///
    /// Empty or duplicate variant names and an empty theme list return
    /// [`ErrorKind::Theme`]. Missing registrations and assets retain the same
    /// error categories as [`Self::highlight`]. Optional token types and scope
    /// paths come from the first requested theme.
    pub fn highlight_with_themes_and_options(
        &mut self,
        code: &str,
        language: &str,
        themes: &[(&str, &str)],
        options: &TokenizeOptions,
    ) -> Result<HighlightTokensWithThemesResult> {
        let mut names = BTreeSet::new();
        if themes.is_empty()
            || themes
                .iter()
                .any(|(name, _)| name.is_empty() || !names.insert(*name))
        {
            return Err(Error::new(
                ErrorKind::Theme,
                "At least one theme with non-empty, unique variant names is required.",
            ));
        }
        let themes: Vec<_> = themes
            .iter()
            .map(|(name, theme)| ((*name).to_owned(), (*theme).to_owned()))
            .collect();
        let mut result = self
            .core
            .tokenize_with_themes(code, language, &themes, options)?;
        convert_token_offsets_to_utf8(
            result
                .tokens
                .iter_mut()
                .flatten()
                .map(|token| &mut token.offset),
            code,
        )?;
        Ok(result)
    }

    /// Highlights directly to escaped, balanced inner HTML per source line.
    ///
    /// This avoids converting token offsets when a Markdown renderer only
    /// needs line fragments. It does not include code-block or line wrappers.
    pub fn highlight_html_lines(
        &mut self,
        code: &str,
        language: &str,
        theme: &str,
        render_options: &RenderOptions,
    ) -> Result<HighlightedLines> {
        let tokens = self.core.tokenize(
            code,
            language,
            theme,
            &TokenizeOptions::default().with_preserve_scope_boundaries(
                render_options.style_mode == crate::StyleMode::Classes,
            ),
        )?;
        let lines = render_html_lines(&tokens, render_options);
        Ok(HighlightedLines {
            lines,
            foreground: tokens.foreground,
            background: tokens.background,
            theme_name: tokens.theme_name,
        })
    }

    /// Highlights nested scope classes and extracts CSS from an unchanged TextMate theme.
    pub fn highlight_html_with_css(
        &mut self,
        code: &str,
        language: &str,
        theme: &str,
        render_options: &RenderOptions,
    ) -> Result<crate::HtmlWithCss> {
        let result = self.highlight_with_options(
            code,
            language,
            theme,
            &TokenizeOptions::default().with_preserve_scope_boundaries(true),
        )?;
        Ok(crate::render_html_with_css(&result, render_options))
    }

    /// Loads a standard language and its dependencies; returns false if absent.
    pub fn load_language(&mut self, language: &str) -> Result<bool> {
        self.core
            .load_standard_language(language)
            .map(|scope| scope.is_some())
    }

    /// Returns advisory regex backtracking warnings from the current compiled
    /// grammar instance for a language.
    ///
    /// Scanners compile lazily during tokenization, so an empty list can mean
    /// that no risky pattern has been found in the scanners built so far. The
    /// list does not cover rules that tokenized input has not reached. Returns
    /// `None` when the language is not registered. Registering a grammar or
    /// injection can invalidate the compiled grammar cache and reset this
    /// snapshot; tokenizing the language again rediscovers warnings as its
    /// scanners compile. Activating a different theme keeps the compiled
    /// grammar and its warnings. This is not a complete grammar lint.
    pub fn backtracking_warnings(
        &mut self,
        language: &str,
    ) -> Result<Option<Vec<BacktrackingWarning>>> {
        let Some(grammar) = self.core.grammar_for_language(language)? else {
            return Ok(None);
        };
        Ok(Some(grammar.backtracking_warnings()))
    }

    /// Loads a standard theme; returns false if absent.
    pub fn load_theme(&mut self, theme: &str) -> Result<bool> {
        self.core.load_standard_theme(theme)
    }

    /// Registers a custom grammar in Ferriki's JSON registration format.
    pub fn register_language_json(&mut self, source: &str) -> Result<()> {
        self.core.load_custom_language(source).map(|_| ())
    }

    /// Registers a typed custom grammar without a JSON intermediary.
    pub fn register_language(&mut self, registration: LanguageRegistration) -> Result<()> {
        self.core.register_language(registration).map(|_| ())
    }

    /// Registers a custom theme in Ferriki's JSON registration format.
    pub fn register_theme_json(&mut self, source: &str) -> Result<()> {
        self.core.load_custom_theme(source).map(|_| ())
    }

    /// Registers a parsed custom theme without a JSON intermediary.
    pub fn register_theme(&mut self, theme: ThemeData) -> Result<()> {
        self.core.register_theme(theme)
    }
}

impl HighlighterBuilder {
    /// Enables automatic regex prefiltering (the default), or disables it to
    /// avoid prefilter construction for one-shot highlighting. This choice
    /// applies to all grammars loaded by the highlighter.
    pub fn with_regex_prefilter(mut self, enabled: bool) -> Self {
        self.regex_prefilter = Some(enabled);
        self
    }

    /// Uses portable asset catalogs prepared by a filesystem or memory source.
    pub fn with_assets(mut self, assets: StandardAssetCatalogs) -> Self {
        self.assets = Some(assets);
        self
    }

    /// Eagerly loads these languages while building the highlighter.
    pub fn load_languages<I, S>(mut self, languages: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        self.languages.extend(
            languages
                .into_iter()
                .map(|language| language.as_ref().to_owned()),
        );
        self
    }

    /// Eagerly loads these themes while building the highlighter.
    pub fn load_themes<I, S>(mut self, themes: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        self.themes
            .extend(themes.into_iter().map(|theme| theme.as_ref().to_owned()));
        self
    }

    /// Builds one reusable highlighter. Unknown requested assets are errors.
    pub fn build(self) -> Result<Highlighter> {
        let mut core =
            HighlighterCore::new_with_regex_prefilter(self.regex_prefilter.unwrap_or(true))?;
        core.standard_assets = self.assets;
        for language in self.languages {
            if core.load_standard_language(&language)?.is_none() {
                return Err(Error::new(
                    ErrorKind::UnknownLanguage,
                    format!("Unknown language `{language}`."),
                ));
            }
        }
        for theme in self.themes {
            if !core.load_standard_theme(&theme)? {
                return Err(Error::new(
                    ErrorKind::UnknownTheme,
                    format!("Unknown theme `{theme}`."),
                ));
            }
        }
        Ok(Highlighter { core })
    }
}

impl HighlighterCore {
    pub fn new() -> Result<Self> {
        Self::new_with_regex_prefilter(true)
    }

    pub fn new_with_regex_prefilter(regex_prefilter: bool) -> Result<Self> {
        Ok(Self {
            standard_assets: None,
            registry: SyncRegistry::new(None, None).map_err(theme_error)?,
            aliases: BTreeMap::new(),
            loaded_language_ids: BTreeSet::new(),
            loaded_custom_language_ids: BTreeSet::new(),
            loaded_language_order: Vec::new(),
            language_aliases: BTreeMap::new(),
            injections: BTreeMap::new(),
            themes: BTreeMap::new(),
            active_theme: None,
            regex_prefilter,
        })
    }

    pub fn with_standard_assets(root: &Path) -> Result<Self> {
        Self::with_assets(StandardAssetCatalogs::load_from_root(root)?)
    }

    pub fn with_assets(assets: StandardAssetCatalogs) -> Result<Self> {
        Self::with_assets_and_regex_prefilter(assets, true)
    }

    pub fn with_assets_and_regex_prefilter(
        assets: StandardAssetCatalogs,
        regex_prefilter: bool,
    ) -> Result<Self> {
        let mut highlighter = Self::new_with_regex_prefilter(regex_prefilter)?;
        highlighter.standard_assets = Some(assets);
        Ok(highlighter)
    }

    pub fn dispose(&mut self) {
        self.registry.dispose();
        self.standard_assets = None;
        self.aliases.clear();
        self.loaded_language_ids.clear();
        self.loaded_custom_language_ids.clear();
        self.loaded_language_order.clear();
        self.language_aliases.clear();
        self.injections.clear();
        self.themes.clear();
        self.active_theme = None;
    }

    pub fn load_standard_theme(&mut self, theme_id: &str) -> Result<bool> {
        if self.themes.contains_key(theme_id) {
            return Ok(true);
        }
        let Some(catalogs) = self.standard_assets.as_ref() else {
            return Ok(false);
        };
        let Some(asset) = catalogs.themes.load_asset(theme_id)? else {
            return Ok(false);
        };
        let theme = parse_theme_data(&asset.id, &asset.theme_json).map_err(|error| {
            Error::new(
                ErrorKind::AssetFormat,
                format!("Failed to parse standard theme `{}`: {error}", asset.id),
            )
        })?;
        self.themes.insert(asset.id.clone(), theme);
        Ok(true)
    }

    pub fn activate_theme(&mut self, theme_id: &str) -> Result<Option<&ThemeData>> {
        if !self.load_standard_theme(theme_id)? {
            return Ok(None);
        }
        if self.active_theme.as_deref() != Some(theme_id) {
            let theme = self
                .themes
                .get(theme_id)
                .ok_or_else(|| Error::from_reason("Loaded Ferriki theme disappeared."))?;
            self.registry
                .set_theme(Some(theme.raw_theme.clone()), None)
                .map_err(theme_error)?;
            self.active_theme = Some(theme_id.to_owned());
        }
        Ok(self.themes.get(theme_id))
    }

    pub fn load_standard_language(&mut self, requested: &str) -> Result<Option<String>> {
        let mut visiting = BTreeSet::new();
        self.load_standard_language_inner(requested, &mut visiting)
    }

    /// Register a user-provided TextMate grammar without pretending it is a
    /// bundled asset. The JSON shape intentionally stays close to Shiki's
    /// LanguageRegistration: the raw grammar fields are parsed by the same
    /// decoder as shipped grammars, while aliases and injection metadata are
    /// kept in the shared lookup model.
    pub fn load_custom_language(&mut self, source: &str) -> Result<Option<String>> {
        let value: Value = serde_json::from_str(source).map_err(|error| {
            Error::new(
                ErrorKind::InvalidRegistration,
                format!("Failed to parse custom language registration: {error}"),
            )
        })?;
        let object = value.as_object().ok_or_else(|| {
            Error::new(
                ErrorKind::InvalidRegistration,
                "Language registration must be a JSON object.",
            )
        })?;
        let id = object
            .get("name")
            .and_then(Value::as_str)
            .filter(|name| !name.is_empty())
            .ok_or_else(|| {
                Error::new(
                    ErrorKind::InvalidRegistration,
                    "Language registration requires a non-empty `name`.",
                )
            })?
            .to_owned();
        let grammar = parse_raw_grammar(source, Some("custom-grammar.json")).map_err(|error| {
            Error::new(
                ErrorKind::InvalidRegistration,
                format!("Failed to parse custom grammar `{id}`: {error}"),
            )
        })?;
        if grammar.scope_name.is_empty() {
            return Err(Error::new(
                ErrorKind::InvalidRegistration,
                format!("Language registration `{id}` requires a non-empty `scopeName`."),
            ));
        }
        let aliases = string_array(object, "aliases")?;
        let inject_to = string_array(object, "injectTo")?;

        self.register_language(LanguageRegistration {
            id,
            grammar,
            aliases,
            inject_to,
        })
        .map(Some)
    }

    pub fn register_language(&mut self, registration: LanguageRegistration) -> Result<String> {
        let LanguageRegistration {
            id,
            grammar,
            aliases,
            inject_to,
        } = registration;
        if id.is_empty() || grammar.scope_name.is_empty() {
            return Err(Error::new(
                ErrorKind::InvalidRegistration,
                "Language registration requires non-empty `id` and `scopeName`.",
            ));
        }
        if self.loaded_custom_language_ids.contains(&id) {
            return Ok(grammar.scope_name);
        }
        let scope_name = grammar.scope_name.clone();
        self.registry.add_grammar(
            grammar,
            self.injections
                .get(&scope_name)
                .cloned()
                .unwrap_or_default(),
        );
        self.aliases.insert(id.clone(), scope_name.clone());
        for alias in &aliases {
            // A user registration must never steal a key that belongs to the
            // standard catalog. The standard resolver can still load that
            // language later, regardless of registration order.
            if self
                .standard_assets
                .as_ref()
                .and_then(|catalogs| catalogs.languages.resolve_id(alias))
                .is_some()
            {
                continue;
            }
            self.aliases.insert(alias.clone(), scope_name.clone());
        }
        for target in &inject_to {
            {
                let injections = self.injections.entry(target.clone()).or_default();
                if !injections.contains(&scope_name) {
                    injections.push(scope_name.clone());
                }
            }
            self.refresh_injections(target);
        }
        self.language_aliases.insert(id.clone(), aliases);
        self.loaded_custom_language_ids.insert(id.clone());
        self.loaded_language_order.push(id);
        Ok(scope_name)
    }

    /// Register a user-provided theme using the same parser and TextMate
    /// theme cache as bundled themes. A simple `include` name is supported so
    /// custom themes can layer on a previously loaded standard/custom theme.
    pub fn load_custom_theme(&mut self, source: &str) -> Result<bool> {
        let value: Value = serde_json::from_str(source).map_err(|error| {
            Error::new(
                ErrorKind::InvalidRegistration,
                format!("Failed to parse custom theme registration: {error}"),
            )
        })?;
        let object = value.as_object().ok_or_else(|| {
            Error::new(
                ErrorKind::InvalidRegistration,
                "Theme registration must be a JSON object.",
            )
        })?;
        let id = object
            .get("name")
            .and_then(Value::as_str)
            .filter(|name| !name.is_empty())
            .ok_or_else(|| {
                Error::new(
                    ErrorKind::InvalidRegistration,
                    "Theme registration requires a non-empty `name`.",
                )
            })?
            .to_owned();
        let mut theme = parse_theme_data(&id, source)?;
        if let Some(include) = object.get("include") {
            let include = include.as_str().ok_or_else(|| {
                Error::new(
                    ErrorKind::InvalidRegistration,
                    "Theme registration `include` must be a theme name.",
                )
            })?;
            if !self.load_standard_theme(include)? {
                return Err(Error::new(
                    ErrorKind::InvalidRegistration,
                    format!("Theme `{include}` not found for custom theme include."),
                ));
            }
            let base = self
                .themes
                .get(include)
                .cloned()
                .ok_or_else(|| Error::from_reason("Included Ferriki theme disappeared."))?;
            let explicit_foreground = object.get("fg").and_then(Value::as_str).is_some()
                || object
                    .get("colors")
                    .and_then(Value::as_object)
                    .and_then(|colors| colors.get("editor.foreground"))
                    .and_then(Value::as_str)
                    .is_some();
            let explicit_background = object.get("bg").and_then(Value::as_str).is_some()
                || object
                    .get("colors")
                    .and_then(Value::as_object)
                    .and_then(|colors| colors.get("editor.background"))
                    .and_then(Value::as_str)
                    .is_some();
            if !explicit_foreground {
                theme.foreground = base.foreground.clone();
            }
            if !explicit_background {
                theme.background = base.background.clone();
            }
            let mut settings = base.raw_theme.settings;
            settings.extend(theme.raw_theme.settings);
            theme.raw_theme.settings = settings;
        }
        self.register_theme(theme)?;
        Ok(true)
    }

    pub fn register_theme(&mut self, theme: ThemeData) -> Result<()> {
        if theme.name.is_empty() {
            return Err(Error::new(
                ErrorKind::InvalidRegistration,
                "Theme registration requires a non-empty `name`.",
            ));
        }
        if self.active_theme.as_deref() == Some(&theme.name) {
            self.active_theme = None;
        }
        self.themes.insert(theme.name.clone(), theme);
        Ok(())
    }

    fn load_standard_language_inner(
        &mut self,
        requested: &str,
        visiting: &mut BTreeSet<String>,
    ) -> Result<Option<String>> {
        if self.registry.lookup(requested).is_some() {
            return Ok(Some(requested.to_owned()));
        }
        if let Some(scope_name) = self.aliases.get(requested) {
            return Ok(Some(scope_name.clone()));
        }

        let Some(catalogs) = self.standard_assets.as_ref() else {
            return Ok(None);
        };
        let Some(asset) = catalogs.languages.load_asset(requested)? else {
            return Ok(None);
        };
        let asset = (*asset).clone();
        if self.loaded_language_ids.contains(&asset.id) {
            return Ok(Some(asset.scope_name));
        }
        if !visiting.insert(asset.id.clone()) {
            return Ok(Some(asset.scope_name));
        }

        for dependency in &asset.embedded_langs {
            self.load_standard_language_inner(dependency, visiting)?;
        }

        let raw_grammar =
            parse_raw_grammar(&asset.grammar_json, Some("grammar.json")).map_err(|error| {
                Error::new(
                    ErrorKind::AssetFormat,
                    format!("Failed to parse standard grammar `{}`: {error}", asset.id),
                )
            })?;
        self.register_loaded_grammar(&asset, raw_grammar);

        self.refresh_injections(&asset.scope_name);

        visiting.remove(&asset.id);
        Ok(Some(asset.scope_name))
    }

    fn register_loaded_grammar(
        &mut self,
        asset: &ferriki_asset_gen::LanguageAsset,
        grammar: RawGrammar,
    ) {
        let scope_name = asset.scope_name.clone();
        self.registry.add_grammar(
            grammar,
            self.injections
                .get(&scope_name)
                .cloned()
                .unwrap_or_default(),
        );
        self.aliases.insert(asset.id.clone(), scope_name.clone());
        for alias in &asset.aliases {
            self.aliases.insert(alias.clone(), scope_name.clone());
        }
        for target in &asset.inject_to {
            let injections = self.injections.entry(target.clone()).or_default();
            if !injections.contains(&scope_name) {
                injections.push(scope_name.clone());
            }
            self.refresh_injections(target);
        }
        if self.loaded_language_ids.insert(asset.id.clone()) {
            self.loaded_language_order.push(asset.id.clone());
        }
        self.language_aliases
            .insert(asset.id.clone(), asset.aliases.clone());
    }

    fn refresh_injections(&mut self, target_scope: &str) {
        let injections = self
            .injections
            .iter()
            .filter(|(target, _)| scope_matches(target, target_scope))
            .flat_map(|(_, injections)| injections.iter().cloned())
            .collect::<BTreeSet<_>>()
            .into_iter()
            .collect();
        self.registry.set_injections(target_scope, injections);
    }

    pub fn resolve_scope(&self, requested: &str) -> Option<String> {
        if self.registry.lookup(requested).is_some() {
            return Some(requested.to_owned());
        }
        self.aliases.get(requested).cloned()
    }

    pub fn loaded_scopes(&self) -> Vec<String> {
        self.loaded_language_order
            .iter()
            .filter_map(|id| self.aliases.get(id).cloned())
            .collect()
    }

    pub fn loaded_languages(&self) -> Vec<String> {
        self.loaded_language_order
            .iter()
            .cloned()
            .chain(
                self.loaded_language_order
                    .iter()
                    .flat_map(|id| self.language_aliases.get(id).into_iter().flatten().cloned()),
            )
            .collect()
    }

    pub fn grammar_for_language(
        &mut self,
        requested: &str,
    ) -> Result<Option<std::rc::Rc<ferriki_textmate::Grammar>>> {
        let Some(scope_name) = self.load_standard_language(requested)? else {
            return Ok(None);
        };
        self.registry
            .grammar_for_scope_name(
                &scope_name,
                GrammarConfiguration::default()
                    .with_regex_prefilter(self.regex_prefilter)
                    .with_initial_language_id(1)
                    .with_balanced_bracket_selectors(Some(vec!["*".to_owned()])),
            )
            .map_err(theme_error)
    }

    pub fn tokenize(
        &mut self,
        code: &str,
        language: &str,
        theme_id: &str,
        options: &TokenizeOptions,
    ) -> Result<HighlightTokensResult> {
        let theme = self
            .activate_theme(theme_id)?
            .ok_or_else(|| {
                Error::new(
                    ErrorKind::UnknownTheme,
                    format!("Unknown theme `{theme_id}`."),
                )
            })?
            .clone();
        if is_plain_language(language) || language == "ansi" {
            let tokens = split_lines(code)
                .into_iter()
                .map(|(line, offset)| {
                    if line.is_empty() {
                        Vec::new()
                    } else {
                        vec![HighlightToken {
                            content: line.to_owned(),
                            offset,
                            color: None,
                            font_style: None,
                            token_type: None,
                            scope_names: None,
                        }]
                    }
                })
                .collect();
            return Ok(HighlightTokensResult {
                tokens,
                foreground: theme.foreground,
                background: theme.background,
                theme_name: theme.name,
            });
        }
        let grammar = self.grammar_for_language(language)?.ok_or_else(|| {
            Error::new(
                ErrorKind::UnknownLanguage,
                format!("Unknown language `{language}`."),
            )
        })?;
        let color_map = self.registry.get_color_map();
        let mut state = None;
        let mut output_lines = Vec::new();

        for (line, line_offset) in split_lines(code) {
            if line.is_empty() {
                output_lines.push(Vec::new());
                continue;
            }
            let line_length = line.encode_utf16().count();
            if options.max_line_length > 0 && line_length >= options.max_line_length {
                output_lines.push(vec![HighlightToken {
                    content: line.to_owned(),
                    offset: line_offset,
                    color: Some(String::new()),
                    font_style: Some(FontStyle::NONE),
                    token_type: options
                        .include_token_type
                        .then_some(StandardTokenType::Other),
                    scope_names: None,
                }]);
                continue;
            }

            let (scope_tokens, binary_tokens, rule_stack) =
                if options.include_scopes || options.preserve_scope_boundaries {
                    let result = grammar
                        .tokenize_line_with_scopes(line, state, options.time_limit_millis)
                        .map_err(|error| {
                            Error::new(
                                ErrorKind::Tokenization,
                                format!("Failed to tokenize `{language}` line scopes: {error}"),
                            )
                        })?;
                    (Some(result.tokens), result.binary_tokens, result.rule_stack)
                } else {
                    let result = grammar
                        .tokenize_line2(line, state, options.time_limit_millis)
                        .map_err(|error| {
                            Error::new(
                                ErrorKind::Tokenization,
                                format!("Failed to tokenize `{language}` line: {error}"),
                            )
                        })?;
                    (None, result.tokens, result.rule_stack)
                };
            let utf16_map = utf16_to_byte_map(line);
            let mut line_tokens = Vec::with_capacity(binary_tokens.len() / 2);
            let mut boundaries: Vec<usize> = binary_tokens
                .as_chunks::<2>()
                .0
                .iter()
                .map(|token| token[0] as usize)
                .filter(|start| *start < line_length)
                .collect();
            boundaries.push(line_length);
            if options.preserve_scope_boundaries {
                for token in scope_tokens
                    .as_ref()
                    .expect("scope boundaries were requested")
                {
                    boundaries.push(token.start_index.min(line_length));
                    boundaries.push(token.end_index.min(line_length));
                }
                boundaries.sort_unstable();
                boundaries.dedup();
            }
            let mut token_index = 0;
            for range in boundaries.windows(2) {
                let start_index = range[0];
                let end_index = range[1];
                while binary_tokens
                    .get(token_index * 2 + 2)
                    .is_some_and(|next| *next as usize <= start_index)
                {
                    token_index += 1;
                }
                if let Some(token) = token_from_metadata(
                    line,
                    &utf16_map,
                    start_index..end_index,
                    line_offset,
                    binary_tokens[token_index * 2 + 1],
                    &color_map,
                    options.include_token_type,
                    scope_tokens.as_ref().and_then(|scopes| {
                        scopes
                            .iter()
                            .find(|token| {
                                token.start_index <= start_index && token.end_index >= end_index
                            })
                            .map(|token| token.scopes.clone())
                    }),
                ) {
                    line_tokens.push(token);
                }
            }
            state = Some(rule_stack);
            output_lines.push(line_tokens);
        }

        Ok(HighlightTokensResult {
            tokens: output_lines,
            foreground: theme.foreground,
            background: theme.background,
            theme_name: theme.name,
        })
    }

    pub fn tokenize_with_themes(
        &mut self,
        code: &str,
        language: &str,
        themes: &[(String, String)],
        options: &TokenizeOptions,
    ) -> Result<HighlightTokensWithThemesResult> {
        if themes.is_empty() {
            return Err(Error::from_reason("At least one theme is required."));
        }

        // Use each theme's encoded token boundaries, just like Shiki. Scope
        // tokens alone split equal metadata (strings, comments and whitespace)
        // and cannot reproduce theme-dependent boundaries or token overrides.
        let results = themes
            .iter()
            .map(|(_, theme)| self.tokenize(code, language, theme, options))
            .collect::<Result<Vec<_>>>()?;
        let mut output_lines = Vec::new();
        for line_index in 0..results[0].tokens.len() {
            let lines: Vec<_> = results
                .iter()
                .map(|result| &result.tokens[line_index])
                .collect();
            let mut boundaries = BTreeSet::new();
            for line in &lines {
                for token in *line {
                    boundaries.insert(token.offset);
                    boundaries.insert(token.offset + token.content.encode_utf16().count());
                }
            }
            let boundaries: Vec<_> = boundaries.into_iter().collect();
            let mut cursors = vec![0; lines.len()];
            let mut tokens = Vec::new();
            for range in boundaries.windows(2) {
                let start = range[0];
                let end = range[1];
                let mut variants = BTreeMap::new();
                let mut base = None;
                for (index, line) in lines.iter().enumerate() {
                    while line[cursors[index]].offset
                        + line[cursors[index]].content.encode_utf16().count()
                        <= start
                    {
                        cursors[index] += 1;
                    }
                    let token = &line[cursors[index]];
                    base.get_or_insert(token);
                    variants.insert(
                        themes[index].0.clone(),
                        HighlightThemeTokenStyle {
                            color: token.color.clone(),
                            font_style: token.font_style,
                        },
                    );
                }
                let token = base.expect("a token boundary has a source token");
                let map = utf16_to_byte_map(&token.content);
                tokens.push(HighlightThemeToken {
                    content: token.content[map[start - token.offset]..map[end - token.offset]]
                        .to_owned(),
                    offset: start,
                    variants,
                    token_type: token.token_type,
                    scope_names: token.scope_names.clone(),
                });
            }
            output_lines.push(tokens);
        }
        Ok(HighlightTokensWithThemesResult {
            tokens: output_lines,
            themes: themes
                .iter()
                .zip(results)
                .map(|((color, _), result)| HighlightThemeMetadata {
                    color: color.clone(),
                    name: result.theme_name,
                    foreground: result.foreground,
                    background: result.background,
                })
                .collect(),
        })
    }
}

fn scope_matches(candidate: &str, scope_name: &str) -> bool {
    candidate == scope_name
        || scope_name
            .strip_prefix(candidate)
            .is_some_and(|suffix| suffix.starts_with('.'))
}

fn string_array(object: &serde_json::Map<String, Value>, key: &str) -> Result<Vec<String>> {
    let Some(value) = object.get(key) else {
        return Ok(Vec::new());
    };
    let values = value.as_array().ok_or_else(|| {
        Error::new(
            ErrorKind::InvalidRegistration,
            format!("Language registration `{key}` must be an array."),
        )
    })?;
    values
        .iter()
        .map(|value| {
            value.as_str().map(str::to_owned).ok_or_else(|| {
                Error::new(
                    ErrorKind::InvalidRegistration,
                    format!("Language registration `{key}` must contain strings."),
                )
            })
        })
        .collect()
}

fn is_plain_language(language: &str) -> bool {
    matches!(language, "text" | "txt" | "plain" | "plaintext")
}

fn theme_error(error: ferriki_textmate::ThemeError) -> Error {
    Error::new(
        ErrorKind::Theme,
        format!("Failed to resolve TextMate theme: {error}"),
    )
}

impl LanguageRegistration {
    /// Registers a parsed grammar under a language ID, without aliases or injections.
    pub fn new(id: impl Into<String>, grammar: RawGrammar) -> Self {
        Self {
            id: id.into(),
            grammar,
            aliases: Vec::new(),
            inject_to: Vec::new(),
        }
    }
    #[must_use]
    pub fn with_aliases(mut self, aliases: impl IntoIterator<Item = String>) -> Self {
        self.aliases = aliases.into_iter().collect();
        self
    }
    #[must_use]
    pub fn with_injections(mut self, scopes: impl IntoIterator<Item = String>) -> Self {
        self.inject_to = scopes.into_iter().collect();
        self
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn standard_highlighter() -> HighlighterCore {
        let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/standard");
        HighlighterCore::with_standard_assets(&root).expect("highlighter")
    }

    fn unlimited_tokenize_options() -> TokenizeOptions {
        TokenizeOptions {
            time_limit_millis: 0,
            ..TokenizeOptions::default()
        }
    }

    #[test]
    fn retained_scope_boundaries_are_independent_of_theme_merging() {
        let mut highlighter = standard_highlighter();
        let code = "{\"message\":\"hello😀\"}";
        let options = unlimited_tokenize_options().with_preserve_scope_boundaries(true);
        let monokai = highlighter
            .tokenize(code, "json", "monokai", &options)
            .expect("tokens");
        let nord = highlighter
            .tokenize(code, "json", "nord", &options)
            .expect("tokens");
        let paths = |result: &HighlightTokensResult| {
            result.tokens[0]
                .iter()
                .map(|token| (token.content.clone(), token.scope_names.clone()))
                .collect::<Vec<_>>()
        };
        assert_eq!(paths(&monokai), paths(&nord));
        assert_eq!(
            monokai.tokens[0]
                .iter()
                .map(|token| token.content.as_str())
                .collect::<String>(),
            code
        );
        let value = monokai.tokens[0]
            .iter()
            .find(|token| token.content == "hello😀")
            .expect("JSON value");
        assert_eq!(value.color.as_deref(), Some("#CFCFC2"));
        assert!(
            value
                .scope_names
                .as_ref()
                .expect("scopes")
                .iter()
                .any(|scope| scope == "meta.structure.dictionary.json")
        );
    }

    #[test]
    fn loads_standard_language_dependencies_and_aliases() {
        let mut highlighter = standard_highlighter();

        assert_eq!(
            highlighter.load_standard_language("vue").expect("language"),
            Some("text.html.vue".to_owned())
        );
        assert_eq!(
            highlighter.resolve_scope("vue"),
            Some("text.html.vue".to_owned())
        );
        assert!(
            highlighter
                .loaded_scopes()
                .contains(&"source.js".to_owned())
        );
        assert!(
            highlighter
                .loaded_scopes()
                .contains(&"text.html.basic".to_owned())
        );
        assert!(
            highlighter
                .loaded_languages()
                .contains(&"javascript".to_owned())
        );
        assert!(highlighter.loaded_languages().contains(&"js".to_owned()));
        assert!(!highlighter.loaded_languages().contains(&"scss".to_owned()));
        assert!(
            highlighter
                .grammar_for_language("vue")
                .expect("grammar")
                .is_some()
        );
    }

    #[test]
    fn activates_standard_theme_in_textmate_registry() {
        let mut highlighter = standard_highlighter();
        let theme = highlighter
            .activate_theme("nord")
            .expect("theme")
            .expect("nord");

        assert_eq!(theme.name, "nord");
        assert_eq!(theme.foreground.to_ascii_uppercase(), "#D8DEE9FF");
        assert_eq!(theme.background.to_ascii_uppercase(), "#2E3440FF");
    }

    #[test]
    fn external_injections_are_loaded_for_target_scope() {
        let mut highlighter = standard_highlighter();
        highlighter
            .load_standard_language("typescript")
            .expect("typescript");
        highlighter
            .load_standard_language("es-tag-css")
            .expect("css injection");
        highlighter
            .load_standard_language("es-tag-html")
            .expect("html injection");

        let injections = highlighter.registry.injections("source.ts");
        assert!(injections.contains(&"inline.es6-css".to_owned()));
        assert!(injections.contains(&"inline.es6-html".to_owned()));
    }

    #[test]
    fn tokenizes_vue_directives_with_embedded_typescript() {
        let mut highlighter = standard_highlighter();
        let grammar = highlighter
            .grammar_for_language("vue")
            .expect("grammar")
            .expect("vue grammar");
        let line = r#"<h1 v-if="count == 1 ? true : 'str'.toUpperCase()"></h1>"#;

        let result = grammar.tokenize_line(line, None, 0).expect("tokens");
        let actual = result
            .tokens
            .iter()
            .map(|token| {
                (
                    &line[token.start_index..token.end_index],
                    token.scopes.last().map(String::as_str),
                )
            })
            .collect::<Vec<_>>();

        assert_eq!(
            actual,
            [
                ("<", Some("punctuation.definition.tag.begin.html.vue")),
                ("h1", Some("entity.name.tag.h1.html.vue")),
                (" ", Some("meta.tag-stuff")),
                ("v-if", Some("keyword.control.conditional.vue")),
                ("=", Some("punctuation.separator.key-value.html.vue")),
                ("\"", Some("punctuation.definition.string.begin.html.vue")),
                ("count", Some("variable.other.readwrite.ts")),
                (" ", Some("source.ts.embedded.html.vue")),
                ("==", Some("keyword.operator.comparison.ts")),
                (" ", Some("source.ts.embedded.html.vue")),
                ("1", Some("constant.numeric.decimal.ts")),
                (" ", Some("source.ts.embedded.html.vue")),
                ("?", Some("keyword.operator.ternary.ts")),
                (" ", Some("source.ts.embedded.html.vue")),
                ("true", Some("constant.language.boolean.true.ts")),
                (" ", Some("source.ts.embedded.html.vue")),
                (":", Some("keyword.operator.ternary.ts")),
                (" ", Some("source.ts.embedded.html.vue")),
                ("'", Some("punctuation.definition.string.begin.ts")),
                ("str", Some("string.quoted.single.ts")),
                ("'", Some("punctuation.definition.string.end.ts")),
                (".", Some("punctuation.accessor.ts")),
                ("toUpperCase", Some("entity.name.function.ts")),
                ("(", Some("meta.brace.round.ts")),
                (")", Some("meta.brace.round.ts")),
                ("\"", Some("punctuation.definition.string.end.html.vue")),
                (">", Some("punctuation.definition.tag.end.html.vue")),
                ("</", Some("punctuation.definition.tag.begin.html.vue")),
                ("h1", Some("entity.name.tag.h1.html.vue")),
                (">", Some("punctuation.definition.tag.end.html.vue")),
            ]
        );
    }

    #[test]
    fn tokenizes_standard_javascript_with_textmate_theme_metadata() {
        let mut highlighter = standard_highlighter();
        let result = highlighter
            .tokenize(
                "console.log(\"Hi\")",
                "javascript",
                "nord",
                &unlimited_tokenize_options(),
            )
            .expect("tokens");

        let pairs = result.tokens[0]
            .iter()
            .map(|token| {
                (
                    token.content.as_str(),
                    token.color.as_deref().expect("styled token"),
                )
            })
            .collect::<Vec<_>>();
        assert_eq!(
            pairs,
            [
                ("console", "#D8DEE9"),
                (".", "#ECEFF4"),
                ("log", "#88C0D0"),
                ("(", "#D8DEE9FF"),
                ("\"", "#ECEFF4"),
                ("Hi", "#A3BE8C"),
                ("\"", "#ECEFF4"),
                (")", "#D8DEE9FF"),
            ]
        );
        assert_eq!(result.theme_name, "nord");
        assert_eq!(result.tokens[0][0].offset, 0);
        assert_eq!(result.tokens[0][1].offset, 7);
    }

    #[test]
    fn tokenizes_multiline_utf16_offsets_and_optional_types() {
        let mut highlighter = standard_highlighter();
        let result = highlighter
            .tokenize(
                "\"😀\"\r\n// x\n",
                "javascript",
                "nord",
                &TokenizeOptions {
                    time_limit_millis: 0,
                    include_token_type: true,
                    ..TokenizeOptions::default()
                },
            )
            .expect("tokens");

        assert_eq!(result.tokens.len(), 3);
        assert_eq!(result.tokens[1][0].offset, 6);
        assert_eq!(
            result.tokens[1][0].token_type,
            Some(StandardTokenType::Comment)
        );
        assert!(result.tokens[2].is_empty());
    }

    #[test]
    fn retains_scope_paths_when_requested_for_explanations() {
        let mut highlighter = standard_highlighter();
        let result = highlighter
            .tokenize(
                "console.log(1)",
                "javascript",
                "nord",
                &TokenizeOptions {
                    time_limit_millis: 0,
                    include_scopes: true,
                    ..TokenizeOptions::default()
                },
            )
            .expect("tokens");

        let console = &result.tokens[0][0];
        assert_eq!(console.content, "console");
        assert!(
            console
                .scope_names
                .as_ref()
                .is_some_and(|scopes| scopes.iter().any(|scope| scope == "source.js"))
        );
    }

    #[test]
    fn emits_unstyled_plaintext_tokens_without_loading_a_grammar() {
        let mut highlighter = standard_highlighter();
        let result = highlighter
            .tokenize("plain\n", "text", "nord", &TokenizeOptions::default())
            .expect("tokens");

        assert_eq!(result.tokens.len(), 2);
        assert_eq!(result.tokens[0][0].content, "plain");
        assert_eq!(result.tokens[0][0].color, None);
        assert_eq!(result.tokens[0][0].font_style, None);
        assert!(result.tokens[1].is_empty());
    }

    #[test]
    fn dispose_releases_assets_registrations_and_compiled_state() {
        let mut highlighter = standard_highlighter();
        highlighter
            .load_standard_language("javascript")
            .expect("javascript");
        highlighter.load_standard_theme("nord").expect("nord");
        highlighter
            .grammar_for_language("javascript")
            .expect("grammar")
            .expect("javascript grammar");
        assert!(highlighter.standard_assets.is_some());
        assert!(!highlighter.themes.is_empty());

        highlighter.dispose();

        assert!(highlighter.standard_assets.is_none());
        assert!(highlighter.themes.is_empty());
        assert!(highlighter.loaded_languages().is_empty());
        assert!(highlighter.registry.lookup("source.js").is_none());
    }
}