ferriki 0.4.1

use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;

use ferriki_textmate::{
    GrammarConfiguration, RawGrammar, ScopeStack, SyncRegistry, Theme, parse_raw_grammar,
};
use serde_json::Value;

use crate::asset_catalog::StandardAssetCatalogs;
use crate::render::{RenderOptions, render_html_lines};
use crate::theme_data::{ThemeData, parse_theme_data};
use crate::tokens::{
    HighlightThemeMetadata, HighlightThemeToken, HighlightThemeTokenStyle, HighlightToken,
    HighlightTokensResult, HighlightTokensWithThemesResult, TokenizeOptions,
    convert_token_offsets_to_utf8, split_lines, token_from_metadata, utf16_to_byte_map,
};
use crate::{Error, ErrorKind, Result};

/// Compatibility bridge for the N-API host. Its token offsets are UTF-16.
/// Rust consumers should use [`Highlighter`] for UTF-8 byte offsets.
#[doc(hidden)]
pub struct HighlighterCore {
    standard_assets: Option<StandardAssetCatalogs>,
    registry: SyncRegistry,
    aliases: BTreeMap<String, String>,
    loaded_language_ids: BTreeSet<String>,
    loaded_custom_language_ids: BTreeSet<String>,
    loaded_language_order: Vec<String>,
    language_aliases: BTreeMap<String, Vec<String>>,
    injections: BTreeMap<String, Vec<String>>,
    themes: BTreeMap<String, ThemeData>,
    active_theme: Option<String>,
}

/// Reusable, synchronous highlighter for Rust consumers.
///
/// A highlighter lazily loads grammars and themes from its asset catalogs.
/// Its mutable API makes cache updates explicit; instances are not shared
/// across threads. Create one instance and reuse it across many documents.
///
/// ```no_run
/// use std::path::Path;
/// use ferriki::{Highlighter, RenderOptions, StandardAssetCatalogs};
///
/// fn main() -> Result<(), Box<dyn std::error::Error>> {
///     let assets = StandardAssetCatalogs::load_from_root(Path::new("assets/shiki"))?;
///     let mut highlighter = Highlighter::builder().with_assets(assets).build()?;
///     let highlighted = highlighter.highlight_html_lines(
///         "fn main() {}", "rust", "nord", &RenderOptions::default(),
///     )?;
///     assert_eq!(highlighted.lines.len(), 1);
///     Ok(())
/// }
/// ```
pub struct Highlighter {
    core: HighlighterCore,
}

/// Escaped, balanced inner HTML and the theme colors for one code block.
///
/// Each entry in `lines` corresponds to one source line. A Markdown renderer
/// owns the outer markup and should apply `foreground` and `background` to its
/// code-block wrapper.
#[non_exhaustive]
pub struct HighlightedLines {
    pub lines: Vec<String>,
    pub foreground: String,
    pub background: String,
    pub theme_name: String,
}

/// A typed custom TextMate grammar registration.
pub struct LanguageRegistration {
    /// Language identifier used by `highlight`.
    pub id: String,
    /// Parsed TextMate grammar with a non-empty scope name.
    pub grammar: RawGrammar,
    /// Additional language identifiers.
    pub aliases: Vec<String>,
    /// Scopes into which this grammar injects.
    pub inject_to: Vec<String>,
}

/// Configuration for a reusable highlighter.
#[derive(Default)]
pub struct HighlighterBuilder {
    assets: Option<StandardAssetCatalogs>,
    languages: Vec<String>,
    themes: Vec<String>,
}

impl Highlighter {
    /// Starts a highlighter builder. Assets are optional for custom-only use.
    pub fn builder() -> HighlighterBuilder {
        HighlighterBuilder::default()
    }

    /// Highlights source into per-line themed tokens.
    pub fn highlight(
        &mut self,
        code: &str,
        language: &str,
        theme: &str,
    ) -> Result<HighlightTokensResult> {
        self.highlight_with_options(code, language, theme, &TokenizeOptions::default())
    }

    /// Highlights source with explicit tokenizer limits and metadata options.
    pub fn highlight_with_options(
        &mut self,
        code: &str,
        language: &str,
        theme: &str,
        options: &TokenizeOptions,
    ) -> Result<HighlightTokensResult> {
        let mut result = self.core.tokenize(code, language, theme, options)?;
        convert_token_offsets_to_utf8(&mut result, code)?;
        Ok(result)
    }

    /// Highlights directly to escaped, balanced inner HTML per source line.
    ///
    /// This avoids converting token offsets when a Markdown renderer only
    /// needs line fragments. It does not include code-block or line wrappers.
    pub fn highlight_html_lines(
        &mut self,
        code: &str,
        language: &str,
        theme: &str,
        render_options: &RenderOptions,
    ) -> Result<HighlightedLines> {
        let tokens = self
            .core
            .tokenize(code, language, theme, &TokenizeOptions::default())?;
        let lines = render_html_lines(&tokens, render_options);
        Ok(HighlightedLines {
            lines,
            foreground: tokens.foreground,
            background: tokens.background,
            theme_name: tokens.theme_name,
        })
    }

    /// Loads a standard language and its dependencies; returns false if absent.
    pub fn load_language(&mut self, language: &str) -> Result<bool> {
        self.core
            .load_standard_language(language)
            .map(|scope| scope.is_some())
    }

    /// Loads a standard theme; returns false if absent.
    pub fn load_theme(&mut self, theme: &str) -> Result<bool> {
        self.core.load_standard_theme(theme)
    }

    /// Registers a custom grammar in Ferriki's JSON registration format.
    pub fn register_language_json(&mut self, source: &str) -> Result<()> {
        self.core.load_custom_language(source).map(|_| ())
    }

    /// Registers a typed custom grammar without a JSON intermediary.
    pub fn register_language(&mut self, registration: LanguageRegistration) -> Result<()> {
        self.core.register_language(registration).map(|_| ())
    }

    /// Registers a custom theme in Ferriki's JSON registration format.
    pub fn register_theme_json(&mut self, source: &str) -> Result<()> {
        self.core.load_custom_theme(source).map(|_| ())
    }

    /// Registers a parsed custom theme without a JSON intermediary.
    pub fn register_theme(&mut self, theme: ThemeData) -> Result<()> {
        self.core.register_theme(theme)
    }
}

impl HighlighterBuilder {
    /// Uses portable asset catalogs prepared by a filesystem or memory source.
    pub fn with_assets(mut self, assets: StandardAssetCatalogs) -> Self {
        self.assets = Some(assets);
        self
    }

    /// Eagerly loads these languages while building the highlighter.
    pub fn load_languages<I, S>(mut self, languages: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        self.languages.extend(
            languages
                .into_iter()
                .map(|language| language.as_ref().to_owned()),
        );
        self
    }

    /// Eagerly loads these themes while building the highlighter.
    pub fn load_themes<I, S>(mut self, themes: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        self.themes
            .extend(themes.into_iter().map(|theme| theme.as_ref().to_owned()));
        self
    }

    /// Builds one reusable highlighter. Unknown requested assets are errors.
    pub fn build(self) -> Result<Highlighter> {
        let mut core = match self.assets {
            Some(assets) => HighlighterCore::with_assets(assets)?,
            None => HighlighterCore::new()?,
        };
        for language in self.languages {
            if core.load_standard_language(&language)?.is_none() {
                return Err(Error::new(
                    ErrorKind::UnknownLanguage,
                    format!("Unknown language `{language}`."),
                ));
            }
        }
        for theme in self.themes {
            if !core.load_standard_theme(&theme)? {
                return Err(Error::new(
                    ErrorKind::UnknownTheme,
                    format!("Unknown theme `{theme}`."),
                ));
            }
        }
        Ok(Highlighter { core })
    }
}

impl HighlighterCore {
    pub fn new() -> Result<Self> {
        Ok(Self {
            standard_assets: None,
            registry: SyncRegistry::new(None, None).map_err(theme_error)?,
            aliases: BTreeMap::new(),
            loaded_language_ids: BTreeSet::new(),
            loaded_custom_language_ids: BTreeSet::new(),
            loaded_language_order: Vec::new(),
            language_aliases: BTreeMap::new(),
            injections: BTreeMap::new(),
            themes: BTreeMap::new(),
            active_theme: None,
        })
    }

    pub fn with_standard_assets(root: &Path) -> Result<Self> {
        Self::with_assets(StandardAssetCatalogs::load_from_root(root)?)
    }

    pub fn with_assets(assets: StandardAssetCatalogs) -> Result<Self> {
        let mut highlighter = Self::new()?;
        highlighter.standard_assets = Some(assets);
        Ok(highlighter)
    }

    pub fn dispose(&mut self) {
        self.registry.dispose();
        self.standard_assets = None;
        self.aliases.clear();
        self.loaded_language_ids.clear();
        self.loaded_custom_language_ids.clear();
        self.loaded_language_order.clear();
        self.language_aliases.clear();
        self.injections.clear();
        self.themes.clear();
        self.active_theme = None;
    }

    pub fn load_standard_theme(&mut self, theme_id: &str) -> Result<bool> {
        if self.themes.contains_key(theme_id) {
            return Ok(true);
        }
        let Some(catalogs) = self.standard_assets.as_ref() else {
            return Ok(false);
        };
        let Some(asset) = catalogs.themes.load_asset(theme_id)? else {
            return Ok(false);
        };
        let theme = parse_theme_data(&asset.id, &asset.theme_json).map_err(|error| {
            Error::new(
                ErrorKind::AssetFormat,
                format!("Failed to parse standard theme `{}`: {error}", asset.id),
            )
        })?;
        self.themes.insert(asset.id.clone(), theme);
        Ok(true)
    }

    pub fn activate_theme(&mut self, theme_id: &str) -> Result<Option<&ThemeData>> {
        if !self.load_standard_theme(theme_id)? {
            return Ok(None);
        }
        if self.active_theme.as_deref() != Some(theme_id) {
            let theme = self
                .themes
                .get(theme_id)
                .ok_or_else(|| Error::from_reason("Loaded Ferriki theme disappeared."))?;
            self.registry
                .set_theme(Some(theme.raw_theme.clone()), None)
                .map_err(theme_error)?;
            self.active_theme = Some(theme_id.to_owned());
        }
        Ok(self.themes.get(theme_id))
    }

    pub fn load_standard_language(&mut self, requested: &str) -> Result<Option<String>> {
        let mut visiting = BTreeSet::new();
        self.load_standard_language_inner(requested, &mut visiting)
    }

    /// Register a user-provided TextMate grammar without pretending it is a
    /// bundled asset. The JSON shape intentionally stays close to Shiki's
    /// LanguageRegistration: the raw grammar fields are parsed by the same
    /// decoder as shipped grammars, while aliases and injection metadata are
    /// kept in the shared lookup model.
    pub fn load_custom_language(&mut self, source: &str) -> Result<Option<String>> {
        let value: Value = serde_json::from_str(source).map_err(|error| {
            Error::new(
                ErrorKind::InvalidRegistration,
                format!("Failed to parse custom language registration: {error}"),
            )
        })?;
        let object = value.as_object().ok_or_else(|| {
            Error::new(
                ErrorKind::InvalidRegistration,
                "Language registration must be a JSON object.",
            )
        })?;
        let id = object
            .get("name")
            .and_then(Value::as_str)
            .filter(|name| !name.is_empty())
            .ok_or_else(|| {
                Error::new(
                    ErrorKind::InvalidRegistration,
                    "Language registration requires a non-empty `name`.",
                )
            })?
            .to_owned();
        let grammar = parse_raw_grammar(source, Some("custom-grammar.json")).map_err(|error| {
            Error::new(
                ErrorKind::InvalidRegistration,
                format!("Failed to parse custom grammar `{id}`: {error}"),
            )
        })?;
        if grammar.scope_name.is_empty() {
            return Err(Error::new(
                ErrorKind::InvalidRegistration,
                format!("Language registration `{id}` requires a non-empty `scopeName`."),
            ));
        }
        let aliases = string_array(object, "aliases")?;
        let inject_to = string_array(object, "injectTo")?;

        self.register_language(LanguageRegistration {
            id,
            grammar,
            aliases,
            inject_to,
        })
        .map(Some)
    }

    pub fn register_language(&mut self, registration: LanguageRegistration) -> Result<String> {
        let LanguageRegistration {
            id,
            grammar,
            aliases,
            inject_to,
        } = registration;
        if id.is_empty() || grammar.scope_name.is_empty() {
            return Err(Error::new(
                ErrorKind::InvalidRegistration,
                "Language registration requires non-empty `id` and `scopeName`.",
            ));
        }
        if self.loaded_custom_language_ids.contains(&id) {
            return Ok(grammar.scope_name);
        }
        let scope_name = grammar.scope_name.clone();
        self.registry.add_grammar(
            grammar,
            self.injections
                .get(&scope_name)
                .cloned()
                .unwrap_or_default(),
        );
        self.aliases.insert(id.clone(), scope_name.clone());
        for alias in &aliases {
            // A user registration must never steal a key that belongs to the
            // standard catalog. The standard resolver can still load that
            // language later, regardless of registration order.
            if self
                .standard_assets
                .as_ref()
                .and_then(|catalogs| catalogs.languages.resolve_id(alias))
                .is_some()
            {
                continue;
            }
            self.aliases.insert(alias.clone(), scope_name.clone());
        }
        for target in &inject_to {
            {
                let injections = self.injections.entry(target.clone()).or_default();
                if !injections.contains(&scope_name) {
                    injections.push(scope_name.clone());
                }
            }
            self.refresh_injections(target);
        }
        self.language_aliases.insert(id.clone(), aliases);
        self.loaded_custom_language_ids.insert(id.clone());
        self.loaded_language_order.push(id);
        Ok(scope_name)
    }

    /// Register a user-provided theme using the same parser and TextMate
    /// theme cache as bundled themes. A simple `include` name is supported so
    /// custom themes can layer on a previously loaded standard/custom theme.
    pub fn load_custom_theme(&mut self, source: &str) -> Result<bool> {
        let value: Value = serde_json::from_str(source).map_err(|error| {
            Error::new(
                ErrorKind::InvalidRegistration,
                format!("Failed to parse custom theme registration: {error}"),
            )
        })?;
        let object = value.as_object().ok_or_else(|| {
            Error::new(
                ErrorKind::InvalidRegistration,
                "Theme registration must be a JSON object.",
            )
        })?;
        let id = object
            .get("name")
            .and_then(Value::as_str)
            .filter(|name| !name.is_empty())
            .ok_or_else(|| {
                Error::new(
                    ErrorKind::InvalidRegistration,
                    "Theme registration requires a non-empty `name`.",
                )
            })?
            .to_owned();
        let mut theme = parse_theme_data(&id, source)?;
        if let Some(include) = object.get("include") {
            let include = include.as_str().ok_or_else(|| {
                Error::new(
                    ErrorKind::InvalidRegistration,
                    "Theme registration `include` must be a theme name.",
                )
            })?;
            if !self.load_standard_theme(include)? {
                return Err(Error::new(
                    ErrorKind::InvalidRegistration,
                    format!("Theme `{include}` not found for custom theme include."),
                ));
            }
            let base = self
                .themes
                .get(include)
                .cloned()
                .ok_or_else(|| Error::from_reason("Included Ferriki theme disappeared."))?;
            let explicit_foreground = object.get("fg").and_then(Value::as_str).is_some()
                || object
                    .get("colors")
                    .and_then(Value::as_object)
                    .and_then(|colors| colors.get("editor.foreground"))
                    .and_then(Value::as_str)
                    .is_some();
            let explicit_background = object.get("bg").and_then(Value::as_str).is_some()
                || object
                    .get("colors")
                    .and_then(Value::as_object)
                    .and_then(|colors| colors.get("editor.background"))
                    .and_then(Value::as_str)
                    .is_some();
            if !explicit_foreground {
                theme.foreground = base.foreground.clone();
            }
            if !explicit_background {
                theme.background = base.background.clone();
            }
            let mut settings = base.raw_theme.settings;
            settings.extend(theme.raw_theme.settings);
            theme.raw_theme.settings = settings;
        }
        self.register_theme(theme)?;
        Ok(true)
    }

    pub fn register_theme(&mut self, theme: ThemeData) -> Result<()> {
        if theme.name.is_empty() {
            return Err(Error::new(
                ErrorKind::InvalidRegistration,
                "Theme registration requires a non-empty `name`.",
            ));
        }
        if self.active_theme.as_deref() == Some(&theme.name) {
            self.active_theme = None;
        }
        self.themes.insert(theme.name.clone(), theme);
        Ok(())
    }

    fn load_standard_language_inner(
        &mut self,
        requested: &str,
        visiting: &mut BTreeSet<String>,
    ) -> Result<Option<String>> {
        if self.registry.lookup(requested).is_some() {
            return Ok(Some(requested.to_owned()));
        }
        if let Some(scope_name) = self.aliases.get(requested) {
            return Ok(Some(scope_name.clone()));
        }

        let Some(catalogs) = self.standard_assets.as_ref() else {
            return Ok(None);
        };
        let Some(asset) = catalogs.languages.load_asset(requested)? else {
            return Ok(None);
        };
        let asset = (*asset).clone();
        if self.loaded_language_ids.contains(&asset.id) {
            return Ok(Some(asset.scope_name));
        }
        if !visiting.insert(asset.id.clone()) {
            return Ok(Some(asset.scope_name));
        }

        for dependency in &asset.embedded_langs {
            self.load_standard_language_inner(dependency, visiting)?;
        }

        let raw_grammar =
            parse_raw_grammar(&asset.grammar_json, Some("grammar.json")).map_err(|error| {
                Error::new(
                    ErrorKind::AssetFormat,
                    format!("Failed to parse standard grammar `{}`: {error}", asset.id),
                )
            })?;
        self.register_loaded_grammar(&asset, raw_grammar);

        self.refresh_injections(&asset.scope_name);

        visiting.remove(&asset.id);
        Ok(Some(asset.scope_name))
    }

    fn register_loaded_grammar(
        &mut self,
        asset: &ferriki_asset_gen::LanguageAsset,
        grammar: RawGrammar,
    ) {
        let scope_name = asset.scope_name.clone();
        self.registry.add_grammar(
            grammar,
            self.injections
                .get(&scope_name)
                .cloned()
                .unwrap_or_default(),
        );
        self.aliases.insert(asset.id.clone(), scope_name.clone());
        for alias in &asset.aliases {
            self.aliases.insert(alias.clone(), scope_name.clone());
        }
        for target in &asset.inject_to {
            let injections = self.injections.entry(target.clone()).or_default();
            if !injections.contains(&scope_name) {
                injections.push(scope_name.clone());
            }
            self.refresh_injections(target);
        }
        if self.loaded_language_ids.insert(asset.id.clone()) {
            self.loaded_language_order.push(asset.id.clone());
        }
        self.language_aliases
            .insert(asset.id.clone(), asset.aliases.clone());
    }

    fn refresh_injections(&mut self, target_scope: &str) {
        let injections = self
            .injections
            .iter()
            .filter(|(target, _)| scope_matches(target, target_scope))
            .flat_map(|(_, injections)| injections.iter().cloned())
            .collect::<BTreeSet<_>>()
            .into_iter()
            .collect();
        self.registry.set_injections(target_scope, injections);
    }

    pub fn resolve_scope(&self, requested: &str) -> Option<String> {
        if self.registry.lookup(requested).is_some() {
            return Some(requested.to_owned());
        }
        self.aliases.get(requested).cloned()
    }

    pub fn loaded_scopes(&self) -> Vec<String> {
        self.loaded_language_order
            .iter()
            .filter_map(|id| self.aliases.get(id).cloned())
            .collect()
    }

    pub fn loaded_languages(&self) -> Vec<String> {
        self.loaded_language_order
            .iter()
            .cloned()
            .chain(
                self.loaded_language_order
                    .iter()
                    .flat_map(|id| self.language_aliases.get(id).into_iter().flatten().cloned()),
            )
            .collect()
    }

    pub fn grammar_for_language(
        &mut self,
        requested: &str,
    ) -> Result<Option<std::rc::Rc<ferriki_textmate::Grammar>>> {
        let Some(scope_name) = self.load_standard_language(requested)? else {
            return Ok(None);
        };
        self.registry
            .grammar_for_scope_name(
                &scope_name,
                GrammarConfiguration {
                    initial_language_id: 1,
                    balanced_bracket_selectors: Some(vec!["*".to_owned()]),
                    ..GrammarConfiguration::default()
                },
            )
            .map_err(theme_error)
    }

    pub fn tokenize(
        &mut self,
        code: &str,
        language: &str,
        theme_id: &str,
        options: &TokenizeOptions,
    ) -> Result<HighlightTokensResult> {
        let theme = self
            .activate_theme(theme_id)?
            .ok_or_else(|| {
                Error::new(
                    ErrorKind::UnknownTheme,
                    format!("Unknown theme `{theme_id}`."),
                )
            })?
            .clone();
        if is_plain_language(language) || language == "ansi" {
            let tokens = split_lines(code)
                .into_iter()
                .map(|(line, offset)| {
                    if line.is_empty() {
                        Vec::new()
                    } else {
                        vec![HighlightToken {
                            content: line.to_owned(),
                            offset,
                            color: None,
                            font_style: None,
                            token_type: None,
                            scope_names: None,
                        }]
                    }
                })
                .collect();
            return Ok(HighlightTokensResult {
                tokens,
                foreground: theme.foreground,
                background: theme.background,
                theme_name: theme.name,
            });
        }
        let grammar = self.grammar_for_language(language)?.ok_or_else(|| {
            Error::new(
                ErrorKind::UnknownLanguage,
                format!("Unknown language `{language}`."),
            )
        })?;
        let color_map = self.registry.get_color_map();
        let mut state = None;
        let mut output_lines = Vec::new();

        for (line, line_offset) in split_lines(code) {
            if line.is_empty() {
                output_lines.push(Vec::new());
                continue;
            }
            let line_length = line.encode_utf16().count();
            if options.max_line_length > 0 && line_length >= options.max_line_length {
                output_lines.push(vec![HighlightToken {
                    content: line.to_owned(),
                    offset: line_offset,
                    color: Some(String::new()),
                    font_style: Some(0),
                    token_type: options.include_token_type.then_some(0),
                    scope_names: None,
                }]);
                continue;
            }

            let scope_result = if options.include_scopes {
                Some(
                    grammar
                        .tokenize_line(line, state.clone(), options.time_limit_millis)
                        .map_err(|error| {
                            Error::new(
                                ErrorKind::Tokenization,
                                format!("Failed to tokenize `{language}` line scopes: {error}"),
                            )
                        })?,
                )
            } else {
                None
            };

            let result = grammar
                .tokenize_line2(line, state, options.time_limit_millis)
                .map_err(|error| {
                    Error::new(
                        ErrorKind::Tokenization,
                        format!("Failed to tokenize `{language}` line: {error}"),
                    )
                })?;
            let utf16_map = utf16_to_byte_map(line);
            let mut line_tokens = Vec::with_capacity(result.tokens.len() / 2);
            for token_index in 0..result.tokens.len() / 2 {
                let start_index = result.tokens[token_index * 2] as usize;
                let end_index = result
                    .tokens
                    .get(token_index * 2 + 2)
                    .copied()
                    .map_or(line_length, |value| value as usize);
                if let Some(token) = token_from_metadata(
                    line,
                    &utf16_map,
                    start_index..end_index,
                    line_offset,
                    result.tokens[token_index * 2 + 1],
                    &color_map,
                    options.include_token_type,
                    scope_result.as_ref().and_then(|scopes| {
                        scopes
                            .tokens
                            .iter()
                            .find(|token| {
                                token.start_index <= start_index && token.end_index >= end_index
                            })
                            .map(|token| token.scopes.clone())
                    }),
                ) {
                    line_tokens.push(token);
                }
            }
            state = Some(result.rule_stack);
            output_lines.push(line_tokens);
        }

        Ok(HighlightTokensResult {
            tokens: output_lines,
            foreground: theme.foreground,
            background: theme.background,
            theme_name: theme.name,
        })
    }

    pub fn tokenize_with_themes(
        &mut self,
        code: &str,
        language: &str,
        themes: &[(String, String)],
        options: &TokenizeOptions,
    ) -> Result<HighlightTokensWithThemesResult> {
        if themes.is_empty() {
            return Err(Error::from_reason("At least one theme is required."));
        }

        let mut theme_data = Vec::with_capacity(themes.len());
        for (color, theme_id) in themes {
            let data = if theme_id == "none" {
                None
            } else {
                if !self.load_standard_theme(theme_id)? {
                    return Err(Error::new(
                        ErrorKind::UnknownTheme,
                        format!("Unknown theme `{theme_id}`."),
                    ));
                }
                Some(
                    self.themes
                        .get(theme_id)
                        .cloned()
                        .ok_or_else(|| Error::from_reason("Loaded Ferriki theme disappeared."))?,
                )
            };
            theme_data.push((color.clone(), theme_id.clone(), data));
        }

        let themes = theme_data
            .into_iter()
            .map(|(color, name, data)| {
                let theme = data
                    .as_ref()
                    .map(|data| {
                        Theme::create_from_raw_theme(Some(&data.raw_theme), None)
                            .map_err(theme_error)
                    })
                    .transpose()?;
                Ok((
                    HighlightThemeMetadata {
                        color,
                        name,
                        foreground: data
                            .as_ref()
                            .map(|data| data.foreground.clone())
                            .unwrap_or_else(|| "inherit".to_owned()),
                        background: data
                            .as_ref()
                            .map(|data| data.background.clone())
                            .unwrap_or_else(|| "inherit".to_owned()),
                    },
                    theme,
                ))
            })
            .collect::<Result<Vec<_>>>()?;

        if is_plain_language(language) || language == "ansi" {
            let mut output_lines = Vec::new();
            for (line, line_offset) in split_lines(code) {
                if line.is_empty() {
                    output_lines.push(Vec::new());
                } else {
                    let utf16_map = utf16_to_byte_map(line);
                    let line_length = line.encode_utf16().count();
                    output_lines.push(vec![theme_token_for_range(
                        line,
                        0..line_length,
                        line_offset,
                        &[],
                        &themes,
                        &utf16_map,
                    )?]);
                }
            }
            return Ok(HighlightTokensWithThemesResult {
                tokens: output_lines,
                themes: themes.into_iter().map(|(metadata, _)| metadata).collect(),
            });
        }

        let grammar_theme = themes
            .iter()
            .find_map(|(metadata, theme)| {
                (metadata.name != "none" && theme.is_some()).then_some(metadata.name.as_str())
            })
            .unwrap_or("nord");
        self.activate_theme(grammar_theme)?;
        let grammar = self.grammar_for_language(language)?.ok_or_else(|| {
            Error::new(
                ErrorKind::UnknownLanguage,
                format!("Unknown language `{language}`."),
            )
        })?;

        let mut state = None;
        let mut output_lines = Vec::new();
        for (line, line_offset) in split_lines(code) {
            if line.is_empty() {
                output_lines.push(Vec::new());
                continue;
            }
            let line_length = line.encode_utf16().count();
            let utf16_map = utf16_to_byte_map(line);
            if options.max_line_length > 0 && line_length >= options.max_line_length {
                output_lines.push(vec![theme_token_for_range(
                    line,
                    0..line_length,
                    line_offset,
                    &[],
                    &themes,
                    &utf16_map,
                )?]);
                continue;
            }

            let result = grammar
                .tokenize_line(line, state, options.time_limit_millis)
                .map_err(|error| {
                    Error::new(
                        ErrorKind::Tokenization,
                        format!("Failed to tokenize `{language}` line: {error}"),
                    )
                })?;
            state = Some(result.rule_stack);
            output_lines.push(
                result
                    .tokens
                    .into_iter()
                    .filter_map(|token| {
                        let end = token.end_index.min(line_length);
                        let start = token.start_index.min(end);
                        (start < end).then(|| {
                            theme_token_for_range(
                                line,
                                start..end,
                                line_offset,
                                &token.scopes,
                                &themes,
                                &utf16_map,
                            )
                        })
                    })
                    .collect::<Result<Vec<_>>>()?,
            );
        }

        Ok(HighlightTokensWithThemesResult {
            tokens: output_lines,
            themes: themes.into_iter().map(|(metadata, _)| metadata).collect(),
        })
    }
}

fn theme_token_for_range(
    line: &str,
    range: std::ops::Range<usize>,
    line_offset: usize,
    scopes: &[String],
    themes: &[(HighlightThemeMetadata, Option<Theme>)],
    utf16_map: &[usize],
) -> Result<HighlightThemeToken> {
    let start_byte = *utf16_map
        .get(range.start)
        .ok_or_else(|| Error::from_reason("Token start is outside the UTF-16 line map."))?;
    let end_byte = *utf16_map
        .get(range.end)
        .ok_or_else(|| Error::from_reason("Token end is outside the UTF-16 line map."))?;
    let content = line
        .get(start_byte..end_byte)
        .ok_or_else(|| Error::from_reason("Token range is not on a UTF-8 boundary."))?
        .to_owned();
    let scope_path = ScopeStack::from(scopes.iter().cloned());
    let variants = themes
        .iter()
        .map(|(metadata, theme)| {
            let style = theme.as_ref().and_then(|theme| {
                theme
                    .match_scope(scope_path.as_deref())
                    .or_else(|| Some(theme.get_defaults().clone()))
            });
            let (color, font_style) = style
                .map(|style| {
                    let defaults = theme.as_ref().map(|theme| theme.get_defaults().clone());
                    let foreground_id = if style.foreground_id == 0 {
                        defaults
                            .as_ref()
                            .map_or(0, |defaults| defaults.foreground_id)
                    } else {
                        style.foreground_id
                    };
                    let font_style = if style.font_style.bits() < 0 {
                        defaults
                            .as_ref()
                            .map_or(0, |defaults| defaults.font_style.bits())
                    } else {
                        style.font_style.bits()
                    };
                    let color = theme.as_ref().and_then(|theme| {
                        let color_map = theme.get_color_map();
                        color_map
                            .get(foreground_id as usize)
                            .filter(|color| !color.is_empty())
                            .cloned()
                    });
                    (color, Some(font_style))
                })
                .unwrap_or((None, None));
            (
                metadata.color.clone(),
                HighlightThemeTokenStyle { color, font_style },
            )
        })
        .collect();
    Ok(HighlightThemeToken {
        content,
        offset: line_offset + range.start,
        variants,
        token_type: None,
        scope_names: (!scopes.is_empty()).then(|| scopes.to_vec()),
    })
}

fn scope_matches(candidate: &str, scope_name: &str) -> bool {
    candidate == scope_name
        || scope_name
            .strip_prefix(candidate)
            .is_some_and(|suffix| suffix.starts_with('.'))
}

fn string_array(object: &serde_json::Map<String, Value>, key: &str) -> Result<Vec<String>> {
    let Some(value) = object.get(key) else {
        return Ok(Vec::new());
    };
    let values = value.as_array().ok_or_else(|| {
        Error::new(
            ErrorKind::InvalidRegistration,
            format!("Language registration `{key}` must be an array."),
        )
    })?;
    values
        .iter()
        .map(|value| {
            value.as_str().map(str::to_owned).ok_or_else(|| {
                Error::new(
                    ErrorKind::InvalidRegistration,
                    format!("Language registration `{key}` must contain strings."),
                )
            })
        })
        .collect()
}

fn is_plain_language(language: &str) -> bool {
    matches!(language, "text" | "txt" | "plain" | "plaintext")
}

fn theme_error(error: ferriki_textmate::ThemeError) -> Error {
    Error::new(
        ErrorKind::Theme,
        format!("Failed to resolve TextMate theme: {error}"),
    )
}

#[cfg(test)]
mod tests {
    use super::*;

    fn standard_highlighter() -> HighlighterCore {
        let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/shiki");
        HighlighterCore::with_standard_assets(&root).expect("highlighter")
    }

    fn unlimited_tokenize_options() -> TokenizeOptions {
        TokenizeOptions {
            time_limit_millis: 0,
            ..TokenizeOptions::default()
        }
    }

    #[test]
    fn loads_standard_language_dependencies_and_aliases() {
        let mut highlighter = standard_highlighter();

        assert_eq!(
            highlighter.load_standard_language("vue").expect("language"),
            Some("text.html.vue".to_owned())
        );
        assert_eq!(
            highlighter.resolve_scope("vue"),
            Some("text.html.vue".to_owned())
        );
        assert!(
            highlighter
                .loaded_scopes()
                .contains(&"source.js".to_owned())
        );
        assert!(
            highlighter
                .loaded_scopes()
                .contains(&"text.html.basic".to_owned())
        );
        assert!(
            highlighter
                .loaded_languages()
                .contains(&"javascript".to_owned())
        );
        assert!(highlighter.loaded_languages().contains(&"js".to_owned()));
        assert!(!highlighter.loaded_languages().contains(&"scss".to_owned()));
        assert!(
            highlighter
                .grammar_for_language("vue")
                .expect("grammar")
                .is_some()
        );
    }

    #[test]
    fn activates_standard_theme_in_textmate_registry() {
        let mut highlighter = standard_highlighter();
        let theme = highlighter
            .activate_theme("nord")
            .expect("theme")
            .expect("nord");

        assert_eq!(theme.name, "nord");
        assert_eq!(theme.foreground.to_ascii_uppercase(), "#D8DEE9FF");
        assert_eq!(theme.background.to_ascii_uppercase(), "#2E3440FF");
    }

    #[test]
    fn external_injections_are_loaded_for_target_scope() {
        let mut highlighter = standard_highlighter();
        highlighter
            .load_standard_language("typescript")
            .expect("typescript");
        highlighter
            .load_standard_language("es-tag-css")
            .expect("css injection");
        highlighter
            .load_standard_language("es-tag-html")
            .expect("html injection");

        let injections = highlighter.registry.injections("source.ts");
        assert!(injections.contains(&"inline.es6-css".to_owned()));
        assert!(injections.contains(&"inline.es6-html".to_owned()));
    }

    #[test]
    fn tokenizes_vue_directives_with_embedded_typescript() {
        let mut highlighter = standard_highlighter();
        let grammar = highlighter
            .grammar_for_language("vue")
            .expect("grammar")
            .expect("vue grammar");
        let line = r#"<h1 v-if="count == 1 ? true : 'str'.toUpperCase()"></h1>"#;

        let result = grammar.tokenize_line(line, None, 0).expect("tokens");
        let actual = result
            .tokens
            .iter()
            .map(|token| {
                (
                    &line[token.start_index..token.end_index],
                    token.scopes.last().map(String::as_str),
                )
            })
            .collect::<Vec<_>>();

        assert_eq!(
            actual,
            [
                ("<", Some("punctuation.definition.tag.begin.html.vue")),
                ("h1", Some("entity.name.tag.h1.html.vue")),
                (" ", Some("meta.tag-stuff")),
                ("v-if", Some("keyword.control.conditional.vue")),
                ("=", Some("punctuation.separator.key-value.html.vue")),
                ("\"", Some("punctuation.definition.string.begin.html.vue")),
                ("count", Some("variable.other.readwrite.ts")),
                (" ", Some("source.ts.embedded.html.vue")),
                ("==", Some("keyword.operator.comparison.ts")),
                (" ", Some("source.ts.embedded.html.vue")),
                ("1", Some("constant.numeric.decimal.ts")),
                (" ", Some("source.ts.embedded.html.vue")),
                ("?", Some("keyword.operator.ternary.ts")),
                (" ", Some("source.ts.embedded.html.vue")),
                ("true", Some("constant.language.boolean.true.ts")),
                (" ", Some("source.ts.embedded.html.vue")),
                (":", Some("keyword.operator.ternary.ts")),
                (" ", Some("source.ts.embedded.html.vue")),
                ("'", Some("punctuation.definition.string.begin.ts")),
                ("str", Some("string.quoted.single.ts")),
                ("'", Some("punctuation.definition.string.end.ts")),
                (".", Some("punctuation.accessor.ts")),
                ("toUpperCase", Some("entity.name.function.ts")),
                ("(", Some("meta.brace.round.ts")),
                (")", Some("meta.brace.round.ts")),
                ("\"", Some("punctuation.definition.string.end.html.vue")),
                (">", Some("punctuation.definition.tag.end.html.vue")),
                ("</", Some("punctuation.definition.tag.begin.html.vue")),
                ("h1", Some("entity.name.tag.h1.html.vue")),
                (">", Some("punctuation.definition.tag.end.html.vue")),
            ]
        );
    }

    #[test]
    fn tokenizes_standard_javascript_with_textmate_theme_metadata() {
        let mut highlighter = standard_highlighter();
        let result = highlighter
            .tokenize(
                "console.log(\"Hi\")",
                "javascript",
                "nord",
                &unlimited_tokenize_options(),
            )
            .expect("tokens");

        let pairs = result.tokens[0]
            .iter()
            .map(|token| {
                (
                    token.content.as_str(),
                    token.color.as_deref().expect("styled token"),
                )
            })
            .collect::<Vec<_>>();
        assert_eq!(
            pairs,
            [
                ("console", "#D8DEE9"),
                (".", "#ECEFF4"),
                ("log", "#88C0D0"),
                ("(", "#D8DEE9FF"),
                ("\"", "#ECEFF4"),
                ("Hi", "#A3BE8C"),
                ("\"", "#ECEFF4"),
                (")", "#D8DEE9FF"),
            ]
        );
        assert_eq!(result.theme_name, "nord");
        assert_eq!(result.tokens[0][0].offset, 0);
        assert_eq!(result.tokens[0][1].offset, 7);
    }

    #[test]
    fn tokenizes_multiline_utf16_offsets_and_optional_types() {
        let mut highlighter = standard_highlighter();
        let result = highlighter
            .tokenize(
                "\"😀\"\r\n// x\n",
                "javascript",
                "nord",
                &TokenizeOptions {
                    time_limit_millis: 0,
                    include_token_type: true,
                    ..TokenizeOptions::default()
                },
            )
            .expect("tokens");

        assert_eq!(result.tokens.len(), 3);
        assert_eq!(result.tokens[1][0].offset, 6);
        assert_eq!(result.tokens[1][0].token_type, Some(1));
        assert!(result.tokens[2].is_empty());
    }

    #[test]
    fn retains_scope_paths_when_requested_for_explanations() {
        let mut highlighter = standard_highlighter();
        let result = highlighter
            .tokenize(
                "console.log(1)",
                "javascript",
                "nord",
                &TokenizeOptions {
                    time_limit_millis: 0,
                    include_scopes: true,
                    ..TokenizeOptions::default()
                },
            )
            .expect("tokens");

        let console = &result.tokens[0][0];
        assert_eq!(console.content, "console");
        assert!(
            console
                .scope_names
                .as_ref()
                .is_some_and(|scopes| scopes.iter().any(|scope| scope == "source.js"))
        );
    }

    #[test]
    fn emits_unstyled_plaintext_tokens_without_loading_a_grammar() {
        let mut highlighter = standard_highlighter();
        let result = highlighter
            .tokenize("plain\n", "text", "nord", &TokenizeOptions::default())
            .expect("tokens");

        assert_eq!(result.tokens.len(), 2);
        assert_eq!(result.tokens[0][0].content, "plain");
        assert_eq!(result.tokens[0][0].color, None);
        assert_eq!(result.tokens[0][0].font_style, None);
        assert!(result.tokens[1].is_empty());
    }

    #[test]
    fn dispose_releases_assets_registrations_and_compiled_state() {
        let mut highlighter = standard_highlighter();
        highlighter
            .load_standard_language("javascript")
            .expect("javascript");
        highlighter.load_standard_theme("nord").expect("nord");
        highlighter
            .grammar_for_language("javascript")
            .expect("grammar")
            .expect("javascript grammar");
        assert!(highlighter.standard_assets.is_some());
        assert!(!highlighter.themes.is_empty());

        highlighter.dispose();

        assert!(highlighter.standard_assets.is_none());
        assert!(highlighter.themes.is_empty());
        assert!(highlighter.loaded_languages().is_empty());
        assert!(highlighter.registry.lookup("source.js").is_none());
    }
}