splintr 0.19.1

use super::backend::{compile_pattern, RegexBackend};
use super::cache::ChunkCache;
use super::error::TokenizerError;
use super::types::{ByteFallback, Tokenizer};
use crate::core::added::{AddedTokenSet, AddedTokens};
use crate::core::bpe::BytePairRanks;
use crate::core::decode_table::Decoder;
use crate::core::encoder::{encoder_from_owned, Encoder};
use crate::core::vocab::{
    build_decoder, load_packed_bpe_borrowed, load_tiktoken_bpe, load_tiktoken_bpe_file,
};
use regexr::RegexBuilder;
use rustc_hash::FxHashMap;
use std::sync::{Arc, OnceLock};

/// Default number of encoded chunks held by the cache.
///
/// Work falls monotonically as this grows, so the choice is bounded by memory.
/// The cache caps at this many entries and fills lazily: a tokenizer that never
/// encodes costs nothing, and one that does settles at a fixed size however much
/// text it sees.
const DEFAULT_CACHE_SIZE: usize = 65536;

impl Tokenizer {
    /// Create a new tokenizer from encoder map, special tokens, and regex pattern.
    ///
    /// Uses regexr as the default regex backend.
    ///
    /// # Arguments
    /// * `encoder` - Map of byte sequences to token IDs
    /// * `special_tokens` - The added tokens: an [`AddedTokenSet`] when the
    ///   `lstrip`/`rstrip` flags matter (a `tokenizer.json`), or a plain name→id
    ///   map when they cannot be declared at all (tiktoken vocabularies, GGUF)
    /// * `pattern` - Regex pattern for tokenization
    pub fn new(
        encoder: FxHashMap<Vec<u8>, u32>,
        special_tokens: impl Into<AddedTokenSet>,
        pattern: &str,
    ) -> Result<Self, TokenizerError> {
        Self::with_options(encoder, special_tokens, pattern, DEFAULT_CACHE_SIZE, false)
    }

    /// Create a new tokenizer with ByteLevel encoding enabled.
    ///
    /// ByteLevel encoding is required for GPT-2, Llama, DeepSeek, and similar tokenizers
    /// that use a byte-to-unicode mapping for handling arbitrary byte sequences.
    pub fn new_byte_level(
        encoder: FxHashMap<Vec<u8>, u32>,
        special_tokens: impl Into<AddedTokenSet>,
        pattern: &str,
    ) -> Result<Self, TokenizerError> {
        Self::with_options(encoder, special_tokens, pattern, DEFAULT_CACHE_SIZE, true)
    }

    /// Create a ByteLevel tokenizer whose pre-tokenizer is a SEQUENCE of
    /// expressions applied in order, llama.cpp's `regex_exprs` list.
    ///
    /// Each expression subdivides the pieces the previous one produced rather
    /// than re-reading the whole text, and the gaps a pass leaves unmatched stay
    /// as pieces of their own — see `subdivide` for the exact semantics and
    /// their source. A one-element list is exactly [`Tokenizer::new_byte_level`]
    /// and keeps the single-regex fast path, so callers can pass a list
    /// unconditionally without paying for the general machinery.
    ///
    /// Vocabularies that need this cannot be expressed as one alternation:
    /// `falcon` splits punctuation runs, then applies the GPT-2 split to the
    /// pieces, then cuts digit runs into groups of three.
    pub fn new_byte_level_chain(
        encoder: FxHashMap<Vec<u8>, u32>,
        special_tokens: impl Into<AddedTokenSet>,
        patterns: &[&str],
    ) -> Result<Self, TokenizerError> {
        let (first, rest) = Self::split_chain_patterns(patterns)?;
        let mut tokenizer = Self::new_byte_level(encoder, special_tokens, first)?;
        tokenizer.set_chain(rest)?;
        Ok(tokenizer)
    }

    /// Create a tokenizer whose pre-tokenizer is a SEQUENCE of expressions
    /// applied in order, llama.cpp's `regex_exprs` list.
    ///
    /// Identical to [`Tokenizer::new_byte_level_chain`] except the head
    /// tokenizer is built with [`Tokenizer::new`] rather than
    /// [`Tokenizer::new_byte_level`], for vocabularies that do not use
    /// ByteLevel encoding.
    pub fn new_chain(
        encoder: FxHashMap<Vec<u8>, u32>,
        special_tokens: impl Into<AddedTokenSet>,
        patterns: &[&str],
    ) -> Result<Self, TokenizerError> {
        let (first, rest) = Self::split_chain_patterns(patterns)?;
        let mut tokenizer = Self::new(encoder, special_tokens, first)?;
        tokenizer.set_chain(rest)?;
        Ok(tokenizer)
    }

    /// Split a pre-tokenizer pattern list into its head (compiled as the
    /// primary regex) and the remaining passes (installed via [`Self::set_chain`]).
    fn split_chain_patterns<'a>(
        patterns: &'a [&'a str],
    ) -> Result<(&'a str, &'a [&'a str]), TokenizerError> {
        patterns
            .split_first()
            .map(|(first, rest)| (*first, rest))
            .ok_or(TokenizerError::EmptyPatternList)
    }

    /// Compile and install the later pre-tokenizer passes on the current backend.
    fn set_chain(&mut self, patterns: &[&str]) -> Result<(), TokenizerError> {
        let compiled = patterns
            .iter()
            .map(|p| compile_pattern(p, self.use_pcre2, self.use_jit))
            .collect::<Result<Vec<_>, _>>()?;
        self.chain = Arc::from(compiled);
        self.chain_patterns = patterns.iter().map(|p| (*p).to_owned()).collect();
        Ok(())
    }

    /// Recompile the later pre-tokenizer passes after a backend or JIT change.
    fn rebuild_chain(&mut self) -> Result<(), TokenizerError> {
        if self.chain_patterns.is_empty() {
            return Ok(());
        }
        let compiled = self
            .chain_patterns
            .iter()
            .map(|p| compile_pattern(p, self.use_pcre2, self.use_jit))
            .collect::<Result<Vec<_>, _>>()?;
        self.chain = Arc::from(compiled);
        Ok(())
    }

    /// Create a new tokenizer with a metaspace (▁) decoder enabled.
    ///
    /// This is byte-level BPE, not SentencePiece — despite the historical name,
    /// it is the linked-list BPE algorithm in this file with a decode-time
    /// ▁ (U+2581) → space substitution bolted on. It is required for Mistral,
    /// Gemma, and similar tokenizers that use ▁ as a word-boundary marker in
    /// their vocab. For real SentencePiece (Unigram, Viterbi decoding) use
    /// [`crate::core::sentencepiece::SentencePieceTokenizer`]; for SPM-BPE
    /// (merge-by-rank) vocabularies use [`crate::core::spm::SpmTokenizer`].
    pub fn new_with_metaspace_decoder(
        encoder: FxHashMap<Vec<u8>, u32>,
        special_tokens: impl Into<AddedTokenSet>,
        pattern: &str,
    ) -> Result<Self, TokenizerError> {
        Self::with_full_options(
            encoder,
            special_tokens,
            pattern,
            DEFAULT_CACHE_SIZE,
            false,
            true,
        )
    }

    /// Create a new tokenizer with custom cache size.
    pub fn with_cache_size(
        encoder: FxHashMap<Vec<u8>, u32>,
        special_tokens: impl Into<AddedTokenSet>,
        pattern: &str,
        cache_size: usize,
    ) -> Result<Self, TokenizerError> {
        Self::with_options(encoder, special_tokens, pattern, cache_size, false)
    }

    /// Create a new tokenizer with full configuration options.
    ///
    /// # Arguments
    /// * `encoder` - Map of byte sequences to token IDs
    /// * `special_tokens` - Map of special token strings to token IDs
    /// * `pattern` - Regex pattern for tokenization
    /// * `cache_size` - Size of the LRU cache for encoded chunks
    /// * `use_byte_level` - Enable ByteLevel encoding for GPT-2/Llama/DeepSeek style tokenizers
    pub fn with_options(
        encoder: FxHashMap<Vec<u8>, u32>,
        special_tokens: impl Into<AddedTokenSet>,
        pattern: &str,
        cache_size: usize,
        use_byte_level: bool,
    ) -> Result<Self, TokenizerError> {
        Self::with_full_options(
            encoder,
            special_tokens,
            pattern,
            cache_size,
            use_byte_level,
            false,
        )
    }

    /// Create a new tokenizer with all configuration options including the
    /// metaspace decoder.
    ///
    /// # Arguments
    /// * `encoder` - Map of byte sequences to token IDs
    /// * `special_tokens` - Map of special token strings to token IDs
    /// * `pattern` - Regex pattern for tokenization
    /// * `cache_size` - Size of the LRU cache for encoded chunks
    /// * `use_byte_level` - Enable ByteLevel encoding for GPT-2/Llama/DeepSeek style tokenizers
    /// * `use_metaspace_decoder` - Enable the metaspace decoder (▁ → space during decode);
    ///   NOT SentencePiece, see [`Tokenizer::new_with_metaspace_decoder`]
    pub fn with_full_options(
        encoder: FxHashMap<Vec<u8>, u32>,
        special_tokens: impl Into<AddedTokenSet>,
        pattern: &str,
        cache_size: usize,
        use_byte_level: bool,
        use_metaspace_decoder: bool,
    ) -> Result<Self, TokenizerError> {
        Self::with_encoder(
            encoder_from_owned(encoder),
            special_tokens,
            pattern,
            cache_size,
            use_byte_level,
            use_metaspace_decoder,
        )
    }

    /// [`Tokenizer::with_full_options`] over a vocabulary already in the
    /// internal representation.
    ///
    /// The bundled vocabularies come this way: their keys borrow from the
    /// embedded payload rather than owning copies, and routing them through the
    /// owned-map form would allocate every token just to hand it back.
    pub(crate) fn with_encoder(
        encoder: Encoder,
        special_tokens: impl Into<AddedTokenSet>,
        pattern: &str,
        cache_size: usize,
        use_byte_level: bool,
        use_metaspace_decoder: bool,
    ) -> Result<Self, TokenizerError> {
        // Build decoder maps
        let decoder = build_decoder(&encoder);

        // Compile regex with regexr (default backend)
        let regex = Arc::new(compile_pattern(pattern, false, true)?);

        // Build the special-token matcher (shared with the other backends) from
        // the declared set — the only place the `lstrip`/`rstrip` flags are
        // consulted — then reduce the set to the plain name→id map the decode
        // tables speak. Reducing *after* building means the flags never have to
        // be carried in a second field that could drift out of step with it.
        let added: AddedTokenSet = special_tokens.into();
        let special_matcher = AddedTokens::new(&added)?;
        let special_tokens = added.into_id_map();
        let special_tokens_decoder: FxHashMap<u32, String> = special_tokens
            .iter()
            .map(|(k, v)| (*v, k.clone()))
            .collect();

        let chunk_cache = ChunkCache::new(cache_size);
        // Ranks come from the encoder until `with_merge_ranks` replaces the source.
        let byte_pair_ranks = Arc::new(BytePairRanks::build(&encoder));
        // Built on first use, not here: a vocabulary that never runs a merge
        // never pays for it, and one whose ranks are replaced by
        // `with_merge_ranks` would otherwise build it twice.
        let pair_ranks = Arc::new(OnceLock::new());
        let raw_space = Arc::new(OnceLock::new());

        Ok(Self {
            raw_encoder: None,
            encoder,
            merge_ranks: None,
            byte_pair_ranks,
            pair_ranks,
            raw_space,
            metaspace_run_split: Arc::new(OnceLock::new()),
            decoder: Arc::new(decoder),
            special_tokens,
            special_tokens_decoder: Arc::new(special_tokens_decoder),
            regex,
            pattern: pattern.to_string(),
            chain: Arc::from(Vec::new()),
            chain_patterns: Arc::from(Vec::new()),
            special_matcher,
            chunk_cache,
            use_byte_level,
            use_metaspace_decoder,
            metaspace_split: true,
            add_prefix_space: false,
            pre_tokenizer: None,
            match_added_tokens: false,
            special_decode_ids: Arc::new(rustc_hash::FxHashSet::default()),
            normalizer: None,
            cache_size,
            use_jit: true,
            use_pcre2: false,
            byte_fallback: None,
            end_of_word_suffix: None,
        })
    }

    /// Attach a separate merge-priority map (bytes → merge rank) so BPE merges
    /// by this order rather than by token id. Use for HuggingFace BPE models
    /// whose ids don't follow merge order (e.g. RoBERTa).
    /// Supply the vocabulary re-keyed by the RAW bytes each token stands for,
    /// so a ByteLevel tokenizer can resolve a pre-token without mapping it into
    /// ByteLevel space first.
    ///
    /// Taken from the caller rather than derived here because the `tokenizer.json`
    /// loader already decodes every token once, to validate it — deriving it a
    /// second time measured at 11-14% of load. The map may be partial: a miss
    /// falls through to the mapped lookup and the same answer, so only a wrong
    /// entry could change ids.
    ///
    /// Ignored unless this tokenizer is ByteLevel, which is the only case where
    /// the two spaces differ.
    pub(crate) fn with_raw_encoder(mut self, raw_encoder: Encoder) -> Self {
        if self.use_byte_level {
            self.raw_encoder = Some(raw_encoder);
        }
        self
    }

    /// Replace the id → bytes table with one covering ids the encode table
    /// deliberately omits.
    ///
    /// The two tables are normally the same vocabulary read in both directions,
    /// which is why the decode table is derived from the encoder. They part
    /// company when a vocabulary states entries BPE can never produce: those
    /// must not be *encodable* — encoding into one contradicts what merging the
    /// same bytes gives — but every id a caller may hold must still *decode*.
    /// See `vocab::orphan_ids`.
    pub fn with_decode_table(mut self, decoder: Decoder) -> Self {
        self.decoder = Arc::new(decoder);
        self
    }

    pub fn with_merge_ranks(mut self, merge_ranks: Encoder) -> Self {
        // The rank source just changed, so the two-byte index derived from it
        // has to be rebuilt — leaving the encoder-derived one in place would
        // answer with the wrong ranks.
        self.byte_pair_ranks = Arc::new(BytePairRanks::build(&merge_ranks));
        self.pair_ranks = Arc::new(OnceLock::new());
        self.raw_space = Arc::new(OnceLock::new());
        self.merge_ranks = Some(merge_ranks);
        self
    }

    /// Enable HF ByteLevel `add_prefix_space`: a leading space is prepended to
    /// the input before tokenizing (unless it already starts with whitespace).
    pub fn with_prefix_space(mut self, add_prefix_space: bool) -> Self {
        self.add_prefix_space = add_prefix_space;
        self
    }

    /// Attach a multi-stage pre-tokenizer pipeline. When the pipeline contains a
    /// `ByteLevel` stage the engine byte-encodes the pieces itself and `encode`
    /// skips re-encoding, so this tokenizer's `use_byte_level` governs only
    /// `decode` — set it to match the engine. An empty pipeline is treated as
    /// absent.
    pub fn with_pre_tokenizer(mut self, pt: crate::core::pretokenizer::PreTokenizer) -> Self {
        self.pre_tokenizer = (!pt.is_empty()).then(|| std::sync::Arc::new(pt));
        self
    }

    /// Make `encode` recognize `special_tokens` (added tokens) in the input,
    /// matching HuggingFace, which always recognizes added tokens.
    pub fn with_added_token_matching(mut self, enabled: bool) -> Self {
        self.match_added_tokens = enabled;
        self
    }

    /// Set the ids of `special=true` added tokens to drop on decode (HF default
    /// `skip_special_tokens=true`). Non-special added tokens stay rendered.
    pub fn with_special_decode_ids(mut self, ids: rustc_hash::FxHashSet<u32>) -> Self {
        self.special_decode_ids = Arc::new(ids);
        self
    }

    /// Attach a text normalizer (HF `normalizer`, e.g. NFC) applied to content
    /// before splitting. An empty normalizer is treated as absent.
    pub fn with_normalizer(mut self, normalizer: crate::core::normalizer::Normalizer) -> Self {
        self.normalizer = (!normalizer.is_empty()).then(|| std::sync::Arc::new(normalizer));
        self
    }

    /// Switch to PCRE2 regex backend.
    ///
    /// PCRE2 is an alternative regex backend. Requires the `pcre2` feature
    /// to be enabled at compile time.
    ///
    /// # Example
    /// ```rust
    /// use splintr::{from_pretrained, Backend};
    ///
    /// let any = from_pretrained("cl100k_base")?;
    /// let Backend::Bpe(tokenizer) = any.into_backend() else {
    ///     unreachable!("cl100k_base loads as a BPE backend");
    /// };
    /// let tokenizer = tokenizer.pcre2(true)?;
    /// # Ok::<(), splintr::TokenizerError>(())
    /// ```
    ///
    /// # Errors
    /// Returns an error if `pcre2` feature is not enabled or regex compilation fails.
    #[cfg(feature = "pcre2")]
    pub fn pcre2(mut self, use_pcre2: bool) -> Result<Self, TokenizerError> {
        self.use_pcre2 = use_pcre2;
        self.regex = Arc::new(compile_pattern(&self.pattern, use_pcre2, self.use_jit)?);
        self.rebuild_chain()?;
        Ok(self)
    }

    /// Switch to PCRE2 regex backend (stub when feature not enabled).
    #[cfg(not(feature = "pcre2"))]
    pub fn pcre2(self, use_pcre2: bool) -> Result<Self, TokenizerError> {
        if use_pcre2 {
            Err(TokenizerError::Pcre2NotEnabled)
        } else {
            Ok(self)
        }
    }

    /// Enable or disable JIT compilation for the regex backend.
    ///
    /// JIT (Just-In-Time) compilation can significantly improve regex matching
    /// performance. JIT availability depends on platform support (e.g., x86-64)
    /// and crate feature flags. When enabled, JIT will be used if available.
    ///
    /// # Arguments
    /// * `use_jit` - Whether to try using JIT compilation
    ///
    /// # Example
    /// ```rust
    /// use splintr::{from_pretrained, Backend};
    ///
    /// let any = from_pretrained("cl100k_base")?;
    /// let Backend::Bpe(tokenizer) = any.into_backend() else {
    ///     unreachable!("cl100k_base loads as a BPE backend");
    /// };
    /// let tokenizer = tokenizer.jit(false)?;
    /// # Ok::<(), splintr::TokenizerError>(())
    /// ```
    #[cfg(feature = "pcre2")]
    pub fn jit(mut self, use_jit: bool) -> Result<Self, TokenizerError> {
        self.use_jit = use_jit;
        self.regex = Arc::new(compile_pattern(&self.pattern, self.use_pcre2, use_jit)?);
        self.rebuild_chain()?;
        Ok(self)
    }

    /// Enable or disable JIT compilation (non-pcre2 version).
    #[cfg(not(feature = "pcre2"))]
    pub fn jit(mut self, use_jit: bool) -> Result<Self, TokenizerError> {
        self.use_jit = use_jit;
        self.regex = Arc::new(compile_pattern(&self.pattern, self.use_pcre2, use_jit)?);
        self.rebuild_chain()?;
        Ok(self)
    }

    /// Create a tokenizer from a tiktoken vocabulary file.
    pub fn from_file(
        vocab_path: &str,
        pattern: &str,
        special_tokens: impl Into<AddedTokenSet>,
    ) -> Result<Self, TokenizerError> {
        let encoder = load_tiktoken_bpe_file(vocab_path)?;
        Self::new(encoder, special_tokens, pattern)
    }

    /// Create a tokenizer from raw vocabulary bytes.
    pub fn from_bytes(
        vocab_data: &[u8],
        pattern: &str,
        special_tokens: impl Into<AddedTokenSet>,
    ) -> Result<Self, TokenizerError> {
        let encoder = load_tiktoken_bpe(vocab_data)?;
        Self::new(encoder, special_tokens, pattern)
    }

    /// Create a tokenizer from raw vocabulary bytes with ByteLevel encoding.
    pub fn from_bytes_byte_level(
        vocab_data: &[u8],
        pattern: &str,
        special_tokens: impl Into<AddedTokenSet>,
    ) -> Result<Self, TokenizerError> {
        let encoder = load_tiktoken_bpe(vocab_data)?;
        Self::new_byte_level(encoder, special_tokens, pattern)
    }

    /// Create a tokenizer from raw vocabulary bytes with a chained pre-tokenizer
    /// pattern sequence. See [`Tokenizer::new_chain`].
    pub fn from_bytes_chain(
        vocab_data: &[u8],
        patterns: &[&str],
        special_tokens: impl Into<AddedTokenSet>,
    ) -> Result<Self, TokenizerError> {
        let encoder = load_tiktoken_bpe(vocab_data)?;
        Self::new_chain(encoder, special_tokens, patterns)
    }

    /// Create a tokenizer from raw vocabulary bytes with ByteLevel encoding and
    /// a chained pre-tokenizer pattern sequence. See [`Tokenizer::new_byte_level_chain`].
    pub fn from_bytes_byte_level_chain(
        vocab_data: &[u8],
        patterns: &[&str],
        special_tokens: impl Into<AddedTokenSet>,
    ) -> Result<Self, TokenizerError> {
        let encoder = load_tiktoken_bpe(vocab_data)?;
        Self::new_byte_level_chain(encoder, special_tokens, patterns)
    }

    /// [`Tokenizer::new`] over a vocabulary already in the internal
    /// representation, so a loader that can build it directly does not pay for
    /// a second 100k-200k entry map.
    pub(crate) fn from_encoder(
        encoder: Encoder,
        special_tokens: impl Into<AddedTokenSet>,
        pattern: &str,
        use_byte_level: bool,
    ) -> Result<Self, TokenizerError> {
        Self::with_encoder(
            encoder,
            special_tokens,
            pattern,
            DEFAULT_CACHE_SIZE,
            use_byte_level,
            false,
        )
    }

    /// [`Tokenizer::new_with_metaspace_decoder`] over an already-built encoder.
    pub(crate) fn from_encoder_with_metaspace_decoder(
        encoder: Encoder,
        special_tokens: impl Into<AddedTokenSet>,
        pattern: &str,
    ) -> Result<Self, TokenizerError> {
        Self::with_encoder(
            encoder,
            special_tokens,
            pattern,
            DEFAULT_CACHE_SIZE,
            false,
            true,
        )
    }

    /// Create a tokenizer from a **packed** vocabulary with a chained
    /// pre-tokenizer pattern sequence.
    ///
    /// The bundled-vocabulary counterpart to [`Tokenizer::from_bytes_chain`]:
    /// same result, reading the binary form
    /// [`load_packed_bpe_borrowed`](crate::core::load_packed_bpe_borrowed)
    /// parses instead of `.tiktoken` text, and borrowing its token bytes rather
    /// than copying them. See that function for why the crate embeds the packed
    /// form.
    pub fn from_packed_chain(
        vocab_data: &'static [u8],
        patterns: &[&str],
        special_tokens: impl Into<AddedTokenSet>,
    ) -> Result<Self, TokenizerError> {
        let encoder = load_packed_bpe_borrowed(vocab_data)?;
        let (first, rest) = Self::split_chain_patterns(patterns)?;
        let mut tokenizer = Self::with_encoder(
            encoder,
            special_tokens,
            first,
            DEFAULT_CACHE_SIZE,
            false,
            false,
        )?;
        tokenizer.set_chain(rest)?;
        Ok(tokenizer)
    }

    /// Packed counterpart to [`Tokenizer::from_bytes_byte_level_chain`], for a
    /// vocabulary that keeps the ByteLevel spelling.
    pub fn from_packed_byte_level_chain(
        vocab_data: &'static [u8],
        patterns: &[&str],
        special_tokens: impl Into<AddedTokenSet>,
    ) -> Result<Self, TokenizerError> {
        let encoder = load_packed_bpe_borrowed(vocab_data)?;
        let (first, rest) = Self::split_chain_patterns(patterns)?;
        let mut tokenizer = Self::with_encoder(
            encoder,
            special_tokens,
            first,
            DEFAULT_CACHE_SIZE,
            true,
            false,
        )?;
        tokenizer.set_chain(rest)?;
        Ok(tokenizer)
    }

    /// Create a tokenizer from raw vocabulary bytes with the metaspace decoder.
    ///
    /// This is byte-level BPE, not SentencePiece — see
    /// [`Tokenizer::new_with_metaspace_decoder`]. It converts ▁ (U+2581) to
    /// space during decoding. Used for Mistral, Gemma, and similar tokenizers.
    pub fn from_bytes_with_metaspace_decoder(
        vocab_data: &[u8],
        pattern: &str,
        special_tokens: impl Into<AddedTokenSet>,
    ) -> Result<Self, TokenizerError> {
        let encoder = load_tiktoken_bpe(vocab_data)?;
        Self::new_with_metaspace_decoder(encoder, special_tokens, pattern)
    }

    /// Create a metaspace-decoder BPE tokenizer (see
    /// [`Tokenizer::new_with_metaspace_decoder`] — NOT SentencePiece) with an
    /// explicit decoder to preserve all token IDs.
    ///
    /// This is used for vocabs with duplicate byte sequences (like Mistral V2 where byte fallback
    /// tokens may duplicate BPE merges). The decoder preserves ALL token IDs, while the encoder
    /// only keeps the lowest ID for each byte sequence.
    pub fn from_bytes_with_metaspace_decoder_preserving_ids(
        vocab_data: &[u8],
        pattern: &str,
        special_tokens: impl Into<AddedTokenSet>,
    ) -> Result<Self, TokenizerError> {
        use crate::core::vocab::load_tiktoken_bpe_with_decoder;
        let (encoder, decoder) = load_tiktoken_bpe_with_decoder(vocab_data)?;
        // This vocabulary was read at runtime, so its tokens are owned; the
        // borrowed representation is only for the embedded ones.
        let encoder = encoder_from_owned(encoder);
        let mut decoder: crate::core::decode_table::Decoder = decoder.into_iter().collect();

        // Compile regex
        let regex = RegexBuilder::new(pattern).jit(true).build()?;

        // Build the special-token matcher (shared with the other backends) from
        // the declared set, then reduce it to the plain name→id map the decode
        // tables speak — see `with_full_options`.
        let added: AddedTokenSet = special_tokens.into();
        let special_matcher = AddedTokens::new(&added)?;
        let special_tokens = added.into_id_map();

        // Add special tokens to decoder
        for (token_str, id) in &special_tokens {
            decoder.insert(*id, token_str.as_bytes());
        }

        // Build the tokenizer manually with explicit decoder
        let special_tokens_decoder: FxHashMap<u32, String> = special_tokens
            .iter()
            .map(|(k, v)| (*v, k.clone()))
            .collect();

        let chunk_cache = ChunkCache::new(DEFAULT_CACHE_SIZE);
        // Ranks come from the encoder until `with_merge_ranks` replaces the source.
        let byte_pair_ranks = Arc::new(BytePairRanks::build(&encoder));
        // Built on first use, not here: a vocabulary that never runs a merge
        // never pays for it, and one whose ranks are replaced by
        // `with_merge_ranks` would otherwise build it twice.
        let pair_ranks = Arc::new(OnceLock::new());
        let raw_space = Arc::new(OnceLock::new());

        Ok(Self {
            encoder,
            // Metaspace, not ByteLevel — nothing to re-key.
            raw_encoder: None,
            merge_ranks: None,
            byte_pair_ranks,
            pair_ranks,
            raw_space,
            metaspace_run_split: Arc::new(OnceLock::new()),
            decoder: Arc::new(decoder),
            special_tokens,
            special_tokens_decoder: Arc::new(special_tokens_decoder),
            regex: Arc::new(RegexBackend::Regexr(Box::new(regex))),
            pattern: pattern.to_string(),
            chain: Arc::from(Vec::new()),
            chain_patterns: Arc::from(Vec::new()),
            special_matcher,
            chunk_cache,
            use_byte_level: false,
            use_metaspace_decoder: true,
            metaspace_split: true,
            add_prefix_space: false,
            pre_tokenizer: None,
            match_added_tokens: false,
            special_decode_ids: Arc::new(rustc_hash::FxHashSet::default()),
            normalizer: None,
            cache_size: DEFAULT_CACHE_SIZE,
            use_jit: true,
            use_pcre2: false,
            byte_fallback: None,
            end_of_word_suffix: None,
        })
    }

    /// Set `Metaspace.split`, which decides whether the metaspace fork splits a
    /// content gap on `▁` or hands the model the whole thing as one piece.
    ///
    /// Only meaningful on a tokenizer built for the metaspace fork; every other
    /// path ignores it. Defaults to true, as HuggingFace's node does.
    pub fn with_metaspace_split(mut self, split: bool) -> Self {
        self.metaspace_split = split;
        self
    }

    /// Attach the [`ByteFallback`] resolution, so a BPE piece the merge cannot
    /// represent is emitted through its `<0xNN>`/`<unk>` ids instead of being
    /// dropped. `None` disables it — the correct choice for any vocabulary that
    /// declares no byte fallback, and *required* for every ByteLevel BPE model:
    /// `Tokenizer::bpe` discards `byte_fallback` outright whenever
    /// `use_byte_level` is true (the `<0xNN>` table is keyed by RAW byte value,
    /// the wrong space once input has already been byte-level-encoded), so a
    /// `Some` here would never fire and callers should not build one (see
    /// `build_bpe` in `src/core/hf_json/loader.rs`, which skips the call).
    pub fn with_byte_fallback(mut self, byte_fallback: Option<ByteFallback>) -> Self {
        self.byte_fallback = byte_fallback;
        self
    }

    /// Attach `model.end_of_word_suffix`, the marker appended to the last symbol
    /// of every word before merging (CLIP's `</w>`).
    ///
    /// An empty suffix is `None`: a BPE model that declares `""` declares no
    /// suffix, which is how every non-CLIP file in the corpus spells its absence.
    pub(crate) fn with_end_of_word_suffix(mut self, suffix: Option<&str>) -> Self {
        self.end_of_word_suffix = suffix.filter(|s| !s.is_empty()).map(Arc::from);
        self
    }

    /// Derive a [`ByteFallback`] from an encoder and a resolved `unk` id, by
    /// looking up the 256 `<0xNN>` token spellings HuggingFace byte-fallback
    /// vocabularies declare. Mirrors `SpmTokenizer::new`'s identical lookup
    /// over its own vocab (see `src/core/spm.rs`) so the two backends agree on
    /// the table's construction.
    ///
    /// A partial set is kept as a partial set: HuggingFace resolves fallback
    /// per character, emitting `<0xNN>` where the entry exists and the unk id
    /// where it does not, so a vocabulary declaring only some `<0xNN>` entries
    /// is a valid file. Returns `None` only when neither half exists (no byte
    /// entries and no unk id), where there is nothing to fall back *to* and
    /// dropping — the no-fallback behavior — is already the answer.
    ///
    /// `declares_byte_fallback` is the model's own `byte_fallback` flag and
    /// gates the `<0xNN>` table ONLY. When it is false the table is left empty
    /// even if the vocabulary spells `<0xNN>` tokens — HuggingFace's BPE model
    /// consults them only under the flag — while `unk_id` still applies, since
    /// the unk branch is not gated on the flag at all (measured against
    /// `tokenizers` 0.22.1; see `build_bpe` in `src/core/hf_json/loader.rs`).
    /// The `<0xNN>` table, over any way of resolving one of those spellings to
    /// its id.
    ///
    /// Deliberately not "from the encode table". A `<0xNN>` piece is not
    /// encodable from its own literal spelling — HuggingFace spells `<0x1D>`
    /// out as characters, and an encode table holding it would answer with one
    /// id where BPE gives six — so the encode table declines it, while the
    /// fallback still has to emit it for the raw byte `0x1D`. The two roles are
    /// separate and so are their sources: callers pass the whole vocabulary.
    pub(crate) fn byte_fallback_from(
        id_of: impl Fn(&[u8]) -> Option<u32>,
        unk_id: Option<u32>,
        declares_byte_fallback: bool,
    ) -> Option<ByteFallback> {
        let mut byte_ids = [None; 256];
        let mut any = false;
        if declares_byte_fallback {
            for (b, slot) in byte_ids.iter_mut().enumerate() {
                *slot = id_of(format!("<0x{b:02X}>").as_bytes());
                any |= slot.is_some();
            }
        }
        (any || unk_id.is_some()).then(|| ByteFallback::new(byte_ids, unk_id))
    }
}