llm_tokenizer/

stop.rs

use std::{collections::HashSet, sync::Arc};

use aho_corasick::AhoCorasick;
use anyhow::Result;

use crate::{
    sequence::Sequence,
    traits::{self, TokenIdType},
};

/// Output from the sequence decoder
#[derive(Debug, Clone, PartialEq)]
pub enum SequenceDecoderOutput {
    /// Normal text output
    Text(String),
    /// Text is being held due to partial stop sequence match
    Held,
    /// Stop sequence matched (hidden - not included in output)
    Stopped,
    /// Stop sequence matched with text (visible - included in output)
    StoppedWithText(String),
}

/// Configuration for stop sequences
#[derive(Debug, Clone, Default)]
pub struct StopSequenceConfig {
    /// Token IDs that trigger a stop
    pub stop_tokens: HashSet<TokenIdType>,
    /// String sequences that trigger a stop
    pub stop_sequences: Vec<String>,
    /// Token IDs for visible stops (included in output)
    pub visible_stop_tokens: HashSet<TokenIdType>,
    /// String sequences for visible stops (included in output)
    pub visible_stop_sequences: Vec<String>,
}

impl StopSequenceConfig {
    /// Builder pattern - add a stop token
    pub fn with_stop_token(mut self, token_id: TokenIdType) -> Self {
        self.stop_tokens.insert(token_id);
        self
    }

    /// Builder pattern - add a stop sequence
    pub fn with_stop_sequence(mut self, sequence: impl Into<String>) -> Self {
        self.stop_sequences.push(sequence.into());
        self
    }

    /// Builder pattern - add a visible stop token
    pub fn with_visible_stop_token(mut self, token_id: TokenIdType) -> Self {
        self.visible_stop_tokens.insert(token_id);
        self
    }

    /// Builder pattern - add a visible stop sequence
    pub fn with_visible_stop_sequence(mut self, sequence: impl Into<String>) -> Self {
        self.visible_stop_sequences.push(sequence.into());
        self
    }
}

/// Decoder that handles stop sequences
pub struct StopSequenceDecoder {
    /// Sequence for incremental decoding (replaces token_buffer + offsets)
    sequence: Sequence,
    config: StopSequenceConfig,
    /// Aho-Corasick automaton for O(N) stop sequence matching
    aho_corasick: Option<AhoCorasick>,
    /// Index boundary: patterns [0..visible_boundary_idx) are hidden,
    /// patterns [visible_boundary_idx..) are visible
    visible_boundary_idx: usize,
    /// Buffer for partial matches (the "jail")
    jail_buffer: String,
    /// Whether we've stopped
    stopped: bool,
}

impl StopSequenceDecoder {
    /// Create a new stop sequence decoder
    pub fn new(
        tokenizer: Arc<dyn traits::Tokenizer>,
        config: StopSequenceConfig,
        skip_special_tokens: bool,
    ) -> Self {
        // Build Aho-Corasick automaton from all stop sequences
        // Hidden sequences come first, then visible sequences
        let mut patterns: Vec<&str> = config
            .stop_sequences
            .iter()
            .filter(|s| !s.is_empty())
            .map(|s| s.as_str())
            .collect();
        let visible_boundary_idx = patterns.len();
        patterns.extend(
            config
                .visible_stop_sequences
                .iter()
                .filter(|s| !s.is_empty())
                .map(|s| s.as_str()),
        );

        let aho_corasick = if patterns.is_empty() {
            None
        } else {
            // AhoCorasick::new is infallible for non-empty, pre-filtered string patterns.
            // Failure would indicate a bug in the Aho-Corasick library itself.
            #[expect(
                clippy::expect_used,
                reason = "AhoCorasick::new with pre-filtered non-empty &str patterns is practically infallible"
            )]
            Some(AhoCorasick::new(patterns).expect("Failed to build Aho-Corasick automaton"))
        };

        StopSequenceDecoder {
            sequence: Sequence::new_with_options(tokenizer, skip_special_tokens),
            config,
            aho_corasick,
            visible_boundary_idx,
            jail_buffer: String::new(),
            stopped: false,
        }
    }

    /// Process a single token
    pub fn process_token(&mut self, token_id: TokenIdType) -> Result<SequenceDecoderOutput> {
        if self.stopped {
            return Ok(SequenceDecoderOutput::Stopped);
        }

        // Check for token-level stops first
        if self.config.stop_tokens.contains(&token_id) {
            self.stopped = true;

            // Flush any jailed text before stopping - use mem::take to avoid clone
            if !self.jail_buffer.is_empty() {
                return Ok(SequenceDecoderOutput::StoppedWithText(std::mem::take(
                    &mut self.jail_buffer,
                )));
            }
            return Ok(SequenceDecoderOutput::Stopped);
        }

        if self.config.visible_stop_tokens.contains(&token_id) {
            self.stopped = true;

            // Include jailed text plus the stop token
            let stop_text = self
                .sequence
                .tokenizer()
                .decode(&[token_id], self.sequence.skip_special_tokens())?;
            let output = format!("{}{}", self.jail_buffer, stop_text);
            self.jail_buffer.clear();
            return Ok(SequenceDecoderOutput::StoppedWithText(output));
        }

        // Use Sequence for incremental decoding
        let new_text = self.sequence.append_token(token_id)?;

        self.jail_buffer.push_str(&new_text);

        // Check for stop sequences using Aho-Corasick (O(N) single-pass)
        if let Some(ac) = &self.aho_corasick {
            if let Some(mat) = ac.find(&self.jail_buffer) {
                self.stopped = true;
                let is_visible = mat.pattern().as_usize() >= self.visible_boundary_idx;

                if is_visible {
                    // Visible stop sequence: include it in output
                    let output = self.jail_buffer[..mat.end()].to_string();
                    self.jail_buffer.clear();
                    return Ok(SequenceDecoderOutput::StoppedWithText(output));
                } else {
                    // Hidden stop sequence: exclude it from output
                    let output = self.jail_buffer[..mat.start()].to_string();
                    self.jail_buffer.clear();
                    return Ok(if output.is_empty() {
                        SequenceDecoderOutput::Stopped
                    } else {
                        SequenceDecoderOutput::StoppedWithText(output)
                    });
                }
            }
        }

        // Check for partial matches: is the end of jail_buffer the start of any stop_seq?
        // This handles stop sequences split across tokens
        let buffer_len = self.jail_buffer.len();
        let mut best_split_pos: Option<usize> = None;

        for stop_seq in self
            .config
            .stop_sequences
            .iter()
            .chain(&self.config.visible_stop_sequences)
        {
            let stop_len = stop_seq.len();

            if stop_len <= 1 || buffer_len == 0 {
                continue;
            }

            let max_len = buffer_len.min(stop_len - 1);

            for len in (1..=max_len).rev() {
                let suffix_start = buffer_len - len;

                if !self.jail_buffer.is_char_boundary(suffix_start) {
                    continue;
                }

                let suffix = &self.jail_buffer[suffix_start..];

                if stop_seq.starts_with(suffix)
                    && best_split_pos.is_none_or(|current| suffix_start < current)
                {
                    best_split_pos = Some(suffix_start);
                    break;
                }
            }
        }

        if let Some(split_pos) = best_split_pos {
            // Hold the partial match, flush the rest
            // Use split_off for zero-copy: keeps [0..split_pos] in place, returns [split_pos..]
            // Then swap so we output the prefix and keep the suffix
            let suffix = self.jail_buffer.split_off(split_pos);
            let to_output = std::mem::replace(&mut self.jail_buffer, suffix);

            if to_output.is_empty() {
                Ok(SequenceDecoderOutput::Held)
            } else {
                Ok(SequenceDecoderOutput::Text(to_output))
            }
        } else {
            // No partial matches - flush everything
            let output = std::mem::take(&mut self.jail_buffer);
            if output.is_empty() {
                Ok(SequenceDecoderOutput::Held)
            } else {
                Ok(SequenceDecoderOutput::Text(output))
            }
        }
    }

    /// Process multiple tokens
    pub fn process_tokens(
        &mut self,
        token_ids: &[TokenIdType],
    ) -> Result<Vec<SequenceDecoderOutput>> {
        // Pre-allocate with exact capacity to avoid reallocations
        let mut outputs = Vec::with_capacity(token_ids.len());
        for &token_id in token_ids {
            outputs.push(self.process_token(token_id)?);
        }
        Ok(outputs)
    }

    /// Flush any held text
    pub fn flush(&mut self) -> SequenceDecoderOutput {
        if self.jail_buffer.is_empty() {
            SequenceDecoderOutput::Text(String::new())
        } else {
            // Use mem::take to avoid clone - transfers ownership and leaves empty string
            SequenceDecoderOutput::Text(std::mem::take(&mut self.jail_buffer))
        }
    }

    /// Check if decoding has stopped
    pub fn is_stopped(&self) -> bool {
        self.stopped
    }

    /// Reset the decoder state
    pub fn reset(&mut self) {
        self.jail_buffer.clear();
        self.sequence.clear();
        self.stopped = false;
    }
}

/// Builder for StopSequenceDecoder
pub struct StopSequenceDecoderBuilder {
    tokenizer: Arc<dyn traits::Tokenizer>,
    config: StopSequenceConfig,
    skip_special_tokens: bool,
}

impl StopSequenceDecoderBuilder {
    pub fn new(tokenizer: Arc<dyn traits::Tokenizer>) -> Self {
        StopSequenceDecoderBuilder {
            tokenizer,
            config: StopSequenceConfig::default(),
            skip_special_tokens: true,
        }
    }

    pub fn stop_token(mut self, token_id: TokenIdType) -> Self {
        self.config.stop_tokens.insert(token_id);
        self
    }

    pub fn stop_sequence(mut self, sequence: impl Into<String>) -> Self {
        self.config.stop_sequences.push(sequence.into());
        self
    }

    pub fn visible_stop_token(mut self, token_id: TokenIdType) -> Self {
        self.config.visible_stop_tokens.insert(token_id);
        self
    }

    pub fn visible_stop_sequence(mut self, sequence: impl Into<String>) -> Self {
        self.config.visible_stop_sequences.push(sequence.into());
        self
    }

    pub fn skip_special_tokens(mut self, skip: bool) -> Self {
        self.skip_special_tokens = skip;
        self
    }

    pub fn build(self) -> StopSequenceDecoder {
        StopSequenceDecoder::new(self.tokenizer, self.config, self.skip_special_tokens)
    }
}

#[cfg(test)]
mod tests {
    use std::sync::Arc;

    use super::StopSequenceDecoderBuilder;
    use crate::{
        mock::MockTokenizer, SequenceDecoderOutput, StopSequenceConfig, StopSequenceDecoder,
    };

    #[test]
    fn test_stop_token_detection() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_token(999); // <eos> token

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process tokens before stop
        let result = decoder.process_token(1).unwrap(); // "Hello"
        assert!(matches!(result, SequenceDecoderOutput::Text(_)));

        // Process stop token
        let result = decoder.process_token(999).unwrap(); // <eos>
        assert_eq!(result, SequenceDecoderOutput::Stopped);

        // Further tokens should also return Stopped
        let result = decoder.process_token(2).unwrap();
        assert_eq!(result, SequenceDecoderOutput::Stopped);
    }

    #[test]
    fn test_visible_stop_token() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_visible_stop_token(999);

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        let result = decoder.process_token(999).unwrap();
        assert!(matches!(result, SequenceDecoderOutput::StoppedWithText(_)));
    }

    #[test]
    fn test_builder_pattern() {
        let tokenizer = Arc::new(MockTokenizer::new());

        let decoder = StopSequenceDecoderBuilder::new(tokenizer)
            .stop_token(999)
            .stop_sequence("STOP")
            .visible_stop_token(1000)
            .skip_special_tokens(true)
            .build();

        assert!(!decoder.is_stopped());
    }

    #[test]
    fn test_incremental_decoding_no_repetition() {
        // This test verifies the critical fix: no repeated output
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default();
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process tokens one by one and collect outputs
        let mut outputs = Vec::new();

        // Token 1: "Hello"
        let result = decoder.process_token(1).unwrap();
        if let SequenceDecoderOutput::Text(text) = result {
            outputs.push(text.clone());
        }

        // Token 2: "world"
        let result = decoder.process_token(2).unwrap();
        if let SequenceDecoderOutput::Text(text) = result {
            outputs.push(text.clone());
        }

        // Token 3: "test"
        let result = decoder.process_token(3).unwrap();
        if let SequenceDecoderOutput::Text(text) = result {
            outputs.push(text.clone());
        }

        // CRITICAL: Each output should be unique (no accumulation)
        // The fix ensures we only output NEW text, not accumulated text
        assert_eq!(outputs.len(), 3);

        for i in 0..outputs.len() {
            for j in i + 1..outputs.len() {
                // No output should contain another (no accumulation)
                assert!(!outputs[j].contains(&outputs[i]));
            }
        }
    }

    #[test]
    fn test_stop_sequence_detection() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_sequence("test");
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process "Hello world"
        decoder.process_token(1).unwrap(); // "Hello"
        decoder.process_token(2).unwrap(); // "world"

        // Process "test" which should trigger stop
        let result = decoder.process_token(3).unwrap(); // "test"

        // Should stop when we hit "test"
        assert!(matches!(
            result,
            SequenceDecoderOutput::Stopped | SequenceDecoderOutput::StoppedWithText(_)
        ));
    }

    #[test]
    fn test_flush_after_partial() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_sequence("NEVER_MATCH");
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process a token
        decoder.process_token(1).unwrap(); // "Hello"

        // Flush should return any remaining text in jail
        let result = decoder.flush();

        // After processing, flush should work
        assert!(matches!(result, SequenceDecoderOutput::Text(_)));
    }

    #[test]
    fn test_reset_functionality() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_token(999);
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process and stop
        decoder.process_token(1).unwrap();
        decoder.process_token(999).unwrap();
        assert!(decoder.is_stopped());

        // Reset should clear everything
        decoder.reset();
        assert!(!decoder.is_stopped());

        // Should be able to process again
        let result = decoder.process_token(2).unwrap();
        assert!(matches!(result, SequenceDecoderOutput::Text(_)));
    }

    #[test]
    fn test_visible_stop_sequence() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_visible_stop_sequence("world");
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process "Hello"
        decoder.process_token(1).unwrap();

        // Process "world" - should include it in output
        let result = decoder.process_token(2).unwrap();

        if let SequenceDecoderOutput::StoppedWithText(text) = result {
            // Should include "world" in the output
            assert!(text.contains("world"));
        } else {
            panic!("Expected StoppedWithText with visible stop sequence");
        }
    }

    #[test]
    fn test_multiple_tokens_processing() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default();
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process multiple tokens at once
        let results = decoder.process_tokens(&[1, 2, 3]).unwrap();

        // Should get results for each token
        assert_eq!(results.len(), 3);

        // Each result should be Text (no stops configured)
        for result in results {
            assert!(matches!(
                result,
                SequenceDecoderOutput::Text(_) | SequenceDecoderOutput::Held
            ));
        }
    }

    /// Test that the jail buffer correctly handles a stop sequence that arrives
    /// across 2+ tokens.  The MockTokenizer decodes token 1 as "Hello" and
    /// token 2's incremental contribution as " world", so the jail buffer
    /// progressively becomes "Hello" then "Hello world".
    ///
    /// With stop sequence "Hello world":
    ///   - Token 1: jail = "Hello" — partial prefix match → Held (or Text of
    ///     the portion before the potential match, which is empty here)
    ///   - Token 2: jail = "Hello world" — full match → Stopped
    #[test]
    fn test_stop_sequence_spanning_multiple_tokens() {
        let tokenizer = Arc::new(MockTokenizer::new());

        // "Hello world" spans token 1 ("Hello") and token 2 (" world")
        let config = StopSequenceConfig::default().with_stop_sequence("Hello world");
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Token 1 ("Hello"): The jail buffer now contains "Hello".
        // "Hello" is a prefix of stop sequence "Hello world", so the text
        // must be held — we should NOT see it emitted as Text yet.
        let result1 = decoder.process_token(1).unwrap();
        assert!(
            matches!(result1, SequenceDecoderOutput::Held),
            "Expected Held while jail buffer is a prefix of the stop sequence, got {result1:?}"
        );
        assert!(
            !decoder.is_stopped(),
            "Decoder should not be stopped after a partial match"
        );

        // Token 2 (" world"): The jail buffer now contains "Hello world",
        // which fully matches the stop sequence. The decoder should stop.
        let result2 = decoder.process_token(2).unwrap();
        assert_eq!(
            result2,
            SequenceDecoderOutput::Stopped,
            "Expected Stopped when jail buffer matches the hidden stop sequence"
        );
        assert!(
            decoder.is_stopped(),
            "Decoder should be stopped after the full stop sequence match"
        );

        // Any further tokens should also return Stopped
        let result3 = decoder.process_token(3).unwrap();
        assert_eq!(result3, SequenceDecoderOutput::Stopped);
    }

    /// Same as above but with a *visible* stop sequence.  When the stop
    /// sequence "Hello world" is visible, the matched text should be included
    /// in the output via StoppedWithText.
    #[test]
    fn test_visible_stop_sequence_spanning_multiple_tokens() {
        let tokenizer = Arc::new(MockTokenizer::new());

        let config = StopSequenceConfig::default().with_visible_stop_sequence("Hello world");
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Token 1 ("Hello"): partial match, should be held
        let result1 = decoder.process_token(1).unwrap();
        assert!(
            matches!(result1, SequenceDecoderOutput::Held),
            "Expected Held for partial visible stop sequence match, got {result1:?}"
        );

        // Token 2 (" world"): completes "Hello world" — visible stop
        let result2 = decoder.process_token(2).unwrap();
        match &result2 {
            SequenceDecoderOutput::StoppedWithText(text) => {
                assert!(
                    text.contains("Hello world"),
                    "Visible stop output should contain the full stop sequence, got: {text:?}"
                );
            }
            other => panic!("Expected StoppedWithText for visible stop sequence, got {other:?}"),
        }
        assert!(decoder.is_stopped());
    }

    /// Test a stop sequence that spans 3 tokens, with preceding text that
    /// should be emitted before the jailed portion.
    ///
    /// Tokens: 3 ("test"), 1 ("Hello"), 2 ("world")
    /// Stop sequence: "Hello world"
    ///
    /// - Token 3: produces "test" — no overlap with "Hello world" → Text("test")
    /// - Token 1: produces " Hello" — "Hello" is a prefix of "Hello world"
    ///   so " " is flushed as Text and "Hello" is held
    /// - Token 2: produces " world" — jail now "Hello world" → Stopped
    #[test]
    fn test_stop_sequence_spanning_tokens_with_preceding_text() {
        let tokenizer = Arc::new(MockTokenizer::new());

        let config = StopSequenceConfig::default().with_stop_sequence("Hello world");
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Token 3 ("test"): no overlap with "Hello world" at all
        let result1 = decoder.process_token(3).unwrap();
        assert!(
            matches!(result1, SequenceDecoderOutput::Text(_)),
            "Expected Text for token with no stop sequence overlap, got {result1:?}"
        );

        // Token 1 ("Hello"): the incremental text is " Hello" (because mock
        // tokenizer joins with spaces). The tail "Hello" is a prefix of the
        // stop sequence, so the decoder should split: emit the non-matching
        // prefix as Text (the space " ") and hold "Hello".
        let result2 = decoder.process_token(1).unwrap();
        match &result2 {
            SequenceDecoderOutput::Text(text) => {
                // The emitted text should be the portion before the partial match
                assert!(
                    !text.contains("Hello"),
                    "Partially-matched 'Hello' should be jailed, not emitted. Got: {text:?}"
                );
            }
            SequenceDecoderOutput::Held => {
                // Also acceptable if the entire chunk is held (implementation detail)
            }
            other => panic!("Expected Text (prefix before partial match) or Held, got {other:?}"),
        }

        // Token 2 ("world"): completes the stop sequence
        let result3 = decoder.process_token(2).unwrap();
        assert!(
            matches!(
                result3,
                SequenceDecoderOutput::Stopped | SequenceDecoderOutput::StoppedWithText(_)
            ),
            "Expected Stopped or StoppedWithText when stop sequence completes, got {result3:?}"
        );
        assert!(decoder.is_stopped());
    }

    #[test]
    fn test_utf8_multibyte_character_boundaries() {
        // This test verifies the fix for the UTF-8 boundary panic
        // The panic occurred when trying to slice jail_buffer at a byte index
        // that was in the middle of a multi-byte UTF-8 character (e.g., '×')
        use crate::mock::MockTokenizer;

        let tokenizer = Arc::new(MockTokenizer::new());

        // Configure stop sequence with a multi-byte character
        let config = StopSequenceConfig::default().with_stop_sequence(" ×");

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Simulate the scenario: jail_buffer will contain " ×" (space + multiplication sign)
        // The '×' character is UTF-8 encoded as bytes [0xC3, 0x97] (2 bytes)
        // When checking for partial matches, we must not slice in the middle of these bytes

        // This should not panic - the fix ensures we only slice at char boundaries
        let result = decoder.process_token(1); // Will add some text to jail_buffer
        assert!(result.is_ok());

        // Even with multi-byte UTF-8 characters in the buffer, processing should work
        let result = decoder.process_token(2);
        assert!(result.is_ok());
    }

    #[test]
    fn test_utf8_multibyte_delta_character() {
        // Test for: byte index 1 is not a char boundary; it is inside 'Δ' (bytes 0..2) of `Δ`
        // 'Δ' (U+0394 GREEK CAPITAL LETTER DELTA) is encoded as [0xCE, 0x94] (2 bytes)
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_sequence("Δ");

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process tokens - should not panic when checking partial matches
        let result = decoder.process_token(1);
        assert!(result.is_ok());
        let result = decoder.process_token(2);
        assert!(result.is_ok());
    }

    #[test]
    fn test_utf8_multibyte_degree_character() {
        // Test for: byte index 1 is not a char boundary; it is inside '°' (bytes 0..2) of `°`
        // '°' (U+00B0 DEGREE SIGN) is encoded as [0xC2, 0xB0] (2 bytes)
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_sequence("°");

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process tokens - should not panic when checking partial matches
        let result = decoder.process_token(1);
        assert!(result.is_ok());
        let result = decoder.process_token(2);
        assert!(result.is_ok());
    }

    #[test]
    fn test_utf8_multibyte_triangle_character() {
        // Test for: byte index 4 is not a char boundary; it is inside '∆' (bytes 2..5) of ` (∆`
        // '∆' (U+2206 INCREMENT) is encoded as [0xE2, 0x88, 0x86] (3 bytes)
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_sequence(" (∆");

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process tokens - should not panic when checking partial matches
        let result = decoder.process_token(1);
        assert!(result.is_ok());
        let result = decoder.process_token(2);
        assert!(result.is_ok());
        let result = decoder.process_token(3);
        assert!(result.is_ok());
    }

    #[test]
    fn test_utf8_multibyte_en_dash_character() {
        // Test for: byte index 3 is not a char boundary; it is inside '–' (bytes 1..4) of ` –`
        // '–' (U+2013 EN DASH) is encoded as [0xE2, 0x80, 0x93] (3 bytes)
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_sequence(" –");

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process tokens - should not panic when checking partial matches
        let result = decoder.process_token(1);
        assert!(result.is_ok());
        let result = decoder.process_token(2);
        assert!(result.is_ok());
        let result = decoder.process_token(3);
        assert!(result.is_ok());
    }

    #[test]
    fn test_utf8_multibyte_various_characters() {
        // Comprehensive test with multiple multi-byte UTF-8 characters
        // Tests 2-byte, 3-byte, and 4-byte UTF-8 sequences
        let test_cases = vec![
            ("×", "multiplication sign - 2 bytes"),
            ("Δ", "Greek Delta - 2 bytes"),
            ("°", "degree sign - 2 bytes"),
            ("∆", "increment - 3 bytes"),
            ("–", "en dash - 3 bytes"),
            ("€", "euro sign - 3 bytes"),
            ("中", "Chinese character - 3 bytes"),
            ("🚀", "rocket emoji - 4 bytes"),
            ("💡", "lightbulb emoji - 4 bytes"),
        ];

        for (stop_char, description) in test_cases {
            let tokenizer = Arc::new(MockTokenizer::new());
            let config = StopSequenceConfig::default().with_stop_sequence(stop_char);

            let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

            // Process multiple tokens - should not panic
            for token_id in 1..=5 {
                let result = decoder.process_token(token_id);
                assert!(
                    result.is_ok(),
                    "Failed on {description} with token {token_id}"
                );
            }
        }
    }
}