llm_tokenizer/

stop.rs

use std::{collections::HashSet, sync::Arc};

use aho_corasick::{AhoCorasick, Input};
use anyhow::Result;

use crate::{
    sequence::Sequence,
    traits::{self, TokenIdType},
};

/// Output from the sequence decoder
#[derive(Debug, Clone, PartialEq)]
pub enum SequenceDecoderOutput {
    /// Normal text output
    Text(String),
    /// Text is being held due to partial stop sequence match
    Held,
    /// Stop sequence matched (hidden - not included in output)
    Stopped,
    /// Stop sequence matched with text (visible - included in output)
    StoppedWithText(String),
}

/// Configuration for stop sequences
#[derive(Debug, Clone, Default)]
pub struct StopSequenceConfig {
    /// Token IDs that trigger a stop
    pub stop_tokens: HashSet<TokenIdType>,
    /// String sequences that trigger a stop
    pub stop_sequences: Vec<String>,
    /// Token IDs for visible stops (included in output)
    pub visible_stop_tokens: HashSet<TokenIdType>,
    /// String sequences for visible stops (included in output)
    pub visible_stop_sequences: Vec<String>,
}

impl StopSequenceConfig {
    /// Builder pattern - add a stop token
    pub fn with_stop_token(mut self, token_id: TokenIdType) -> Self {
        self.stop_tokens.insert(token_id);
        self
    }

    /// Builder pattern - add a stop sequence
    pub fn with_stop_sequence(mut self, sequence: impl Into<String>) -> Self {
        self.stop_sequences.push(sequence.into());
        self
    }

    /// Builder pattern - add a visible stop token
    pub fn with_visible_stop_token(mut self, token_id: TokenIdType) -> Self {
        self.visible_stop_tokens.insert(token_id);
        self
    }

    /// Builder pattern - add a visible stop sequence
    pub fn with_visible_stop_sequence(mut self, sequence: impl Into<String>) -> Self {
        self.visible_stop_sequences.push(sequence.into());
        self
    }
}

/// Decoder that handles stop sequences
pub struct StopSequenceDecoder {
    /// Sequence for incremental decoding (replaces token_buffer + offsets)
    sequence: Sequence,
    config: StopSequenceConfig,
    /// Aho-Corasick automaton for O(N) stop sequence matching
    aho_corasick: Option<AhoCorasick>,
    /// Index boundary: patterns [0..visible_boundary_idx) are hidden,
    /// patterns [visible_boundary_idx..) are visible
    visible_boundary_idx: usize,
    /// Buffer for partial matches (the "jail")
    jail_buffer: String,
    /// Maximum bytes to retain in jail_buffer — equal to the longest stop sequence.
    /// Text beyond this window cannot participate in a future match and is safe to drain.
    jail_max_bytes: usize,
    /// Whether we've stopped
    stopped: bool,
    /// True when there are no string stop sequences (only token-level stops).
    /// In this mode the jail buffer is bypassed entirely for lower overhead.
    token_only: bool,
}

impl StopSequenceDecoder {
    /// Create a new stop sequence decoder
    pub fn new(
        tokenizer: Arc<dyn traits::Tokenizer>,
        config: StopSequenceConfig,
        skip_special_tokens: bool,
    ) -> Self {
        // Build Aho-Corasick automaton from all stop sequences
        // Hidden sequences come first, then visible sequences
        let mut patterns: Vec<&str> = config
            .stop_sequences
            .iter()
            .filter(|s| !s.is_empty())
            .map(|s| s.as_str())
            .collect();
        let visible_boundary_idx = patterns.len();
        patterns.extend(
            config
                .visible_stop_sequences
                .iter()
                .filter(|s| !s.is_empty())
                .map(|s| s.as_str()),
        );

        // Precompute the maximum stop sequence length in bytes.
        // The jail buffer only needs to retain this many bytes — any text older than
        // this window cannot be part of a future match and is safe to emit.
        let jail_max_bytes = config
            .stop_sequences
            .iter()
            .chain(&config.visible_stop_sequences)
            .map(|s| s.len())
            .max()
            .unwrap_or(0);

        let aho_corasick = if patterns.is_empty() {
            None
        } else {
            // AhoCorasick::new is infallible for non-empty, pre-filtered string patterns.
            // Failure would indicate a bug in the Aho-Corasick library itself.
            #[expect(
                clippy::expect_used,
                reason = "AhoCorasick::new with pre-filtered non-empty &str patterns is practically infallible"
            )]
            Some(AhoCorasick::new(patterns).expect("Failed to build Aho-Corasick automaton"))
        };

        let token_only = aho_corasick.is_none();

        StopSequenceDecoder {
            sequence: Sequence::new_with_options(tokenizer, skip_special_tokens),
            config,
            aho_corasick,
            visible_boundary_idx,
            jail_buffer: String::new(),
            jail_max_bytes,
            stopped: false,
            token_only,
        }
    }

    /// Process a single token
    pub fn process_token(&mut self, token_id: TokenIdType) -> Result<SequenceDecoderOutput> {
        if self.stopped {
            return Ok(SequenceDecoderOutput::Stopped);
        }

        // Check for token-level stops first
        if self.config.stop_tokens.contains(&token_id) {
            self.stopped = true;

            // Flush any jailed text before stopping - use mem::take to avoid clone
            if !self.jail_buffer.is_empty() {
                return Ok(SequenceDecoderOutput::StoppedWithText(std::mem::take(
                    &mut self.jail_buffer,
                )));
            }
            return Ok(SequenceDecoderOutput::Stopped);
        }

        if self.config.visible_stop_tokens.contains(&token_id) {
            self.stopped = true;

            // Include jailed text plus the stop token
            let stop_text = self
                .sequence
                .tokenizer()
                .decode(&[token_id], self.sequence.skip_special_tokens())?;
            let output = format!("{}{}", self.jail_buffer, stop_text);
            self.jail_buffer.clear();
            return Ok(SequenceDecoderOutput::StoppedWithText(output));
        }

        // Use Sequence for incremental decoding
        let new_text = self.sequence.append_token(token_id)?;

        // Optimization #6: fast path when only token-level stops are configured.
        // No string stop sequences means no jail needed — emit text immediately.
        if self.token_only {
            if new_text.is_empty() {
                return Ok(SequenceDecoderOutput::Held);
            }
            return Ok(SequenceDecoderOutput::Text(new_text));
        }

        let old_len = self.jail_buffer.len();
        self.jail_buffer.push_str(&new_text);

        // Check for stop sequences using Aho-Corasick.
        // Optimization #3: scope the search to avoid rescanning old text.
        // A match can start at earliest at `old_len - jail_max_bytes + 1` because
        // any match starting earlier would have been found on a previous call.
        if let Some(ac) = &self.aho_corasick {
            let search_start = if old_len >= self.jail_max_bytes {
                // Walk forward to a char boundary (we must not start mid-codepoint)
                let raw = old_len + 1 - self.jail_max_bytes;
                let mut start = raw;
                while start < self.jail_buffer.len() && !self.jail_buffer.is_char_boundary(start) {
                    start += 1;
                }
                start
            } else {
                0
            };

            let input = Input::new(&self.jail_buffer).span(search_start..self.jail_buffer.len());
            if let Some(mat) = ac.find(input) {
                self.stopped = true;
                let is_visible = mat.pattern().as_usize() >= self.visible_boundary_idx;

                if is_visible {
                    // Visible stop sequence: include it in output
                    let output = self.jail_buffer[..mat.end()].to_string();
                    self.jail_buffer.clear();
                    return Ok(SequenceDecoderOutput::StoppedWithText(output));
                } else {
                    // Hidden stop sequence: exclude it from output
                    let output = self.jail_buffer[..mat.start()].to_string();
                    self.jail_buffer.clear();
                    return Ok(if output.is_empty() {
                        SequenceDecoderOutput::Stopped
                    } else {
                        SequenceDecoderOutput::StoppedWithText(output)
                    });
                }
            }
        }

        // Drain the jail buffer down to at most jail_max_bytes, emitting safe text.
        // Any text older than the window cannot be part of a future stop sequence match.
        if self.jail_buffer.len() > self.jail_max_bytes {
            // Find a char-safe drain point: we want to keep the last jail_max_bytes,
            // but must not split a multi-byte UTF-8 character.
            let mut drain_to = self.jail_buffer.len() - self.jail_max_bytes;
            while drain_to > 0 && !self.jail_buffer.is_char_boundary(drain_to) {
                // Move backward to retain at least jail_max_bytes (safe: retains more, not less)
                drain_to -= 1;
            }

            if drain_to > 0 {
                let suffix = self.jail_buffer.split_off(drain_to);
                let to_output = std::mem::replace(&mut self.jail_buffer, suffix);
                return Ok(SequenceDecoderOutput::Text(to_output));
            }
        }

        // Buffer is within the window — hold everything for potential partial match
        Ok(SequenceDecoderOutput::Held)
    }

    /// Process multiple tokens.
    ///
    /// Early-exits after a `Stopped` result — remaining tokens are not processed.
    pub fn process_tokens(
        &mut self,
        token_ids: &[TokenIdType],
    ) -> Result<Vec<SequenceDecoderOutput>> {
        let mut outputs = Vec::with_capacity(token_ids.len());
        for &token_id in token_ids {
            let output = self.process_token(token_id)?;
            let done = matches!(
                output,
                SequenceDecoderOutput::Stopped | SequenceDecoderOutput::StoppedWithText(_)
            );
            outputs.push(output);
            if done {
                break;
            }
        }
        Ok(outputs)
    }

    /// Flush any held text. Returns `Held` if the buffer is empty.
    pub fn flush(&mut self) -> SequenceDecoderOutput {
        if self.jail_buffer.is_empty() {
            SequenceDecoderOutput::Held
        } else {
            // Use mem::take to avoid clone - transfers ownership and leaves empty string
            SequenceDecoderOutput::Text(std::mem::take(&mut self.jail_buffer))
        }
    }

    /// Check if decoding has stopped
    pub fn is_stopped(&self) -> bool {
        self.stopped
    }

    /// Reset the decoder state
    pub fn reset(&mut self) {
        self.jail_buffer.clear();
        self.sequence.clear();
        self.stopped = false;
    }
}

/// Builder for StopSequenceDecoder
pub struct StopSequenceDecoderBuilder {
    tokenizer: Arc<dyn traits::Tokenizer>,
    config: StopSequenceConfig,
    skip_special_tokens: bool,
}

impl StopSequenceDecoderBuilder {
    pub fn new(tokenizer: Arc<dyn traits::Tokenizer>) -> Self {
        StopSequenceDecoderBuilder {
            tokenizer,
            config: StopSequenceConfig::default(),
            skip_special_tokens: true,
        }
    }

    pub fn stop_token(mut self, token_id: TokenIdType) -> Self {
        self.config.stop_tokens.insert(token_id);
        self
    }

    pub fn stop_sequence(mut self, sequence: impl Into<String>) -> Self {
        self.config.stop_sequences.push(sequence.into());
        self
    }

    pub fn visible_stop_token(mut self, token_id: TokenIdType) -> Self {
        self.config.visible_stop_tokens.insert(token_id);
        self
    }

    pub fn visible_stop_sequence(mut self, sequence: impl Into<String>) -> Self {
        self.config.visible_stop_sequences.push(sequence.into());
        self
    }

    pub fn skip_special_tokens(mut self, skip: bool) -> Self {
        self.skip_special_tokens = skip;
        self
    }

    pub fn build(self) -> StopSequenceDecoder {
        StopSequenceDecoder::new(self.tokenizer, self.config, self.skip_special_tokens)
    }
}

#[cfg(test)]
mod tests {
    use std::sync::Arc;

    use super::StopSequenceDecoderBuilder;
    use crate::{
        mock::MockTokenizer, SequenceDecoderOutput, StopSequenceConfig, StopSequenceDecoder,
    };

    #[test]
    fn test_stop_token_detection() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_token(999); // <eos> token

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process tokens before stop
        let result = decoder.process_token(1).unwrap(); // "Hello"
        assert!(matches!(result, SequenceDecoderOutput::Text(_)));

        // Process stop token
        let result = decoder.process_token(999).unwrap(); // <eos>
        assert_eq!(result, SequenceDecoderOutput::Stopped);

        // Further tokens should also return Stopped
        let result = decoder.process_token(2).unwrap();
        assert_eq!(result, SequenceDecoderOutput::Stopped);
    }

    #[test]
    fn test_visible_stop_token() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_visible_stop_token(999);

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        let result = decoder.process_token(999).unwrap();
        assert!(matches!(result, SequenceDecoderOutput::StoppedWithText(_)));
    }

    #[test]
    fn test_builder_pattern() {
        let tokenizer = Arc::new(MockTokenizer::new());

        let decoder = StopSequenceDecoderBuilder::new(tokenizer)
            .stop_token(999)
            .stop_sequence("STOP")
            .visible_stop_token(1000)
            .skip_special_tokens(true)
            .build();

        assert!(!decoder.is_stopped());
    }

    #[test]
    fn test_incremental_decoding_no_repetition() {
        // This test verifies the critical fix: no repeated output
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default();
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process tokens one by one and collect outputs
        let mut outputs = Vec::new();

        // Token 1: "Hello"
        let result = decoder.process_token(1).unwrap();
        if let SequenceDecoderOutput::Text(text) = result {
            outputs.push(text.clone());
        }

        // Token 2: "world"
        let result = decoder.process_token(2).unwrap();
        if let SequenceDecoderOutput::Text(text) = result {
            outputs.push(text.clone());
        }

        // Token 3: "test"
        let result = decoder.process_token(3).unwrap();
        if let SequenceDecoderOutput::Text(text) = result {
            outputs.push(text.clone());
        }

        // CRITICAL: Each output should be unique (no accumulation)
        // The fix ensures we only output NEW text, not accumulated text
        assert_eq!(outputs.len(), 3);

        for i in 0..outputs.len() {
            for j in i + 1..outputs.len() {
                // No output should contain another (no accumulation)
                assert!(!outputs[j].contains(&outputs[i]));
            }
        }
    }

    #[test]
    fn test_stop_sequence_detection() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_sequence("test");
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process "Hello world"
        decoder.process_token(1).unwrap(); // "Hello"
        decoder.process_token(2).unwrap(); // "world"

        // Process "test" which should trigger stop
        let result = decoder.process_token(3).unwrap(); // "test"

        // Should stop when we hit "test"
        assert!(matches!(
            result,
            SequenceDecoderOutput::Stopped | SequenceDecoderOutput::StoppedWithText(_)
        ));
    }

    #[test]
    fn test_flush_after_partial() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_sequence("NEVER_MATCH");
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process a token
        decoder.process_token(1).unwrap(); // "Hello"

        // Flush should return any remaining text in jail
        let result = decoder.flush();

        // After processing, flush should work
        assert!(matches!(result, SequenceDecoderOutput::Text(_)));
    }

    #[test]
    fn test_reset_functionality() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_token(999);
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process and stop
        decoder.process_token(1).unwrap();
        decoder.process_token(999).unwrap();
        assert!(decoder.is_stopped());

        // Reset should clear everything
        decoder.reset();
        assert!(!decoder.is_stopped());

        // Should be able to process again
        let result = decoder.process_token(2).unwrap();
        assert!(matches!(result, SequenceDecoderOutput::Text(_)));
    }

    #[test]
    fn test_visible_stop_sequence() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_visible_stop_sequence("world");
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process "Hello"
        decoder.process_token(1).unwrap();

        // Process "world" - should include it in output
        let result = decoder.process_token(2).unwrap();

        if let SequenceDecoderOutput::StoppedWithText(text) = result {
            // Should include "world" in the output
            assert!(text.contains("world"));
        } else {
            panic!("Expected StoppedWithText with visible stop sequence");
        }
    }

    #[test]
    fn test_multiple_tokens_processing() {
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default();
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process multiple tokens at once
        let results = decoder.process_tokens(&[1, 2, 3]).unwrap();

        // Should get results for each token
        assert_eq!(results.len(), 3);

        // Each result should be Text (no stops configured)
        for result in results {
            assert!(matches!(
                result,
                SequenceDecoderOutput::Text(_) | SequenceDecoderOutput::Held
            ));
        }
    }

    /// Test that the jail buffer correctly handles a stop sequence that arrives
    /// across 2+ tokens.  The MockTokenizer decodes token 1 as "Hello" and
    /// token 2's incremental contribution as " world", so the jail buffer
    /// progressively becomes "Hello" then "Hello world".
    ///
    /// With stop sequence "Hello world":
    ///   - Token 1: jail = "Hello" — partial prefix match → Held (or Text of
    ///     the portion before the potential match, which is empty here)
    ///   - Token 2: jail = "Hello world" — full match → Stopped
    #[test]
    fn test_stop_sequence_spanning_multiple_tokens() {
        let tokenizer = Arc::new(MockTokenizer::new());

        // "Hello world" spans token 1 ("Hello") and token 2 (" world")
        let config = StopSequenceConfig::default().with_stop_sequence("Hello world");
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Token 1 ("Hello"): The jail buffer now contains "Hello".
        // "Hello" is a prefix of stop sequence "Hello world", so the text
        // must be held — we should NOT see it emitted as Text yet.
        let result1 = decoder.process_token(1).unwrap();
        assert!(
            matches!(result1, SequenceDecoderOutput::Held),
            "Expected Held while jail buffer is a prefix of the stop sequence, got {result1:?}"
        );
        assert!(
            !decoder.is_stopped(),
            "Decoder should not be stopped after a partial match"
        );

        // Token 2 (" world"): The jail buffer now contains "Hello world",
        // which fully matches the stop sequence. The decoder should stop.
        let result2 = decoder.process_token(2).unwrap();
        assert_eq!(
            result2,
            SequenceDecoderOutput::Stopped,
            "Expected Stopped when jail buffer matches the hidden stop sequence"
        );
        assert!(
            decoder.is_stopped(),
            "Decoder should be stopped after the full stop sequence match"
        );

        // Any further tokens should also return Stopped
        let result3 = decoder.process_token(3).unwrap();
        assert_eq!(result3, SequenceDecoderOutput::Stopped);
    }

    /// Same as above but with a *visible* stop sequence.  When the stop
    /// sequence "Hello world" is visible, the matched text should be included
    /// in the output via StoppedWithText.
    #[test]
    fn test_visible_stop_sequence_spanning_multiple_tokens() {
        let tokenizer = Arc::new(MockTokenizer::new());

        let config = StopSequenceConfig::default().with_visible_stop_sequence("Hello world");
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Token 1 ("Hello"): partial match, should be held
        let result1 = decoder.process_token(1).unwrap();
        assert!(
            matches!(result1, SequenceDecoderOutput::Held),
            "Expected Held for partial visible stop sequence match, got {result1:?}"
        );

        // Token 2 (" world"): completes "Hello world" — visible stop
        let result2 = decoder.process_token(2).unwrap();
        match &result2 {
            SequenceDecoderOutput::StoppedWithText(text) => {
                assert!(
                    text.contains("Hello world"),
                    "Visible stop output should contain the full stop sequence, got: {text:?}"
                );
            }
            other => panic!("Expected StoppedWithText for visible stop sequence, got {other:?}"),
        }
        assert!(decoder.is_stopped());
    }

    /// Test a stop sequence that spans 3 tokens, with preceding text that
    /// should be emitted before the jailed portion.
    ///
    /// Tokens: 3 ("test"), 1 ("Hello"), 2 ("world")
    /// Stop sequence: "Hello world" (11 bytes)
    ///
    /// With the bounded jail window, all text is held until the jail exceeds
    /// jail_max_bytes (11). The jail accumulates:
    ///   - Token 3: jail = "test" (4 bytes ≤ 11) → Held
    ///   - Token 1: jail = "test Hello" (10 bytes ≤ 11) → Held
    ///   - Token 2: jail = "test Hello world" → Aho-Corasick matches "Hello world"
    ///     → StoppedWithText("test ") (text before the hidden stop sequence)
    #[test]
    fn test_stop_sequence_spanning_tokens_with_preceding_text() {
        let tokenizer = Arc::new(MockTokenizer::new());

        let config = StopSequenceConfig::default().with_stop_sequence("Hello world");
        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Token 3 ("test"): jail = "test" (4 bytes), within the 11-byte window → Held
        let result1 = decoder.process_token(3).unwrap();
        assert!(
            matches!(result1, SequenceDecoderOutput::Held),
            "Expected Held for token within jail window, got {result1:?}"
        );

        // Token 1 ("Hello"): jail = "test Hello" (10 bytes), still within window → Held
        let result2 = decoder.process_token(1).unwrap();
        assert!(
            matches!(result2, SequenceDecoderOutput::Held),
            "Expected Held for token within jail window, got {result2:?}"
        );

        // Token 2 ("world"): jail = "test Hello world" — Aho-Corasick matches
        // "Hello world", so we stop. Text before the match ("test ") is emitted.
        let result3 = decoder.process_token(2).unwrap();
        assert!(
            matches!(
                result3,
                SequenceDecoderOutput::Stopped | SequenceDecoderOutput::StoppedWithText(_)
            ),
            "Expected Stopped or StoppedWithText when stop sequence completes, got {result3:?}"
        );
        assert!(decoder.is_stopped());

        // Verify that any text before the stop sequence is preserved
        if let SequenceDecoderOutput::StoppedWithText(text) = &result3 {
            assert!(
                !text.contains("Hello world"),
                "Hidden stop sequence should not appear in output, got: {text:?}"
            );
        }
    }

    #[test]
    fn test_utf8_multibyte_character_boundaries() {
        // This test verifies the fix for the UTF-8 boundary panic
        // The panic occurred when trying to slice jail_buffer at a byte index
        // that was in the middle of a multi-byte UTF-8 character (e.g., '×')
        use crate::mock::MockTokenizer;

        let tokenizer = Arc::new(MockTokenizer::new());

        // Configure stop sequence with a multi-byte character
        let config = StopSequenceConfig::default().with_stop_sequence(" ×");

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Simulate the scenario: jail_buffer will contain " ×" (space + multiplication sign)
        // The '×' character is UTF-8 encoded as bytes [0xC3, 0x97] (2 bytes)
        // When checking for partial matches, we must not slice in the middle of these bytes

        // This should not panic - the fix ensures we only slice at char boundaries
        let result = decoder.process_token(1); // Will add some text to jail_buffer
        assert!(result.is_ok());

        // Even with multi-byte UTF-8 characters in the buffer, processing should work
        let result = decoder.process_token(2);
        assert!(result.is_ok());
    }

    #[test]
    fn test_utf8_multibyte_delta_character() {
        // Test for: byte index 1 is not a char boundary; it is inside 'Δ' (bytes 0..2) of `Δ`
        // 'Δ' (U+0394 GREEK CAPITAL LETTER DELTA) is encoded as [0xCE, 0x94] (2 bytes)
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_sequence("Δ");

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process tokens - should not panic when checking partial matches
        let result = decoder.process_token(1);
        assert!(result.is_ok());
        let result = decoder.process_token(2);
        assert!(result.is_ok());
    }

    #[test]
    fn test_utf8_multibyte_degree_character() {
        // Test for: byte index 1 is not a char boundary; it is inside '°' (bytes 0..2) of `°`
        // '°' (U+00B0 DEGREE SIGN) is encoded as [0xC2, 0xB0] (2 bytes)
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_sequence("°");

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process tokens - should not panic when checking partial matches
        let result = decoder.process_token(1);
        assert!(result.is_ok());
        let result = decoder.process_token(2);
        assert!(result.is_ok());
    }

    #[test]
    fn test_utf8_multibyte_triangle_character() {
        // Test for: byte index 4 is not a char boundary; it is inside '∆' (bytes 2..5) of ` (∆`
        // '∆' (U+2206 INCREMENT) is encoded as [0xE2, 0x88, 0x86] (3 bytes)
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_sequence(" (∆");

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process tokens - should not panic when checking partial matches
        let result = decoder.process_token(1);
        assert!(result.is_ok());
        let result = decoder.process_token(2);
        assert!(result.is_ok());
        let result = decoder.process_token(3);
        assert!(result.is_ok());
    }

    #[test]
    fn test_utf8_multibyte_en_dash_character() {
        // Test for: byte index 3 is not a char boundary; it is inside '–' (bytes 1..4) of ` –`
        // '–' (U+2013 EN DASH) is encoded as [0xE2, 0x80, 0x93] (3 bytes)
        let tokenizer = Arc::new(MockTokenizer::new());
        let config = StopSequenceConfig::default().with_stop_sequence(" –");

        let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

        // Process tokens - should not panic when checking partial matches
        let result = decoder.process_token(1);
        assert!(result.is_ok());
        let result = decoder.process_token(2);
        assert!(result.is_ok());
        let result = decoder.process_token(3);
        assert!(result.is_ok());
    }

    #[test]
    fn test_utf8_multibyte_various_characters() {
        // Comprehensive test with multiple multi-byte UTF-8 characters
        // Tests 2-byte, 3-byte, and 4-byte UTF-8 sequences
        let test_cases = vec![
            ("×", "multiplication sign - 2 bytes"),
            ("Δ", "Greek Delta - 2 bytes"),
            ("°", "degree sign - 2 bytes"),
            ("∆", "increment - 3 bytes"),
            ("–", "en dash - 3 bytes"),
            ("€", "euro sign - 3 bytes"),
            ("中", "Chinese character - 3 bytes"),
            ("🚀", "rocket emoji - 4 bytes"),
            ("💡", "lightbulb emoji - 4 bytes"),
        ];

        for (stop_char, description) in test_cases {
            let tokenizer = Arc::new(MockTokenizer::new());
            let config = StopSequenceConfig::default().with_stop_sequence(stop_char);

            let mut decoder = StopSequenceDecoder::new(tokenizer, config, false);

            // Process multiple tokens - should not panic
            for token_id in 1..=5 {
                let result = decoder.process_token(token_id);
                assert!(
                    result.is_ok(),
                    "Failed on {description} with token {token_id}"
                );
            }
        }
    }
}