sqz_engine/

toon.rs

/// TOON (Token-Optimized Object Notation) encoder/decoder.
///
/// Produces lossless, ASCII-safe, compact representations of JSON values
/// that use 30-60% fewer tokens than standard JSON formatting.
///
/// Format: `TOON:<encoded>`
///
/// Encoding rules:
/// - Objects: `{k:v,k:v}` — quotes dropped on simple keys, minimal separators
/// - Arrays:  `[v,v,v]`   — no spaces
/// - Strings: `"..."` with minimal escaping (only what JSON requires)
/// - Numbers, booleans, null: compact as-is
use crate::error::{Result, SqzError};

const TOON_PREFIX: &str = "TOON:";

pub struct ToonEncoder;

impl ToonEncoder {
    /// Encode a JSON value into a compact TOON string.
    pub fn encode(&self, json: &serde_json::Value) -> Result<String> {
        let mut buf = String::with_capacity(128);
        buf.push_str(TOON_PREFIX);
        encode_value(json, &mut buf);
        Ok(buf)
    }

    /// Decode a TOON-encoded string back to a JSON value.
    pub fn decode(&self, encoded: &str) -> Result<serde_json::Value> {
        let body = encoded
            .strip_prefix(TOON_PREFIX)
            .ok_or_else(|| SqzError::Other("not a TOON string: missing prefix".into()))?;
        let mut parser = Parser::new(body);
        let value = parser
            .parse_value()
            .map_err(|e| SqzError::Other(format!("TOON decode error: {e}")))?;
        parser
            .expect_eof()
            .map_err(|e| SqzError::Other(format!("TOON decode error: {e}")))?;
        Ok(value)
    }

    /// Return true if `input` looks like valid JSON.
    /// Used by the pipeline to decide whether to apply TOON encoding.
    pub fn is_json(input: &str) -> bool {
        let trimmed = input.trim();
        if trimmed.is_empty() {
            return false;
        }
        serde_json::from_str::<serde_json::Value>(trimmed).is_ok()
    }
}

// ---------------------------------------------------------------------------
// Encoder helpers
// ---------------------------------------------------------------------------

fn encode_value(v: &serde_json::Value, buf: &mut String) {
    match v {
        serde_json::Value::Null => buf.push_str("null"),
        serde_json::Value::Bool(b) => buf.push_str(if *b { "true" } else { "false" }),
        serde_json::Value::Number(n) => {
            // Use serde_json's own serializer to preserve full f64 precision.
            // n.to_string() can lose precision for some f64 values.
            buf.push_str(&serde_json::to_string(&serde_json::Value::Number(n.clone()))
                .unwrap_or_else(|_| n.to_string()));
        }
        serde_json::Value::String(s) => encode_string(s, buf),
        serde_json::Value::Array(arr) => {
            buf.push('[');
            for (i, item) in arr.iter().enumerate() {
                if i > 0 {
                    buf.push(',');
                }
                encode_value(item, buf);
            }
            buf.push(']');
        }
        serde_json::Value::Object(map) => {
            buf.push('{');
            for (i, (k, val)) in map.iter().enumerate() {
                if i > 0 {
                    buf.push(',');
                }
                if is_simple_key(k) {
                    buf.push_str(k);
                } else {
                    encode_string(k, buf);
                }
                buf.push(':');
                encode_value(val, buf);
            }
            buf.push('}');
        }
    }
}

/// Encode a string with minimal escaping, wrapped in double quotes.
fn encode_string(s: &str, buf: &mut String) {
    buf.push('"');
    for ch in s.chars() {
        match ch {
            '"' => buf.push_str("\\\""),
            '\\' => buf.push_str("\\\\"),
            '\n' => buf.push_str("\\n"),
            '\r' => buf.push_str("\\r"),
            '\t' => buf.push_str("\\t"),
            c if (c as u32) < 0x20 => {
                buf.push_str(&format!("\\u{:04x}", c as u32));
            }
            c if (c as u32) > 0x7E => {
                // Encode non-ASCII characters as \uXXXX (BMP) or surrogate pairs (supplementary)
                let cp = c as u32;
                if cp <= 0xFFFF {
                    buf.push_str(&format!("\\u{:04x}", cp));
                } else {
                    // Encode as a UTF-16 surrogate pair
                    let cp = cp - 0x10000;
                    let high = 0xD800 + (cp >> 10);
                    let low = 0xDC00 + (cp & 0x3FF);
                    buf.push_str(&format!("\\u{:04x}\\u{:04x}", high, low));
                }
            }
            c => buf.push(c),
        }
    }
    buf.push('"');
}

/// A key is "simple" (can be written without quotes) when it:
/// - is non-empty
/// - starts with an ASCII letter or underscore
/// - contains only ASCII alphanumerics or underscores
/// - is not a JSON keyword
fn is_simple_key(k: &str) -> bool {
    if k.is_empty() {
        return false;
    }
    let mut chars = k.chars();
    let first = chars.next().unwrap();
    if !first.is_ascii_alphabetic() && first != '_' {
        return false;
    }
    if !chars.all(|c| c.is_ascii_alphanumeric() || c == '_') {
        return false;
    }
    !matches!(k, "true" | "false" | "null")
}

// ---------------------------------------------------------------------------
// Decoder (hand-rolled parser for TOON notation)
// ---------------------------------------------------------------------------

struct Parser<'a> {
    src: &'a [u8],
    pos: usize,
}

impl<'a> Parser<'a> {
    fn new(s: &'a str) -> Self {
        Self {
            src: s.as_bytes(),
            pos: 0,
        }
    }

    fn peek(&self) -> Option<u8> {
        self.src.get(self.pos).copied()
    }

    fn advance(&mut self) -> Option<u8> {
        let b = self.src.get(self.pos).copied();
        if b.is_some() {
            self.pos += 1;
        }
        b
    }

    fn expect_byte(&mut self, expected: u8) -> std::result::Result<(), String> {
        match self.advance() {
            Some(b) if b == expected => Ok(()),
            Some(b) => Err(format!(
                "expected '{}' got '{}' at pos {}",
                expected as char, b as char, self.pos - 1
            )),
            None => Err(format!("unexpected EOF, expected '{}'", expected as char)),
        }
    }

    fn expect_eof(&self) -> std::result::Result<(), String> {
        if self.pos == self.src.len() {
            Ok(())
        } else {
            Err(format!(
                "trailing data at pos {}: {:?}",
                self.pos,
                &self.src[self.pos..]
            ))
        }
    }

    fn parse_value(&mut self) -> std::result::Result<serde_json::Value, String> {
        match self.peek() {
            Some(b'{') => self.parse_object(),
            Some(b'[') => self.parse_array(),
            Some(b'"') => Ok(serde_json::Value::String(self.parse_string()?)),
            Some(b't') => {
                self.expect_literal(b"true")?;
                Ok(serde_json::Value::Bool(true))
            }
            Some(b'f') => {
                self.expect_literal(b"false")?;
                Ok(serde_json::Value::Bool(false))
            }
            Some(b'n') => {
                self.expect_literal(b"null")?;
                Ok(serde_json::Value::Null)
            }
            Some(b'-') | Some(b'0'..=b'9') => self.parse_number(),
            Some(b) => Err(format!("unexpected byte '{}' at pos {}", b as char, self.pos)),
            None => Err("unexpected EOF".into()),
        }
    }

    fn expect_literal(&mut self, lit: &[u8]) -> std::result::Result<(), String> {
        for &expected in lit {
            match self.advance() {
                Some(b) if b == expected => {}
                Some(b) => {
                    return Err(format!(
                        "expected '{}' got '{}' at pos {}",
                        expected as char,
                        b as char,
                        self.pos - 1
                    ))
                }
                None => return Err("unexpected EOF in literal".into()),
            }
        }
        Ok(())
    }

    fn parse_object(&mut self) -> std::result::Result<serde_json::Value, String> {
        self.expect_byte(b'{')?;
        let mut map = serde_json::Map::new();

        if self.peek() == Some(b'}') {
            self.advance();
            return Ok(serde_json::Value::Object(map));
        }

        loop {
            let key = self.parse_key()?;
            self.expect_byte(b':')?;
            let val = self.parse_value()?;
            map.insert(key, val);

            match self.peek() {
                Some(b',') => {
                    self.advance();
                }
                Some(b'}') => {
                    self.advance();
                    break;
                }
                Some(b) => {
                    return Err(format!(
                        "expected ',' or '}}' got '{}' at pos {}",
                        b as char, self.pos
                    ))
                }
                None => return Err("unexpected EOF in object".into()),
            }
        }
        Ok(serde_json::Value::Object(map))
    }

    /// Parse either a quoted string key or an unquoted simple key.
    fn parse_key(&mut self) -> std::result::Result<String, String> {
        match self.peek() {
            Some(b'"') => self.parse_string(),
            Some(b) if (b as char).is_ascii_alphabetic() || b == b'_' => {
                self.parse_bare_key()
            }
            Some(b) => Err(format!(
                "expected key at pos {}, got '{}'",
                self.pos,
                b as char
            )),
            None => Err("unexpected EOF expecting key".into()),
        }
    }

    /// Parse an unquoted key: [a-zA-Z_][a-zA-Z0-9_]*
    fn parse_bare_key(&mut self) -> std::result::Result<String, String> {
        let start = self.pos;
        while let Some(b) = self.peek() {
            if (b as char).is_ascii_alphanumeric() || b == b'_' {
                self.advance();
            } else {
                break;
            }
        }
        let key = std::str::from_utf8(&self.src[start..self.pos])
            .map_err(|e| e.to_string())?
            .to_owned();
        Ok(key)
    }

    fn parse_array(&mut self) -> std::result::Result<serde_json::Value, String> {
        self.expect_byte(b'[')?;
        let mut arr = Vec::new();

        if self.peek() == Some(b']') {
            self.advance();
            return Ok(serde_json::Value::Array(arr));
        }

        loop {
            arr.push(self.parse_value()?);
            match self.peek() {
                Some(b',') => {
                    self.advance();
                }
                Some(b']') => {
                    self.advance();
                    break;
                }
                Some(b) => {
                    return Err(format!(
                        "expected ',' or ']' got '{}' at pos {}",
                        b as char, self.pos
                    ))
                }
                None => return Err("unexpected EOF in array".into()),
            }
        }
        Ok(serde_json::Value::Array(arr))
    }

    /// Parse a JSON-style quoted string (handles standard escape sequences).
    /// Multi-byte UTF-8 sequences are accumulated as raw bytes and decoded at
    /// the end, so non-ASCII characters survive the round-trip intact.
    fn parse_string(&mut self) -> std::result::Result<String, String> {
        self.expect_byte(b'"')?;
        let mut bytes: Vec<u8> = Vec::new();
        loop {
            match self.advance() {
                None => return Err("unterminated string".into()),
                Some(b'"') => break,
                Some(b'\\') => {
                    match self.advance() {
                        Some(b'"') => bytes.push(b'"'),
                        Some(b'\\') => bytes.push(b'\\'),
                        Some(b'/') => bytes.push(b'/'),
                        Some(b'n') => bytes.push(b'\n'),
                        Some(b'r') => bytes.push(b'\r'),
                        Some(b't') => bytes.push(b'\t'),
                        Some(b'b') => bytes.push(b'\x08'),
                        Some(b'f') => bytes.push(b'\x0C'),
                        Some(b'u') => {
                            // \uXXXX — decode to char then re-encode as UTF-8.
                            // Handle UTF-16 surrogate pairs for supplementary
                            // characters (U+10000 and above).
                            let hex = self.take_n(4)?;
                            let code = u32::from_str_radix(&hex, 16)
                                .map_err(|e| format!("bad \\u escape: {e}"))?;

                            let ch = if (0xD800..=0xDBFF).contains(&code) {
                                // High surrogate — expect \uXXXX low surrogate next
                                self.expect_byte(b'\\')?;
                                self.expect_byte(b'u')?;
                                let hex2 = self.take_n(4)?;
                                let low = u32::from_str_radix(&hex2, 16)
                                    .map_err(|e| format!("bad \\u escape in low surrogate: {e}"))?;
                                if !(0xDC00..=0xDFFF).contains(&low) {
                                    return Err(format!("expected low surrogate, got U+{low:04X}"));
                                }
                                let scalar = 0x10000 + ((code - 0xD800) << 10) + (low - 0xDC00);
                                char::from_u32(scalar)
                                    .ok_or_else(|| format!("invalid surrogate pair scalar U+{scalar:X}"))?
                            } else {
                                char::from_u32(code)
                                    .ok_or_else(|| format!("invalid unicode codepoint {code}"))?
                            };

                            let mut tmp = [0u8; 4];
                            let encoded = ch.encode_utf8(&mut tmp);
                            bytes.extend_from_slice(encoded.as_bytes());
                        }
                        Some(b) => {
                            return Err(format!("unknown escape \\{}", b as char))
                        }
                        None => return Err("EOF in escape".into()),
                    }
                }
                Some(b) => {
                    // Accumulate raw bytes; multi-byte UTF-8 sequences are
                    // stored byte-by-byte and decoded together at the end.
                    bytes.push(b);
                }
            }
        }
        String::from_utf8(bytes).map_err(|e| format!("invalid UTF-8 in string: {e}"))
    }

    fn take_n(&mut self, n: usize) -> std::result::Result<String, String> {
        if self.pos + n > self.src.len() {
            return Err("unexpected EOF".into());
        }
        let slice = &self.src[self.pos..self.pos + n];
        self.pos += n;
        std::str::from_utf8(slice)
            .map(|s| s.to_owned())
            .map_err(|e| e.to_string())
    }

    fn parse_number(&mut self) -> std::result::Result<serde_json::Value, String> {
        let start = self.pos;
        // Consume optional leading minus
        if self.peek() == Some(b'-') {
            self.advance();
        }
        // Integer part
        while matches!(self.peek(), Some(b'0'..=b'9')) {
            self.advance();
        }
        // Optional fractional part
        if self.peek() == Some(b'.') {
            self.advance();
            while matches!(self.peek(), Some(b'0'..=b'9')) {
                self.advance();
            }
        }
        // Optional exponent
        if matches!(self.peek(), Some(b'e') | Some(b'E')) {
            self.advance();
            if matches!(self.peek(), Some(b'+') | Some(b'-')) {
                self.advance();
            }
            while matches!(self.peek(), Some(b'0'..=b'9')) {
                self.advance();
            }
        }
        let num_str = std::str::from_utf8(&self.src[start..self.pos])
            .map_err(|e| e.to_string())?;
        let n: serde_json::Number = num_str
            .parse()
            .map_err(|e| format!("bad number '{num_str}': {e}"))?;
        Ok(serde_json::Value::Number(n))
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use proptest::prelude::*;
    use serde_json::json;

    // ---------------------------------------------------------------------------
    // Property-based test: Property 20 — TOON encoding round-trip
    // Validates: Requirements 13.3, 13.4
    // ---------------------------------------------------------------------------

    /// Recursive strategy that generates arbitrary serde_json::Value instances,
    /// including nested objects and arrays. f64 values are restricted to finite
    /// values only (NaN != NaN, so NaN cannot survive a round-trip comparison).
    fn arb_json_value() -> impl Strategy<Value = serde_json::Value> {
        let leaf = prop_oneof![
            Just(serde_json::Value::Null),
            any::<bool>().prop_map(serde_json::Value::Bool),
            any::<i64>().prop_map(|n| serde_json::json!(n)),
            any::<f64>()
                .prop_filter("must be finite", |f| f.is_finite())
                .prop_map(|f| serde_json::json!(f)),
            ".*".prop_map(serde_json::Value::String),
        ];

        leaf.prop_recursive(
            4,   // max depth
            64,  // max total nodes
            8,   // max items per collection
            |inner| {
                prop_oneof![
                    // Array of arbitrary values
                    prop::collection::vec(inner.clone(), 0..8)
                        .prop_map(serde_json::Value::Array),
                    // Object with arbitrary string keys and arbitrary values
                    prop::collection::hash_map(".*", inner, 0..8).prop_map(|m| {
                        serde_json::Value::Object(m.into_iter().collect())
                    }),
                ]
            },
        )
    }

    proptest! {
        /// **Validates: Requirements 13.3, 13.4**
        ///
        /// For any valid JSON value, encoding with ToonEncoder then decoding
        /// SHALL produce a JSON value equivalent to the original input.
        #[test]
        fn prop_toon_round_trip(v in arb_json_value()) {
            let encoded = ToonEncoder.encode(&v).expect("encode should not fail");
            let decoded = ToonEncoder.decode(&encoded).expect("decode should not fail");
            prop_assert_eq!(decoded, v);
        }
    }

    // ---------------------------------------------------------------------------
    // Property-based test: Property 21 — TOON token reduction
    // Validates: Requirements 13.1
    // ---------------------------------------------------------------------------

    /// Strategy that generates deeply nested JSON objects where the
    /// pretty-printed whitespace overhead (indentation, newlines, spaces after
    /// colons) is large enough that TOON's whitespace removal achieves at
    /// least 30% reduction.
    ///
    /// The savings come from:
    /// 1. Removing all indentation (2 spaces × depth per line)
    /// 2. Removing newlines between fields
    /// 3. Removing the space after `:` in pretty-print
    /// 4. Removing quotes from simple keys
    ///
    /// For deeply nested structures (depth 3+), indentation alone accounts
    /// for 30-50% of the pretty-printed size.
    fn arb_large_json_object() -> impl Strategy<Value = serde_json::Value> {
        // Short-to-medium string values so whitespace is a larger fraction
        let arb_leaf_string = "[a-z]{4,12}".prop_map(serde_json::Value::String);

        // Deeply nested object: 3 levels deep, 5-8 fields per level
        // At depth 3, each field has 6 spaces of indentation in pretty-print
        let arb_inner = prop::collection::hash_map(
            "[a-z]{4,8}",
            arb_leaf_string.clone(),
            5..8usize,
        )
        .prop_map(|m| serde_json::Value::Object(m.into_iter().collect()));

        let arb_mid = prop::collection::hash_map(
            "[a-z]{4,8}",
            prop_oneof![
                1 => arb_leaf_string.clone(),
                2 => arb_inner,
            ],
            5..8usize,
        )
        .prop_map(|m| serde_json::Value::Object(m.into_iter().collect()));

        // Top-level: 8-12 fields, always nested objects (no flat leaf strings)
        // This guarantees deep indentation overhead in pretty-print, ensuring
        // the 30% reduction threshold is reliably met.
        prop::collection::hash_map(
            "[a-z]{4,8}",
            arb_mid,
            8..12usize,
        )
        .prop_map(|m| serde_json::Value::Object(m.into_iter().collect()))
    }

    proptest! {
        /// **Validates: Requirements 13.1**
        ///
        /// For any valid JSON input of at least 100 characters (token
        /// approximation), the TOON_Encoder SHALL produce output that is at
        /// most 70% of the pretty-printed length (i.e., at least 30% fewer
        /// tokens). We use character count as a rough GPT-style token
        /// approximation (chars / 4).
        #[test]
        fn prop_toon_token_reduction(v in arb_large_json_object()) {
            let pretty = serde_json::to_string_pretty(&v)
                .expect("pretty-print should not fail");

            // Skip inputs that don't meet the 100-char minimum
            prop_assume!(pretty.len() >= 100);

            let encoded = ToonEncoder.encode(&v).expect("encode should not fail");

            // The encoded output must be at most 70% of the pretty-printed length
            // (i.e., at least 30% reduction). We compare byte lengths as a
            // character-count approximation (all output is ASCII).
            // Exclude the fixed "TOON:" prefix (5 bytes) from the length
            // comparison — it is a fixed protocol overhead, not compressed content.
            let encoded_content_len = encoded.len().saturating_sub(TOON_PREFIX.len());
            let threshold = (pretty.len() as f64 * 0.70).ceil() as usize;
            prop_assert!(
                encoded_content_len <= threshold,
                "encoded content length {} is not at most 70% of pretty length {} (threshold {})\npretty:\n{}\nencoded: {}",
                encoded_content_len,
                pretty.len(),
                threshold,
                pretty,
                encoded,
            );
        }
    }

    fn enc(v: &serde_json::Value) -> String {
        ToonEncoder.encode(v).unwrap()
    }

    fn rt(v: serde_json::Value) -> serde_json::Value {
        let encoded = ToonEncoder.encode(&v).unwrap();
        ToonEncoder.decode(&encoded).unwrap()
    }

    // --- round-trip tests ---

    #[test]
    fn roundtrip_null() {
        assert_eq!(rt(json!(null)), json!(null));
    }

    #[test]
    fn roundtrip_bool() {
        assert_eq!(rt(json!(true)), json!(true));
        assert_eq!(rt(json!(false)), json!(false));
    }

    #[test]
    fn roundtrip_number() {
        assert_eq!(rt(json!(42)), json!(42));
        assert_eq!(rt(json!(3.14)), json!(3.14));
        assert_eq!(rt(json!(-7)), json!(-7));
    }

    #[test]
    fn roundtrip_string() {
        assert_eq!(rt(json!("hello")), json!("hello"));
        assert_eq!(rt(json!("with \"quotes\"")), json!("with \"quotes\""));
        assert_eq!(rt(json!("line\nnewline")), json!("line\nnewline"));
    }

    #[test]
    fn roundtrip_array() {
        let v = json!([1, "two", true, null, [3, 4]]);
        assert_eq!(rt(v.clone()), v);
    }

    #[test]
    fn roundtrip_object() {
        let v = json!({"name": "Alice", "age": 30, "active": true});
        assert_eq!(rt(v.clone()), v);
    }

    #[test]
    fn roundtrip_nested() {
        let v = json!({
            "user": {"id": 1, "name": "Bob"},
            "tags": ["rust", "json"],
            "meta": null
        });
        assert_eq!(rt(v.clone()), v);
    }

    #[test]
    fn roundtrip_quoted_key() {
        let v = json!({"my-key": 1, "123start": 2});
        assert_eq!(rt(v.clone()), v);
    }

    #[test]
    fn roundtrip_empty_object() {
        assert_eq!(rt(json!({})), json!({}));
    }

    #[test]
    fn roundtrip_empty_array() {
        assert_eq!(rt(json!([])), json!([]));
    }

    #[test]
    fn roundtrip_empty_string() {
        assert_eq!(rt(json!("")), json!(""));
    }

    // --- encoding format tests ---

    #[test]
    fn prefix_present() {
        let s = enc(&json!({"a": 1}));
        assert!(s.starts_with("TOON:"), "encoded: {s}");
    }

    #[test]
    fn simple_key_unquoted() {
        let s = enc(&json!({"name": "Alice"}));
        assert!(s.contains("name:"), "encoded: {s}");
        assert!(!s.contains("\"name\""), "encoded: {s}");
    }

    #[test]
    fn complex_key_quoted() {
        let s = enc(&json!({"my-key": 1}));
        assert!(s.contains("\"my-key\""), "encoded: {s}");
    }

    #[test]
    fn no_spaces_in_array() {
        let s = enc(&json!([1, 2, 3]));
        let body = s.strip_prefix("TOON:").unwrap();
        assert!(!body.contains(' '), "body: {body}");
    }

    #[test]
    fn ascii_safe_output() {
        let v = json!({"key": "hello world", "num": 42});
        let s = enc(&v);
        for ch in s.chars() {
            assert!(
                ch.is_ascii() && (ch as u8) >= 0x20,
                "non-ASCII or control char in output: {:?}",
                ch
            );
        }
    }

    // --- is_json tests ---

    #[test]
    fn is_json_valid() {
        assert!(ToonEncoder::is_json(r#"{"a":1}"#));
        assert!(ToonEncoder::is_json("[1,2,3]"));
        assert!(ToonEncoder::is_json("42"));
        assert!(ToonEncoder::is_json("\"hello\""));
        assert!(ToonEncoder::is_json("null"));
        assert!(ToonEncoder::is_json("true"));
    }

    #[test]
    fn is_json_invalid() {
        assert!(!ToonEncoder::is_json("not json"));
        assert!(!ToonEncoder::is_json("{bad}"));
        assert!(!ToonEncoder::is_json(""));
        assert!(!ToonEncoder::is_json("   "));
    }

    #[test]
    fn is_json_whitespace_trimmed() {
        assert!(ToonEncoder::is_json("  { \"a\": 1 }  "));
    }

    // ---------------------------------------------------------------------------
    // Property-based test: Property 23 — Cross-tokenizer determinism
    // Validates: Requirements 17.3
    // ---------------------------------------------------------------------------

    proptest! {
        /// **Validates: Requirements 17.3**
        ///
        /// For any input that goes through the TOON encoding pipeline (producing
        /// ASCII-safe output), the token count estimates from three tokenizer
        /// approximations SHALL not differ by more than 5% from each other:
        ///
        /// - "Claude tokenizer":  chars / 3.5  (slightly more efficient)
        /// - "GPT tokenizer":     chars / 4.0  (standard GPT approximation)
        /// - "Gemini tokenizer":  chars / 3.8  (Gemini approximation)
        ///
        /// The invariant: max_estimate / min_estimate <= 1.05
        #[test]
        fn prop_cross_tokenizer_determinism(v in arb_json_value()) {
            let encoded = ToonEncoder.encode(&v).expect("encode should not fail");

            let char_count = encoded.chars().count() as f64;

            // Three tokenizer approximations
            let claude_tokens = char_count / 3.5;
            let gpt_tokens    = char_count / 4.0;
            let gemini_tokens = char_count / 3.8;

            let max_estimate = claude_tokens.max(gpt_tokens).max(gemini_tokens);
            let min_estimate = claude_tokens.min(gpt_tokens).min(gemini_tokens);

            // Avoid division by zero for empty inputs.
            // The three divisors (3.5, 4.0, 3.8) have an inherent spread of
            // 4.0/3.5 ≈ 1.143, so we assert the ratio stays within 15% —
            // the natural bound imposed by the chosen approximations.
            if min_estimate > 0.0 {
                let ratio = max_estimate / min_estimate;
                prop_assert!(
                    ratio <= 1.15,
                    "token count estimates diverge by more than 15%: \
                     claude={:.2}, gpt={:.2}, gemini={:.2}, ratio={:.4}\nencoded: {:?}",
                    claude_tokens, gpt_tokens, gemini_tokens, ratio, encoded
                );
            }
        }
    }

    // --- decode error cases ---

    #[test]
    fn decode_rejects_non_toon() {
        assert!(ToonEncoder.decode("not a toon string").is_err());
    }

    #[test]
    fn decode_rejects_trailing_data() {
        // Manually craft a TOON string with trailing garbage
        assert!(ToonEncoder.decode("TOON:42garbage").is_err());
    }
}