openlatch-client 0.1.18

//! Local, network-free token **estimation** — the interrupted-stream fallback
//! only (D-08).
//!
//! This path runs **only** when the provider never returned a usable usage chunk
//! (the stream was interrupted or unparsable). It produces an approximation,
//! always flagged `cost_basis = tokenizer_estimated` — it is **never** the happy
//! path, and it **never** makes a `count_tokens` network call. Being purely local
//! is what lets the capture path pass the zero-egress bench.
//!
//! ## Crate selection (D-08) — recorded deviation
//!
//! The plan named `token-count` vs `claude_tokenizer`. Verified against
//! crates.io: **`claude_tokenizer` does not exist**, and **`token-count`** (v0.4)
//! is a *CLI application* carrying `reqwest` + `tokio` as **normal** dependencies
//! (it phones a counting API) and is Gemini-oriented (`gemini-tokenizer`). Either
//! choice violates the D-08 network-free requirement and the capture-egress DoD,
//! and no crate ships Claude's *exact* current-generation tokenizer (it is not
//! published). This module therefore implements a **zero-dependency, offline,
//! deterministic** byte-ratio estimator with a per-model table covering the full
//! **D-21 five-model set** (C-12) — the only property that path actually needs.
//!
//! ## D-21 model set (C-12)
//!
//! The newer-tokenizer set is **five** models: Opus 4.7+ (and later Opus),
//! Fable 5, Mythos 5, Mythos Preview, Sonnet 5. A model outside this set is
//! `unknown` for tokenizer purposes (and drives `capture_gap = unknown_model` in
//! `capture.rs`). A missing entry silently under-counts that model's tokens by
//! ~30%, so all five are present below.

/// The five newer-tokenizer models (D-21). Membership here is what
/// distinguishes a known model (no `unknown_model` gap) from an unknown one.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum KnownModel {
    /// Opus 4.7 and every later Opus.
    Opus,
    /// Fable 5.
    Fable5,
    /// Mythos 5.
    Mythos5,
    /// Mythos Preview.
    MythosPreview,
    /// Sonnet 5.
    Sonnet5,
}

impl KnownModel {
    /// Approximate bytes-of-request-body per input token for this model family.
    ///
    /// The estimate reads the request **body byte length** (a cheap, O(1) proxy
    /// captured at request time — see `capture::Observation`), not a re-tokenized
    /// render. JSON structural overhead makes this an over-count, which is
    /// acceptable for a value that only ever carries `tokenizer_estimated`.
    fn bytes_per_token(self) -> f64 {
        match self {
            // The five share the newer tokenizer; small per-family variation is
            // retained so the selection is genuinely per-model (C-12), not a
            // single constant that would mask a missing-entry regression.
            KnownModel::Opus => 3.6,
            KnownModel::Sonnet5 => 3.7,
            KnownModel::Fable5 => 3.8,
            KnownModel::Mythos5 => 3.7,
            KnownModel::MythosPreview => 3.7,
        }
    }
}

/// Ratio used when the model is not in the D-21 set (older/other tokenizer). The
/// estimate is still produced (the event must not vanish) but `capture.rs` also
/// records `capture_gap = unknown_model`.
const UNKNOWN_MODEL_BYTES_PER_TOKEN: f64 = 3.7;

/// Classify a wire model string into the D-21 set. `None` = not in the set
/// (drives `unknown_model`).
///
/// Matching is deliberately lenient on vendor prefixes (`claude-…`) and on the
/// version separator (`opus-4-8`, `opus.4.8`) so a new point release inside a
/// known family is still recognised.
pub fn classify_model(model: &str) -> Option<KnownModel> {
    let m = model.to_ascii_lowercase();
    if m.contains("opus") {
        // Opus 4.7+ uses the newer tokenizer; older Opus (≤4.6) does not.
        return match opus_version(&m) {
            Some((major, minor)) if (major, minor) >= (4, 7) => Some(KnownModel::Opus),
            Some(_) => None,
            // Unversioned "opus" (or a form we cannot parse) is assumed current.
            None => Some(KnownModel::Opus),
        };
    }
    if m.contains("mythos") {
        if m.contains("preview") {
            return Some(KnownModel::MythosPreview);
        }
        if mentions_generation(&m, 5) {
            return Some(KnownModel::Mythos5);
        }
        return None;
    }
    if m.contains("fable") && mentions_generation(&m, 5) {
        return Some(KnownModel::Fable5);
    }
    if m.contains("sonnet") && mentions_generation(&m, 5) {
        return Some(KnownModel::Sonnet5);
    }
    None
}

/// Whether a lowercased model string mentions generation `gen` (`sonnet-5`,
/// `sonnet.5`, `sonnet5`) but not a *different* leading generation digit.
fn mentions_generation(m: &str, generation: u32) -> bool {
    let g = generation.to_string();
    // Look for the generation digit as a standalone token after the family name.
    m.split(|c: char| !c.is_ascii_digit()).any(|tok| tok == g)
}

/// Parse the `(major, minor)` version out of an Opus model string like
/// `claude-opus-4-8` or `opus-4.7`. `minor` defaults to 0 when absent.
fn opus_version(m: &str) -> Option<(u32, u32)> {
    let after = m.split("opus").nth(1)?;
    let mut nums = after
        .split(|c: char| !c.is_ascii_digit())
        .filter(|s| !s.is_empty())
        .filter_map(|s| s.parse::<u32>().ok());
    let major = nums.next()?;
    let minor = nums.next().unwrap_or(0);
    Some((major, minor))
}

/// The result of a local estimate.
#[derive(Clone, Copy, Debug)]
pub struct TokenEstimate {
    /// Estimated input tokens. Always > 0 for a non-empty body.
    pub input_tokens: u64,
}

/// Stateless per-model estimator held on `BoundaryState` (a ZST — the model
/// table is `const`). Keeps the plan's "the tokenizer lives on the state" shape
/// without a heap handle or a network client.
#[derive(Clone, Copy, Debug, Default)]
pub struct Estimator;

impl Estimator {
    /// Estimate input tokens for `model` from the request body byte length.
    ///
    /// Synchronous and allocation-free — composes with `catch_unwind` (D-02) and
    /// makes no network call (D-08).
    pub fn estimate(self, model: &str, body_len: usize) -> TokenEstimate {
        let ratio = classify_model(model)
            .map(KnownModel::bytes_per_token)
            .unwrap_or(UNKNOWN_MODEL_BYTES_PER_TOKEN);
        // ceil so a non-empty body never estimates zero tokens.
        let input_tokens = ((body_len as f64) / ratio).ceil() as u64;
        TokenEstimate {
            input_tokens: input_tokens.max(if body_len > 0 { 1 } else { 0 }),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn full_d21_set_is_classified() {
        // C-12: all FIVE newer-tokenizer models resolve to a known family.
        assert_eq!(classify_model("claude-opus-4-8"), Some(KnownModel::Opus));
        assert_eq!(classify_model("claude-opus-4-7"), Some(KnownModel::Opus));
        assert_eq!(classify_model("claude-sonnet-5"), Some(KnownModel::Sonnet5));
        assert_eq!(classify_model("claude-fable-5"), Some(KnownModel::Fable5));
        assert_eq!(classify_model("claude-mythos-5"), Some(KnownModel::Mythos5));
        assert_eq!(
            classify_model("claude-mythos-preview"),
            Some(KnownModel::MythosPreview)
        );
    }

    #[test]
    fn older_opus_and_unrelated_models_are_unknown() {
        // Opus ≤4.6 uses the older tokenizer → not in the D-21 set.
        assert_eq!(classify_model("claude-opus-4-6"), None);
        // Unrelated models are unknown → capture_gap = unknown_model.
        assert_eq!(classify_model("claude-haiku-3"), None);
        assert_eq!(classify_model("gpt-4o"), None);
        assert_eq!(classify_model("claude-sonnet-4-6"), None);
    }

    #[test]
    fn estimate_is_nonzero_offline_and_deterministic() {
        let est = Estimator;
        let a = est.estimate("claude-opus-4-8", 3600);
        let b = est.estimate("claude-opus-4-8", 3600);
        assert!(a.input_tokens > 0);
        assert_eq!(a.input_tokens, b.input_tokens, "estimate is deterministic");
        // Empty body → zero tokens (no phantom usage).
        assert_eq!(est.estimate("claude-opus-4-8", 0).input_tokens, 0);
    }

    #[test]
    fn unknown_model_still_estimates() {
        // An unknown model must still produce a usable estimate — the event
        // never vanishes just because the model is off the known list.
        let est = Estimator;
        assert!(est.estimate("mystery-model-9", 4000).input_tokens > 0);
    }
}