toktier 0.2.7

Rust API preview for TokTier's exact, stateful tokenization runtime
Documentation
/// Runtime certification policy.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub enum Policy {
    /// Everything [`Policy::Certified`] admits, and in addition a device
    /// or compiler toolchain the certification campaigns have not judged,
    /// as long as the shipped kernel loads and runs on it and the build
    /// itself is the judged one. Such a route is labelled
    /// [`Certification::Supported`] rather than certified, and
    /// [`Certification::LocallyVerified`] once a local check has compared
    /// it against the reference engine on this machine.
    ///
    /// The default since 0.2.6. What it does **not** admit is any
    /// disagreement inside the certified core: a build that is not the
    /// judged one, a fatbin that does not match its digest, or a
    /// world-writable compiler component still routes to the reference
    /// engine.
    #[default]
    Supported,
    /// Admit only evidence-bound accelerated routes; fall back exactly.
    /// Unchanged in meaning: a device architecture or a compiler triple
    /// the registry has not judged is not eligible here.
    Certified,
    /// Force the frozen Hugging Face reference implementation.
    Reference,
    /// Require at least one certified accelerated backend at construction.
    /// Per-input safety guards may still fall back to the reference.
    RequireAccelerated,
    /// Permit explicitly requested experimental delivery while labelling it.
    Experimental,
}

impl Policy {
    /// Whether this policy admits a device or toolchain no campaign has
    /// judged, on a build whose own certified core is intact.
    pub(crate) fn admits_unjudged_device(self) -> bool {
        matches!(self, Self::Supported | Self::RequireAccelerated)
    }
}

/// Requested execution device.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub enum Device {
    #[default]
    Auto,
    Cpu,
    Cuda(u32),
}

/// CUDA kernel delivery preference.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub enum GpuDelivery {
    #[default]
    Auto,
    Prebuilt,
    Jit,
    Disabled,
}

/// Backend that actually produced token IDs.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub enum Backend {
    HuggingFace,
    FastCpu,
    Gpu,
}

impl Backend {
    pub(crate) fn from_internal(value: &str) -> Self {
        match value {
            "gpu" => Self::Gpu,
            "fast_cpu" => Self::FastCpu,
            _ => Self::HuggingFace,
        }
    }
}

/// Stable route/fallback reason identifier.
///
/// The run-time variants are this crate's spelling of the frozen `R_*`
/// namespace in `docs/contracts/routing.md` Section 5: the router records
/// those codes in its own ledger, and the ones reachable from a Rust
/// execution are named here rather than in a second vocabulary. The
/// plan-time variants describe admission decisions this crate makes for
/// itself and are not claimed to be the same codes.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub enum ReasonCode {
    /// `R_INPUT_BELOW_GPU_THRESHOLD`
    InputBelowGpuThreshold,
    /// `R_INPUT_ADDED_TOKEN`
    InputAddedToken,
    /// `R_INPUT_GUARD_ROUTED`
    InputGuardRouted,
    /// `R_EXEC_FAULT`
    ExecutionFault,
    /// `R_INPUT_POSTPROCESS_ROUTED`
    InputPostprocessRouted,
    /// `R_SESSION_NO_SAFE_CUT`
    SessionNoSafeCut,
    /// `R_INVALID_PRIOR_STATE`
    ///
    /// A stored session tail did not describe itself consistently, so
    /// the accumulated text was re-encoded from the reference. Named in
    /// the frozen namespace since 0.2.7; before that it travelled as an
    /// `Other` string that was in neither vocabulary.
    InvalidPriorState,
    GpuUnavailable,
    GpuUncertified,
    CpuUncertified,
    RuntimeBuildUncertified,
    ReferenceRequested,
    /// A reason the router named that this release has no frozen code
    /// for. The string is the router's own token, passed through rather
    /// than mapped onto a code that does not mean it.
    Other(String),
}

/// The frozen codes for the two reasons the session repair names in its
/// path rather than in a ledger entry.
///
/// The router's crate carries no constant for either, because no router
/// path records them; these are the spelling on the Rust face, and they
/// are the same strings `docs/contracts/routing.md` Section 5.2 lists.
pub(crate) const R_SESSION_NO_SAFE_CUT: &str = "R_SESSION_NO_SAFE_CUT";
pub(crate) const R_INVALID_PRIOR_STATE: &str = "R_INVALID_PRIOR_STATE";

impl ReasonCode {
    /// The reason behind one routed execution, from the code the router
    /// recorded for it. Read rather than inferred, so this answer and the
    /// ledger cannot come to disagree.
    pub(crate) fn from_ledger_code(code: &str) -> Self {
        match code {
            toktier_routing_core::R_INPUT_BELOW_GPU_THRESHOLD => Self::InputBelowGpuThreshold,
            toktier_routing_core::R_INPUT_ADDED_TOKEN => Self::InputAddedToken,
            toktier_routing_core::R_INPUT_GUARD_ROUTED => Self::InputGuardRouted,
            toktier_routing_core::R_EXEC_FAULT => Self::ExecutionFault,
            toktier_routing_core::R_INPUT_POSTPROCESS_ROUTED => Self::InputPostprocessRouted,
            R_SESSION_NO_SAFE_CUT => Self::SessionNoSafeCut,
            R_INVALID_PRIOR_STATE => Self::InvalidPriorState,
            other => Self::Other(other.to_owned()),
        }
    }
}

/// Certification status attached to one admitted route.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub enum Certification {
    Certified,
    CertifiedSource,
    /// The engines are the judged ones and they run here, but this
    /// device or compiler toolchain is outside the judged list, so no
    /// campaign has measured this combination. Admitted by default under
    /// [`Policy::Supported`]; `Policy::Certified` routes it to the
    /// reference engine instead.
    Supported,
    /// A `Supported` route that a local check has compared against this
    /// binary's reference engine on this machine. The record is not a
    /// certificate: it says what was compared, by whom, and on what, and
    /// it expires when the driver, toolchain, kernel or source identity
    /// changes.
    LocallyVerified,
    Experimental,
    Reference,
}

/// Immutable route selected at tokenizer construction.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct RoutePlan {
    pub family: String,
    pub artifact_sha256: String,
    pub backends: Vec<Backend>,
    pub gpu_min_bytes: u64,
    pub certification: Certification,
    pub reasons: Vec<ReasonCode>,
}

/// Facts for one completed encode or append operation.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct ExecutionFacts {
    pub backend: Backend,
    pub path: String,
    pub source: Option<String>,
    pub input_bytes: u64,
    pub certification: Certification,
    /// Why this execution ran where it did, when a routing decision
    /// moved it off the first admitted backend. `None` means the
    /// admitted route ran and there was nothing to record; why the
    /// admitted route is what it is belongs to
    /// [`RoutePlan::reasons`](crate::RoutePlan), not to one execution.
    pub reason: Option<ReasonCode>,
}

/// Build/runtime facts that can be emitted by a control plane without
/// parsing human-readable output.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct DoctorFacts {
    pub crate_version: String,
    pub oracle: String,
    pub registry_verified: bool,
    pub python_required: bool,
    pub sqlite_compiled: bool,
    pub prebuilt_gpu_compiled: bool,
    pub jit_compiled: bool,
    /// Whether this build can acquire an artifact over the network. The
    /// `network` feature is not in the default set, so a build that has
    /// to fetch and does not carry it answers `NETWORK_DISABLED`.
    pub network_compiled: bool,
    pub runtime_build: RuntimeBuildFacts,
    pub cuda: Option<CudaFacts>,
}

/// Exact build identity embedded by the Python-free Rust facade.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct RuntimeBuildFacts {
    pub source_digest: String,
    pub fast_cpu_source_digest: String,
    pub native_host_source_digest: String,
    pub toolchain: String,
    pub build_flags: Vec<String>,
    /// How the whole compiled closure of this build compares with the
    /// judged one (three-way shape unchanged since 0.2.4). Since 0.2.6
    /// this is a reading, not a gate: `certified` looks at
    /// `core_closure`. Readers who want the 0.2.4 and 0.2.5 rule can
    /// require `dependency_closure == "verified"` themselves.
    pub dependency_closure: String,
    /// Why the build flags of this build are not the judged ones, when a
    /// judged entry agrees with it on every other axis. `None` when the
    /// flags match, or when something else is the disagreement.
    pub build_flag_divergence: Option<String>,
    /// `true` when the certified core of this build -- TokTier's own
    /// crates, the packages they call directly, and the text-semantics
    /// libraries beneath them -- is the one the certification evidence
    /// was taken on, together with the source identities, toolchain and
    /// build flags of a judged entry. Packages outside that core are
    /// compared as well and reported in `dependency_advisory`; they do
    /// not decide this field.
    pub certified: bool,
    /// The same comparison as `dependency_closure`, restricted to the
    /// certified core. Anything other than `"verified"` here holds
    /// `certified` at `false` and says what differs and what aligns it.
    pub core_closure: String,
    /// What this build compiles that the judged build did not, where the
    /// difference is reported rather than gating: packages outside the
    /// certified core, and core packages whose package version moved
    /// while the behaviour version this binary reads did not. `None`
    /// when there is nothing to report. It never says that the packages
    /// it names cannot influence behaviour; it says which packages the
    /// certificate covers and reports the rest.
    pub dependency_advisory: Option<String>,
    /// The version of the text semantics this binary carries, per unit,
    /// beside the version the evidence was taken on.
    pub behavior_versions: Vec<BehaviorVersion>,
}

/// The version of one text-semantics unit, as this binary reads it and
/// as the certification evidence recorded it.
///
/// A unit is the thing whose version can move ids: the Unicode tables a
/// regex engine carries, the Oniguruma library a binding links, the
/// segmentation tables a crate ships. `observed` is `None` for a unit
/// this binary has no way to ask, in which case the package versions of
/// that unit are compared exactly instead.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct BehaviorVersion {
    pub unit: String,
    pub observed: Option<String>,
    pub judged: String,
    pub source: String,
}

/// Non-invasive CUDA probe result. Probe failure is data, not a panic.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct CudaFacts {
    pub device_ordinal: u32,
    pub available: bool,
    pub architecture: Option<String>,
    /// The CUDA driver API version the device was opened through.
    ///
    /// An environment fact, not a certificate premise: it is reported
    /// because a reader wants to know it, and no certification claim
    /// rests on the number. Where a registry row binds a driver floor,
    /// that floor is checked separately as a precondition for the
    /// kernel loading at all.
    pub driver_api_version: Option<i32>,
    pub error: Option<String>,
}

/// Public snapshot of the native state-store counters.
#[derive(Debug, Clone, PartialEq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct StoreStats {
    pub format: String,
    pub session_count: u64,
    pub node_count: u64,
    pub puts: u64,
    pub appends: u64,
    pub forks: u64,
    pub lookups: u64,
    pub lookup_hits: u64,
    pub lookup_misses: u64,
    pub checksum_rejects: u64,
    pub sessions_evicted: u64,
    pub import_rejects: u64,
    pub revision_conflicts: u64,
    pub path_counts: BTreeMap<String, u64>,
}

impl From<toktier_store_core::StatsSnapshot> for StoreStats {
    fn from(value: toktier_store_core::StatsSnapshot) -> Self {
        Self {
            format: value.format.to_owned(),
            session_count: value.session_count,
            node_count: value.node_count,
            puts: value.puts,
            appends: value.extends,
            forks: value.forks,
            lookups: value.lookups,
            lookup_hits: value.lookup_hits,
            lookup_misses: value.lookup_misses,
            checksum_rejects: value.checksum_rejects,
            sessions_evicted: value.sessions_evicted,
            import_rejects: value.import_rejects,
            revision_conflicts: value.revision_conflicts,
            path_counts: value.path_counts,
        }
    }
}
use std::collections::BTreeMap;

#[cfg(test)]
mod tests {
    use super::*;

    /// Every run-time code the router can record maps onto this crate's
    /// name for it, and an unrecognised one is passed through rather
    /// than turned into a code that does not mean it.
    #[test]
    fn every_recorded_reason_maps_onto_the_frozen_namespace() {
        for (code, expected) in [
            (
                toktier_routing_core::R_INPUT_BELOW_GPU_THRESHOLD,
                ReasonCode::InputBelowGpuThreshold,
            ),
            (
                toktier_routing_core::R_INPUT_ADDED_TOKEN,
                ReasonCode::InputAddedToken,
            ),
            (
                toktier_routing_core::R_INPUT_GUARD_ROUTED,
                ReasonCode::InputGuardRouted,
            ),
            (
                toktier_routing_core::R_EXEC_FAULT,
                ReasonCode::ExecutionFault,
            ),
            (
                toktier_routing_core::R_INPUT_POSTPROCESS_ROUTED,
                ReasonCode::InputPostprocessRouted,
            ),
        ] {
            assert_eq!(ReasonCode::from_ledger_code(code), expected, "for {code}");
        }
        assert_eq!(
            ReasonCode::from_ledger_code(R_INVALID_PRIOR_STATE),
            ReasonCode::InvalidPriorState
        );
        assert_eq!(
            ReasonCode::from_ledger_code("R_SOMETHING_LATER"),
            ReasonCode::Other("R_SOMETHING_LATER".to_owned())
        );
    }
}