use std::collections::BTreeMap;
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, Instant};
use toktier_routing_core::{
default_chunk_workers, BpeSyncBoundary, ChunkEncoder, ChunkedIds, FastEncodeSource,
CHUNKED_DEFAULT_SCAN_BUDGET, R_CHUNK_FAMILY_UNSUPPORTED,
};
use crate::session::TokenizerInner;
use crate::{Backend, Certification, Error, ErrorCode, Policy, ReasonCode, Result, TokenBuffer};
pub const DEFAULT_SCAN_BUDGET: usize = CHUNKED_DEFAULT_SCAN_BUDGET;
pub const MIN_CHARS: usize = 256_000;
pub const MIN_BYTES_PER_WORKER: usize = 64 * 1024;
const CHUNK_TABLE: [(Option<usize>, usize); 4] = [
(Some(MIN_CHARS), 1),
(Some(1_000_000), 4),
(Some(4_000_000), 16),
(None, 32),
];
const EVIDENCE_ADVERSARIAL: &str = "adversarial";
const EVIDENCE_CORPUS: &str = "corpus";
const STATUS_CERTIFIED: &str = "certified";
const LABEL_NATIVE_EVIDENCE: &str = "native_evidence";
const REFUSAL_ROSTER: &str = "artifact_not_in_certified_repair_roster";
const REFUSAL_NO_EVIDENCE: &str = "artifact_without_chunked_evidence";
const REFUSAL_INCOMPLETE: &str = "chunked_evidence_incomplete";
const REFUSAL_FACE: &str = "loader_face_differs_from_chunked_evidence";
pub fn default_chunks(chars: usize) -> usize {
let mut selected = 1;
for (bound, chunks) in CHUNK_TABLE {
if bound.is_none_or(|bound| chars < bound) {
selected = chunks;
break;
}
}
let useful = chars.div_ceil(MIN_BYTES_PER_WORKER);
selected.min(useful).max(1)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct ChunkOptions {
pub k: Option<usize>,
pub min_chunk: Option<usize>,
pub scan_budget: usize,
}
impl Default for ChunkOptions {
fn default() -> Self {
Self {
k: None,
min_chunk: None,
scan_budget: DEFAULT_SCAN_BUDGET,
}
}
}
impl ChunkOptions {
pub fn new() -> Self {
Self::default()
}
pub fn k(mut self, k: usize) -> Self {
self.k = Some(k);
self
}
pub fn min_chunk(mut self, chars: usize) -> Self {
self.min_chunk = Some(chars);
self
}
pub fn scan_budget(mut self, chars: usize) -> Self {
self.scan_budget = chars;
self
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct ChunkPlan {
pub spans: Vec<(usize, usize)>,
pub byte_spans: Vec<(usize, usize)>,
pub k: usize,
pub chars: usize,
pub cut_misses: usize,
pub literal_rejects: usize,
pub norm_rejects: usize,
}
impl ChunkPlan {
pub fn cuts(&self) -> Vec<usize> {
self.spans.iter().skip(1).map(|span| span.0).collect()
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct RowSources {
pub gigatoken: usize,
pub hf_added_token: usize,
pub hf_engine_guard: usize,
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct ChunkedEncoding {
ids: TokenBuffer,
pub chunks: usize,
pub k_requested: usize,
pub cuts: Vec<usize>,
pub cut_misses: usize,
pub literal_rejects: usize,
pub norm_rejects: usize,
pub serial_reason: Option<ReasonCode>,
pub below_threshold: bool,
pub sources: RowSources,
pub certification: Certification,
pub evidence_label: Option<String>,
pub plan_time: Duration,
pub encode_time: Duration,
pub concat_time: Duration,
pub cold_forks: usize,
}
impl ChunkedEncoding {
pub fn ids(&self) -> &[u32] {
self.ids.as_slice()
}
pub fn token_buffer(&self) -> &TokenBuffer {
&self.ids
}
pub fn into_token_buffer(self) -> TokenBuffer {
self.ids
}
fn serial(
ids: Vec<u32>,
k_requested: usize,
reason: ReasonCode,
certification: Certification,
encode_time: Duration,
) -> Self {
Self {
ids: TokenBuffer::from(ids),
chunks: 1,
k_requested,
cuts: Vec::new(),
cut_misses: 0,
literal_rejects: 0,
norm_rejects: 0,
serial_reason: Some(reason),
below_threshold: false,
sources: RowSources::default(),
certification,
evidence_label: None,
plan_time: Duration::ZERO,
encode_time,
concat_time: Duration::ZERO,
cold_forks: 0,
}
}
fn from_native(
encoded: ChunkedIds,
certification: Certification,
evidence_label: Option<&str>,
) -> Self {
let cut = encoded.serial_reason.is_none() && !encoded.below_threshold;
let evidence_label = evidence_label.filter(|_| cut).map(str::to_owned);
let mut sources = RowSources::default();
for source in &encoded.sources {
match source {
FastEncodeSource::Gigatoken => sources.gigatoken += 1,
FastEncodeSource::ReferenceAddedToken => sources.hf_added_token += 1,
FastEncodeSource::ReferenceEngineGuard => sources.hf_engine_guard += 1,
}
}
Self {
ids: TokenBuffer::from(encoded.ids),
chunks: encoded.chunks,
k_requested: encoded.k_requested,
cuts: encoded.cuts,
cut_misses: encoded.cut_misses,
literal_rejects: encoded.literal_rejects,
norm_rejects: encoded.norm_rejects,
serial_reason: encoded.serial_reason.map(ReasonCode::from_ledger_code),
below_threshold: encoded.below_threshold,
sources,
certification,
evidence_label,
plan_time: Duration::from_nanos(encoded.plan_ns),
encode_time: Duration::from_nanos(encoded.encode_ns),
concat_time: Duration::from_nanos(encoded.concat_ns),
cold_forks: encoded.cold_forks,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum AdmissionGate {
EvidenceTable,
LiftedByExperimentalPolicy,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct Admission {
pub admitted: bool,
pub refusal: Option<String>,
pub family: String,
pub artifact_sha256: String,
pub cpu_route: bool,
pub normalizer: Option<String>,
pub literals: usize,
pub evidence_id: Option<String>,
pub evidence_status: Option<String>,
pub native_evidence: Vec<String>,
pub evidence_label: Option<String>,
pub gate: AdmissionGate,
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
#[non_exhaustive]
pub struct ChunkedStats {
pub calls: u64,
pub applied: u64,
pub serial: u64,
pub below_threshold: u64,
pub chunks_total: u64,
pub cut_misses: u64,
pub literal_rejects: u64,
pub norm_rejects: u64,
pub fallback_counts: BTreeMap<String, u64>,
pub workers: usize,
pub default_workers: usize,
pub evidence_label: Option<String>,
}
#[derive(Debug, Default)]
struct Counters {
calls: u64,
applied: u64,
serial: u64,
below_threshold: u64,
chunks_total: u64,
cut_misses: u64,
literal_rejects: u64,
norm_rejects: u64,
fallback_counts: BTreeMap<String, u64>,
}
impl Counters {
fn record(&mut self, encoded: &ChunkedEncoding) {
self.calls += 1;
if let Some(reason) = &encoded.serial_reason {
self.serial += 1;
*self
.fallback_counts
.entry(reason_spelling(reason))
.or_insert(0) += 1;
} else if encoded.below_threshold {
self.below_threshold += 1;
} else {
self.applied += 1;
}
self.chunks_total += encoded.chunks as u64;
self.cut_misses += encoded.cut_misses as u64;
self.literal_rejects += encoded.literal_rejects as u64;
self.norm_rejects += encoded.norm_rejects as u64;
}
}
fn reason_spelling(reason: &ReasonCode) -> String {
match reason {
ReasonCode::ChunkNoCut => toktier_routing_core::R_CHUNK_NO_CUT.to_owned(),
ReasonCode::ChunkFamilyUnsupported => R_CHUNK_FAMILY_UNSUPPORTED.to_owned(),
ReasonCode::ChunkRouteNotFastCpu => {
toktier_routing_core::R_CHUNK_ROUTE_NOT_FAST_CPU.to_owned()
}
ReasonCode::Other(code) => code.clone(),
other => format!("{other:?}"),
}
}
#[derive(Debug, Default)]
pub(crate) struct State {
admission: OnceLock<std::result::Result<Admission, (ErrorCode, String)>>,
counters: Mutex<Counters>,
}
#[derive(Debug, Clone, Copy)]
pub struct Chunked<'a> {
inner: &'a Arc<TokenizerInner>,
}
impl<'a> Chunked<'a> {
pub(crate) fn new(inner: &'a Arc<TokenizerInner>) -> Self {
Self { inner }
}
pub fn admission(&self) -> Result<Admission> {
self.inner
.chunked
.admission
.get_or_init(|| {
decide(self.inner).map_err(|error| (error.code(), error.message().to_owned()))
})
.clone()
.map_err(|(code, message)| {
Error::new(code, message).with_family(&self.inner.artifact.identity().family)
})
}
pub fn plan(&self, text: &str, options: &ChunkOptions) -> Result<ChunkPlan> {
let (k, budget) = resolve(options, text)?;
let admission = self.admission()?;
if let Some(refusal) = &admission.refusal {
return Err(Error::new(
ErrorCode::UncertifiedTokenizer,
format!(
"{} has no certified chunk-cut rule: {refusal}",
admission.family
),
)
.with_family(&admission.family));
}
if !admission.cpu_route {
return Err(no_cpu_route(&admission.family));
}
let plan = self
.inner
.router
.chunked_plan(text, k, options.min_chunk, budget)
.map_err(|error| Error::new(ErrorCode::BackendExecutionFault, error.to_string()))?;
Ok(ChunkPlan {
spans: plan.spans,
byte_spans: plan.byte_spans,
k: plan.k,
chars: plan.chars,
cut_misses: plan.cut_misses,
literal_rejects: plan.literal_rejects,
norm_rejects: plan.norm_rejects,
})
}
pub fn encode(&self, text: &str, options: &ChunkOptions) -> Result<ChunkedEncoding> {
let (k, budget) = resolve(options, text)?;
let admission = self.admission()?;
let certification = self.inner.plan.certification;
let encoded = if admission.admitted {
let encoded = self
.inner
.router
.chunked_encode_ids(text, k, options.min_chunk, budget)
.map_err(|error| Error::new(ErrorCode::BackendExecutionFault, error.to_string()))?;
ChunkedEncoding::from_native(
encoded,
certification,
admission.evidence_label.as_deref(),
)
} else {
let started = Instant::now();
let routed =
self.inner.router.encode_ids(text, false).map_err(|error| {
Error::new(ErrorCode::BackendExecutionFault, error.to_string())
})?;
ChunkedEncoding::serial(
routed.ids,
k,
ReasonCode::ChunkFamilyUnsupported,
certification,
started.elapsed(),
)
};
self.inner
.chunked
.counters
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.record(&encoded);
Ok(encoded)
}
pub fn warm(&self, workers: Option<usize>, text: Option<&str>) -> Result<usize> {
let admission = self.admission()?;
if !admission.cpu_route {
return Err(no_cpu_route(&admission.family));
}
self.inner
.router
.chunked_warm(workers.unwrap_or_else(default_chunk_workers), text)
.map_err(|error| Error::new(ErrorCode::BackendExecutionFault, error.to_string()))
}
pub fn stats(&self) -> ChunkedStats {
let evidence_label = self
.admission()
.ok()
.and_then(|admission| admission.evidence_label);
let counters = self
.inner
.chunked
.counters
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
ChunkedStats {
calls: counters.calls,
applied: counters.applied,
serial: counters.serial,
below_threshold: counters.below_threshold,
chunks_total: counters.chunks_total,
cut_misses: counters.cut_misses,
literal_rejects: counters.literal_rejects,
norm_rejects: counters.norm_rejects,
fallback_counts: counters.fallback_counts.clone(),
workers: self
.inner
.router
.chunk_encoder()
.map(ChunkEncoder::worker_count)
.unwrap_or(0),
default_workers: default_chunk_workers(),
evidence_label,
}
}
}
fn no_cpu_route(family: &str) -> Error {
Error::new(
ErrorCode::UncertifiedTokenizer,
format!("family {family} has no corrected CPU route on this runtime to chunk on"),
)
.with_family(family)
}
fn resolve(options: &ChunkOptions, text: &str) -> Result<(usize, usize)> {
if options.scan_budget == 0 {
return Err(Error::new(
ErrorCode::InvalidArgument,
"scan_budget must be a positive integer",
));
}
let k = match options.k {
Some(0) => {
return Err(Error::new(
ErrorCode::InvalidArgument,
"k must be a positive integer or None",
))
}
Some(k) => k,
None => default_chunks(text.chars().count()),
};
Ok((k, options.scan_budget))
}
pub(crate) fn levels_status(levels: &[String]) -> Option<String> {
let adversarial = levels.iter().any(|level| level == EVIDENCE_ADVERSARIAL);
let corpus = levels.iter().any(|level| level == EVIDENCE_CORPUS);
match (adversarial, corpus) {
(true, true) => Some(STATUS_CERTIFIED.to_owned()),
(true, false) => Some(format!("{EVIDENCE_ADVERSARIAL}_only")),
(false, true) => Some(format!("{EVIDENCE_CORPUS}_only")),
(false, false) => None,
}
}
fn native_evidence_label(levels: &[String]) -> Option<String> {
match levels_status(levels).as_deref() {
Some(STATUS_CERTIFIED) => None,
Some(status) => Some(format!("{LABEL_NATIVE_EVIDENCE}:{status}")),
None => Some(format!("{LABEL_NATIVE_EVIDENCE}:none")),
}
}
fn decide(inner: &Arc<TokenizerInner>) -> Result<Admission> {
let identity = inner.artifact.identity();
let family = identity.family.clone();
let artifact_sha256 = identity.tokenizer_sha256.clone();
let cpu_route = inner.plan.backends.contains(&Backend::FastCpu);
let gate = if inner.policy == Policy::Experimental {
AdmissionGate::LiftedByExperimentalPolicy
} else {
AdmissionGate::EvidenceTable
};
let roster = inner
.registry
.repair(&family)
.is_some_and(|row| row.artifact_sha256 == artifact_sha256);
let (normalizer, literals, mut refusal) = if !roster {
(None, 0, Some(REFUSAL_ROSTER.to_owned()))
} else {
match inner.router.chunk_encoder() {
Ok(encoder) => structural(encoder),
Err(_) => {
let boundary = BpeSyncBoundary::new(inner.registry.pclass())
.map_err(|error| Error::new(ErrorCode::RegistryInvalid, error.to_string()))?;
let encoder = ChunkEncoder::new(
&inner.artifact.bytes(),
boundary,
Arc::clone(&inner.reference),
)
.map_err(|error| {
Error::new(
ErrorCode::Internal,
format!("chunk planner construction failed: {error}"),
)
})?;
structural(&encoder)
}
}
};
let row = inner
.registry
.chunked_admission(&family)
.filter(|row| row.artifact_sha256 == artifact_sha256);
let evidence_id = row.map(|row| row.evidence_id.clone());
let native_evidence = row
.map(|row| row.evidence.native.clone())
.unwrap_or_default();
let evidence_status = levels_status(&native_evidence);
if refusal.is_none() && gate == AdmissionGate::EvidenceTable {
match row {
None => refusal = Some(REFUSAL_NO_EVIDENCE.to_owned()),
Some(row) if row.status != STATUS_CERTIFIED => {
refusal = Some(format!("{REFUSAL_INCOMPLETE}:{}", row.status));
}
Some(row) => {
if row
.loader_face_sha256
.as_deref()
.is_some_and(|face| face != artifact_sha256)
{
refusal = Some(format!(
"{REFUSAL_FACE}:{}",
&artifact_sha256[..artifact_sha256.len().min(12)]
));
}
}
}
}
let evidence_label = native_evidence_label(&native_evidence);
Ok(Admission {
admitted: refusal.is_none(),
refusal,
family,
artifact_sha256,
cpu_route,
normalizer,
literals,
evidence_id,
evidence_status,
native_evidence,
evidence_label,
gate,
})
}
fn structural(encoder: &ChunkEncoder) -> (Option<String>, usize, Option<String>) {
(
encoder.normalizer().map(|mode| mode.as_str().to_owned()),
encoder.literal_count(),
encoder.refusal().map(str::to_owned),
)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn default_chunks_follows_the_python_table() {
assert_eq!(default_chunks(0), 1);
assert_eq!(default_chunks(255_999), 1);
assert_eq!(default_chunks(256_000), 4);
assert_eq!(default_chunks(999_999), 4);
assert_eq!(default_chunks(1_000_000), 16);
assert_eq!(default_chunks(3_999_999), 16);
assert_eq!(default_chunks(4_000_000), 32);
assert_eq!(default_chunks(100_000_000), 32);
assert_eq!(default_chunks(262_144), 4);
assert_eq!(
default_chunks(1_000_000),
16.min(1_000_000usize.div_ceil(65_536))
);
}
#[test]
fn options_refuse_zero_bounds() {
let text = "x".repeat(300);
let zero_budget = ChunkOptions::default().scan_budget(0);
assert_eq!(
resolve(&zero_budget, &text).unwrap_err().code(),
ErrorCode::InvalidArgument
);
let zero_k = ChunkOptions::default().k(0);
assert_eq!(
resolve(&zero_k, &text).unwrap_err().code(),
ErrorCode::InvalidArgument
);
assert_eq!(
resolve(&ChunkOptions::default(), &text).unwrap(),
(1, DEFAULT_SCAN_BUDGET)
);
assert_eq!(
resolve(&ChunkOptions::default().k(7).scan_budget(40), &text).unwrap(),
(7, 40)
);
let long = "\u{1f642}".repeat(300_000);
assert_eq!(resolve(&ChunkOptions::default(), &long).unwrap().0, 4);
}
fn levels(items: &[&str]) -> Vec<String> {
items.iter().map(|item| (*item).to_owned()).collect()
}
#[test]
fn levels_status_spells_the_levels_like_the_table_generator() {
assert_eq!(levels_status(&levels(&[])), None);
assert_eq!(
levels_status(&levels(&["adversarial"])).as_deref(),
Some("adversarial_only")
);
assert_eq!(
levels_status(&levels(&["corpus"])).as_deref(),
Some("corpus_only")
);
assert_eq!(
levels_status(&levels(&["adversarial", "corpus"])).as_deref(),
Some("certified")
);
assert_eq!(
levels_status(&levels(&["corpus", "adversarial"])).as_deref(),
Some("certified")
);
assert_eq!(levels_status(&levels(&["later"])), None);
}
#[test]
fn the_native_evidence_label_is_a_function_of_the_levels_alone() {
assert_eq!(
native_evidence_label(&levels(&["adversarial"])).as_deref(),
Some("native_evidence:adversarial_only")
);
assert_eq!(
native_evidence_label(&levels(&["corpus"])).as_deref(),
Some("native_evidence:corpus_only")
);
assert_eq!(
native_evidence_label(&levels(&[])).as_deref(),
Some("native_evidence:none")
);
assert_eq!(
native_evidence_label(&levels(&["later"])).as_deref(),
Some("native_evidence:none")
);
assert_eq!(
native_evidence_label(&levels(&["adversarial", "corpus"])),
None
);
assert_eq!(
native_evidence_label(&levels(&["corpus", "adversarial"])),
None
);
}
#[test]
fn only_a_result_that_was_cut_carries_the_evidence_label() {
let native = |serial_reason: Option<&'static str>, below_threshold: bool| ChunkedIds {
ids: vec![1, 2, 3],
chunks: if serial_reason.is_none() && !below_threshold {
2
} else {
1
},
k_requested: 2,
cuts: Vec::new(),
cut_misses: 0,
literal_rejects: 0,
norm_rejects: 0,
serial_reason,
below_threshold,
sources: Vec::new(),
row_bytes: Vec::new(),
plan_ns: 0,
encode_ns: 0,
concat_ns: 0,
worker_ns_max: 0,
worker_ns_min: 0,
cold_forks: 0,
};
let label = Some("native_evidence:adversarial_only");
let cut =
ChunkedEncoding::from_native(native(None, false), Certification::Supported, label);
assert_eq!(cut.evidence_label.as_deref(), label);
let fully_evidenced =
ChunkedEncoding::from_native(native(None, false), Certification::Supported, None);
assert_eq!(fully_evidenced.evidence_label, None);
for (reason, below) in [
(None, true),
(Some(toktier_routing_core::R_CHUNK_NO_CUT), false),
(
Some(toktier_routing_core::R_CHUNK_ROUTE_NOT_FAST_CPU),
false,
),
] {
let serial = ChunkedEncoding::from_native(
native(reason, below),
Certification::Supported,
label,
);
assert_eq!(serial.evidence_label, None, "{reason:?} {below}");
}
}
#[test]
fn counters_follow_the_python_record_rule() {
let mut counters = Counters::default();
let chunked = ChunkedEncoding {
ids: TokenBuffer::from(vec![1, 2]),
chunks: 3,
k_requested: 4,
cuts: vec![10, 20],
cut_misses: 1,
literal_rejects: 2,
norm_rejects: 3,
serial_reason: None,
below_threshold: false,
sources: RowSources::default(),
certification: Certification::Reference,
evidence_label: None,
plan_time: Duration::ZERO,
encode_time: Duration::ZERO,
concat_time: Duration::ZERO,
cold_forks: 0,
};
counters.record(&chunked);
let mut short = chunked.clone();
short.chunks = 1;
short.below_threshold = true;
counters.record(&short);
let serial = ChunkedEncoding::serial(
vec![1],
2,
ReasonCode::ChunkFamilyUnsupported,
Certification::Reference,
Duration::ZERO,
);
counters.record(&serial);
let mut no_cut = chunked.clone();
no_cut.chunks = 1;
no_cut.serial_reason = Some(ReasonCode::ChunkNoCut);
counters.record(&no_cut);
assert_eq!(counters.calls, 4);
assert_eq!(counters.applied, 1);
assert_eq!(counters.below_threshold, 1);
assert_eq!(counters.serial, 2);
assert_eq!(counters.chunks_total, 3 + 1 + 1 + 1);
assert_eq!(counters.cut_misses, 3);
assert_eq!(
counters.fallback_counts,
BTreeMap::from([
("R_CHUNK_FAMILY_UNSUPPORTED".to_owned(), 1),
("R_CHUNK_NO_CUT".to_owned(), 1),
])
);
}
}