use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use sha2::{Digest, Sha256};
use toktier_routing_core::{FastCpuEngine, FastRepairSpec, NativeRouter, ReferenceEngine};
use crate::artifact::{ArtifactManager, Revision};
use crate::diagnostics::{
Backend, Certification, CudaFacts, DoctorFacts, ExecutionFacts, ReasonCode, RoutePlan,
RuntimeBuildFacts, StoreStats,
};
use crate::manifest::{LocalArtifact, Registry};
use crate::session::{SessionStoreState, TokenizerInner};
#[cfg(feature = "jit")]
use crate::JitCompiler;
use crate::{
Device, Encoding, Error, ErrorCode, GpuDelivery, Policy, RaggedEncoding, Result, Session,
TokenBuffer,
};
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct EncodeOptions {
pub add_special_tokens: bool,
pub offsets: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DecodeOptions {
pub skip_special_tokens: bool,
}
impl Default for DecodeOptions {
fn default() -> Self {
Self {
skip_special_tokens: true,
}
}
}
#[derive(Debug, Clone)]
struct RuntimeConfig {
home: Option<PathBuf>,
artifact_cache: PathBuf,
artifact_manager: Option<ArtifactManager>,
explicit_artifacts: HashMap<String, PathBuf>,
policy: Policy,
device: Device,
gpu_delivery: GpuDelivery,
gpu_min_bytes: u64,
diagnostics: bool,
persistent_sessions: bool,
seed_digest_overlap: bool,
#[cfg(feature = "jit")]
jit_compiler: Option<JitCompiler>,
}
#[derive(Debug, Clone)]
pub struct RuntimeBuilder {
config: RuntimeConfig,
}
impl Default for RuntimeBuilder {
fn default() -> Self {
let artifact_cache =
crate::fsutil::default_cache_directory("TOKTIER_ARTIFACT_CACHE", "artifacts");
Self {
config: RuntimeConfig {
home: None,
artifact_cache,
artifact_manager: None,
explicit_artifacts: HashMap::new(),
policy: Policy::default(),
device: Device::Auto,
gpu_delivery: GpuDelivery::Auto,
gpu_min_bytes: 64 * 1024,
diagnostics: false,
persistent_sessions: false,
seed_digest_overlap: false,
#[cfg(feature = "jit")]
jit_compiler: None,
},
}
}
}
impl RuntimeBuilder {
pub fn home(mut self, path: impl Into<PathBuf>) -> Self {
self.config.home = Some(path.into());
self.config.persistent_sessions = true;
self
}
pub fn artifact_cache(mut self, path: impl Into<PathBuf>) -> Self {
let path = path.into();
self.config.artifact_cache = path.clone();
if let Some(manager) = &self.config.artifact_manager {
self.config.artifact_manager = Some(manager.with_cache(path));
}
self
}
pub fn artifacts(mut self, manager: ArtifactManager) -> Self {
self.config.artifact_cache = manager.cache_root().to_path_buf();
self.config.artifact_manager = Some(manager);
self
}
pub fn artifact_directory(
mut self,
family: impl Into<String>,
path: impl Into<PathBuf>,
) -> Self {
self.config
.explicit_artifacts
.insert(family.into(), path.into());
self
}
pub fn policy(mut self, policy: Policy) -> Self {
self.config.policy = policy;
self
}
pub fn device(mut self, device: Device) -> Self {
self.config.device = device;
self
}
pub fn gpu_delivery(mut self, delivery: GpuDelivery) -> Self {
self.config.gpu_delivery = delivery;
self
}
pub fn gpu_min_bytes(mut self, bytes: u64) -> Self {
self.config.gpu_min_bytes = bytes;
self
}
pub fn diagnostics(mut self, enabled: bool) -> Self {
self.config.diagnostics = enabled;
self
}
pub fn persistent_sessions(mut self, enabled: bool) -> Self {
self.config.persistent_sessions = enabled;
self
}
pub fn seed_digest_overlap(mut self, enabled: bool) -> Self {
self.config.seed_digest_overlap = enabled;
self
}
#[cfg(feature = "jit")]
pub fn jit_compiler(mut self, compiler: JitCompiler) -> Self {
self.config.jit_compiler = Some(compiler);
self
}
#[cfg(feature = "jit")]
pub fn accept_uncertified_jit(mut self, accept: bool) -> Result<Self> {
let compiler = JitCompiler::builder()
.accept_uncertified_jit(accept)
.build()?;
self.config.jit_compiler = Some(compiler);
Ok(self)
}
pub fn build(mut self) -> Result<Runtime> {
if self.config.artifact_cache.as_os_str().is_empty() {
return Err(Error::new(
ErrorCode::ConfigInvalid,
"artifact_cache must not be empty",
));
}
if self.config.persistent_sessions && self.config.home.is_none() {
self.config.home = crate::fsutil::default_state_directory();
if self.config.home.is_none() {
return Err(Error::new(
ErrorCode::ConfigInvalid,
"persistent sessions require RuntimeBuilder::home, TOKTIER_HOME, \
XDG_STATE_HOME, or HOME",
));
}
}
if matches!(self.config.device, Device::Cpu)
&& !matches!(
self.config.gpu_delivery,
GpuDelivery::Auto | GpuDelivery::Disabled
)
{
return Err(Error::new(
ErrorCode::ConfigInvalid,
"a CPU device cannot request GPU kernel delivery",
));
}
let registry = Registry::load()?;
let artifacts = match &self.config.artifact_manager {
Some(manager) => manager.with_cache(self.config.artifact_cache.clone()),
None => ArtifactManager::builder()
.cache(self.config.artifact_cache.clone())
.build()?,
};
Ok(Runtime {
inner: Arc::new(RuntimeInner {
registry,
artifacts,
config: self.config,
}),
})
}
}
#[derive(Debug)]
struct RuntimeInner {
registry: Arc<Registry>,
artifacts: ArtifactManager,
config: RuntimeConfig,
}
#[derive(Debug, Clone)]
pub struct Runtime {
inner: Arc<RuntimeInner>,
}
impl Runtime {
pub fn builder() -> RuntimeBuilder {
RuntimeBuilder::default()
}
pub fn shutdown(self) {}
pub fn doctor(&self) -> DoctorFacts {
let standing = self.inner.registry.rust_api_build_standing();
let runtime_build = RuntimeBuildFacts {
source_digest: env!("TOKTIER_RUST_API_SOURCE_SHA256").to_owned(),
fast_cpu_source_digest: env!("TOKTIER_RUST_API_FAST_CPU_SOURCE_SHA256").to_owned(),
native_host_source_digest: env!("TOKTIER_RUST_API_NATIVE_HOST_SOURCE_SHA256")
.to_owned(),
toolchain: env!("TOKTIER_RUST_API_TOOLCHAIN").to_owned(),
build_flags: env!("TOKTIER_RUST_API_BUILD_FLAGS")
.split('\x1f')
.map(str::to_owned)
.collect(),
dependency_closure: crate::DEPENDENCY_CLOSURE.to_owned(),
build_flag_divergence: self.inner.registry.rust_api_build_flag_divergence(),
certified: self.inner.registry.rust_api_build_certified(),
source_identity_judged: standing.source_identity_judged,
recipe_judged: standing.recipe_judged,
toolchain_divergence: self.inner.registry.rust_api_toolchain_divergence(),
core_closure: crate::behavior_version::core_closure().to_owned(),
dependency_advisory: crate::behavior_version::dependency_advisory().map(str::to_owned),
behavior_versions: crate::behavior_version::facts(),
reference_versions: crate::behavior_version::reference_facts(),
reference_certification_reason:
crate::behavior_version::reference_certification_reason().map(str::to_owned),
};
let ordinal = match self.inner.config.device {
Device::Cuda(ordinal) => ordinal,
Device::Auto => 0,
Device::Cpu => {
return DoctorFacts {
crate_version: env!("CARGO_PKG_VERSION").to_owned(),
oracle: crate::ORACLE.to_owned(),
registry_verified: true,
python_required: false,
sqlite_compiled: cfg!(feature = "sqlite"),
prebuilt_gpu_compiled: cfg!(feature = "prebuilt-gpu"),
jit_compiled: cfg!(feature = "jit"),
network_compiled: cfg!(feature = "network"),
runtime_build,
cuda: None,
};
}
};
let cuda = cuda_facts(ordinal);
DoctorFacts {
crate_version: env!("CARGO_PKG_VERSION").to_owned(),
oracle: crate::ORACLE.to_owned(),
registry_verified: true,
python_required: false,
sqlite_compiled: cfg!(feature = "sqlite"),
prebuilt_gpu_compiled: cfg!(feature = "prebuilt-gpu"),
jit_compiled: cfg!(feature = "jit"),
network_compiled: cfg!(feature = "network"),
runtime_build,
cuda: Some(cuda),
}
}
pub fn load(&self, family: &str) -> Result<Tokenizer> {
let explicit = self.inner.config.explicit_artifacts.get(family);
if explicit.is_none() {
self.inner.artifacts.ensure(family, &self.inner.registry)?;
}
let artifact = self.inner.registry.verify_local(
family,
&self.inner.config.artifact_cache,
explicit.map(PathBuf::as_path),
)?;
self.build_tokenizer(artifact)
}
pub fn load_local(&self, family: &str, directory: impl AsRef<Path>) -> Result<Tokenizer> {
let artifact = self.inner.registry.verify_local(
family,
&self.inner.config.artifact_cache,
Some(directory.as_ref()),
)?;
self.build_tokenizer(artifact)
}
pub fn family_for_sha256(&self, sha256_hex: &str) -> Option<String> {
self.inner
.registry
.family_for_tokenizer_sha256(sha256_hex)
.map(str::to_owned)
}
pub fn load_repository(&self, repo_id: &str, revision: &str) -> Result<Tokenizer> {
let family = self.family_for_repo(repo_id, revision)?;
self.load(&family)
}
pub fn from_pretrained(&self, repo_id: &str, revision: Revision) -> Result<Tokenizer> {
self.load_repository(repo_id, revision.as_str())
}
pub fn artifacts(&self) -> ArtifactManager {
self.inner.artifacts.clone()
}
pub(crate) fn family_for_repo(&self, repo_id: &str, revision: &str) -> Result<String> {
self.inner.registry.resolve_repo(repo_id, revision)
}
fn build_tokenizer(&self, artifact: LocalArtifact) -> Result<Tokenizer> {
let bytes = artifact.bytes();
let reference = Arc::new(ReferenceEngine::from_bytes(&bytes).map_err(|error| {
Error::new(ErrorCode::Internal, error.to_string())
.with_path(artifact.tokenizer_json())
.with_family(&artifact.identity().family)
})?);
let family = artifact.identity().family.clone();
let mut reasons = Vec::new();
let policy = self.inner.config.policy;
let runtime_build_certified = self.inner.registry.rust_api_build_certified();
let accelerated_build_allowed = runtime_build_certified || policy == Policy::Experimental;
let fast_cpu_build_allowed =
accelerated_build_allowed || self.inner.registry.rust_api_sources_judged();
if !fast_cpu_build_allowed && policy != Policy::Reference {
reasons.push(ReasonCode::RuntimeBuildUncertified);
}
let fast_cpu_route = if policy == Policy::Reference {
reasons.push(ReasonCode::ReferenceRequested);
None
} else if !fast_cpu_build_allowed {
None
} else {
self.build_fast_cpu(&artifact, Arc::clone(&reference), &mut reasons)?
};
let fast_cpu_certification = fast_cpu_route.as_ref().map(|route| {
if runtime_build_certified && route.toolchain_judged {
Certification::CertifiedSource
} else if crate::verify_local::is_locally_verified(&crate::verify_local::cpu_key(
&family,
&artifact.identity().tokenizer_sha256,
)) {
Certification::LocallyVerified
} else {
Certification::Supported
}
});
let fast_cpu = fast_cpu_route.map(|route| route.engine);
#[allow(unused_mut)]
let mut gpu = None;
#[allow(unused_mut)]
let mut gpu_detail = None;
#[allow(unused_mut)]
let mut gpu_assurance = None;
#[allow(unused_mut)]
let mut gpu_verification_key = None;
#[cfg(feature = "jit")]
let mut jit_detail = None;
let wants_gpu = !matches!(self.inner.config.device, Device::Cpu)
&& !matches!(self.inner.config.gpu_delivery, GpuDelivery::Disabled)
&& self.inner.config.policy != Policy::Reference;
let gpu_build = gpu_build_decision(
classify_build(&BuildReading::of(&self.inner.registry)),
policy,
);
#[allow(unused_variables)]
let cap_gpu_label = matches!(gpu_build, GpuBuildDecision::Admit { cap_label: true });
if let (
true,
GpuBuildDecision::Refuse {
reasons: why,
message,
},
) = (wants_gpu, &gpu_build)
{
if matches!(self.inner.config.device, Device::Cuda(_)) {
return Err(Error::new(ErrorCode::UncertifiedRuntime, message.clone()));
}
reasons.extend(why.iter().cloned());
} else if wants_gpu {
#[cfg(feature = "prebuilt-gpu")]
{
let ordinal = match self.inner.config.device {
Device::Cuda(value) => i32::try_from(value).map_err(|_| {
Error::new(ErrorCode::InvalidArgument, "CUDA ordinal exceeds i32")
})?,
Device::Auto => 0,
Device::Cpu => unreachable!(),
};
let built = if self.inner.config.gpu_delivery == GpuDelivery::Jit {
#[cfg(feature = "jit")]
{
let compiler = match &self.inner.config.jit_compiler {
Some(compiler) => compiler.clone(),
None => JitCompiler::builder().build()?,
};
compiler
.compile_product(
&family,
&self.inner.registry,
ordinal,
self.inner.config.policy,
)
.and_then(|product| {
let facts = product.facts.clone();
crate::gpu_data::build_jit(
&self.inner.registry,
&artifact,
Arc::clone(&reference),
ordinal,
self.inner.config.policy,
&product,
)
.map(|built| (built, Some(facts)))
})
}
#[cfg(not(feature = "jit"))]
{
Err(Error::new(
ErrorCode::KernelIncompatible,
"GpuDelivery::Jit requires the crate's `jit` feature",
))
}
} else {
let prebuilt = crate::gpu_data::build_prebuilt(
&self.inner.registry,
&artifact,
Arc::clone(&reference),
ordinal,
self.inner.config.policy,
);
#[cfg(feature = "jit")]
{
prebuilt.map(|built| (built, None))
}
#[cfg(not(feature = "jit"))]
{
prebuilt.map(|built| (built, ()))
}
};
match built {
Ok((built, _maybe_facts)) => {
let locally_verified =
if gpu_label_reads_local_record(built.assurance, cap_gpu_label) {
let key = crate::verify_local::gpu_key(
&built,
&family,
&artifact.identity().tokenizer_sha256,
);
let verified = crate::verify_local::is_locally_verified(&key);
gpu_verification_key = Some(key);
verified
} else {
false
};
gpu_assurance =
Some(gpu_label(built.assurance, cap_gpu_label, locally_verified));
gpu_detail = Some((built.architecture, built.driver_api_version));
gpu = Some(built.engine);
#[cfg(feature = "jit")]
{
jit_detail = _maybe_facts;
}
}
Err(error) if matches!(self.inner.config.device, Device::Auto) => {
reasons.push(
if matches!(
error.code(),
ErrorCode::UncertifiedTokenizer | ErrorCode::UncertifiedRuntime
) {
ReasonCode::GpuUncertified
} else {
ReasonCode::GpuUnavailable
},
);
}
Err(error) => return Err(error),
}
}
#[cfg(not(feature = "prebuilt-gpu"))]
{
if matches!(self.inner.config.device, Device::Cuda(_)) {
return Err(Error::new(
ErrorCode::KernelIncompatible,
"the crate was built without the prebuilt-gpu feature",
));
}
reasons.push(ReasonCode::GpuUnavailable);
}
}
let mut chain = Vec::new();
let mut thresholds = Vec::new();
let mut backends = Vec::new();
if gpu.is_some() {
chain.push("gpu".to_owned());
thresholds.push(self.inner.config.gpu_min_bytes);
backends.push(Backend::Gpu);
}
if fast_cpu.is_some() {
chain.push("fast_cpu".to_owned());
thresholds.push(0);
backends.push(Backend::FastCpu);
}
chain.push("hf".to_owned());
thresholds.push(0);
backends.push(Backend::HuggingFace);
if self.inner.config.policy == Policy::RequireAccelerated
&& fast_cpu.is_none()
&& gpu.is_none()
{
return Err(Error::new(
ErrorCode::UncertifiedTokenizer,
format!("family {family} has no certified accelerated route on this runtime"),
));
}
let special_tokens = special_token_plan(&bytes, &reference);
let postprocessor_adds_tokens = special_tokens.kind != PostprocessorKind::Identity;
let repair_fast_cpu = fast_cpu.is_some();
let router = Arc::new(
NativeRouter::new(
chain,
thresholds,
Arc::clone(&reference),
fast_cpu.clone(),
repair_fast_cpu,
gpu.map(toktier_routing_core::NativeGpuSource::Open),
postprocessor_adds_tokens,
self.inner.config.diagnostics,
)
.map_err(|error| Error::new(ErrorCode::ConfigInvalid, error.to_string()))?,
);
let fingerprint = semantic_fingerprint(
artifact.identity().tokenizer_sha256.as_bytes(),
repair_fast_cpu,
);
let literal_guard = reference
.added_tokens()
.iter()
.map(|(_, content, _)| content.chars().count())
.max()
.unwrap_or(0);
let fast_guard = fast_cpu
.as_ref()
.map(|engine| engine.minimum_seal_tail_chars())
.unwrap_or(0);
let seal_guard = u64::try_from(literal_guard.max(fast_guard))
.map_err(|_| Error::new(ErrorCode::ConfigInvalid, "session seal guard exceeds u64"))?;
let store_path = if self.inner.config.persistent_sessions {
let home = self
.inner
.config
.home
.as_ref()
.expect("builder validated persistent home");
Some(home.join("sessions").join(format!("{family}.sqlite3")))
} else {
None
};
let store = SessionStoreState::new(
Arc::clone(&router),
fingerprint,
seal_guard,
store_path.as_deref(),
self.inner.config.seed_digest_overlap,
)?;
let plan = RoutePlan {
family: family.clone(),
artifact_sha256: artifact.identity().tokenizer_sha256.clone(),
backends,
gpu_min_bytes: self.inner.config.gpu_min_bytes,
certification: if self.inner.config.policy == Policy::Experimental {
Certification::Experimental
} else if self.inner.config.policy == Policy::Reference {
Certification::Reference
} else {
match (gpu_assurance, fast_cpu_certification) {
(Some(gpu), Some(cpu)) if label_rank(cpu) < label_rank(gpu) => cpu,
(Some(gpu), _) => gpu,
(None, Some(cpu)) => cpu,
(None, None) => Certification::Reference,
}
},
reasons,
postprocessor_class: special_tokens.kind.word().to_owned(),
special_token_prefix: special_tokens.kind.prefix().to_vec(),
postprocessor_self_check: special_tokens.self_check,
literal_segmentation: {
let facts = router.literal_segmentation();
(facts["served"] != true).then(|| {
facts["reason"]
.as_str()
.unwrap_or("the artifact is outside the served class")
.to_owned()
})
},
};
Ok(Tokenizer {
inner: Arc::new(TokenizerInner {
artifact,
reference,
router,
plan,
policy: self.inner.config.policy,
registry: Arc::clone(&self.inner.registry),
chunked: crate::chunked::State::default(),
store: std::sync::Mutex::new(store),
gpu_detail,
gpu_verification_key,
#[cfg(feature = "jit")]
jit_detail,
}),
})
}
fn build_fast_cpu(
&self,
artifact: &LocalArtifact,
reference: Arc<ReferenceEngine>,
reasons: &mut Vec<ReasonCode>,
) -> Result<Option<FastCpuRoute>> {
let family = &artifact.identity().family;
if let Some(examined) = self.inner.registry.examined(family) {
return self.build_examined_fast_cpu(artifact, reference, reasons, examined);
}
let Some(repair) = self.inner.registry.repair(family) else {
reasons.push(ReasonCode::CpuUncertified);
return Ok(None);
};
let support = self.inner.registry.support(family).ok_or_else(|| {
Error::new(
ErrorCode::RegistryInvalid,
format!("repair family {family} has no support-registry row"),
)
})?;
let fast_support = support.backends.get("fast_cpu");
let fast_status = fast_support
.and_then(|value| value.get("status"))
.and_then(serde_json::Value::as_str);
let fast_source = fast_support
.and_then(|value| value.get("source_digest"))
.and_then(serde_json::Value::as_str);
let fast_toolchain = fast_support
.and_then(|value| value.get("toolchain"))
.and_then(serde_json::Value::as_str);
let source_bound = fast_source == Some(env!("TOKTIER_RUST_API_FAST_CPU_SOURCE_SHA256"))
|| self.inner.config.policy == Policy::Experimental;
if !matches!(fast_status, Some("certified") | Some("certified_source"))
|| support.artifact_sha256 != artifact.identity().tokenizer_sha256
|| repair.artifact_sha256 != artifact.identity().tokenizer_sha256
|| !source_bound
{
reasons.push(ReasonCode::CpuUncertified);
return Ok(None);
}
let toolchain_judged = fast_toolchain == Some(env!("TOKTIER_RUST_API_TOOLCHAIN"));
let spec = FastRepairSpec::new(
family.clone(),
repair.artifact_sha256.clone(),
repair.margin,
repair.effective_l_max,
repair.has_normalizer,
);
let engine = FastCpuEngine::from_reference(
&artifact.bytes(),
reference,
spec,
self.inner.registry.pclass(),
)
.map_err(|error| {
Error::new(
ErrorCode::BackendExecutionFault,
format!("certified corrected-CPU construction failed: {error}"),
)
})?;
Ok(Some(FastCpuRoute {
engine: Arc::new(engine),
toolchain_judged,
}))
}
}
impl Runtime {
fn build_examined_fast_cpu(
&self,
artifact: &LocalArtifact,
reference: Arc<ReferenceEngine>,
reasons: &mut Vec<ReasonCode>,
examined: &crate::manifest::ExaminedRow,
) -> Result<Option<FastCpuRoute>> {
let family = &artifact.identity().family;
reasons.push(ReasonCode::ExaminedArtifact);
if self.inner.config.policy == Policy::Certified
|| examined.artifact_sha256 != artifact.identity().tokenizer_sha256
{
reasons.push(ReasonCode::CpuUncertified);
return Ok(None);
}
let bound = self
.inner
.registry
.pipeline_peers(&examined.pipeline_id)
.into_iter()
.any(|peer| {
let fast = peer.backends.get("fast_cpu");
let status = fast
.and_then(|value| value.get("status"))
.and_then(serde_json::Value::as_str);
let source = fast
.and_then(|value| value.get("source_digest"))
.and_then(serde_json::Value::as_str);
matches!(status, Some("certified") | Some("certified_source"))
&& (source == Some(env!("TOKTIER_RUST_API_FAST_CPU_SOURCE_SHA256"))
|| self.inner.config.policy == Policy::Experimental)
});
if !bound {
reasons.push(ReasonCode::CpuUncertified);
return Ok(None);
}
let spec = FastRepairSpec::new(
family.clone(),
examined.artifact_sha256.clone(),
examined.repair.margin,
examined.repair.effective_l_max,
examined.repair.has_normalizer,
);
let engine = FastCpuEngine::from_reference(
&artifact.bytes(),
reference,
spec,
self.inner.registry.pclass(),
)
.map_err(|error| {
Error::new(
ErrorCode::BackendExecutionFault,
format!("corrected-CPU construction failed for an examined artifact: {error}"),
)
})?;
Ok(Some(FastCpuRoute {
engine: Arc::new(engine),
toolchain_judged: false,
}))
}
}
struct FastCpuRoute {
engine: Arc<FastCpuEngine>,
toolchain_judged: bool,
}
#[cfg(feature = "prebuilt-gpu")]
fn cuda_facts(ordinal: u32) -> CudaFacts {
match i32::try_from(ordinal) {
Ok(ordinal_i32) => match toktier_cuda_driver::CudaContext::new(ordinal_i32) {
Ok(context) => {
let (major, minor) = context.architecture();
CudaFacts {
device_ordinal: ordinal,
available: true,
architecture: Some(format!("sm_{major}{minor}")),
driver_api_version: Some(context.driver_version()),
error: None,
}
}
Err(error) => CudaFacts {
device_ordinal: ordinal,
available: false,
architecture: None,
driver_api_version: None,
error: Some(error.to_string()),
},
},
Err(_) => CudaFacts {
device_ordinal: ordinal,
available: false,
architecture: None,
driver_api_version: None,
error: Some("CUDA ordinal exceeds i32".to_owned()),
},
}
}
#[cfg(not(feature = "prebuilt-gpu"))]
fn cuda_facts(ordinal: u32) -> CudaFacts {
CudaFacts {
device_ordinal: ordinal,
available: false,
architecture: None,
driver_api_version: None,
error: Some("CUDA support was not compiled into this feature set".to_owned()),
}
}
#[derive(Debug, Clone)]
pub struct Tokenizer {
pub(crate) inner: Arc<TokenizerInner>,
}
impl Tokenizer {
pub(crate) fn verification_key(&self) -> Option<&crate::verify_local::VerificationKey> {
self.inner.gpu_verification_key.as_ref()
}
pub fn family(&self) -> &str {
&self.inner.artifact.identity().family
}
pub fn artifact(&self) -> &LocalArtifact {
&self.inner.artifact
}
pub fn plan(&self) -> &RoutePlan {
&self.inner.plan
}
pub fn chunked(&self) -> crate::chunked::Chunked<'_> {
crate::chunked::Chunked::new(&self.inner)
}
pub fn store_stats(&self) -> StoreStats {
self.inner
.store
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.store
.stats()
.into()
}
pub fn gpu_facts(&self) -> Option<(&str, i32)> {
self.inner
.gpu_detail
.as_ref()
.map(|(architecture, version)| (architecture.as_str(), *version))
}
#[cfg(feature = "jit")]
pub fn jit_facts(&self) -> Option<&crate::JitArtifact> {
self.inner.jit_detail.as_ref()
}
pub fn encode(&self, text: &str) -> Result<Encoding> {
self.encode_with(text, EncodeOptions::default())
}
pub fn encode_with(&self, text: &str, options: EncodeOptions) -> Result<Encoding> {
if options.offsets && options.add_special_tokens {
return Err(Error::new(
ErrorCode::InvalidArgument,
"offsets with post-processor special tokens are not exposed by Rust API v1",
));
}
let (router_flag, prefix) = self.special_token_service(options.add_special_tokens);
let mut routed = self
.inner
.router
.encode_ids(text, router_flag)
.map_err(|error| Error::new(ErrorCode::BackendExecutionFault, error.to_string()))?;
if let Some(prefix) = prefix {
routed.ids = prepend(prefix, routed.ids);
}
let offsets = options
.offsets
.then(|| {
self.inner
.reference
.spans_soa_for_ids(text, &routed.ids)
.and_then(|spans| {
self.inner
.reference
.require_normalization_stable(text)
.map(|()| spans)
})
.map(|(starts, ends)| starts.into_iter().zip(ends).collect::<Vec<_>>())
})
.transpose()
.map_err(|error| Error::new(ErrorCode::BackendExecutionFault, error.to_string()))?;
let facts = facts_from_routed(
&routed.backend,
routed.path.as_deref(),
routed.source,
text.len(),
self.inner.plan.certification,
routed.reason,
);
Ok(Encoding::new(routed.ids, offsets, facts))
}
pub fn encode_batch<S: AsRef<str>>(&self, documents: &[S]) -> Result<RaggedEncoding> {
self.encode_batch_with(documents, EncodeOptions::default())
}
pub fn encode_batch_with<S: AsRef<str>>(
&self,
documents: &[S],
options: EncodeOptions,
) -> Result<RaggedEncoding> {
if options.offsets {
return Err(Error::new(
ErrorCode::InvalidArgument,
"batch offsets are not part of the ragged ID-only API",
));
}
let texts = documents.iter().map(AsRef::as_ref).collect::<Vec<_>>();
let (router_flag, prefix) = self.special_token_service(options.add_special_tokens);
let routed = self
.inner
.router
.encode_batch_ids(&texts, router_flag)
.map_err(|error| Error::new(ErrorCode::BackendExecutionFault, error.to_string()))?;
let rows = routed
.into_iter()
.zip(texts)
.map(|(row, text)| {
let facts = facts_from_routed(
&row.backend,
row.path.as_deref(),
row.source,
text.len(),
self.inner.plan.certification,
row.reason,
);
let ids = match prefix {
Some(prefix) => prepend(prefix, row.ids),
None => row.ids,
};
(ids, facts)
})
.collect();
RaggedEncoding::from_rows(rows)
}
fn special_token_service(&self, add_special_tokens: bool) -> (bool, Option<&[u32]>) {
if !add_special_tokens {
return (false, None);
}
match self.inner.plan.postprocessor_class.as_str() {
CLASS_IDENTITY => (false, None),
CLASS_PREFIX => (false, Some(self.inner.plan.special_token_prefix.as_slice())),
_ => (true, None),
}
}
pub fn decode(&self, ids: impl AsRef<[u32]>, options: DecodeOptions) -> Result<String> {
self.inner
.reference
.decode(ids.as_ref(), options.skip_special_tokens)
.map_err(|error| Error::new(ErrorCode::BackendExecutionFault, error.to_string()))
}
pub fn open_session(&self, name: impl Into<String>) -> Result<Session> {
Session::open(Arc::clone(&self.inner), name.into())
}
pub fn lookup(&self, text: &str) -> Result<Option<Encoding>> {
let mut state = self
.inner
.store
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.ensure_healthy()?;
if !state.lookup_has_a_slot() {
return Ok(None);
}
let key = state.key;
let Some(hit) = state.store.lookup(key, text, self.inner.router.as_ref())? else {
return Ok(None);
};
let matched_chars = usize::try_from(hit.matched_chars).map_err(|_| {
Error::new(
ErrorCode::Internal,
"content-prefix length does not fit this platform",
)
})?;
let total_chars = text.chars().count();
if matched_chars > total_chars {
return Err(Error::new(
ErrorCode::StoreCorrupt,
"content-prefix lookup returned a length beyond the query",
));
}
let suffix_byte = if matched_chars == total_chars {
text.len()
} else {
text.char_indices()
.nth(matched_chars)
.map(|(offset, _)| offset)
.ok_or_else(|| {
Error::new(
ErrorCode::StoreCorrupt,
"content-prefix lookup returned a non-existent character boundary",
)
})?
};
if suffix_byte < text.len() {
state.store.append_patch(
hit.handle,
&text[suffix_byte..],
hit.revision,
self.inner.router.as_ref(),
)?;
}
let ids = state.store.shared_all_ids(hit.handle)?;
state.store.evict(hit.handle);
let facts = ExecutionFacts {
backend: Backend::FastCpu,
path: "content_lookup".to_owned(),
source: Some("native_store".to_owned()),
input_bytes: text.len() as u64,
certification: self.inner.plan.certification,
reason: None,
};
Ok(Some(Encoding::from_buffer(
TokenBuffer::from_shared(ids),
facts,
)))
}
pub fn shutdown(self) {}
}
pub(crate) fn facts_from_routed(
backend: &str,
path: Option<&str>,
source: Option<String>,
input_bytes: usize,
certification: Certification,
reason: Option<&str>,
) -> ExecutionFacts {
ExecutionFacts {
backend: Backend::from_internal(backend),
path: path.unwrap_or("unknown").to_owned(),
source,
input_bytes: input_bytes as u64,
certification,
reason: reason.map(ReasonCode::from_ledger_code),
}
}
fn label_rank(label: Certification) -> u8 {
match label {
Certification::Certified | Certification::CertifiedSource => 4,
Certification::LocallyVerified => 3,
Certification::Supported => 2,
Certification::Experimental => 1,
_ => 0,
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct BuildReading {
recipe_certified: bool,
sources_judged: bool,
core_closure: String,
reference_reason: Option<String>,
toolchain_divergence: Option<String>,
flag_divergence: Option<String>,
}
impl BuildReading {
fn of(registry: &Registry) -> Self {
Self {
recipe_certified: registry.rust_api_build_certified(),
sources_judged: registry.rust_api_build_standing().source_identity_judged,
core_closure: crate::behavior_version::core_closure().to_owned(),
reference_reason: crate::behavior_version::reference_certification_reason()
.map(str::to_owned),
toolchain_divergence: registry.rust_api_toolchain_divergence(),
flag_divergence: registry.rust_api_build_flag_divergence(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum BuildState {
JudgedRecipe,
OtherToolchain { divergence: String },
OtherFlags { divergence: String },
UncertifiedReference,
UnlocatedClosure { closure: String },
ClosureMismatched { closure: String },
SourcesUnjudged,
}
fn classify_build(reading: &BuildReading) -> BuildState {
if reading.recipe_certified {
return BuildState::JudgedRecipe;
}
if !reading.sources_judged {
return BuildState::SourcesUnjudged;
}
if reading.core_closure.starts_with("unlocated") {
return BuildState::UnlocatedClosure {
closure: reading.core_closure.clone(),
};
}
if reading.core_closure != "verified" {
return BuildState::ClosureMismatched {
closure: reading.core_closure.clone(),
};
}
if reading.reference_reason.is_some() {
return BuildState::UncertifiedReference;
}
match (&reading.flag_divergence, &reading.toolchain_divergence) {
(Some(flags), _) => BuildState::OtherFlags {
divergence: flags.clone(),
},
(None, Some(toolchain)) => BuildState::OtherToolchain {
divergence: toolchain.clone(),
},
(None, None) => BuildState::OtherFlags {
divergence: format!(
"no judged row records the toolchain `{}` together with the build flags of \
this build",
env!("TOKTIER_RUST_API_TOOLCHAIN")
),
},
}
}
#[cfg(feature = "prebuilt-gpu")]
fn gpu_label_reads_local_record(assurance: crate::gpu_data::Assurance, cap_label: bool) -> bool {
use crate::gpu_data::Assurance;
match assurance {
Assurance::Judged => cap_label,
Assurance::Unjudged => true,
Assurance::Experimental => false,
}
}
#[cfg(feature = "prebuilt-gpu")]
fn gpu_label(
assurance: crate::gpu_data::Assurance,
cap_label: bool,
locally_verified: bool,
) -> Certification {
use crate::gpu_data::Assurance;
match assurance {
Assurance::Judged if !cap_label => Certification::Certified,
Assurance::Experimental => Certification::Experimental,
Assurance::Judged | Assurance::Unjudged => {
if locally_verified {
Certification::LocallyVerified
} else {
Certification::Supported
}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum GpuBuildDecision {
Admit { cap_label: bool },
Refuse {
reasons: Vec<ReasonCode>,
message: String,
},
}
fn gpu_build_decision(state: BuildState, policy: Policy) -> GpuBuildDecision {
let digests = format!(
"rust_api {}, fast_cpu {}, native_host {}",
env!("TOKTIER_RUST_API_SOURCE_SHA256"),
env!("TOKTIER_RUST_API_FAST_CPU_SOURCE_SHA256"),
env!("TOKTIER_RUST_API_NATIVE_HOST_SOURCE_SHA256"),
);
let refuse = |reasons: Vec<ReasonCode>, why: String| GpuBuildDecision::Refuse {
reasons,
message: format!(
"the GPU route is not admitted for this Rust build: {why}. Runtime::doctor() \
reports the build's standing; a registered release build, or an explicit \
Policy::Experimental, runs it"
),
};
match state {
BuildState::JudgedRecipe => GpuBuildDecision::Admit { cap_label: false },
_ if policy == Policy::Experimental => GpuBuildDecision::Admit { cap_label: false },
BuildState::OtherToolchain { .. } | BuildState::OtherFlags { .. }
if policy.admits_unjudged_device() =>
{
GpuBuildDecision::Admit { cap_label: true }
}
BuildState::OtherToolchain { divergence } | BuildState::OtherFlags { divergence } => {
refuse(
vec![
ReasonCode::GpuUncertified,
ReasonCode::GpuBuildRecipeUnjudged,
],
format!(
"its sources are judged and its recipe is not ({divergence}); policy \
{policy:?} admits the GPU route only on the judged recipe, \
Policy::Supported runs it labelled supported"
),
)
}
BuildState::UncertifiedReference => refuse(
vec![ReasonCode::GpuUncertified],
"a reference library of this build is outside its certified versions \
(uncertified_reference)"
.to_owned(),
),
BuildState::UnlocatedClosure { closure } => refuse(
vec![ReasonCode::GpuUncertified],
format!("its dependency closure could not be compared ({closure})"),
),
BuildState::ClosureMismatched { closure } => refuse(
vec![ReasonCode::GpuUncertified],
format!("a sibling crate is not the judged copy ({closure})"),
),
BuildState::SourcesUnjudged => refuse(
vec![ReasonCode::GpuUncertified],
format!(
"no judged row of the shipped runtime-build registry records its source \
identities ({digests})"
),
),
}
}
const CLASS_IDENTITY: &str = "identity";
const CLASS_PREFIX: &str = "prefix";
const CLASS_REFERENCE: &str = "reference";
#[derive(Debug, Clone, PartialEq, Eq)]
enum PostprocessorKind {
Identity,
Prefix(Vec<u32>),
Reference,
}
impl PostprocessorKind {
fn word(&self) -> &'static str {
match self {
Self::Identity => CLASS_IDENTITY,
Self::Prefix(_) => CLASS_PREFIX,
Self::Reference => CLASS_REFERENCE,
}
}
fn prefix(&self) -> &[u32] {
match self {
Self::Prefix(ids) => ids,
Self::Identity | Self::Reference => &[],
}
}
}
struct SpecialTokenPlan {
kind: PostprocessorKind,
self_check: Option<String>,
}
fn special_token_plan(bytes: &[u8], reference: &ReferenceEngine) -> SpecialTokenPlan {
let document = serde_json::from_slice::<serde_json::Value>(bytes).ok();
let kind = document
.as_ref()
.map_or(PostprocessorKind::Reference, classify_postprocessor);
if kind == PostprocessorKind::Reference {
return SpecialTokenPlan {
kind,
self_check: None,
};
}
let first_added = document.as_ref().and_then(first_added_literal);
let mut probes = vec![
String::new(),
"a".to_owned(),
"hello world".to_owned(),
"caf\u{e9} \u{4e2d}\u{6587} \u{1f600} na\u{ef}ve".to_owned(),
];
if let Some(literal) = first_added {
probes.push(literal.clone());
probes.push(format!("a {literal} b"));
}
for (index, probe) in probes.iter().enumerate() {
let agrees = match (
reference.encode_ids(probe, true),
reference.encode_ids(probe, false),
) {
(Ok(with), Ok(core)) => with == prepend(kind.prefix(), core),
_ => false,
};
if !agrees {
return SpecialTokenPlan {
kind: PostprocessorKind::Reference,
self_check: Some(format!(
"probe {index} ({} bytes) answered otherwise than the {} class on the \
reference engine",
probe.len(),
kind.word()
)),
};
}
}
SpecialTokenPlan {
kind,
self_check: None,
}
}
fn first_added_literal(document: &serde_json::Value) -> Option<String> {
document
.get("added_tokens")?
.as_array()?
.iter()
.filter_map(|row| row.get("content").and_then(serde_json::Value::as_str))
.find(|content| !content.is_empty())
.map(str::to_owned)
}
fn classify_postprocessor(document: &serde_json::Value) -> PostprocessorKind {
let absent = |key: &str| document.get(key).is_none_or(serde_json::Value::is_null);
if !absent("truncation") || !absent("padding") {
return PostprocessorKind::Reference;
}
match document.get("post_processor") {
None | Some(serde_json::Value::Null) => PostprocessorKind::Identity,
Some(processor) => match processor_type(processor) {
Some("Sequence") => {
let Some(members) = processor
.get("processors")
.and_then(serde_json::Value::as_array)
else {
return PostprocessorKind::Reference;
};
let mut template = None;
for member in members {
match processor_type(member) {
Some("ByteLevel") => {}
Some("TemplateProcessing") if template.is_none() => {
template = Some(classify_template(member));
}
_ => return PostprocessorKind::Reference,
}
}
template.unwrap_or(PostprocessorKind::Identity)
}
Some("ByteLevel") => PostprocessorKind::Identity,
Some("TemplateProcessing") => classify_template(processor),
_ => PostprocessorKind::Reference,
},
}
}
fn processor_type(processor: &serde_json::Value) -> Option<&str> {
processor.get("type").and_then(serde_json::Value::as_str)
}
fn classify_template(template: &serde_json::Value) -> PostprocessorKind {
let Some(single) = template.get("single").and_then(serde_json::Value::as_array) else {
return PostprocessorKind::Reference;
};
match single.as_slice() {
[sequence] if template_piece(sequence, "Sequence") == Some("A") => {
PostprocessorKind::Identity
}
[special, sequence] if template_piece(sequence, "Sequence") == Some("A") => {
let Some(key) = template_piece(special, "SpecialToken") else {
return PostprocessorKind::Reference;
};
let ids = template
.get("special_tokens")
.and_then(|tokens| tokens.get(key))
.filter(|entry| entry.get("id").and_then(serde_json::Value::as_str) == Some(key))
.and_then(|entry| entry.get("ids"))
.and_then(serde_json::Value::as_array);
match ids.map(Vec::as_slice) {
Some([id]) => id
.as_u64()
.and_then(|id| u32::try_from(id).ok())
.map_or(PostprocessorKind::Reference, |id| {
PostprocessorKind::Prefix(vec![id])
}),
_ => PostprocessorKind::Reference,
}
}
_ => PostprocessorKind::Reference,
}
}
fn template_piece<'a>(piece: &'a serde_json::Value, kind: &str) -> Option<&'a str> {
let object = piece.as_object().filter(|object| object.len() == 1)?;
let inner = object.get(kind)?;
if inner.get("type_id").and_then(serde_json::Value::as_u64) != Some(0) {
return None;
}
inner.get("id").and_then(serde_json::Value::as_str)
}
fn prepend(prefix: &[u32], ids: Vec<u32>) -> Vec<u32> {
if prefix.is_empty() {
return ids;
}
let mut out = Vec::with_capacity(prefix.len() + ids.len());
out.extend_from_slice(prefix);
out.extend(ids);
out
}
pub(crate) fn semantic_fingerprint(artifact_sha256: &[u8], fast_cpu: bool) -> [u8; 32] {
semantic_fingerprint_for(
artifact_sha256,
fast_cpu,
env!("TOKTIER_RUST_API_SOURCE_SHA256"),
)
}
pub(crate) fn semantic_fingerprint_for(
artifact_sha256: &[u8],
fast_cpu: bool,
source_identity: &str,
) -> [u8; 32] {
let mut digest = Sha256::new();
digest.update(b"toktier.rust.fingerprint.v1\0");
for component in [
artifact_sha256,
b"tokenizers" as &[u8],
b"0.22.2",
b"toktier-rust-serving-v2",
source_identity.as_bytes(),
if fast_cpu { b"fast_cpu" } else { b"hf" },
if fast_cpu {
b"toktier-fast-repair-v1" as &[u8]
} else {
b""
},
] {
digest.update((component.len() as u32).to_le_bytes());
digest.update(component);
}
digest.finalize().into()
}
#[allow(dead_code)]
fn _assert_send_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<Runtime>();
assert_send_sync::<Tokenizer>();
assert_send_sync::<TokenBuffer>();
}
#[cfg(test)]
mod tests {
use super::{
classify_build, classify_postprocessor, gpu_build_decision, label_rank, special_token_plan,
BuildReading, BuildState, GpuBuildDecision, PostprocessorKind,
};
use crate::{Certification, Policy, ReasonCode};
use serde_json::json;
use toktier_routing_core::ReferenceEngine;
#[cfg(feature = "prebuilt-gpu")]
#[test]
fn the_gpu_label_cap_holds_in_every_cell() {
use super::{gpu_label, gpu_label_reads_local_record};
use crate::gpu_data::Assurance;
use Assurance::{Experimental as X, Judged as J, Unjudged as U};
use Certification::{Certified, Experimental, LocallyVerified as Local, Supported};
let cases = [
(J, false, false, Certified, false),
(J, false, true, Certified, false),
(J, true, false, Supported, true),
(J, true, true, Local, true),
(U, false, false, Supported, true),
(U, false, true, Local, true),
(U, true, false, Supported, true),
(U, true, true, Local, true),
(X, false, false, Experimental, false),
(X, true, true, Experimental, false),
];
for (assurance, capped, verified, label, reads) in cases {
assert_eq!(
gpu_label(assurance, capped, verified),
label,
"{assurance:?} capped={capped} verified={verified}"
);
assert_eq!(
gpu_label_reads_local_record(assurance, capped),
reads,
"{assurance:?} capped={capped}"
);
}
for assurance in [
Assurance::Judged,
Assurance::Unjudged,
Assurance::Experimental,
] {
for verified in [false, true] {
assert_ne!(
gpu_label(assurance, true, verified),
Certification::Certified
);
}
}
}
const POLICIES: [Policy; 5] = [
Policy::Supported,
Policy::Certified,
Policy::Reference,
Policy::RequireAccelerated,
Policy::Experimental,
];
fn judged() -> BuildReading {
BuildReading {
recipe_certified: true,
sources_judged: true,
core_closure: "verified".to_owned(),
reference_reason: None,
toolchain_divergence: None,
flag_divergence: None,
}
}
fn moved(change: impl FnOnce(&mut BuildReading)) -> BuildReading {
let mut reading = judged();
reading.recipe_certified = false;
change(&mut reading);
reading
}
fn admitted(decision: &GpuBuildDecision) -> Option<bool> {
match decision {
GpuBuildDecision::Admit { cap_label } => Some(*cap_label),
GpuBuildDecision::Refuse { .. } => None,
}
}
#[test]
fn the_judged_recipe_admits_the_gpu_route_as_before() {
assert_eq!(classify_build(&judged()), BuildState::JudgedRecipe);
for policy in POLICIES {
assert_eq!(
admitted(&gpu_build_decision(BuildState::JudgedRecipe, policy)),
Some(false),
"{policy:?}"
);
}
}
fn assert_widened(state: BuildState) {
for policy in POLICIES {
let decision = gpu_build_decision(state.clone(), policy);
match policy {
Policy::Supported | Policy::RequireAccelerated => {
assert_eq!(admitted(&decision), Some(true), "{state:?} {policy:?}");
}
Policy::Experimental => {
assert_eq!(admitted(&decision), Some(false), "{state:?} {policy:?}");
}
_ => {
let GpuBuildDecision::Refuse { reasons, message } = decision else {
panic!("{state:?} admitted under {policy:?}");
};
assert_eq!(
reasons,
vec![
ReasonCode::GpuUncertified,
ReasonCode::GpuBuildRecipeUnjudged
]
);
assert!(message.contains("recipe is not"), "{message}");
assert!(message.contains("doctor()"), "{message}");
}
}
}
}
fn assert_not_widened(state: BuildState, axis: &str) {
for policy in POLICIES {
let decision = gpu_build_decision(state.clone(), policy);
if policy == Policy::Experimental {
assert_eq!(admitted(&decision), Some(false), "{state:?}");
continue;
}
let GpuBuildDecision::Refuse { reasons, message } = decision else {
panic!("{state:?} admitted under {policy:?}");
};
assert_eq!(reasons, vec![ReasonCode::GpuUncertified], "{state:?}");
assert!(message.contains(axis), "{state:?}: {message}");
assert!(message.contains("doctor()"), "{message}");
}
}
#[test]
fn state_a_another_toolchain_is_admitted_capped_outside_certified() {
let reading = moved(|reading| {
reading.toolchain_divergence = Some("rustc 1.94.0 is not judged".to_owned());
});
let state = classify_build(&reading);
assert!(
matches!(state, BuildState::OtherToolchain { .. }),
"{state:?}"
);
assert_widened(state);
}
#[test]
fn state_b_other_flags_are_admitted_capped_outside_certified() {
for toolchain in [None, Some("rustc 1.94.0 is not judged".to_owned())] {
let reading = moved(|reading| {
reading.flag_divergence = Some("profile=dev is not judged".to_owned());
reading.toolchain_divergence = toolchain;
});
let state = classify_build(&reading);
assert!(matches!(state, BuildState::OtherFlags { .. }), "{state:?}");
assert_widened(state);
}
let state = classify_build(&moved(|_| {}));
assert!(matches!(state, BuildState::OtherFlags { .. }), "{state:?}");
assert_widened(state);
}
#[test]
fn state_c_uncertified_reference_is_not_widened() {
for flags in [None, Some("profile=dev".to_owned())] {
let reading = moved(|reading| {
reading.reference_reason = Some("uncertified_reference".to_owned());
reading.flag_divergence = flags;
});
let state = classify_build(&reading);
assert_eq!(state, BuildState::UncertifiedReference);
assert_not_widened(state, "uncertified_reference");
}
}
#[test]
fn state_d_unlocated_closure_is_not_widened() {
let reading = moved(|reading| {
reading.core_closure = "unlocated: no governing lockfile".to_owned();
});
let state = classify_build(&reading);
assert!(
matches!(state, BuildState::UnlocatedClosure { .. }),
"{state:?}"
);
assert_not_widened(state, "could not be compared");
}
#[test]
fn unjudged_sources_and_a_mismatched_closure_stay_refused() {
let state = classify_build(&moved(|reading| {
reading.core_closure = "mismatched: toktier-store-core 0.0.1".to_owned();
}));
assert!(
matches!(state, BuildState::ClosureMismatched { .. }),
"{state:?}"
);
assert_not_widened(state, "judged copy");
let state = classify_build(&moved(|reading| reading.sources_judged = false));
assert_eq!(state, BuildState::SourcesUnjudged);
assert_not_widened(state, env!("TOKTIER_RUST_API_SOURCE_SHA256"));
assert_not_widened(
BuildState::SourcesUnjudged,
env!("TOKTIER_RUST_API_NATIVE_HOST_SOURCE_SHA256"),
);
}
#[test]
fn labels_are_ordered_from_certified_to_experimental() {
let order = [
Certification::Certified,
Certification::LocallyVerified,
Certification::Supported,
Certification::Experimental,
Certification::Reference,
];
for pair in order.windows(2) {
assert!(label_rank(pair[0]) > label_rank(pair[1]), "{pair:?}");
}
assert_eq!(
label_rank(Certification::Certified),
label_rank(Certification::CertifiedSource)
);
}
fn template(single: serde_json::Value, special: serde_json::Value) -> serde_json::Value {
json!({"type": "TemplateProcessing", "single": single, "pair": [], "special_tokens": special})
}
fn special(id: &str) -> serde_json::Value {
json!({"SpecialToken": {"id": id, "type_id": 0}})
}
fn sequence(id: &str) -> serde_json::Value {
json!({"Sequence": {"id": id, "type_id": 0}})
}
fn entry(key: &str, ids: serde_json::Value) -> serde_json::Value {
json!({key: {"id": key, "ids": ids, "tokens": [key]}})
}
fn classify(post_processor: serde_json::Value) -> PostprocessorKind {
classify_postprocessor(&json!({"post_processor": post_processor}))
}
#[test]
fn the_post_processor_classes_follow_the_template_shape() {
let bos = || entry("<s>", json!([1]));
assert_eq!(
classify(serde_json::Value::Null),
PostprocessorKind::Identity
);
assert_eq!(
classify_postprocessor(&json!({})),
PostprocessorKind::Identity
);
assert_eq!(
classify(json!({"type": "ByteLevel", "trim_offsets": false})),
PostprocessorKind::Identity
);
assert_eq!(
classify(template(json!([sequence("A")]), json!({}))),
PostprocessorKind::Identity
);
assert_eq!(
classify(template(json!([special("<s>"), sequence("A")]), bos())),
PostprocessorKind::Prefix(vec![1])
);
let llama = json!({"type": "Sequence", "processors": [
{"type": "ByteLevel"},
template(json!([special("<s>"), sequence("A")]), bos()),
]});
assert_eq!(classify(llama), PostprocessorKind::Prefix(vec![1]));
for refused in [
template(json!([sequence("A"), special("<s>")]), bos()),
template(
json!([special("<s>"), sequence("A"), special("<s>")]),
bos(),
),
template(
json!([special("<s>"), special("<s>"), sequence("A")]),
bos(),
),
template(json!([sequence("A"), sequence("A")]), json!({})),
template(json!([special("<s>")]), bos()),
template(json!([sequence("B")]), json!({})),
template(
json!([special("<s>"), sequence("A")]),
entry("<s>", json!([1, 2])),
),
template(
json!([special("<s>"), sequence("A")]),
entry("<s>", json!([])),
),
template(
json!([special("<s>"), sequence("A")]),
entry("<s>", json!([4_294_967_296_u64])),
),
template(
json!([special("<s>"), sequence("A")]),
entry("<bos>", json!([1])),
),
template(
json!([{"SpecialToken": {"id": "<s>", "type_id": 1}}, sequence("A")]),
bos(),
),
template(
json!([{"SpecialToken": {"id": "<s>", "type_id": 0}, "x": 1}, sequence("A")]),
bos(),
),
json!({"type": "BertProcessing", "sep": ["[SEP]", 102], "cls": ["[CLS]", 101]}),
json!({"type": "RobertaProcessing"}),
json!({"type": "SomethingLater"}),
json!({"no_type": true}),
json!({"type": "Sequence", "processors": [
template(json!([special("<s>"), sequence("A")]), bos()),
template(json!([sequence("A")]), json!({})),
]}),
json!({"type": "Sequence", "processors": [
{"type": "Sequence", "processors": [{"type": "ByteLevel"}]},
]}),
json!({"type": "Sequence", "processors": [{"type": "RobertaProcessing"}]}),
json!({"type": "Sequence"}),
] {
assert_eq!(
classify(refused.clone()),
PostprocessorKind::Reference,
"{refused}"
);
}
for key in ["truncation", "padding"] {
assert_eq!(
classify_postprocessor(&json!({"post_processor": null, key: {"max_length": 8}})),
PostprocessorKind::Reference,
"{key}"
);
}
}
#[test]
fn a_special_token_entry_naming_another_id_is_the_reference_s() {
let other = json!({"<s>": {"id": "<bos>", "ids": [0], "tokens": ["<s>"]}});
assert_eq!(
classify(template(json!([special("<s>"), sequence("A")]), other)),
PostprocessorKind::Reference
);
let same = json!({"<s>": {"id": "<s>", "ids": [0], "tokens": ["<s>"]}});
assert_eq!(
classify(template(json!([special("<s>"), sequence("A")]), same)),
PostprocessorKind::Prefix(vec![0])
);
let unnamed = json!({"<s>": {"ids": [0], "tokens": ["<s>"]}});
assert_eq!(
classify(template(json!([special("<s>"), sequence("A")]), unnamed)),
PostprocessorKind::Reference
);
}
#[test]
fn the_probe_literal_is_the_first_added_token_in_document_order() {
use super::first_added_literal;
let document = json!({"added_tokens": [
{"id": 9, "content": ""},
{"id": 7, "content": "<later_id_first_in_document>"},
{"id": 1, "content": "<lowest_id>"},
]});
assert_eq!(
first_added_literal(&document).as_deref(),
Some("<later_id_first_in_document>")
);
assert_eq!(first_added_literal(&json!({"added_tokens": []})), None);
assert_eq!(first_added_literal(&json!({})), None);
}
fn word_level(post_processor: serde_json::Value) -> Vec<u8> {
serde_json::to_vec(&json!({
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [{"id": 0, "content": "<s>", "single_word": false,
"lstrip": false, "rstrip": false, "normalized": false, "special": true}],
"normalizer": null,
"pre_tokenizer": {"type": "Whitespace"},
"post_processor": post_processor,
"decoder": null,
"model": {"type": "WordLevel", "vocab": {"<s>": 0, "a": 1, "b": 2, "[UNK]": 3},
"unk_token": "[UNK]"}
}))
.unwrap()
}
#[test]
fn the_construction_check_keeps_a_class_the_reference_agrees_with() {
let bytes = word_level(template(
json!([special("<s>"), sequence("A")]),
entry("<s>", json!([0])),
));
let reference = ReferenceEngine::from_bytes(&bytes).unwrap();
let plan = special_token_plan(&bytes, &reference);
assert_eq!(plan.kind, PostprocessorKind::Prefix(vec![0]));
assert_eq!(plan.self_check, None);
let bytes = word_level(serde_json::Value::Null);
let reference = ReferenceEngine::from_bytes(&bytes).unwrap();
let plan = special_token_plan(&bytes, &reference);
assert_eq!(plan.kind, PostprocessorKind::Identity);
assert_eq!(plan.self_check, None);
}
#[test]
fn the_construction_check_demotes_a_class_the_reference_contradicts() {
let executed = word_level(template(
json!([special("<s>"), sequence("A")]),
entry("<s>", json!([0])),
));
let reference = ReferenceEngine::from_bytes(&executed).unwrap();
let described = word_level(serde_json::Value::Null);
let plan = special_token_plan(&described, &reference);
assert_eq!(plan.kind, PostprocessorKind::Reference);
let named = plan.self_check.expect("the failing probe is named");
assert!(named.starts_with("probe 0 "), "{named}");
assert!(named.contains("identity class"), "{named}");
}
}