use std::collections::{BTreeMap, HashMap};
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::sync::{Arc, OnceLock};
use flate2::read::ZlibDecoder;
use serde::Deserialize;
use sha2::{Digest, Sha256};
use crate::{Error, ErrorCode, Result};
pub(crate) const ARTIFACT_MANIFEST_BYTES: &[u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/data/src/toktier/artifacts/tables/artifact_manifest.v1.json"
));
pub(crate) const ARTIFACT_CONVERSIONS_BYTES: &[u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/data/src/toktier/artifacts/tables/artifact_conversions.v1.json"
));
pub(crate) const SUPPORT_REGISTRY_BYTES: &[u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/data/src/toktier/routing/tables/support_registry.v1.json"
));
pub(crate) const CHUNKED_ADMISSION_BYTES: &[u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/data/src/toktier/routing/tables/chunked_admission.v1.json"
));
pub(crate) const EXAMINED_ARTIFACTS_BYTES: &[u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/data/src/toktier/routing/tables/examined_artifacts.v1.json"
));
pub(crate) const SIBLING_ALIASES_BYTES: &[u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/data/src/toktier/artifacts/tables/sibling_aliases.v1.json"
));
pub(crate) const REPAIR_MANIFEST_BYTES: &[u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/data/src/toktier/repair/tables/fast_repair_families.v1.json"
));
pub(crate) const REPAIR_PCLASS_COMPRESSED: &[u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/data/src/toktier/repair/tables/repair_pclass.v1.zlib"
));
pub(crate) const KERNEL_FAMILIES_BYTES: &[u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/data/src/toktier/kernels/tables/kernel_families.v1.json"
));
pub(crate) const PREBUILT_MANIFEST_BYTES: &[u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/data/src/toktier/kernels/prebuilt/build_manifest.json"
));
#[cfg(feature = "prebuilt-gpu")]
pub(crate) const PREBUILT_FATBIN: &[u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/data/src/toktier/kernels/prebuilt/pretok_kernel.fatbin"
));
const ARTIFACT_MANIFEST_SHA256: &str =
"9f2e2edbb6220b31c280503f97a1d9d5be0fb0b7070ebe527a729b198dc35776";
const ARTIFACT_CONVERSIONS_SHA256: &str =
"8fa33251a20e12ab8e35894732e49ea93fc2599f4e3425d1730d81bb1e9f2719";
const SUPPORT_REGISTRY_SHA256: &str = env!("TOKTIER_SUPPORT_REGISTRY_SHA256");
const CHUNKED_ADMISSION_SHA256: &str = env!("TOKTIER_CHUNKED_ADMISSION_SHA256");
const SIBLING_ALIASES_SHA256: &str =
"f61866d6af8fd5268a7c4304ae3a9db8d8d87dc973c156408ad06a4a58e8893d";
const REPAIR_MANIFEST_SHA256: &str =
"8801781427f98456a7dfce6d9cc8f8ddd8dd4ec31c1ff37b71913d83182f3853";
const KERNEL_FAMILIES_SHA256: &str =
"a792fc914e2c8103da0d5e31eb55b4b4cb5163c491924f3c4a648009c41a4628";
const EXAMINED_ARTIFACTS_SHA256: &str =
"c218cf0e73c67dc02854bd88295acf329d4e7aa5a39d3ba9b0fb211c311892d4";
const PREBUILT_MANIFEST_SHA256: &str =
"9f193e6b42408a09ee0796cbff525e283836389c40a91d67aa4ec1271db8acbe";
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
pub struct ArtifactIdentity {
pub family: String,
pub repo_id: String,
pub revision: String,
pub tokenizer_sha256: String,
pub tokenizer_size: u64,
}
#[derive(Debug, Clone)]
pub struct LocalArtifact {
identity: ArtifactIdentity,
directory: PathBuf,
tokenizer_json: PathBuf,
bytes: Arc<[u8]>,
}
impl LocalArtifact {
pub fn identity(&self) -> &ArtifactIdentity {
&self.identity
}
pub fn directory(&self) -> &Path {
&self.directory
}
pub fn tokenizer_json(&self) -> &Path {
&self.tokenizer_json
}
pub(crate) fn bytes(&self) -> Arc<[u8]> {
Arc::clone(&self.bytes)
}
}
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct ArtifactFileRow {
pub(crate) sha256: String,
pub(crate) size: u64,
}
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct ArtifactRow {
pub(crate) repo_id: String,
pub(crate) revision: String,
pub(crate) files: BTreeMap<String, ArtifactFileRow>,
}
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct ConversionRow {
pub(crate) converter: String,
pub(crate) inputs: Vec<ConversionInputRow>,
}
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct ConversionInputRow {
pub(crate) name: String,
}
#[derive(Debug, Deserialize)]
struct ConversionTable {
conversions: BTreeMap<String, ConversionRow>,
}
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct RepairRow {
pub(crate) family: String,
pub(crate) artifact_sha256: String,
pub(crate) margin: usize,
pub(crate) effective_l_max: usize,
pub(crate) has_normalizer: bool,
}
#[derive(Debug, Deserialize)]
struct CarriedRepair {
margin: usize,
effective_l_max: usize,
has_normalizer: bool,
}
#[derive(Debug, Deserialize)]
struct RepairManifest {
schema: String,
pclass: PclassRow,
families: Vec<RepairRow>,
}
#[derive(Debug, Deserialize)]
struct PclassRow {
raw_sha256: String,
compressed_sha256: String,
}
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct SupportArtifact {
pub(crate) family: String,
pub(crate) artifact_sha256: String,
pub(crate) pipeline_id: String,
#[cfg_attr(not(feature = "jit"), allow(dead_code))]
pub(crate) oracle_id: String,
#[cfg_attr(not(feature = "jit"), allow(dead_code))]
pub(crate) evidence_id: String,
pub(crate) backends: BTreeMap<String, serde_json::Value>,
}
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct ChunkedAdmissionRow {
pub(crate) family: String,
pub(crate) artifact_sha256: String,
#[serde(default)]
pub(crate) loader_face_sha256: Option<String>,
pub(crate) status: String,
pub(crate) evidence_id: String,
#[serde(default)]
pub(crate) evidence: ChunkedEvidence,
}
#[derive(Debug, Clone, Default, Deserialize)]
pub(crate) struct ChunkedEvidence {
#[serde(default)]
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) python: Vec<String>,
#[serde(default)]
pub(crate) native: Vec<String>,
}
#[derive(Debug, Deserialize)]
struct ChunkedAdmissionTable {
schema: String,
schema_version: u64,
rows: Vec<ChunkedAdmissionRow>,
}
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct ExaminedRow {
pub(crate) family: String,
pub(crate) artifact_sha256: String,
pub(crate) pipeline_id: String,
pub(crate) repair: ExaminedRepair,
}
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct ExaminedRepair {
pub(crate) margin: usize,
pub(crate) effective_l_max: usize,
pub(crate) has_normalizer: bool,
}
#[derive(Debug, Deserialize)]
struct ExaminedTable {
schema_version: u64,
status: String,
artifacts: Vec<ExaminedRow>,
}
#[derive(Debug, Deserialize)]
struct SupportRegistry {
schema_version: u64,
#[serde(default)]
runtime_builds: Vec<RuntimeBuildRow>,
artifacts: Vec<SupportArtifact>,
}
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct RuntimeBuildRow {
runtime: String,
source_digest: String,
build_flags: Vec<String>,
toolchain: String,
fast_cpu_source_digest: String,
native_host_source_digest: String,
evidence_id: String,
}
#[derive(Debug, Clone, Deserialize)]
#[cfg_attr(not(feature = "prebuilt-gpu"), allow(dead_code))]
pub(crate) struct KernelFamily {
pub(crate) band: String,
pub(crate) ruleset: String,
pub(crate) digits_max: Option<i32>,
pub(crate) class_table: String,
#[serde(default)]
pub(crate) contractions: bool,
}
#[derive(Debug, Clone, Deserialize)]
#[cfg_attr(not(feature = "prebuilt-gpu"), allow(dead_code))]
pub(crate) struct ClassTableRow {
pub(crate) file: String,
pub(crate) shape: Vec<usize>,
pub(crate) dtype: String,
pub(crate) sha256: String,
pub(crate) meta_file: Option<String>,
pub(crate) meta_sha256: Option<String>,
}
#[derive(Debug, Deserialize)]
struct KernelManifest {
schema: String,
class_tables: BTreeMap<String, ClassTableRow>,
bands: BTreeMap<String, KernelBand>,
families: BTreeMap<String, KernelFamily>,
}
#[derive(Debug, Deserialize)]
struct KernelBand {
#[serde(default)]
e2e: bool,
}
#[derive(Debug, Clone, Deserialize)]
#[cfg_attr(not(feature = "prebuilt-gpu"), allow(dead_code))]
pub(crate) struct PrebuiltManifest {
pub(crate) toolchain: String,
pub(crate) architectures: BTreeMap<String, serde_json::Value>,
pub(crate) fatbin: PrebuiltFatbin,
pub(crate) kernels: BTreeMap<String, String>,
}
#[derive(Debug, Clone, Deserialize)]
#[cfg_attr(not(feature = "prebuilt-gpu"), allow(dead_code))]
pub(crate) struct PrebuiltFatbin {
pub(crate) size: u64,
pub(crate) digest: String,
}
#[derive(Debug, Deserialize)]
struct AliasManifest {
schema_version: u64,
aliases: Vec<AliasRow>,
}
#[derive(Debug, Clone, Deserialize)]
pub(crate) struct AliasRow {
pub(crate) repo_id: String,
pub(crate) revision: String,
pub(crate) source_sha256: String,
pub(crate) source_size: u64,
pub(crate) canonical_family: String,
pub(crate) canonical_anchor_sha256: String,
pub(crate) basis: String,
pub(crate) canonical_packaged: bool,
}
pub(crate) fn admitted_alias_basis(basis: &str) -> bool {
matches!(
basis,
"identical"
| "identical_source"
| "equivalent_canonicalisation"
| "equivalent_serialisation"
| "equivalent_loader_face"
)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct BuildStanding {
pub(crate) source_identity_judged: bool,
pub(crate) recipe_judged: bool,
}
#[derive(Debug)]
#[cfg_attr(not(feature = "prebuilt-gpu"), allow(dead_code))]
pub(crate) struct Registry {
artifacts: BTreeMap<String, ArtifactRow>,
conversions: BTreeMap<String, ConversionRow>,
repairs: HashMap<String, RepairRow>,
support: HashMap<String, SupportArtifact>,
examined: HashMap<String, ExaminedRow>,
chunked_admission: HashMap<String, ChunkedAdmissionRow>,
runtime_builds: Vec<RuntimeBuildRow>,
pub(crate) kernel_families: BTreeMap<String, KernelFamily>,
pub(crate) class_tables: BTreeMap<String, ClassTableRow>,
pub(crate) prebuilt: PrebuiltManifest,
pub(crate) aliases: Vec<AliasRow>,
pclass: Vec<u8>,
}
fn nearest_judged_row<'a>(
observed: &[&str],
candidates: &[&'a RuntimeBuildRow],
) -> Option<&'a RuntimeBuildRow> {
candidates.iter().copied().min_by_key(|row| {
let missing = row
.build_flags
.iter()
.filter(|flag| !observed.contains(&flag.as_str()))
.count();
let extra = observed
.iter()
.filter(|flag| !row.build_flags.iter().any(|other| other == *flag))
.count();
(missing + extra, usize::from(names_optional_feature(row)))
})
}
fn names_optional_feature(row: &RuntimeBuildRow) -> bool {
const OPTIONAL: [&str; 2] = ["jit", "network"];
row.build_flags
.iter()
.filter_map(|flag| flag.strip_prefix("features="))
.any(|features| {
features
.split(',')
.any(|feature| OPTIONAL.contains(&feature.trim()))
})
}
fn describe_flag_divergence(observed: &[&str], judged: &[String]) -> Option<String> {
let key = |flag: &str| flag.split_once('=').map(|(key, _)| key.to_owned());
let value = |flag: &str| flag.split_once('=').map(|(_, value)| value.to_owned());
let mut differences = Vec::new();
for flag in observed {
let Some(name) = key(flag) else { continue };
match judged
.iter()
.find(|other| key(other).as_deref() == Some(name.as_str()))
{
Some(other) if other == flag => {}
Some(other) => differences.push(format!(
"{name} is {:?} here and {:?} in the judged build",
value(flag).unwrap_or_default(),
value(other).unwrap_or_default(),
)),
None => differences.push(format!("{name} is not a judged build flag")),
}
}
for other in judged {
let Some(name) = key(other) else { continue };
if !observed
.iter()
.any(|flag| key(flag).as_deref() == Some(name.as_str()))
{
differences.push(format!(
"{name} is judged but this build does not report it"
));
}
}
if differences.is_empty() {
return None;
}
Some(format!(
"the build flags of this build are not the judged ones: {}. Building with the judged \
flags brings it back -- `rustflags` is the one a caller usually sets, through RUSTFLAGS \
or a Cargo configuration file, and the judged builds set none. A build that means to \
keep them can still take the accelerated route by selecting Policy::Experimental, which \
labels the result experimental rather than certified",
differences.join("; ")
))
}
impl Registry {
pub(crate) fn load() -> Result<Arc<Self>> {
static REGISTRY: OnceLock<std::result::Result<Arc<Registry>, (ErrorCode, String)>> =
OnceLock::new();
REGISTRY
.get_or_init(|| {
Self::load_embedded()
.map(Arc::new)
.map_err(|error| (error.code(), error.message().to_owned()))
})
.clone()
.map_err(|(code, message)| Error::new(code, message))
}
fn load_embedded() -> Result<Self> {
verify_embedded(
"artifact manifest",
ARTIFACT_MANIFEST_BYTES,
ARTIFACT_MANIFEST_SHA256,
)?;
verify_embedded(
"support registry",
SUPPORT_REGISTRY_BYTES,
SUPPORT_REGISTRY_SHA256,
)?;
verify_embedded(
"sibling aliases",
SIBLING_ALIASES_BYTES,
SIBLING_ALIASES_SHA256,
)?;
verify_embedded(
"repair manifest",
REPAIR_MANIFEST_BYTES,
REPAIR_MANIFEST_SHA256,
)?;
verify_embedded(
"kernel families",
KERNEL_FAMILIES_BYTES,
KERNEL_FAMILIES_SHA256,
)?;
verify_embedded(
"prebuilt manifest",
PREBUILT_MANIFEST_BYTES,
PREBUILT_MANIFEST_SHA256,
)?;
verify_embedded(
"artifact conversion table",
ARTIFACT_CONVERSIONS_BYTES,
ARTIFACT_CONVERSIONS_SHA256,
)?;
verify_embedded(
"chunked admission table",
CHUNKED_ADMISSION_BYTES,
CHUNKED_ADMISSION_SHA256,
)?;
verify_embedded(
"examined artifact table",
EXAMINED_ARTIFACTS_BYTES,
EXAMINED_ARTIFACTS_SHA256,
)?;
let artifacts: BTreeMap<String, ArtifactRow> =
parse_json("artifact manifest", ARTIFACT_MANIFEST_BYTES)?;
let conversions: ConversionTable =
parse_json("artifact conversion table", ARTIFACT_CONVERSIONS_BYTES)?;
let repair: RepairManifest = parse_json("repair manifest", REPAIR_MANIFEST_BYTES)?;
if repair.schema != "toktier.fast_repair_families.v1" {
return Err(registry_error("unexpected repair manifest schema"));
}
verify_embedded(
"compressed repair property table",
REPAIR_PCLASS_COMPRESSED,
&repair.pclass.compressed_sha256,
)?;
let mut decoder = ZlibDecoder::new(REPAIR_PCLASS_COMPRESSED);
let mut pclass = Vec::with_capacity(0x11_0000);
decoder.read_to_end(&mut pclass).map_err(|error| {
Error::new(
ErrorCode::RegistryInvalid,
format!("cannot decompress repair property table: {error}"),
)
})?;
if pclass.len() != 0x11_0000 || sha256_hex(&pclass) != repair.pclass.raw_sha256 {
return Err(registry_error(
"decompressed repair property table has the wrong identity",
));
}
let support: SupportRegistry = parse_json("support registry", SUPPORT_REGISTRY_BYTES)?;
if support.schema_version != 1 {
return Err(registry_error("unexpected support registry schema"));
}
let kernel: KernelManifest = parse_json("kernel families", KERNEL_FAMILIES_BYTES)?;
if kernel.schema != "toktier.kernel_families.v1" {
return Err(registry_error("unexpected kernel-family schema"));
}
for (family, row) in &kernel.families {
let band = kernel.bands.get(&row.band).ok_or_else(|| {
registry_error(format!("kernel family {family} names an unknown band"))
})?;
if !kernel.class_tables.contains_key(&row.class_table) {
return Err(registry_error(format!(
"kernel family {family} names an unknown class table"
)));
}
if !band.e2e {
return Err(registry_error(format!(
"unexpected non-e2e kernel family {family}"
)));
}
}
let aliases: AliasManifest = parse_json("sibling aliases", SIBLING_ALIASES_BYTES)?;
if aliases.schema_version != 1 {
return Err(registry_error("unexpected sibling-alias schema"));
}
let prebuilt: PrebuiltManifest = parse_json("prebuilt manifest", PREBUILT_MANIFEST_BYTES)?;
let chunked: ChunkedAdmissionTable =
parse_json("chunked admission table", CHUNKED_ADMISSION_BYTES)?;
if chunked.schema != "toktier.chunked_admission.v1" || chunked.schema_version != 1 {
return Err(registry_error("unexpected chunked admission table schema"));
}
let mut chunked_admission = HashMap::with_capacity(chunked.rows.len());
for row in chunked.rows {
if chunked_admission.insert(row.family.clone(), row).is_some() {
return Err(registry_error(
"the chunked admission table repeats an artifact",
));
}
}
let examined_table: ExaminedTable =
parse_json("examined artifact table", EXAMINED_ARTIFACTS_BYTES)?;
if examined_table.schema_version != 1 || examined_table.status != "examined" {
return Err(registry_error("unexpected examined artifact table schema"));
}
let mut examined = HashMap::with_capacity(examined_table.artifacts.len());
for row in examined_table.artifacts {
if repair
.families
.iter()
.any(|certified| certified.artifact_sha256 == row.artifact_sha256)
|| support
.artifacts
.iter()
.any(|certified| certified.artifact_sha256 == row.artifact_sha256)
{
return Err(registry_error(format!(
"examined artifact {} is also a certified one",
row.family
)));
}
if examined.insert(row.family.clone(), row).is_some() {
return Err(registry_error(
"the examined artifact table repeats a family",
));
}
}
let mut repairs: HashMap<String, RepairRow> = repair
.families
.into_iter()
.map(|row| (row.family.clone(), row))
.collect();
for row in &support.artifacts {
let Some(raw) = row
.backends
.get("fast_cpu")
.and_then(|entry| entry.get("repair"))
else {
continue;
};
let carried: CarriedRepair = serde_json::from_value(raw.clone()).map_err(|error| {
registry_error(format!(
"carried repair parameters of {} are malformed: {error}",
row.family
))
})?;
if repairs.contains_key(&row.family)
|| repairs
.values()
.any(|other| other.artifact_sha256 == row.artifact_sha256)
|| examined.contains_key(&row.family)
|| examined
.values()
.any(|other| other.artifact_sha256 == row.artifact_sha256)
{
return Err(registry_error(format!(
"carried repair parameters of {} collide with another repair source",
row.family
)));
}
repairs.insert(
row.family.clone(),
RepairRow {
family: row.family.clone(),
artifact_sha256: row.artifact_sha256.clone(),
margin: carried.margin,
effective_l_max: carried.effective_l_max,
has_normalizer: carried.has_normalizer,
},
);
}
let runtime_builds = support.runtime_builds;
let support = support
.artifacts
.into_iter()
.map(|row| (row.family.clone(), row))
.collect();
Ok(Self {
artifacts,
conversions: conversions.conversions,
repairs,
support,
examined,
chunked_admission,
runtime_builds,
kernel_families: kernel.families,
class_tables: kernel.class_tables,
prebuilt,
aliases: aliases.aliases,
pclass,
})
}
pub(crate) fn artifact(&self, family: &str) -> Result<&ArtifactRow> {
self.artifacts.get(family).ok_or_else(|| {
let mut message = format!("unknown tokenizer family {family:?}");
let suggestions =
crate::suggest::close_matches(family, self.artifacts.keys().map(String::as_str));
if !suggestions.is_empty() {
let rendered = suggestions
.iter()
.map(|candidate| format!("{candidate:?}"))
.collect::<Vec<_>>()
.join(", ");
message.push_str(&format!("; closest valid family IDs: {rendered}"));
}
Error::new(ErrorCode::ArtifactNotFound, message).with_family(family)
})
}
pub(crate) fn conversion(&self, family: &str) -> Option<&ConversionRow> {
self.conversions.get(family)
}
pub(crate) fn artifacts(&self) -> impl Iterator<Item = (&str, &ArtifactRow)> {
self.artifacts
.iter()
.map(|(family, row)| (family.as_str(), row))
}
pub(crate) fn family_for_tokenizer_sha256(&self, sha256: &str) -> Option<&str> {
self.artifacts
.iter()
.find(|(_, row)| {
row.files
.get("tokenizer.json")
.is_some_and(|file| file.sha256 == sha256)
})
.map(|(family, _)| family.as_str())
}
pub(crate) fn repair(&self, family: &str) -> Option<&RepairRow> {
self.repairs.get(family)
}
pub(crate) fn support(&self, family: &str) -> Option<&SupportArtifact> {
self.support.get(family)
}
pub(crate) fn examined(&self, family: &str) -> Option<&ExaminedRow> {
self.examined.get(family)
}
pub(crate) fn pipeline_peers(&self, pipeline_id: &str) -> Vec<&SupportArtifact> {
let mut peers = self
.support
.values()
.filter(|row| row.pipeline_id == pipeline_id)
.collect::<Vec<_>>();
peers.sort_by(|left, right| left.family.cmp(&right.family));
peers
}
pub(crate) fn chunked_admission(&self, family: &str) -> Option<&ChunkedAdmissionRow> {
self.chunked_admission.get(family)
}
pub(crate) fn rust_api_build_flag_divergence(&self) -> Option<String> {
let observed = env!("TOKTIER_RUST_API_BUILD_FLAGS")
.split('\x1f')
.collect::<Vec<_>>();
let candidates = self.judged_rows_for_these_sources().collect::<Vec<_>>();
if candidates.is_empty()
|| candidates
.iter()
.any(|row| row.build_flags.iter().eq(observed.iter().copied()))
{
return None;
}
let closest = nearest_judged_row(&observed, &candidates).expect("a candidate row");
describe_flag_divergence(&observed, &closest.build_flags)
}
fn judged_rows_for_these_sources(&self) -> impl Iterator<Item = &RuntimeBuildRow> {
self.runtime_builds.iter().filter(|row| {
row.runtime == "rust_api"
&& row.source_digest == env!("TOKTIER_RUST_API_SOURCE_SHA256")
&& row.fast_cpu_source_digest == env!("TOKTIER_RUST_API_FAST_CPU_SOURCE_SHA256")
&& row.native_host_source_digest
== env!("TOKTIER_RUST_API_NATIVE_HOST_SOURCE_SHA256")
&& !row.evidence_id.is_empty()
})
}
pub(crate) fn rust_api_build_standing(&self) -> BuildStanding {
let observed_flags = env!("TOKTIER_RUST_API_BUILD_FLAGS")
.split('\x1f')
.map(str::to_owned)
.collect::<Vec<_>>();
let mut rows = self.judged_rows_for_these_sources().peekable();
let source_identity_judged = rows.peek().is_some();
let recipe_judged = rows.any(|row| {
row.build_flags == observed_flags && row.toolchain == env!("TOKTIER_RUST_API_TOOLCHAIN")
});
BuildStanding {
source_identity_judged,
recipe_judged,
}
}
pub(crate) fn rust_api_toolchain_divergence(&self) -> Option<String> {
let mut judged = self
.judged_rows_for_these_sources()
.map(|row| row.toolchain.as_str())
.collect::<Vec<_>>();
judged.sort_unstable();
judged.dedup();
if judged.is_empty() || judged.contains(&env!("TOKTIER_RUST_API_TOOLCHAIN")) {
return None;
}
Some(format!(
"this build was compiled by `{}`; the judged builds from these sources were compiled \
by `{}`. The fast CPU engine is the judged one by source identity, so it is admitted \
under every policy but Reference and labelled Certification::Supported \
(supported_untested) rather than CertifiedSource. `toktier-rust verify-local --engine \
cpu` compares the two engines on this machine",
env!("TOKTIER_RUST_API_TOOLCHAIN"),
judged.join("`, `")
))
}
pub(crate) fn rust_api_build_certified(&self) -> bool {
if !crate::behavior_version::core_closure_verified()
|| crate::behavior_version::reference_certification_reason().is_some()
{
return false;
}
let standing = self.rust_api_build_standing();
standing.source_identity_judged && standing.recipe_judged
}
pub(crate) fn rust_api_sources_judged(&self) -> bool {
self.rust_api_build_standing().source_identity_judged
&& crate::behavior_version::sources_closure_admits()
}
pub(crate) fn pclass(&self) -> Vec<u8> {
self.pclass.clone()
}
pub(crate) fn resolve_repo(&self, repo_id: &str, revision: &str) -> Result<String> {
if let Some((family, _)) = self
.artifacts
.iter()
.find(|(_, row)| row.repo_id == repo_id && row.revision == revision)
{
return Ok(family.clone());
}
let alias = self
.aliases
.iter()
.find(|row| row.repo_id == repo_id && row.revision == revision)
.ok_or_else(|| {
Error::new(
ErrorCode::UncertifiedTokenizer,
format!("{repo_id}@{revision} is not in the shipped verified-repository table"),
)
})?;
if !alias.canonical_packaged
|| alias.source_sha256.len() != 64
|| alias.source_size == 0
|| !admitted_alias_basis(alias.basis.as_str())
{
return Err(Error::new(
ErrorCode::UncertifiedTokenizer,
format!("{repo_id}@{revision} has no admitted canonical execution artifact"),
));
}
let canonical = self.artifact(&alias.canonical_family)?;
let file = canonical
.files
.get("tokenizer.json")
.ok_or_else(|| registry_error("canonical artifact does not bind tokenizer.json"))?;
if file.sha256 != alias.canonical_anchor_sha256 {
return Err(registry_error("sibling alias canonical anchor drifted"));
}
Ok(alias.canonical_family.clone())
}
pub(crate) fn verify_local(
&self,
family: &str,
cache_root: &Path,
explicit: Option<&Path>,
) -> Result<LocalArtifact> {
let row = self.artifact(family)?;
let file = row.files.get("tokenizer.json").ok_or_else(|| {
registry_error(format!("artifact {family} does not bind tokenizer.json"))
})?;
let directory = explicit.map_or_else(
|| cache_root.join(format!("{}-{}", family, &row.revision[..12])),
Path::to_path_buf,
);
let given_is_link = explicit.is_some()
&& fs::symlink_metadata(&directory)
.is_ok_and(|metadata| metadata.file_type().is_symlink())
&& !directory.is_dir();
let tokenizer_json = if directory.is_file() || given_is_link {
directory.clone()
} else {
directory.join("tokenizer.json")
};
let resolved = if explicit.is_some() {
fs::canonicalize(&tokenizer_json).map_err(|error| {
Error::new(
ErrorCode::ArtifactNotFound,
format!("verified tokenizer artifact is unavailable: {error}"),
)
.with_path(&tokenizer_json)
.with_family(family)
})?
} else {
tokenizer_json.clone()
};
let metadata = fs::symlink_metadata(&resolved).map_err(|error| {
Error::new(
ErrorCode::ArtifactNotFound,
format!("verified tokenizer artifact is unavailable: {error}"),
)
.with_path(&tokenizer_json)
.with_family(family)
})?;
if metadata.file_type().is_symlink() || !metadata.file_type().is_file() {
let message = if explicit.is_some() {
"tokenizer artifact must be a regular file"
} else {
"tokenizer artifact must be a regular, non-symlink file"
};
return Err(Error::new(ErrorCode::ArtifactNotFound, message)
.with_path(&tokenizer_json)
.with_family(family));
}
if metadata.len() != file.size {
return Err(Error::new(
ErrorCode::ArtifactSizeMismatch,
format!(
"tokenizer.json has {} bytes; manifest requires {}",
metadata.len(),
file.size
),
)
.with_path(&tokenizer_json)
.with_family(family));
}
let bytes = fs::read(&resolved).map_err(|error| {
Error::new(ErrorCode::Io, error.to_string())
.with_path(&tokenizer_json)
.with_family(family)
})?;
let observed = sha256_hex(&bytes);
if observed != file.sha256 {
return Err(Error::new(
ErrorCode::ArtifactHashMismatch,
format!(
"tokenizer.json digest mismatch: expected {}, observed {observed}",
file.sha256
),
)
.with_path(&tokenizer_json)
.with_family(family));
}
let artifact_directory = tokenizer_json
.parent()
.unwrap_or_else(|| Path::new("."))
.to_path_buf();
Ok(LocalArtifact {
identity: ArtifactIdentity {
family: family.to_owned(),
repo_id: row.repo_id.clone(),
revision: row.revision.clone(),
tokenizer_sha256: file.sha256.clone(),
tokenizer_size: file.size,
},
directory: artifact_directory,
tokenizer_json,
bytes: bytes.into(),
})
}
}
fn parse_json<T: for<'de> Deserialize<'de>>(name: &str, bytes: &[u8]) -> Result<T> {
serde_json::from_slice(bytes).map_err(|error| {
Error::new(
ErrorCode::RegistryInvalid,
format!("cannot parse shipped {name}: {error}"),
)
})
}
fn verify_embedded(name: &str, bytes: &[u8], expected: &str) -> Result<()> {
let observed = sha256_hex(bytes);
if observed == expected.trim_start_matches("sha256:") {
Ok(())
} else {
Err(registry_error(format!(
"shipped {name} digest mismatch: expected {expected}, observed {observed}"
)))
}
}
pub(crate) fn sha256_hex(bytes: &[u8]) -> String {
crate::fsutil::hex(&Sha256::digest(bytes))
}
#[cfg_attr(not(feature = "prebuilt-gpu"), allow(dead_code))]
pub(crate) fn domain_sha256_hex(domain: &[u8], bytes: &[u8]) -> String {
let mut digest = Sha256::new();
digest.update(domain);
digest.update(bytes);
crate::fsutil::hex(&digest.finalize())
}
fn registry_error(message: impl Into<String>) -> Error {
Error::new(ErrorCode::RegistryInvalid, message)
}
#[cfg(test)]
mod tests {
use super::{
admitted_alias_basis, describe_flag_divergence, nearest_judged_row, parse_json,
registry_error, verify_embedded, ArtifactRow, ChunkedAdmissionTable, RuntimeBuildRow,
CHUNKED_ADMISSION_BYTES, SUPPORT_REGISTRY_BYTES, SUPPORT_REGISTRY_SHA256,
};
use crate::ErrorCode;
#[test]
fn every_examined_artifact_has_a_certified_pipeline_peer() {
let registry = super::Registry::load().expect("embedded tables load");
for (family, row) in ®istry.examined {
assert!(registry.support(family).is_none(), "{family} is certified");
assert!(
registry.repair(family).is_none(),
"{family} has a roster row"
);
let pinned = registry
.artifact(family)
.expect("examined family is in the artifact manifest")
.files
.get("tokenizer.json")
.expect("tokenizer.json is pinned")
.sha256
.clone();
assert_eq!(pinned, row.artifact_sha256);
let peers = registry.pipeline_peers(&row.pipeline_id);
assert!(
peers.iter().any(|peer| peer
.backends
.get("fast_cpu")
.and_then(|entry| entry.get("status"))
.and_then(serde_json::Value::as_str)
== Some("certified_source")),
"{family} has no certified pipeline peer"
);
}
}
#[test]
fn the_shipped_examined_table_is_empty_and_rows_still_parse() {
let registry = super::Registry::load().expect("embedded tables load");
assert!(registry.examined.is_empty());
let table: super::ExaminedTable = serde_json::from_str(
r#"{"schema_version": 1, "status": "examined", "artifacts": [{
"family": "examined_fixture",
"artifact_sha256": "eeee",
"pipeline_id": "pipeline.9f6d5117c3680d0e",
"repair": {"margin": 36, "effective_l_max": 128, "has_normalizer": true}
}]}"#,
)
.expect("an examined row parses");
assert_eq!(table.artifacts.len(), 1);
assert_eq!(table.artifacts[0].repair.margin, 36);
}
#[test]
fn carried_records_hold_their_anchors_repair_parameters() {
let registry = super::Registry::load().expect("embedded tables load");
for (family, anchor) in [
("deepseek_v4_1_flash", "deepseek_v4_flash"),
("mimo_v2_6", "qwen3_8b"),
] {
let carried = registry.repair(family).expect("carried repair row");
let support = registry.support(family).expect("certified record");
assert_eq!(carried.artifact_sha256, support.artifact_sha256);
let anchor = registry.repair(anchor).expect("anchor roster row");
assert_eq!(
(
carried.margin,
carried.effective_l_max,
carried.has_normalizer
),
(anchor.margin, anchor.effective_l_max, anchor.has_normalizer)
);
}
}
#[test]
fn every_chunked_row_status_is_the_state_its_python_levels_spell() {
let table: ChunkedAdmissionTable =
parse_json("chunked admission table", CHUNKED_ADMISSION_BYTES).unwrap();
assert!(!table.rows.is_empty());
for row in &table.rows {
assert_eq!(
crate::chunked::levels_status(&row.evidence.python).as_deref(),
Some(row.status.as_str()),
"{}",
row.family
);
}
}
#[test]
fn alias_admission_accepts_every_recorded_basis() {
for basis in [
"identical",
"identical_source",
"equivalent_canonicalisation",
"equivalent_serialisation",
"equivalent_loader_face",
] {
assert!(admitted_alias_basis(basis), "{basis} must be admitted");
}
}
#[test]
fn alias_admission_refuses_an_unrecorded_basis() {
for basis in ["", "equivalent", "loader_face", "equivalent_loader_face2"] {
assert!(!admitted_alias_basis(basis), "{basis:?} must not admit");
}
}
fn judged_row(features: &str) -> RuntimeBuildRow {
RuntimeBuildRow {
runtime: "rust_api".to_owned(),
source_digest: "s".to_owned(),
build_flags: vec![
"profile=release".to_owned(),
"opt-level=3".to_owned(),
"target=x86_64-unknown-linux-gnu".to_owned(),
"debug=false".to_owned(),
"target-features=fxsr,sse,sse2".to_owned(),
"rustflags=".to_owned(),
format!("features={features}"),
],
toolchain: "t".to_owned(),
fast_cpu_source_digest: "f".to_owned(),
native_host_source_digest: "n".to_owned(),
evidence_id: "e".to_owned(),
}
}
fn observed_flags(features: &str) -> Vec<String> {
judged_row(features).build_flags
}
#[test]
fn the_divergence_reference_is_the_plain_recipe_not_an_optional_one() {
let default_row = judged_row("default,prebuilt-gpu,serving,sqlite");
let jit_row = judged_row("default,jit,prebuilt-gpu,serving,sqlite");
let flags = observed_flags("default,network,prebuilt-gpu,serving,sqlite");
let observed = flags.iter().map(String::as_str).collect::<Vec<_>>();
for candidates in [vec![&default_row, &jit_row], vec![&jit_row, &default_row]] {
let chosen = nearest_judged_row(&observed, &candidates).expect("a candidate row");
assert_eq!(
chosen.build_flags.last().map(String::as_str),
Some("features=default,prebuilt-gpu,serving,sqlite"),
"the plain recipe is the reference whatever order the rows arrive in",
);
let message = describe_flag_divergence(&observed, &chosen.build_flags)
.expect("a build with network diverges from the default recipe");
assert!(
message.contains("network") && !message.contains("jit"),
"the reported difference names the feature the reader turned on: {message}",
);
}
}
#[test]
fn the_nearest_row_is_the_one_with_the_smallest_difference() {
let exact = judged_row("default,prebuilt-gpu,serving,sqlite");
let far = judged_row("default,jit,network,serde,serving,sqlite");
let flags = observed_flags("default,prebuilt-gpu,serving,sqlite");
let observed = flags.iter().map(String::as_str).collect::<Vec<_>>();
let candidates = vec![&far, &exact];
let chosen = nearest_judged_row(&observed, &candidates).expect("a candidate row");
assert_eq!(
chosen.build_flags.last().map(String::as_str),
Some("features=default,prebuilt-gpu,serving,sqlite"),
);
assert_eq!(
describe_flag_divergence(&observed, &chosen.build_flags),
None
);
}
#[test]
fn a_shipped_record_that_does_not_hold_together_is_refused() {
verify_embedded(
"support registry",
SUPPORT_REGISTRY_BYTES,
SUPPORT_REGISTRY_SHA256,
)
.expect("the shipped registry verifies");
let error = verify_embedded(
"support registry",
b"not the shipped bytes",
SUPPORT_REGISTRY_SHA256,
)
.expect_err("a digest mismatch");
assert_eq!(error.code(), ErrorCode::RegistryInvalid);
assert_eq!(error.code().as_str(), "REGISTRY_INVALID");
assert!(
error.message().contains("digest mismatch")
&& error.message().contains("support registry"),
"{}",
error.message()
);
let error = parse_json::<ArtifactRow>("artifact manifest", b"{\"repo_id\": 7}")
.expect_err("a shape mismatch");
assert_eq!(error.code(), ErrorCode::RegistryInvalid);
assert!(
error
.message()
.starts_with("cannot parse shipped artifact manifest"),
"{}",
error.message()
);
let error = registry_error("kernel family x names an unknown band");
assert_eq!(error.code(), ErrorCode::RegistryInvalid);
}
fn judged() -> Vec<String> {
[
"profile=release",
"opt-level=3",
"target=x86_64-unknown-linux-gnu",
"debug=false",
"target-features=fxsr,sse,sse2",
"rustflags=",
"features=default,network,prebuilt-gpu,serving,sqlite",
]
.into_iter()
.map(str::to_owned)
.collect()
}
#[test]
fn matching_flags_have_nothing_to_report() {
let judged = judged();
let observed = judged.iter().map(String::as_str).collect::<Vec<_>>();
assert_eq!(describe_flag_divergence(&observed, &judged), None);
}
#[test]
fn a_caller_s_rustflags_are_named_with_both_ways_out() {
let judged = judged();
let mut observed = judged.clone();
observed[5] = "rustflags=-C target-cpu=native".to_owned();
let observed = observed.iter().map(String::as_str).collect::<Vec<_>>();
let reported = describe_flag_divergence(&observed, &judged).expect("a divergence");
assert!(
reported.contains(
"rustflags is \"-C target-cpu=native\" here and \"\" in the judged build"
),
"{reported}"
);
assert!(reported.contains("RUSTFLAGS"), "{reported}");
assert!(reported.contains("Policy::Experimental"), "{reported}");
assert!(!reported.contains('\n'), "{reported}");
}
#[test]
fn only_the_keys_that_differ_are_named() {
let judged = judged();
let mut observed = judged.clone();
observed[1] = "opt-level=2".to_owned();
let observed = observed.iter().map(String::as_str).collect::<Vec<_>>();
let reported = describe_flag_divergence(&observed, &judged).expect("a divergence");
assert!(reported.contains("opt-level is \"2\""), "{reported}");
assert!(!reported.contains("profile is"), "{reported}");
assert!(!reported.contains("rustflags is"), "{reported}");
}
#[test]
fn a_key_on_only_one_side_is_named_as_such() {
let judged = judged();
let observed = ["profile=release", "lto=fat"];
let reported = describe_flag_divergence(&observed, &judged).expect("a divergence");
assert!(
reported.contains("lto is not a judged build flag"),
"{reported}"
);
assert!(
reported.contains("rustflags is judged but this build does not report it"),
"{reported}"
);
}
}