use std::sync::OnceLock;
use crate::diagnostics::{BehaviorVersion, ReferenceVersion};
const FIELD_SEPARATOR: char = '\u{1f}';
const RECORD_SEPARATOR: char = '\u{1e}';
const AGE_CANDIDATES: &[&str] = &[
"23.0", "22.0", "21.0", "20.0", "19.0", "18.0", "17.0", "16.0", "15.1", "15.0", "14.0", "13.0",
"12.1", "12.0", "11.0", "10.0",
];
struct Reading {
unit: String,
judged: String,
source: String,
observed: Option<String>,
}
fn judged_readings() -> Vec<Reading> {
let recorded = env!("TOKTIER_RUST_API_JUDGED_BEHAVIOR");
if recorded.is_empty() {
return Vec::new();
}
recorded
.split(FIELD_SEPARATOR)
.filter_map(|record| {
let mut fields = record.split(RECORD_SEPARATOR);
let unit = fields.next()?;
let judged = fields.next()?;
let source = fields.next()?;
if unit.is_empty() || judged.is_empty() {
return None;
}
Some(Reading {
observed: observe(unit),
unit: unit.to_owned(),
judged: judged.to_owned(),
source: source.to_owned(),
})
})
.collect()
}
fn drift_for(unit: &str) -> Option<&'static str> {
let recorded = env!("TOKTIER_RUST_API_R2_DRIFT");
recorded.split(FIELD_SEPARATOR).find_map(|record| {
let (name, report) = record.split_once(RECORD_SEPARATOR)?;
(name == unit).then_some(report)
})
}
fn observe(unit: &str) -> Option<String> {
match unit {
"regex" => AGE_CANDIDATES
.iter()
.find(|age| regex::Regex::new(&format!(r"\p{{Age={age}}}")).is_ok())
.map(|age| (*age).to_owned()),
"onig" => Some(onig::version()).filter(|version| !version.is_empty()),
"unicode-segmentation" => {
let (major, minor, patch) = unicode_segmentation::UNICODE_VERSION;
Some(format!("{major}.{minor}.{patch}"))
}
_ => None,
}
}
fn carries(unit: &str, observed: &str) -> String {
match unit {
"regex" => format!("carries Unicode {observed} (regex tables)"),
"onig" => format!("links Oniguruma {observed}"),
"unicode-segmentation" => format!("reads segmentation tables {observed}"),
other => format!("reads {other} {observed}"),
}
}
fn unchanged(unit: &str, version: &str) -> String {
match unit {
"regex" => format!("Unicode data version unchanged ({version})"),
"onig" => format!("Oniguruma library version unchanged ({version})"),
"unicode-segmentation" => format!("segmentation table version unchanged ({version})"),
other => format!("{other} behaviour version unchanged ({version})"),
}
}
const COMPARE_HINT: &str = "To compare the accelerated engine with this binary's reference \
engine on your own text, run `toktier-rust verify-local --engine \
cpu --family <family> --input <path>` where that CLI is \
installed (`cargo install --locked toktier` builds it; a crate \
dependency alone brings no binary), or `toktier verify-local` \
with the same options from the Python package";
const ALIGN_HINT: &str = "the `dependency_closure` line of this same report carries the `cargo \
update --precise` command for each package it names, to run in the \
workspace that owns the governing lockfile, which brings the library \
back onto the judged behaviour version";
fn readings() -> &'static Vec<Reading> {
static READINGS: OnceLock<Vec<Reading>> = OnceLock::new();
READINGS.get_or_init(judged_readings)
}
pub(crate) fn core_closure() -> &'static str {
env!("TOKTIER_RUST_API_CORE_CLOSURE_STATIC")
}
pub(crate) fn core_closure_verified() -> bool {
core_closure() == "verified"
}
pub(crate) fn sources_closure_admits() -> bool {
let core = core_closure();
core == "verified" || core.starts_with("unlocated")
}
fn probe_divergence(
readings: &[Reading],
drift: &dyn Fn(&str) -> Option<&'static str>,
) -> Option<String> {
for reading in readings {
match &reading.observed {
Some(observed) if observed != &reading.judged => {
return Some(format!(
"the reference engine in this binary {} where the certified evidence was \
taken on {}; some code points may tokenize differently; the accelerated \
engines run, the route is labelled supported_untested and \
reference_certification_reason reads uncertified_reference. {ALIGN_HINT}. \
{COMPARE_HINT}",
carries(&reading.unit, observed),
reading.judged
));
}
Some(_) => {}
None => {
if let Some(moved) = drift(&reading.unit) {
return Some(format!(
"the behaviour version of {} could not be read in this binary, so the \
package versions are compared instead: {moved}; the accelerated \
engines run, the route is labelled supported_untested and \
reference_certification_reason reads uncertified_reference. \
{ALIGN_HINT}",
reading.unit
));
}
}
}
}
None
}
fn probe_status() -> Option<&'static str> {
static STATUS: OnceLock<Option<String>> = OnceLock::new();
STATUS
.get_or_init(|| probe_divergence(readings(), &drift_for))
.as_deref()
}
pub(crate) fn dependency_advisory() -> Option<&'static str> {
static ADVISORY: OnceLock<Option<String>> = OnceLock::new();
ADVISORY
.get_or_init(|| {
let mut parts = Vec::new();
let outside = env!("TOKTIER_RUST_API_DEPENDENCY_ADVISORY");
if !outside.is_empty() {
parts.push(outside.to_owned());
}
if let Some(divergence) = probe_status() {
parts.push(divergence.to_owned());
}
for reading in readings() {
let Some(observed) = &reading.observed else {
continue;
};
if observed != &reading.judged {
continue;
}
if let Some(drift) = drift_for(&reading.unit) {
parts.push(format!(
"{drift}: {}; the certificate holds. {COMPARE_HINT}",
unchanged(&reading.unit, observed)
));
}
}
(!parts.is_empty()).then(|| parts.join(". "))
})
.as_deref()
}
fn reference_readings() -> &'static Vec<ReferenceVersion> {
static READINGS: OnceLock<Vec<ReferenceVersion>> = OnceLock::new();
READINGS.get_or_init(|| {
let recorded = env!("TOKTIER_RUST_API_REFERENCE_VERSIONS");
if recorded.is_empty() {
return Vec::new();
}
recorded
.split(FIELD_SEPARATOR)
.filter_map(|record| {
let mut fields = record.split(RECORD_SEPARATOR);
let package = fields.next()?;
let judged = fields.next()?;
let resolved = fields.next()?;
let certified = fields.next()?;
let status = fields.next()?;
if package.is_empty() || judged.is_empty() {
return None;
}
Some(ReferenceVersion {
package: package.to_owned(),
judged: judged.to_owned(),
resolved: (!resolved.is_empty()).then(|| resolved.to_owned()),
certified_versions: certified
.split(',')
.filter(|version| !version.is_empty())
.map(str::to_owned)
.collect(),
certified: status == "certified",
})
})
.collect()
})
}
pub(crate) fn reference_facts() -> Vec<ReferenceVersion> {
reference_readings().clone()
}
pub(crate) fn reference_certification_reason() -> Option<&'static str> {
let unspoken_for = !env!("TOKTIER_RUST_API_REFERENCE_UNCERTIFIED").is_empty();
(reference_readings()
.iter()
.any(|reading| !reading.certified)
|| probe_status().is_some()
|| unspoken_for)
.then_some("uncertified_reference")
}
pub(crate) fn facts() -> Vec<BehaviorVersion> {
readings()
.iter()
.map(|reading| BehaviorVersion {
unit: reading.unit.clone(),
observed: reading.observed.clone(),
judged: reading.judged.clone(),
source: reading.source.clone(),
})
.collect()
}
#[cfg(test)]
mod tests {
use super::{
carries, drift_for, observe, probe_divergence, readings, unchanged, Reading,
AGE_CANDIDATES, FIELD_SEPARATOR, RECORD_SEPARATOR,
};
fn reading(unit: &str, judged: &str, observed: Option<&str>) -> Reading {
Reading {
unit: unit.to_owned(),
judged: judged.to_owned(),
source: "probe:regex-age".to_owned(),
observed: observed.map(str::to_owned),
}
}
fn no_drift(_: &str) -> Option<&'static str> {
None
}
#[test]
fn the_probe_reading_has_three_states() {
let moved = probe_divergence(&[reading("regex", "16.0", Some("17.0"))], &no_drift)
.expect("a divergence");
assert!(
moved.starts_with(
"the reference engine in this binary carries Unicode 17.0 (regex tables) where \
the certified evidence was taken on 16.0; some code points may tokenize \
differently; the accelerated engines run, the route is labelled \
supported_untested and reference_certification_reason reads \
uncertified_reference. the `dependency_closure` line"
),
"{moved}"
);
assert!(moved.contains("verify-local --engine cpu"), "{moved}");
for word in ["held on the reference route", "admits no accelerated"] {
assert!(!moved.contains(word), "{moved}");
}
assert_eq!(
probe_divergence(&[reading("regex", "16.0", None)], &no_drift),
None
);
let unread = probe_divergence(&[reading("regex", "16.0", None)], &|unit| {
(unit == "regex").then_some("regex 1.13.1 -> 1.14.0")
})
.expect("a divergence");
assert!(
unread.starts_with(
"the behaviour version of regex could not be read in this binary, so the \
package versions are compared instead: regex 1.13.1 -> 1.14.0; the \
accelerated engines run"
),
"{unread}"
);
assert_eq!(
probe_divergence(
&[
reading("regex", "16.0", Some("16.0")),
reading("onig", "6.9.10", Some("6.9.10")),
],
&no_drift,
),
None
);
}
#[test]
fn a_record_with_no_units_has_nothing_to_report() {
assert_eq!(probe_divergence(&[], &no_drift), None);
}
#[test]
fn every_reference_library_resolves_at_a_certified_version_here() {
let readings = super::reference_facts();
assert!(!readings.is_empty(), "no reference library was recorded");
assert!(readings
.iter()
.any(|reading| reading.package == "tokenizers"));
for reading in &readings {
assert_eq!(
reading.resolved.as_deref(),
Some(reading.judged.as_str()),
"{} does not resolve at its judged version here; a build of this workspace \
with a target directory outside the tree locates no governing lockfile \
unless TOKTIER_CARGO_LOCK is set to the absolute path of the workspace's \
own Cargo.lock",
reading.package
);
assert!(reading.certified_versions.contains(&reading.judged));
assert!(reading.certified, "{} is not certified", reading.package);
}
assert_eq!(super::reference_certification_reason(), None);
}
#[test]
fn every_judged_unit_reads_the_way_the_record_says() {
let readings = readings();
assert!(!readings.is_empty(), "no behaviour unit was recorded");
for reading in readings {
let observed = reading
.observed
.as_deref()
.unwrap_or_else(|| panic!("{} could not be read", reading.unit));
assert_eq!(
observed, reading.judged,
"{} reads {observed} and the record judges {}",
reading.unit, reading.judged
);
}
}
#[test]
fn a_unit_this_release_cannot_ask_reads_as_unreadable() {
assert!(observe("regex").is_some());
assert!(observe("onig").is_some());
assert!(observe("unicode-segmentation").is_some());
assert!(observe("unicode_categories").is_none());
assert!(observe("").is_none());
}
#[test]
fn the_age_probe_answers_from_the_tables_rather_than_the_candidates() {
let observed = observe("regex").expect("a regex age");
assert!(AGE_CANDIDATES.contains(&observed.as_str()));
assert!(
regex::Regex::new(&format!(r"\p{{Age={observed}}}")).is_ok(),
"the answer does not compile"
);
let newer = AGE_CANDIDATES
.iter()
.take_while(|age| **age != observed)
.collect::<Vec<_>>();
for age in newer {
assert!(
regex::Regex::new(&format!(r"\p{{Age={age}}}")).is_err(),
"{age} compiles and is newer than the answer"
);
}
}
#[test]
fn the_unit_sentences_say_which_tables_and_which_version() {
assert_eq!(
carries("regex", "17.0"),
"carries Unicode 17.0 (regex tables)"
);
assert_eq!(carries("onig", "6.9.11"), "links Oniguruma 6.9.11");
assert_eq!(
unchanged("regex", "16.0"),
"Unicode data version unchanged (16.0)"
);
assert_eq!(
unchanged("a-new-unit", "1.0"),
"a-new-unit behaviour version unchanged (1.0)"
);
}
#[test]
fn the_drift_line_is_read_by_unit() {
assert_eq!(FIELD_SEPARATOR, '\u{1f}');
assert_eq!(RECORD_SEPARATOR, '\u{1e}');
assert!(drift_for("regex").is_none());
assert!(drift_for("a-unit-that-is-not-recorded").is_none());
}
}