#[path = "property_guard_table.rs"]
mod table;
use std::fmt;
use toktier_routing_core::NativeRouter;
use toktier_store_core::{
AppendReport, BoundaryCut, Encoding, EngineError, SessionEncoder, TailState, WitnessCategory,
};
pub(crate) use table::{ADMITTED_COPIES, MOVED_CODE_POINTS, MOVED_RANGES};
pub(crate) const PROPERTIES_CRATE: &str = "icu_properties";
pub(crate) const DATA_CRATE: &str = "icu_properties_data";
pub(crate) const CARGO_LOCK_ENV: &str = "TOKTIER_CARGO_LOCK";
pub(crate) const DATA_DIR_ENV: &str = "ICU4X_DATA_DIR";
pub(crate) const RECORD_SEPARATOR: char = '\u{1f}';
pub(crate) const FIELD_SEPARATOR: char = '\u{1e}';
const CODE_POINT_LIMIT: u32 = 0x11_0000;
const BLOCK_SHIFT: u32 = 6;
const BLOCK_WORDS: usize = (CODE_POINT_LIMIT >> BLOCK_SHIFT) as usize / 64;
const fn table_is_well_formed() -> bool {
let mut index = 0;
let mut count: u32 = 0;
while index < MOVED_RANGES.len() {
let (low, high) = MOVED_RANGES[index];
if low > high || high >= CODE_POINT_LIMIT {
return false;
}
if index > 0 && MOVED_RANGES[index - 1].1 + 1 >= low {
return false;
}
count += high - low + 1;
index += 1;
}
count == MOVED_CODE_POINTS
}
const _: () = assert!(
table_is_well_formed(),
"the property guard table is malformed"
);
const MIN_MOVED: u32 = if MOVED_RANGES.is_empty() {
CODE_POINT_LIMIT
} else {
MOVED_RANGES[0].0
};
const MAX_MOVED: u32 = if MOVED_RANGES.is_empty() {
0
} else {
MOVED_RANGES[MOVED_RANGES.len() - 1].1
};
const BLOCKS: [u64; BLOCK_WORDS] = {
let mut blocks = [0u64; BLOCK_WORDS];
let mut index = 0;
while index < MOVED_RANGES.len() {
let (low, high) = MOVED_RANGES[index];
let mut block = low >> BLOCK_SHIFT;
while block <= high >> BLOCK_SHIFT {
blocks[(block / 64) as usize] |= 1 << (block % 64);
block += 1;
}
index += 1;
}
blocks
};
#[inline(always)]
fn block_holds_moved(block: u32) -> bool {
BLOCKS
.get((block / 64) as usize)
.is_some_and(|word| word & (1 << (block % 64)) != 0)
}
const fn lead_span(byte: u8) -> Option<(u32, u32)> {
let byte = byte as u32;
match byte {
0x00..=0x7F => Some((byte, byte)),
0xC2..=0xDF => Some(((byte & 0x1F) << 6, ((byte & 0x1F) << 6) | 0x3F)),
0xE0..=0xEF => Some(((byte & 0x0F) << 12, ((byte & 0x0F) << 12) | 0xFFF)),
0xF0..=0xF4 => {
let low = (byte & 0x07) << 18;
let high = low | 0x3_FFFF;
Some((
low,
if high < CODE_POINT_LIMIT {
high
} else {
CODE_POINT_LIMIT - 1
},
))
}
_ => None,
}
}
const LEAD_MAY_HOLD: [bool; 256] = {
let mut table = [false; 256];
let mut byte = 0;
while byte < 256 {
if let Some((low, high)) = lead_span(byte as u8) {
let mut index = 0;
while index < MOVED_RANGES.len() {
let (moved_low, moved_high) = MOVED_RANGES[index];
if moved_low <= high && low <= moved_high {
table[byte] = true;
}
index += 1;
}
}
byte += 1;
}
table
};
const LEAD_MIN: u8 = {
let mut byte = 0;
while byte < 255 && !LEAD_MAY_HOLD[byte] {
byte += 1;
}
byte as u8
};
const WORD_SKIP: bool = LEAD_MIN >= 0xC0;
const ONES: u64 = 0x0101_0101_0101_0101;
const HIGH_BITS: u64 = 0x8080_8080_8080_8080;
const LOW_BITS: u64 = 0x7F7F_7F7F_7F7F_7F7F;
const LEAD_BIAS: u64 = if WORD_SKIP {
(0x100 - LEAD_MIN as u64) * ONES
} else {
0
};
#[inline(always)]
fn word_reaches_lead_min(word: u64) -> bool {
word & HIGH_BITS & ((word & LOW_BITS) + LEAD_BIAS) != 0
}
#[inline]
pub(crate) fn is_moved(code_point: u32) -> bool {
if !(MIN_MOVED..=MAX_MOVED).contains(&code_point)
|| !block_holds_moved(code_point >> BLOCK_SHIFT)
{
return false;
}
let index = MOVED_RANGES.partition_point(|&(_, high)| high < code_point);
MOVED_RANGES
.get(index)
.is_some_and(|&(low, _)| low <= code_point)
}
#[inline(always)]
fn block_at(bytes: &[u8], start: usize) -> Option<(u32, usize)> {
let lead = u32::from(*bytes.get(start)?);
let next = |offset: usize| {
bytes
.get(start + offset)
.map(|&byte| u32::from(byte) & 0x3F)
};
Some(match lead {
0x00..=0x7F => (lead >> BLOCK_SHIFT, 1),
0xC0..=0xDF => (lead & 0x1F, 2),
0xE0..=0xEF => (((lead & 0x0F) << 6) | next(1)?, 3),
_ => (((lead & 0x07) << 12) | (next(1)? << 6) | next(2)?, 4),
})
}
pub(crate) fn holds_moved(text: &str) -> bool {
let bytes = text.as_bytes();
let mut position = 0;
while position < bytes.len() {
if WORD_SKIP {
while let Some(word) = bytes
.get(position..position + 8)
.and_then(|chunk| <[u8; 8]>::try_from(chunk).ok())
{
if word_reaches_lead_min(u64::from_le_bytes(word)) {
break;
}
position += 8;
}
}
let end = bytes.len().min(position + 8);
let Some(start) = bytes
.get(position..end)
.and_then(|window| {
window
.iter()
.position(|&byte| LEAD_MAY_HOLD[usize::from(byte)])
})
.map(|offset| position + offset)
else {
position = end;
continue;
};
if bytes[start] & 0xF0 == 0xE0 {
position = start;
while let Some(&[lead, next, _]) = bytes.get(position..position + 3) {
if lead & 0xF0 != 0xE0 {
break;
}
let block = (u32::from(lead & 0x0F) << 6) | u32::from(next & 0x3F);
if block_holds_moved(block) {
match text.get(position..).and_then(|rest| rest.chars().next()) {
Some(character) if !is_moved(u32::from(character)) => {}
_ => return true,
}
}
position += 3;
}
if position == start {
return true;
}
continue;
}
let Some((block, width)) = block_at(bytes, start) else {
return true;
};
if block_holds_moved(block) {
match text.get(start..).and_then(|rest| rest.chars().next()) {
Some(character) if !is_moved(u32::from(character)) => {}
_ => return true,
}
}
position = start + width;
}
false
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum LocatedBy {
Named,
OutDir,
Manifest,
None,
}
impl LocatedBy {
pub(crate) fn parse(word: &str) -> Self {
match word {
"named" => Self::Named,
"out_dir" => Self::OutDir,
"manifest" => Self::Manifest,
_ => Self::None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct LockCopy {
pub(crate) name: String,
pub(crate) version: String,
pub(crate) source: Option<String>,
pub(crate) checksum: Option<String>,
}
pub(crate) fn parse_copies(recorded: &str) -> Result<Vec<LockCopy>, String> {
if recorded.is_empty() {
return Ok(Vec::new());
}
recorded
.split(RECORD_SEPARATOR)
.map(|record| {
let fields: Vec<&str> = record.split(FIELD_SEPARATOR).collect();
match fields.as_slice() {
[name, version, source, checksum] if !name.is_empty() && !version.is_empty() => {
Ok(LockCopy {
name: (*name).to_owned(),
version: (*version).to_owned(),
source: (!source.is_empty()).then(|| (*source).to_owned()),
checksum: (!checksum.is_empty()).then(|| (*checksum).to_owned()),
})
}
_ => Err(format!(
"the build script's property-data record {record:?} is unreadable"
)),
}
})
.collect()
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Standing {
Judged,
Guarded {
resolved: String,
},
Untabled {
reason: String,
remedy: String,
},
Contradicted { reason: String },
Unread {
located_by: LocatedBy,
},
}
impl Standing {
#[cfg(test)]
pub(crate) fn admits_fast_route(&self) -> bool {
matches!(self, Self::Judged | Self::Guarded { .. })
}
#[cfg(test)]
pub(crate) fn is_guarded(&self) -> bool {
matches!(self, Self::Guarded { .. })
}
}
fn untabled(reason: String, copies: &[LockCopy]) -> Standing {
Standing::Untabled {
reason,
remedy: remedy_for(copies),
}
}
const LINKED_MAJOR: &str = "2";
fn is_linked_major(copy: &LockCopy) -> bool {
copy.version.split(['.', '-', '+']).next() == Some(LINKED_MAJOR)
}
fn remedy() -> String {
remedy_for(&[])
}
fn remedy_for(copies: &[LockCopy]) -> String {
let judged: Vec<String> = ADMITTED_COPIES
.iter()
.filter(|copy| copy.judged)
.map(|judged| {
let package = match copies.iter().find(|copy| copy.name == judged.name) {
Some(copy) => {
format!("{}@{}", judged.name, copy.version)
}
None => format!("{}@{LINKED_MAJOR}", judged.name),
};
format!(
"`cargo update --package {package} --precise {}`",
judged.version
)
})
.collect();
format!(
"align the lockfile with the judged property data ({}), or set {CARGO_LOCK_ENV} at \
build time to the absolute path of the lockfile that governs this build",
judged.join(" and ")
)
}
const EXPERIMENTAL: &str = "under Policy::Experimental the route runs and only texts holding a \
listed code point go to the reference engine; the list was not \
made for this data, so the other ids are unchecked";
impl fmt::Display for Standing {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Judged => formatter.write_str("judged"),
Self::Contradicted { reason } => write!(
formatter,
"untabled: judged property-data reading contradicted by an encode check: {reason}; \
the fast CPU route is declined, including under Policy::Experimental; \
rebuild with the consuming build's actual lockfile (TOKTIER_CARGO_LOCK must \
be absolute), use --locked with cargo install --path, or align the actual \
property data with the judged copies"
),
Self::Guarded { resolved } => write!(
formatter,
"guarded: the property data resolves to {resolved}; texts holding one of the \
{MOVED_CODE_POINTS} listed code points (every code point whose class differs \
from the judged data's in some listed copy) are encoded on the reference engine"
),
Self::Untabled { reason, remedy } => write!(
formatter,
"untabled: {reason}, so the fast CPU route is not built ({EXPERIMENTAL}); \
{remedy}"
),
Self::Unread { located_by } => {
let how = match located_by {
LocatedBy::Manifest => {
"the only lockfile found lies above this crate's own manifest, and the \
build cannot tell whether that file governs it (it does for a build of \
this crate's own workspace, not for a build that consumes this crate \
from another tree)"
}
_ => {
"no lockfile governing this build was located, or the value of \
TOKTIER_CARGO_LOCK was refused (it must be an absolute path outside this \
crate's own directory)"
}
};
write!(
formatter,
"unread: {how}, so the property data this build resolved is unknown and the \
fast CPU route is not built ({EXPERIMENTAL}); {}",
remedy()
)
}
}
}
}
pub(crate) fn decide_standing(
located_by: LocatedBy,
recorded: &str,
references_certified: bool,
) -> Standing {
let read = match located_by {
LocatedBy::Named | LocatedBy::OutDir => true,
LocatedBy::Manifest | LocatedBy::None => false,
};
if !read {
return Standing::Unread { located_by };
}
let copies = match parse_copies(recorded) {
Ok(copies) => copies,
Err(reason) => return untabled(reason, &[]),
};
if let Some(copy) = copies
.iter()
.find(|copy| copy.name != PROPERTIES_CRATE && copy.name != DATA_CRATE)
{
return untabled(
format!(
"the property-data record names another crate ({} {})",
copy.name, copy.version
),
&[],
);
}
let (linked, others): (Vec<&LockCopy>, Vec<&LockCopy>) =
copies.iter().partition(|copy| is_linked_major(copy));
let linked_copies: Vec<LockCopy> = linked.iter().map(|copy| (*copy).clone()).collect();
for name in [PROPERTIES_CRATE, DATA_CRATE] {
if !linked.iter().any(|copy| copy.name == name) {
return untabled(
format!(
"the governing lockfile holds no copy of {name} at major version \
{LINKED_MAJOR}, the one the engine links"
),
&linked_copies,
);
}
}
let mut all_judged = true;
for copy in &linked {
let label = format!("{} {}", copy.name, copy.version);
if copy.source.as_deref() != Some(table::REGISTRY_SOURCE) {
return untabled(
format!("{label} does not come from the crates.io registry"),
&linked_copies,
);
}
let Some(checksum) = copy.checksum.as_deref() else {
return untabled(
format!("the governing lockfile gives no checksum for {label}"),
&linked_copies,
);
};
let Some(admitted) = ADMITTED_COPIES.iter().find(|admitted| {
admitted.name == copy.name
&& admitted.version == copy.version
&& admitted.checksum == checksum
}) else {
return untabled(
format!(
"{label} (checksum {checksum}) is not among the property-data copies this \
release was judged against"
),
&linked_copies,
);
};
all_judged &= admitted.judged;
}
if all_judged && references_certified {
return Standing::Judged;
}
let label = |copies: &[&LockCopy]| {
let mut labels: Vec<String> = copies
.iter()
.map(|copy| format!("{} {}", copy.name, copy.version))
.collect();
labels.sort();
labels.join(", ")
};
let mut resolved = label(&linked);
if !others.is_empty() {
resolved.push_str(&format!(
" (beside {}, which the engine does not link)",
label(&others)
));
}
Standing::Guarded { resolved }
}
pub(crate) fn decide_build_standing(
data_dir_set: bool,
located_by: LocatedBy,
recorded: &str,
references_certified: bool,
) -> Standing {
if data_dir_set {
return Standing::Untabled {
reason: format!(
"{DATA_DIR_ENV} was set when this crate was built, so the property data \
compiled into it is that directory's and not the data of the copies the \
lockfile names"
),
remedy: format!(
"unset {DATA_DIR_ENV} and rebuild, so the build compiles the data the lockfile \
names"
),
};
}
decide_standing(located_by, recorded, references_certified)
}
pub(crate) const LOCATED_BY_RECORD: &str = env!("TOKTIER_RUST_API_LOCK_LOCATED_BY");
pub(crate) const PROPERTY_DATA_RECORD: &str = env!("TOKTIER_RUST_API_PROPERTY_DATA");
pub(crate) const DATA_DIR_RECORD: &str = env!("TOKTIER_RUST_API_PROPERTY_DATA_DIR");
fn references_certified() -> bool {
let records = crate::behavior_version::reference_facts();
[PROPERTIES_CRATE, DATA_CRATE].iter().all(|name| {
records
.iter()
.any(|record| record.package == *name && record.certified)
})
}
pub(crate) fn standing() -> &'static Standing {
static STANDING: std::sync::OnceLock<Standing> = std::sync::OnceLock::new();
STANDING.get_or_init(|| {
decide_build_standing(
!DATA_DIR_RECORD.is_empty(),
LocatedBy::parse(LOCATED_BY_RECORD),
PROPERTY_DATA_RECORD,
references_certified(),
)
})
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Handling {
NoGuard,
Guard,
Decline(crate::ReasonCode),
}
pub(crate) fn handling(standing: &Standing, policy: crate::Policy) -> Handling {
let experimental = policy == crate::Policy::Experimental;
match standing {
Standing::Judged => Handling::NoGuard,
Standing::Contradicted { .. } => {
Handling::Decline(crate::ReasonCode::PropertyDataContradicted)
}
Standing::Guarded { .. } => Handling::Guard,
Standing::Untabled { .. } if experimental => Handling::Guard,
Standing::Unread { .. } if experimental => Handling::Guard,
Standing::Untabled { .. } => Handling::Decline(crate::ReasonCode::PropertyDataUntabled),
Standing::Unread { .. } => Handling::Decline(crate::ReasonCode::PropertyDataUnread),
}
}
pub(crate) fn router_for<'a>(
ordinary: &'a NativeRouter,
reference_only: Option<&'a NativeRouter>,
text: &str,
) -> (&'a NativeRouter, bool) {
match reference_only {
Some(reference_only) if holds_moved(text) => (reference_only, true),
_ => (ordinary, false),
}
}
pub(crate) struct GuardedEncoder<'a> {
ordinary: &'a NativeRouter,
reference_only: Option<&'a NativeRouter>,
routed: std::cell::Cell<bool>,
}
impl<'a> GuardedEncoder<'a> {
pub(crate) fn new(
ordinary: &'a NativeRouter,
reference_only: Option<&'a NativeRouter>,
) -> Self {
Self {
ordinary,
reference_only,
routed: std::cell::Cell::new(false),
}
}
pub(crate) fn guard_routed(&self) -> bool {
self.routed.get()
}
fn pick(&self, hit: impl FnOnce() -> bool) -> &'a NativeRouter {
match self.reference_only {
Some(reference_only) if hit() => {
self.routed.set(true);
reference_only
}
_ => self.ordinary,
}
}
}
impl SessionEncoder for GuardedEncoder<'_> {
fn encode(&self, text: &str) -> Result<Encoding, EngineError> {
self.pick(|| holds_moved(text)).encode(text)
}
fn append(&self, tail: &mut TailState, delta: &str) -> Result<AppendReport, EngineError> {
self.pick(|| holds_moved(tail.text()) || holds_moved(delta))
.append(tail, delta)
}
fn last_certified_boundary(
&self,
tail: &TailState,
floor_char: u64,
ceil_char: u64,
) -> Result<Option<BoundaryCut>, EngineError> {
if self.reference_only.is_some() && holds_moved(tail.text()) {
return Ok(None);
}
self.ordinary
.last_certified_boundary(tail, floor_char, ceil_char)
}
fn witness_category(&self) -> WitnessCategory {
self.ordinary.witness_category()
}
}
#[cfg(test)]
#[path = "property_guard_tests.rs"]
mod wiring_tests;
#[cfg(test)]
mod tests {
use super::{
decide_build_standing, decide_standing, holds_moved, is_moved, parse_copies, LocatedBy,
Standing, ADMITTED_COPIES, DATA_CRATE, FIELD_SEPARATOR, MOVED_CODE_POINTS, MOVED_RANGES,
PROPERTIES_CRATE, RECORD_SEPARATOR,
};
const REGISTRY: &str = "registry+https://github.com/rust-lang/crates.io-index";
fn naive(code_point: u32) -> bool {
MOVED_RANGES
.iter()
.any(|&(low, high)| (low..=high).contains(&code_point))
}
fn scalar(code_point: u32) -> Option<String> {
char::from_u32(code_point).map(String::from)
}
const HITS: &[&str] = &[
"1\u{11DE1}2345",
"it's\u{323B0}'s \u{323B0}'ll",
"abc \u{20C1}def ghi",
"Hello \u{0295}\u{2022}\u{1D25}\u{2022}\u{0294} World a\u{0295}b A\u{0295}B",
];
const CLEAN: &[&str] = &[
"",
"plain ascii text 12345",
"caf\u{e9} na\u{ef}ve \u{fc}ber stra\u{df}e",
"\u{4e2d}\u{6587}\u{6d4b}\u{8bd5} 12345 \u{30c6}\u{30b9}\u{30c8}",
"\u{1F600} \u{1FAE9} \u{1FAC6} \u{1FABE} \u{1FADC}",
"\u{0294} \u{0296}",
];
#[test]
fn the_table_is_the_generated_one() {
assert!(!MOVED_RANGES.is_empty());
assert_eq!(MOVED_RANGES[0], (0x0295, 0x0295));
let total: u32 = MOVED_RANGES.iter().map(|&(low, high)| high - low + 1).sum();
assert_eq!(total, MOVED_CODE_POINTS);
for pair in MOVED_RANGES.windows(2) {
assert!(pair[0].1 + 1 < pair[1].0, "{pair:?}");
}
}
#[test]
fn every_range_end_is_in_and_every_neighbour_is_out() {
for &(low, high) in MOVED_RANGES {
for code_point in [low, high] {
assert!(is_moved(code_point), "U+{code_point:04X}");
if let Some(text) = scalar(code_point) {
assert!(holds_moved(&text), "U+{code_point:04X}");
}
}
for code_point in [low.wrapping_sub(1), high + 1] {
assert!(!is_moved(code_point), "U+{code_point:04X}");
if let Some(text) = scalar(code_point) {
assert!(!holds_moved(&text), "U+{code_point:04X}");
}
}
}
}
#[test]
fn every_code_point_answers_as_the_ranges_say() {
for code_point in 0..0x11_0000 {
let expected = naive(code_point);
assert_eq!(is_moved(code_point), expected, "U+{code_point:04X}");
if let Some(text) = scalar(code_point) {
assert_eq!(holds_moved(&text), expected, "U+{code_point:04X}");
}
}
}
#[test]
fn a_hit_is_found_at_every_offset_and_after_every_kind_of_prefix() {
let prefixes = ["a", "\u{e9}", "\u{4e2d}", "\u{1F600}", "\u{0294}"];
let mut probes: Vec<u32> = MOVED_RANGES.iter().flat_map(|&(l, h)| [l, h]).collect();
probes.dedup();
for &code_point in &probes {
let Some(hit) = scalar(code_point) else {
continue;
};
for prefix in prefixes {
for repeat in 0..18 {
let before = prefix.repeat(repeat);
for after in ["", "x", "tail text after it", "\u{e9}\u{e9}"] {
let text = format!("{before}{hit}{after}");
assert!(holds_moved(&text), "{text:?}");
let clean = format!("{before}{after}");
assert!(!holds_moved(&clean), "{clean:?}");
}
}
}
}
}
#[test]
fn the_review_texts_are_hits_and_clean_texts_are_not() {
for text in HITS {
assert!(holds_moved(text), "{text:?}");
}
for text in CLEAN {
assert!(!holds_moved(text), "{text:?}");
}
let long_clean = "word ".repeat(200_000);
assert!(!holds_moved(&long_clean));
let long_hit = format!("{long_clean}1\u{11DE1}2345");
assert!(holds_moved(&long_hit));
}
fn judged(name: &str) -> &'static super::table::AdmittedCopy {
ADMITTED_COPIES
.iter()
.find(|copy| copy.name == name && copy.judged)
.expect("a judged copy")
}
fn tabled(name: &str, version: &str) -> &'static super::table::AdmittedCopy {
ADMITTED_COPIES
.iter()
.find(|copy| copy.name == name && copy.version == version)
.expect("an admitted copy")
}
fn record(name: &str, version: &str, source: &str, checksum: &str) -> String {
[name, version, source, checksum].join(&FIELD_SEPARATOR.to_string())
}
fn of(copy: &super::table::AdmittedCopy) -> String {
record(copy.name, copy.version, REGISTRY, copy.checksum)
}
fn records(parts: &[String]) -> String {
parts.join(&RECORD_SEPARATOR.to_string())
}
fn judged_record() -> String {
records(&[of(judged(PROPERTIES_CRATE)), of(judged(DATA_CRATE))])
}
fn guarded_record() -> String {
records(&[
of(tabled(PROPERTIES_CRATE, "2.3.0")),
of(tabled(DATA_CRATE, "2.3.0")),
])
}
fn untabled(standing: &Standing) -> bool {
matches!(standing, Standing::Untabled { .. })
}
#[test]
fn the_judged_copies_are_the_workspace_lockfiles() {
let lock = include_str!("../data/build/judged_dependencies.lock");
for name in [PROPERTIES_CRATE, DATA_CRATE] {
let copy = judged(name);
let block = format!(
"name = \"{}\"\nversion = \"{}\"\nsource = \"{REGISTRY}\"\nchecksum = \"{}\"\n",
copy.name, copy.version, copy.checksum
);
assert!(lock.contains(&block), "{block}");
assert_eq!(
ADMITTED_COPIES
.iter()
.filter(|other| other.name == name && other.judged)
.count(),
1
);
}
}
#[test]
fn a_located_lockfile_with_the_judged_copies_is_judged() {
for located_by in [LocatedBy::Named, LocatedBy::OutDir] {
assert_eq!(
decide_standing(located_by, &judged_record(), true),
Standing::Judged
);
}
}
#[test]
fn the_workspace_fallback_never_counts_as_read() {
for recorded in [judged_record(), guarded_record(), String::new()] {
let standing = decide_standing(LocatedBy::Manifest, &recorded, true);
assert_eq!(
standing,
Standing::Unread {
located_by: LocatedBy::Manifest
}
);
assert!(!standing.admits_fast_route());
assert!(standing.to_string().contains("TOKTIER_CARGO_LOCK"));
}
assert_eq!(LocatedBy::parse("manifest"), LocatedBy::Manifest);
}
#[test]
fn no_located_lockfile_is_unread() {
for recorded in [String::new(), judged_record()] {
let standing = decide_standing(LocatedBy::None, &recorded, true);
assert_eq!(
standing,
Standing::Unread {
located_by: LocatedBy::None
}
);
assert!(!standing.admits_fast_route());
}
assert_eq!(LocatedBy::parse("elsewhere"), LocatedBy::None);
assert_eq!(LocatedBy::parse(""), LocatedBy::None);
}
#[test]
fn every_tabled_pair_is_guarded() {
let standing = decide_standing(LocatedBy::OutDir, &guarded_record(), true);
assert_eq!(
standing,
Standing::Guarded {
resolved: "icu_properties 2.3.0, icu_properties_data 2.3.0".to_owned()
}
);
assert!(standing.admits_fast_route());
let mixed = records(&[
of(judged(PROPERTIES_CRATE)),
of(tabled(DATA_CRATE, "2.0.0")),
]);
assert!(decide_standing(LocatedBy::Named, &mixed, true).is_guarded());
}
#[test]
fn uncertified_reference_records_are_never_judged() {
let standing = decide_standing(LocatedBy::OutDir, &judged_record(), false);
assert!(standing.is_guarded(), "{standing:?}");
}
#[test]
fn two_copies_of_one_crate_are_judged_copy_by_copy() {
let both_tabled = records(&[
of(judged(PROPERTIES_CRATE)),
of(judged(DATA_CRATE)),
of(tabled(DATA_CRATE, "2.4.0")),
]);
assert!(decide_standing(LocatedBy::OutDir, &both_tabled, true).is_guarded());
let one_off = records(&[
of(judged(PROPERTIES_CRATE)),
of(judged(DATA_CRATE)),
record(PROPERTIES_CRATE, "2.9.0", REGISTRY, &"a".repeat(64)),
]);
assert!(untabled(&decide_standing(
LocatedBy::OutDir,
&one_off,
true
)));
}
#[test]
fn only_copies_of_major_version_two_decide() {
let old = [
record(PROPERTIES_CRATE, "1.5.1", REGISTRY, &"a".repeat(64)),
record(DATA_CRATE, "1.5.1", REGISTRY, &"d".repeat(64)),
record(PROPERTIES_CRATE, "3.0.0", "", ""),
];
let mut beside_judged = vec![of(judged(PROPERTIES_CRATE)), of(judged(DATA_CRATE))];
beside_judged.extend(old.iter().cloned());
assert_eq!(
decide_standing(LocatedBy::OutDir, &records(&beside_judged), true),
Standing::Judged
);
let mut beside_newer = vec![
of(tabled(PROPERTIES_CRATE, "2.4.0")),
of(tabled(DATA_CRATE, "2.4.0")),
];
beside_newer.extend(old.iter().cloned());
let standing = decide_standing(LocatedBy::Named, &records(&beside_newer), true);
assert_eq!(
standing,
Standing::Guarded {
resolved: "icu_properties 2.4.0, icu_properties_data 2.4.0 (beside \
icu_properties 1.5.1, icu_properties 3.0.0, icu_properties_data \
1.5.1, which the engine does not link)"
.to_owned()
}
);
for alone in [
records(&old[..2]),
records(&[record(PROPERTIES_CRATE, "3.0.0", REGISTRY, &"e".repeat(64))]),
records(&[
of(judged(PROPERTIES_CRATE)),
record(DATA_CRATE, "1.5.1", REGISTRY, &"d".repeat(64)),
]),
] {
let standing = decide_standing(LocatedBy::OutDir, &alone, true);
assert!(untabled(&standing), "{alone:?}: {standing:?}");
assert!(
standing.to_string().contains("major version 2"),
"{standing}"
);
}
}
#[test]
fn the_remedy_names_the_resolved_copy_by_its_version() {
let newer = records(&[
of(judged(PROPERTIES_CRATE)),
record(DATA_CRATE, "2.9.0", REGISTRY, &"b".repeat(64)),
record(PROPERTIES_CRATE, "1.5.1", REGISTRY, &"a".repeat(64)),
]);
let line = decide_standing(LocatedBy::OutDir, &newer, true).to_string();
let data = judged(DATA_CRATE);
let code = judged(PROPERTIES_CRATE);
assert!(
line.contains(&format!(
"`cargo update --package icu_properties_data@2.9.0 --precise {}`",
data.version
)),
"{line}"
);
assert!(
line.contains(&format!(
"`cargo update --package icu_properties@{} --precise {}`",
code.version, code.version
)),
"{line}"
);
assert!(!line.contains("icu_properties@1.5.1"), "{line}");
}
#[test]
fn copies_off_the_table_are_untabled() {
let data = judged(DATA_CRATE);
let code = of(judged(PROPERTIES_CRATE));
let cases = [
records(&[
code.clone(),
record(DATA_CRATE, "2.5.0", REGISTRY, &"b".repeat(64)),
]),
records(&[
code.clone(),
record(DATA_CRATE, data.version, REGISTRY, &"c".repeat(64)),
]),
records(&[code.clone(), record(DATA_CRATE, data.version, REGISTRY, "")]),
records(&[
code.clone(),
record(
DATA_CRATE,
data.version,
"git+https://example.invalid/icu4x#0",
data.checksum,
),
]),
records(&[code.clone(), record(DATA_CRATE, data.version, "", "")]),
code.clone(),
of(data),
String::new(),
format!("{code}{FIELD_SEPARATOR}extra"),
records(&[
code.clone(),
record("icu_other", "2.0.0", REGISTRY, data.checksum),
]),
];
for recorded in &cases {
for located_by in [LocatedBy::Named, LocatedBy::OutDir] {
let standing = decide_standing(located_by, recorded, true);
assert!(untabled(&standing), "{recorded:?}: {standing:?}");
assert!(!standing.admits_fast_route());
}
}
}
#[test]
fn a_replaced_data_directory_is_untabled_with_its_own_remedy() {
for located_by in [
LocatedBy::Named,
LocatedBy::OutDir,
LocatedBy::Manifest,
LocatedBy::None,
] {
for recorded in [judged_record(), guarded_record(), String::new()] {
let standing = decide_build_standing(true, located_by, &recorded, true);
assert!(untabled(&standing), "{located_by:?}: {standing:?}");
assert!(!standing.admits_fast_route());
let line = standing.to_string();
assert!(
line.starts_with("untabled: ICU4X_DATA_DIR was set"),
"{line}"
);
assert!(line.contains("unset ICU4X_DATA_DIR"), "{line}");
assert!(!line.contains("cargo update"), "{line}");
}
}
assert_eq!(
decide_build_standing(false, LocatedBy::OutDir, &judged_record(), true),
Standing::Judged
);
assert!(
decide_build_standing(false, LocatedBy::Named, &guarded_record(), true).is_guarded()
);
}
#[test]
fn the_record_reads_back_what_the_build_script_writes() {
let copies = parse_copies(&judged_record()).expect("readable");
assert_eq!(copies.len(), 2);
assert_eq!(copies[0].name, PROPERTIES_CRATE);
assert_eq!(copies[0].source.as_deref(), Some(REGISTRY));
assert_eq!(parse_copies("").expect("empty"), Vec::new());
assert!(parse_copies("icu_properties").is_err());
let blank = record(DATA_CRATE, "2.0.1", "", "");
let copies = parse_copies(&blank).expect("readable");
assert_eq!(copies[0].source, None);
assert_eq!(copies[0].checksum, None);
}
#[test]
fn the_standing_names_the_remedy() {
let untabled = decide_standing(LocatedBy::OutDir, "", true).to_string();
let unread = decide_standing(LocatedBy::None, "", true).to_string();
for line in [&untabled, &unread] {
assert!(line.contains("TOKTIER_CARGO_LOCK"), "{line}");
assert!(line.contains("absolute path"), "{line}");
assert!(
line.contains("cargo update --package icu_properties@2 --precise"),
"{line}"
);
assert!(
line.contains("cargo update --package icu_properties_data@2 --precise"),
"{line}"
);
}
assert!(untabled.starts_with("untabled: "));
assert!(unread.starts_with("unread: "));
assert_eq!(Standing::Judged.to_string(), "judged");
let guarded = decide_standing(LocatedBy::OutDir, &guarded_record(), true);
let line = guarded.to_string();
assert!(line.starts_with("guarded: "), "{line}");
assert!(line.contains(&MOVED_CODE_POINTS.to_string()), "{line}");
assert!(line.contains("icu_properties_data 2.3.0"), "{line}");
}
}
pub(crate) fn confirm_encodes(
family: &str,
engine: &toktier_routing_core::FastCpuEngine,
reference: &toktier_routing_core::ReferenceEngine,
) -> Option<Standing> {
check_encodes(family, |text| {
let fast = engine.encode_ids(text).map_err(|error| error.to_string())?;
let expected = reference
.encode_ids(text, false)
.map_err(|error| error.to_string())?;
Ok(fast == expected)
})
}
fn check_encodes(
family: &str,
mut agrees: impl FnMut(&str) -> Result<bool, String>,
) -> Option<Standing> {
for text in [
"1\u{11de1}2345",
"it's\u{323b0}'s \u{323b0}'ll",
"9\u{1e6e0}9\u{1e6e0}87654321 x",
"abc \u{20c1}def ghi",
] {
let reason = match agrees(text) {
Ok(true) => continue,
Ok(false) => format!("family {family}, probe {text:?}: fast and reference ids differ"),
Err(error) => {
format!("family {family}, probe {text:?}: check could not complete ({error})")
}
};
return Some(Standing::Contradicted { reason });
}
None
}
#[cfg(test)]
mod encode_confirmation_tests {
use super::*;
#[test]
fn all_four_probe_contexts_must_agree() {
let mut texts = Vec::new();
assert_eq!(
check_encodes("test", |text| {
texts.push(text.to_owned());
Ok(true)
}),
None
);
assert_eq!(texts.len(), 4);
assert!(texts.iter().all(|text| holds_moved(text)));
for failed in 0..4 {
let mut seen = 0;
let result = check_encodes("test", |_| {
let equal = seen != failed;
seen += 1;
Ok(equal)
});
assert_eq!(seen, failed + 1);
let standing = result.expect("a mismatch must decline");
assert!(standing.to_string().contains("ids differ"));
for policy in [
crate::Policy::Supported,
crate::Policy::Experimental,
crate::Policy::RequireAccelerated,
] {
assert_eq!(
handling(&standing, policy),
Handling::Decline(crate::ReasonCode::PropertyDataContradicted)
);
}
}
}
#[test]
fn a_probe_error_declines_and_names_the_remedy() {
let standing = check_encodes("test", |_| Err("injected fault".into())).unwrap();
let line = standing.to_string();
for needle in [
"untabled:",
"judged",
"contradicted",
"injected fault",
"TOKTIER_CARGO_LOCK",
"--locked",
] {
assert!(line.contains(needle), "{line}");
}
}
}