use super::{Failure, collation_glibc, unsupported};
use crate::conversion::profile::Profile;
use icu_collator::{
Collator as IcuCollator, CollatorBorrowed,
options::{AlternateHandling, CollatorOptions, Strength},
};
use icu_locale_core::Locale;
use std::ffi::OsString;
#[derive(Clone, Copy, Default, PartialEq, Eq)]
pub(super) enum Collation {
#[default]
Bytes,
Language(&'static str),
Unsupported,
}
#[derive(Clone, Copy)]
pub(super) struct Policy {
pub numeric: Profile,
numbers: bool,
pub utf8: bool,
ascii_case: bool,
pub collation: Collation,
}
impl Default for Policy {
fn default() -> Self {
Self {
numeric: Profile::C,
numbers: true,
utf8: false,
ascii_case: true,
collation: Collation::Bytes,
}
}
}
fn recognized(bytes: &[u8]) -> Option<(bool, Profile, Collation)> {
let (name, modifier) = match bytes.iter().position(|&b| b == b'@') {
Some(at) => (&bytes[..at], Some(&bytes[at..])),
None => (bytes, None),
};
let (base, codeset) = match name.iter().position(|&b| b == b'.') {
Some(dot) => (&name[..dot], Some(&name[dot + 1..])),
None => (name, None),
};
let utf8 = match codeset {
Some(codeset) => {
codeset.eq_ignore_ascii_case(b"UTF-8") || codeset.eq_ignore_ascii_case(b"utf8")
}
None => super::numeric_locales::BARE_UTF8
.binary_search(&base)
.is_ok(),
};
if let b"C" | b"POSIX" = base {
return Some((utf8, Profile::C, Collation::Bytes));
}
let numbers = |key: &[u8]| {
let table = super::numeric_locales::NUMERIC;
table
.binary_search_by(|(entry, _)| (*entry).cmp(key))
.ok()
.map(|at| table[at].1)
};
let refused = super::numeric_locales::REFUSED.binary_search(&base).is_ok();
let own = modifier.and_then(|modifier| {
let mut key = base.to_vec();
key.extend_from_slice(modifier);
numbers(&key)
});
let numeric = match own.or_else(|| numbers(base)) {
Some(numeric) => numeric,
None if refused => return None,
None => return Some((false, Profile::C, Collation::Bytes)),
};
if !utf8 && !(numeric.radix().is_ascii() && numeric.thousands().is_ascii()) {
return None;
}
let table = super::collation_locales::COLLATION;
let collation = if utf8 && own.is_none() {
table
.binary_search_by(|(entry, _)| (*entry).cmp(base))
.map_or(Collation::Unsupported, |at| {
Collation::Language(table[at].1)
})
} else {
Collation::Unsupported
};
Some((utf8, numeric, collation))
}
fn turkic_case(name: &[u8]) -> bool {
let mut parts = name.splitn(2, |&b| b == b'@');
let base = parts.next().unwrap_or_default();
let modifier = parts.next();
let language = base.split(|&b| b == b'.').next().unwrap_or_default();
matches!(
language,
b"az_AZ" | b"crh_UA" | b"ku_TR" | b"tr_CY" | b"tr_TR"
) || (language == b"tt_RU" && modifier == Some(b"iqtelif"))
}
fn utf8_codeset(name: &[u8]) -> bool {
let name = name.split(|&b| b == b'@').next().unwrap_or_default();
name.iter().position(|&b| b == b'.').is_some_and(|dot| {
let codeset = &name[dot + 1..];
codeset.eq_ignore_ascii_case(b"UTF-8") || codeset.eq_ignore_ascii_case(b"utf8")
})
}
fn setting(
get: &impl Fn(&str) -> Option<OsString>,
category: &'static str,
) -> (&'static str, OsString) {
["LC_ALL", category, "LANG"]
.into_iter()
.find_map(|key| get(key).filter(|v| !v.is_empty()).map(|v| (key, v)))
.unwrap_or(("", "C".into()))
}
fn refusal(category: &'static str, reason: &str, hint: &str, utf8: bool) -> Failure {
let (source, value) = setting(&|key| std::env::var_os(key), category);
let variable = if source == "LC_ALL" {
"LC_ALL"
} else {
category
};
let mut message = reason.as_bytes().to_vec();
super::named_fields::quote_with(value.as_encoded_bytes(), &mut message, utf8);
message.extend_from_slice(
format!(" from {source}\nhint: {}\n", hint.replace("{}", variable)).as_bytes(),
);
Failure {
status: 77,
message,
hint: None,
}
}
impl Policy {
pub fn from_env() -> Self {
Self::resolve(|key| std::env::var_os(key))
}
pub(super) fn resolve(get: impl Fn(&str) -> Option<OsString>) -> Self {
let category = |name| setting(&get, name).1;
let ctype = category("LC_CTYPE");
let utf8 = recognized(ctype.as_encoded_bytes())
.map_or_else(|| utf8_codeset(ctype.as_encoded_bytes()), |v| v.0);
let numeric = recognized(category("LC_NUMERIC").as_encoded_bytes()).map(|v| v.1);
let collation = recognized(category("LC_COLLATE").as_encoded_bytes())
.map_or(Collation::Unsupported, |v| v.2);
Self {
numeric: numeric.unwrap_or(Profile::C),
numbers: numeric.is_some(),
utf8,
ascii_case: !turkic_case(ctype.as_encoded_bytes()),
collation,
}
}
pub fn numbers(self) -> Result<(), Failure> {
if self.numbers {
return Ok(());
}
Err(refusal(
"LC_NUMERIC",
"unsupported numeric locale ",
"set {}=C.UTF-8, or a UTF-8 locale whose decimal separator is one byte",
self.utf8,
))
}
pub fn sorting(self) -> Result<(), Failure> {
if self.collation != Collation::Unsupported {
return Ok(());
}
Err(refusal(
"LC_COLLATE",
"unsupported sorting locale ",
"set {}=C.UTF-8 for byte order; fastmash --help lists the language locales",
self.utf8,
))
}
pub fn case_folding(self) -> Result<(), Failure> {
if self.ascii_case {
return Ok(());
}
Err(refusal(
"LC_CTYPE",
"unsupported -i under character type ",
"this locale does not fold i and I; set {}=C.UTF-8 or omit -i",
self.utf8,
))
}
pub fn language(self) -> bool {
matches!(self.collation, Collation::Language(_))
}
pub fn collator(self) -> Result<Option<Collator>, Failure> {
self.sorting()?;
let Collation::Language(tag) = self.collation else {
return Ok(None);
};
Collator::new(tag).map(Some)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Weight {
Ignored(u16),
First(u16),
Unweighted,
Other(u8),
}
const LETTER: Weight = Weight::Other(1);
pub(super) struct Collator {
icu: CollatorBorrowed<'static>,
first: &'static [u32],
unweighted: &'static [(u32, u32)],
ascii: [Weight; 128],
blocks: Box<[std::sync::OnceLock<Block>]>,
}
enum Block {
Uniform(Weight),
Mixed(Box<[Weight]>),
Unknown,
}
const SCRATCH_ROOM: usize = 1 << 16;
thread_local! {
static SCRATCH: std::cell::RefCell<(String, Vec<u8>)> =
const { std::cell::RefCell::new((String::new(), Vec::new())) };
}
#[cfg(test)]
thread_local! {
pub(super) static KEY_WRITES_BEFORE_FAILURE: std::cell::Cell<Option<usize>> =
const { std::cell::Cell::new(None) };
}
pub(super) struct KeyTooLarge;
struct KeyBuffer<'a>(&'a mut Vec<u8>);
impl KeyBuffer<'_> {
#[inline]
fn room(&mut self, additional: usize) -> Result<(), KeyTooLarge> {
#[cfg(test)]
if KEY_WRITES_BEFORE_FAILURE.with(|count| match count.get() {
Some(0) => true,
Some(n) => {
count.set(Some(n - 1));
false
}
None => false,
}) {
return Err(KeyTooLarge);
}
if self.0.capacity() - self.0.len() < additional {
self.grow(additional)?;
}
Ok(())
}
#[cold]
#[inline(never)]
fn grow(&mut self, additional: usize) -> Result<(), KeyTooLarge> {
self.0.try_reserve(additional).map_err(|_| KeyTooLarge)
}
fn bytes(&mut self, bytes: &[u8]) -> Result<(), KeyTooLarge> {
self.room(bytes.len())?;
self.0.extend_from_slice(bytes);
Ok(())
}
fn number(&mut self, number: u64) -> Result<(), KeyTooLarge> {
match u16::try_from(number) {
Ok(short) if short < 0xff00 => self.bytes(&short.to_be_bytes()),
_ => {
self.bytes(&[0xff])?;
self.bytes(&number.to_be_bytes())
}
}
}
}
const AFTER_SKIPS: u64 = 1 << 62;
const LETTER_RANK: u16 = u16::MAX;
struct Fourth<'a> {
out: KeyBuffer<'a>,
letters: u64,
skipped: u64,
}
impl Fourth<'_> {
fn letters(&mut self, count: u64) -> Result<(), KeyTooLarge> {
if count != 0 && self.skipped != 0 {
self.after_skips(LETTER_RANK)?;
self.letters = count - 1;
} else {
self.letters += count;
}
Ok(())
}
fn skip(&mut self) {
self.skipped += 1;
}
fn ignored(&mut self, rank: u16) -> Result<(), KeyTooLarge> {
if self.skipped == 0 {
self.out.number(self.letters)?;
self.out.bytes(&rank.to_be_bytes())?;
} else {
self.after_skips(rank)?;
}
self.letters = 0;
Ok(())
}
fn after_skips(&mut self, weight: u16) -> Result<(), KeyTooLarge> {
self.out
.number(AFTER_SKIPS - self.letters.min(AFTER_SKIPS - 1))?;
self.out.number(self.skipped + 1)?;
self.out.bytes(&weight.to_be_bytes())?;
self.skipped = 0;
Ok(())
}
fn end(&mut self) -> Result<(), KeyTooLarge> {
self.out.number(self.letters)
}
}
impl icu_collator::CollationKeySink for KeyBuffer<'_> {
type Error = KeyTooLarge;
type State = ();
type Output = ();
#[inline]
fn write(&mut self, _: &mut (), bytes: &[u8]) -> Result<(), KeyTooLarge> {
self.bytes(bytes)
}
#[inline]
fn write_byte(&mut self, _: &mut (), byte: u8) -> Result<(), KeyTooLarge> {
self.room(1)?;
self.0.push(byte);
Ok(())
}
fn finish(&mut self, _: ()) -> Result<(), KeyTooLarge> {
Ok(())
}
}
impl Collator {
fn new(tag: &'static str) -> Result<Self, Failure> {
let language = Locale::try_from_str(tag).expect("collation tags are valid BCP 47");
let mut options = CollatorOptions::default();
options.alternate_handling = Some(AlternateHandling::NonIgnorable);
options.strength = Some(Strength::Tertiary);
let icu = IcuCollator::try_new(language.into(), options)
.map_err(|_| unsupported("compiled locale data unavailable"))?;
let (first, unweighted) = collation_glibc::LOCALES
.binary_search_by(|(entry, ..)| (*entry).cmp(tag))
.map_or((&[][..], &[][..]), |at| {
let (_, first, unweighted) = collation_glibc::LOCALES[at];
(first, unweighted)
});
let mut blocks = Vec::new();
blocks
.try_reserve_exact(256)
.map_err(|_| unsupported("command memory allocation failed"))?;
blocks.resize_with(256, std::sync::OnceLock::new);
let mut collator = Self {
icu,
first,
unweighted,
ascii: [LETTER; 128],
blocks: blocks.into_boxed_slice(),
};
for byte in 0..128u8 {
collator.ascii[usize::from(byte)] = collator.classify(char::from(byte));
}
Ok(collator)
}
#[inline]
fn weight(&self, character: char) -> Weight {
let point = u32::from(character);
if let Some(&weight) = self.ascii.get(point as usize) {
return weight;
}
let Some(block) = self.blocks.get((point >> 8) as usize) else {
return self.classify(character);
};
match block.get_or_init(|| self.block(point >> 8)) {
Block::Uniform(weight) => *weight,
Block::Mixed(weights) => weights[(point & 0xff) as usize],
Block::Unknown => self.classify(character),
}
}
#[cold]
fn block(&self, block: u32) -> Block {
let weight = |at: u32| char::from_u32(block << 8 | at).map_or(LETTER, |c| self.classify(c));
let uniform = weight(0);
if (1..256).all(|at| weight(at) == uniform) {
return Block::Uniform(uniform);
}
let mut weights = Vec::new();
if weights.try_reserve_exact(256).is_err() {
return Block::Unknown;
}
weights.extend((0..256).map(weight));
Block::Mixed(weights.into_boxed_slice())
}
fn classify(&self, character: char) -> Weight {
let point = u32::from(character);
let ignored = collation_glibc::IGNORED;
let at = ignored.partition_point(|&(first, ..)| first <= point);
if let Some(&(first, last, rank)) = at.checked_sub(1).map(|at| &ignored[at])
&& point <= last
{
return match self.first.iter().position(|&weighed| weighed == point) {
Some(at) => Weight::First(at as u16),
None => Weight::Ignored(rank + (point - first) as u16 + self.first.len() as u16),
};
}
if within(collation_glibc::NO_FOURTH, point) || within(self.unweighted, point) {
return Weight::Unweighted;
}
let decomposed = collation_glibc::DECOMPOSED;
let at = decomposed.partition_point(|&(first, ..)| first <= point);
match at.checked_sub(1).map(|at| decomposed[at]) {
Some((_, last, length)) if point <= last => Weight::Other(length),
_ => LETTER,
}
}
pub(super) fn write_key(&self, text: &str, out: &mut Vec<u8>) -> Result<(), KeyTooLarge> {
let room = text.len().saturating_mul(3).saturating_add(16);
out.try_reserve(room).map_err(|_| KeyTooLarge)?;
let mut key = KeyBuffer(out);
let bytes = text.as_bytes();
let plain = |byte: u8| byte < 0x80 && self.ascii[usize::from(byte)] == LETTER;
let Some(start) = bytes.iter().position(|&byte| !plain(byte)) else {
self.icu.write_sort_key_to(text, &mut key)?;
key.bytes(&[1])?;
return key.number(text.len() as u64);
};
SCRATCH.with_borrow_mut(|(mapped, fourth)| {
mapped.clear();
fourth.clear();
let written = self.map(text, start, mapped, fourth);
let result = written.and_then(|()| {
self.icu.write_sort_key_to(mapped.as_str(), &mut key)?;
key.bytes(&[1])?;
key.bytes(fourth)
});
if mapped.capacity() > SCRATCH_ROOM {
*mapped = String::new();
}
if fourth.capacity() > SCRATCH_ROOM {
*fourth = Vec::new();
}
result
})
}
pub(super) fn write_identity(
&self,
text: &str,
out: &mut Vec<u8>,
) -> Result<bool, KeyTooLarge> {
static PLACED: std::sync::OnceLock<CharSet> = std::sync::OnceLock::new();
let table = collation_glibc::REORDERED;
let placed = PLACED.get_or_init(|| CharSet::new(table.iter().map(|&(letter, _)| letter)));
if text.is_ascii() || !text.chars().any(|c| placed.contains(c)) {
return Ok(false);
}
let mut key = KeyBuffer(out);
for character in text.chars() {
let point = u32::from(character);
if placed.contains(character)
&& let Ok(at) = table.binary_search_by_key(&point, |&(letter, _)| letter)
{
let base = char::from_u32(table[at].1).expect("bases are characters");
key.bytes(base.encode_utf8(&mut [0; 4]).as_bytes())?;
key.bytes(&[0xff])?;
}
key.bytes(character.encode_utf8(&mut [0; 4]).as_bytes())?;
}
Ok(true)
}
fn map(
&self,
text: &str,
start: usize,
mapped: &mut String,
fourth: &mut Vec<u8>,
) -> Result<(), KeyTooLarge> {
let bytes = text.as_bytes();
let plain = |byte: u8| byte < 0x80 && self.ascii[usize::from(byte)] == LETTER;
let mut fourth = Fourth {
out: KeyBuffer(fourth),
letters: start as u64,
skipped: 0,
};
let (mut run, mut at) = (0, start);
while let Some(character) = text[at..].chars().next() {
let end = at + character.len_utf8();
let replacement = match self.weight(character) {
LETTER if character.is_ascii() => {
let letters = bytes[end..].iter().position(|&byte| !plain(byte));
let end = letters.map_or(bytes.len(), |letters| end + letters);
fourth.letters((end - at) as u64)?;
at = end;
continue;
}
Weight::Other(length) => {
fourth.letters(u64::from(length))?;
at = end;
continue;
}
Weight::Unweighted => {
fourth.skip();
at = end;
continue;
}
Weight::Ignored(rank) => {
fourth.ignored(rank)?;
'\0'
}
Weight::First(rank) => {
fourth.ignored(rank)?;
'\u{fffe}'
}
};
let copy = &text[run..at];
mapped
.try_reserve(copy.len() + replacement.len_utf8())
.map_err(|_| KeyTooLarge)?;
mapped.push_str(copy);
mapped.push(replacement);
(run, at) = (end, end);
}
let copy = &text[run..];
mapped.try_reserve(copy.len()).map_err(|_| KeyTooLarge)?;
mapped.push_str(copy);
fourth.end()
}
}
pub(super) fn compose(text: &str, out: &mut String) -> Result<bool, KeyTooLarge> {
if text.is_ascii() {
return Ok(false);
}
let table = collation_glibc::CONTRACTIONS;
static LAST: std::sync::OnceLock<CharSet> = std::sync::OnceLock::new();
let last = LAST.get_or_init(|| {
CharSet::new(
table
.iter()
.filter_map(|(parts, _)| parts.chars().last())
.map(u32::from),
)
});
if !text.chars().any(|character| last.contains(character)) {
return Ok(false);
}
let (mut composed, mut run, mut at) = (false, 0, 0);
while let Some(character) = text[at..].chars().next() {
let first = table.partition_point(|(parts, _)| parts.chars().next() < Some(character));
let longest = table[first..]
.iter()
.take_while(|(parts, _)| parts.starts_with(character))
.filter(|(parts, _)| text[at..].starts_with(parts))
.max_by_key(|(parts, _)| parts.len());
let Some(&(parts, letter)) = longest else {
at += character.len_utf8();
continue;
};
if !composed {
out.clear();
composed = true;
}
let copy = &text[run..at];
out.try_reserve(copy.len() + letter.len_utf8())
.map_err(|_| KeyTooLarge)?;
out.push_str(copy);
out.push(letter);
at += parts.len();
run = at;
}
if composed {
let copy = &text[run..];
out.try_reserve(copy.len()).map_err(|_| KeyTooLarge)?;
out.push_str(copy);
}
Ok(composed)
}
struct CharSet {
blocks: Vec<u16>,
bitmaps: Vec<[u64; 4]>,
}
impl CharSet {
fn new(points: impl IntoIterator<Item = u32>) -> Self {
let mut set = Self {
blocks: Vec::new(),
bitmaps: Vec::new(),
};
for point in points {
let block = (point >> 8) as usize;
if set.blocks.len() <= block {
set.blocks.resize(block + 1, 0);
}
if set.blocks[block] == 0 {
set.bitmaps.push([0; 4]);
set.blocks[block] =
u16::try_from(set.bitmaps.len()).expect("few blocks hold such characters");
}
let bitmap = &mut set.bitmaps[usize::from(set.blocks[block]) - 1];
bitmap[(point as usize >> 6) & 3] |= 1 << (point & 63);
}
set
}
#[inline]
fn contains(&self, character: char) -> bool {
let point = u32::from(character);
match self.blocks.get((point >> 8) as usize) {
Some(&at) if at != 0 => {
self.bitmaps[usize::from(at) - 1][(point as usize >> 6) & 3] >> (point & 63) & 1
!= 0
}
_ => false,
}
}
}
fn within(ranges: &[(u32, u32)], point: u32) -> bool {
let at = ranges.partition_point(|&(first, _)| first <= point);
at.checked_sub(1).is_some_and(|at| point <= ranges[at].1)
}
pub(super) fn text(bytes: &[u8]) -> Result<&str, Failure> {
if bytes.contains(&0) {
return Err(unsupported(
"language sorting requires NUL-free UTF-8 keys; use LC_COLLATE=C for raw bytes",
)
.with_hint(super::failure::hints::LANGUAGE_SORT));
}
std::str::from_utf8(bytes).map_err(|_| {
unsupported("language sorting requires valid UTF-8 keys; use LC_COLLATE=C for raw bytes")
.with_hint(super::failure::hints::LANGUAGE_SORT)
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn categories_precedence_and_unknown_collation_are_independent() {
let policy = Policy::resolve(|key| match key {
"LANG" => Some("de_DE.UTF-8".into()),
"LC_NUMERIC" => Some("en_US.utf8".into()),
"LC_COLLATE" => Some("unknown".into()),
_ => None,
});
assert_eq!(policy.numeric, Profile::EN_NUMERIC);
assert!(policy.utf8);
assert!(policy.collation == Collation::Bytes);
let policy = Policy::resolve(|key| match key {
"LANG" => Some("de_DE.UTF-8".into()),
"LC_COLLATE" => Some("cs_CZ.UTF-8".into()),
_ => None,
});
assert!(policy.sorting().is_err());
let policy = Policy::resolve(|key| match key {
"LC_ALL" => Some("C.UTF-8".into()),
"LANG" => Some("unknown".into()),
_ => None,
});
assert_eq!(policy.numeric, Profile::C);
assert!(policy.utf8);
assert!(policy.sorting().is_ok());
}
fn all(name: &'static str) -> Policy {
Policy::resolve(|key| (key == "LC_ALL").then(|| name.into()))
}
fn collator(tag: &'static str) -> Collator {
let policy = Policy {
collation: Collation::Language(tag),
..Policy::default()
};
policy.collator().ok().unwrap().unwrap()
}
fn sorted(collator: &Collator, words: &[&'static str]) -> Vec<&'static str> {
let mut keyed: Vec<(Vec<u8>, &str)> = words
.iter()
.map(|&word| {
let mut key = Vec::new();
assert!(collator.write_key(word, &mut key).is_ok());
(key, word)
})
.collect();
keyed.sort();
keyed.into_iter().map(|(_, word)| word).collect()
}
fn model_order(collator: &Collator, a: &str, b: &str) -> std::cmp::Ordering {
let read = |text: &str| {
let (mut mapped, mut items, mut skipped) = (String::new(), Vec::new(), 0u64);
for character in text.chars() {
let (replacement, weight, length) = match collator.classify(character) {
Weight::Other(length) => (character, LETTER_RANK, length),
Weight::Unweighted => {
mapped.push(character);
skipped += 1;
continue;
}
Weight::Ignored(rank) => ('\0', rank, 1),
Weight::First(rank) => ('\u{fffe}', rank, 1),
};
mapped.push(replacement);
items.push((skipped + 1, weight));
items.extend((1..length).map(|_| (1, LETTER_RANK)));
skipped = 0;
}
let mut key = Vec::new();
assert!(collator.icu.write_sort_key_to(&mapped, &mut key).is_ok());
(key, items)
};
read(a).cmp(&read(b))
}
#[test]
fn contractions_compose_as_glibc_reads_them() {
let cases = [
("и\u{306}", Some("й")),
("ди\u{306}к", Some("дйк")),
("L\u{b7}l\u{b7}", Some("Ŀŀ")),
("L\u{387}", Some("Ŀ")),
("\u{cc6}\u{cc2}\u{cd5}", Some("\u{ccb}")),
("\u{cc6}\u{cc2}x", Some("\u{cca}x")),
(
"\u{fb2}\u{f71}\u{f80}\u{f71}\u{f72}",
Some("\u{f77}\u{f73}"),
),
("й", None),
("и \u{306}", None),
("\u{306}и", None),
("e\u{301}", None),
("L", None),
("", None),
];
for (text, expected) in cases {
let mut out = String::from("stale");
let composed = compose(text, &mut out).ok().unwrap();
assert_eq!(composed.then_some(out.as_str()), expected, "{text:?}");
}
let table = collation_glibc::CONTRACTIONS;
assert!(table.windows(2).all(|pair| pair[0].0 < pair[1].0));
for &(parts, letter) in table {
let mut out = String::new();
assert!(compose(parts, &mut out).ok().unwrap());
assert_eq!(out, letter.to_string());
}
}
#[test]
fn identities_place_letters_after_their_decomposition() {
let collator = Collator::new("en-US").ok().unwrap();
let identity = |text: &str| {
let mut out = Vec::new();
if !collator.write_identity(text, &mut out).ok().unwrap() {
out = text.as_bytes().to_vec();
}
out
};
let cases = [
("e\u{301}", "\u{e9}"),
("u\u{308}\u{304}", "\u{1d6}"),
("\u{415}\u{308}", "\u{401}"),
("x\u{415}\u{308}y", "x\u{401}y"),
("\u{3b1}\u{301}", "\u{3ac}"),
("\u{415}\u{308}\u{401}", "\u{401}\u{415}\u{308}"),
];
for (before, after) in cases {
assert!(identity(before) < identity(after), "{before:?} {after:?}");
}
let mut out = Vec::new();
assert!(
!collator
.write_identity("\u{e9}t\u{e9}", &mut out)
.ok()
.unwrap()
);
assert!(out.is_empty());
for &(letter, base) in collation_glibc::REORDERED {
let letter = char::from_u32(letter).unwrap().to_string();
let mut out = Vec::new();
assert!(collator.write_identity(&letter, &mut out).ok().unwrap());
let mut expected = char::from_u32(base).unwrap().to_string().into_bytes();
expected.push(0xff);
expected.extend_from_slice(letter.as_bytes());
assert_eq!(out, expected);
}
}
#[test]
fn language_keys_follow_the_model() {
let set = [
"a", "b", "č", "ł", "中", "á", " ", "\u{a0}", "-", "_", "(", "é", "e\u{301}", "1",
];
let mut words = vec![String::new()];
for length in 1..=3 {
let shorter: Vec<String> = words
.iter()
.filter(|w| w.chars().count() < length)
.cloned()
.collect();
for word in shorter {
words.extend(set.iter().map(|piece| format!("{word}{piece}")));
}
}
words.sort();
words.dedup();
for tag in ["en-US", "es-ES", "hr-HR", "pl-PL", "is-IS", "ja-JP"] {
let collator = collator(tag);
for point in 0..=0xffffu32 {
if let Some(character) = char::from_u32(point) {
assert_eq!(collator.weight(character), collator.classify(character));
}
}
let key = |text: &str| {
let mut key = Vec::new();
assert!(collator.write_key(text, &mut key).is_ok());
key
};
let mut by_key = words.clone();
by_key.sort_by_cached_key(|word| key(word));
let mut by_model = words.clone();
by_model.sort_by(|a, b| model_order(&collator, a, b));
assert_eq!(by_key, by_model, "{tag}");
}
}
#[test]
fn language_keys_order_around_unweighted_letters_as_glibc_does() {
for (tag, glibc) in [
(
"en-US",
&[
"(中",
"中(",
"-中a_",
"中-a-",
"中国(北京)",
"中国北京()",
][..],
),
("hr-HR", &["-ča_", "–ča", "č-a-", "č–a"]),
(
"pl-PL",
&["-ła_", "ł-a-", "ul\u{a0}.Dluga", "ul.\u{a0}Dluga"],
),
("is-IS", &["’áb", "á’b"]),
("es-MX", &["Cia\u{a0}Ltda", "Cia.\u{a0}Ltda"]),
(
"es-ES",
&[
"a \u{ad}b",
"a \u{200b}b",
"a\u{a0}-b",
"a\u{ad} b",
"a\u{200b} b",
"a-\u{a0}b",
],
),
("gl-ES", &["a\u{a0}-b", "a-\u{a0}b"]),
] {
let mut words = glibc.to_vec();
words.reverse();
assert_eq!(sorted(&collator(tag), &words), glibc, "{tag}");
}
}
#[test]
fn language_keys_order_ignored_characters_as_glibc_does() {
let words = [
"ab", "a-b", "a+c", "a b", "a0", "a$b", "ac", "Ab", "a_b", "中", "中-", "čb", "c-b",
];
let glibc = [
"a$b", "a0", "a b", "a-b", "a_b", "ab", "Ab", "a+c", "ac", "c-b", "čb", "中", "中-",
];
assert_eq!(sorted(&collator("en-US"), &words), glibc);
assert_eq!(sorted(&collator("hr-HR"), &words), glibc);
let space_first = [
"a b", "a$b", "a0", "a-b", "a_b", "ab", "Ab", "a+c", "ac", "c-b", "čb", "中", "中-",
];
assert_eq!(sorted(&collator("es-ES"), &words), space_first);
assert_eq!(sorted(&collator("pl-PL"), &words), space_first);
}
#[test]
fn unknown_names_behave_as_c_as_in_gnu() {
for name in [
"xx_YY.UTF-8",
"unknown",
"de",
"english",
"en_XX.UTF-8@euro",
] {
let policy = all(name);
assert!(policy.numbers().is_ok(), "{name}");
assert_eq!(policy.numeric, Profile::C, "{name}");
assert!(policy.collation == Collation::Bytes, "{name}");
assert!(!policy.utf8, "{name}");
}
let c = all("C.UTF-8@anything");
assert!(c.utf8 && c.collation == Collation::Bytes && c.numeric == Profile::C);
}
#[test]
fn variants_of_a_known_locale_keep_its_numbers() {
let de = all("de_DE.UTF-8");
assert_eq!(de.numeric.radix(), b',');
for name in ["de_DE", "de_DE.ISO-8859-1", "de_DE.iso88591", "de_DE@euro"] {
let policy = all(name);
assert!(policy.numbers().is_ok(), "{name}");
assert_eq!(policy.numeric, de.numeric, "{name}");
assert!(!policy.utf8, "{name}");
assert!(policy.sorting().is_err(), "{name}");
}
let fr = all("fr_FR.UTF-8");
assert!(!fr.numeric.thousands().is_ascii());
assert!(all("fr_FR.ISO-8859-1").numbers().is_err());
assert!(all("fr_FR@euro").numbers().is_err());
let latin = all("nan_TW.UTF-8@latin");
assert!(latin.utf8 && latin.numbers().is_ok());
assert_eq!(latin.numeric.grouping(), b"\x03");
assert_eq!(all("nan_TW.UTF-8").numeric.grouping(), b"\x04");
assert!(latin.sorting().is_err());
assert_eq!(all("de_DE.UTF-8@euro").numeric, de.numeric);
let en = all("en_US.UTF-8@nothing");
assert_eq!(en.numeric, all("en_US.UTF-8").numeric);
assert!(en.utf8 && en.language());
for name in ["ps_AF.UTF-8", "ps_AF", "ps_AF.UTF-8@nothing"] {
assert!(all(name).numbers().is_err(), "{name}");
}
}
}