pub(crate) mod tables;
pub fn is_identifier_start(c: char) -> bool {
if c.is_ascii() {
return c == '_' || c.is_ascii_alphabetic();
}
let cp = c as u32;
cp <= 0xFFFF && in_ranges(tables::IDENTIFIER_START_RANGES, cp as u16)
}
pub fn is_identifier_part(c: char) -> bool {
if c.is_ascii() {
return c == '_' || c.is_ascii_alphanumeric();
}
let cp = c as u32;
cp <= 0xFFFF && in_ranges(tables::IDENTIFIER_PART_RANGES, cp as u16)
}
fn in_ranges(ranges: &[(u16, u16)], cp: u16) -> bool {
let idx = ranges.partition_point(|&(_, last)| last < cp);
idx < ranges.len() && ranges[idx].0 <= cp
}
#[cfg(test)]
mod tests {
use super::tables::{IDENTIFIER_PART_RANGES, IDENTIFIER_START_RANGES, UNICODE_VERSION};
use super::{is_identifier_part, is_identifier_start};
use sha2::Digest;
#[test]
fn test_generated_unicode_identifier_tables_shape_and_boundaries() {
assert_eq!(UNICODE_VERSION, (13, 0, 0));
assert_eq!(IDENTIFIER_START_RANGES.len(), 383, "ADR-0182 start ranges");
assert_eq!(IDENTIFIER_PART_RANGES.len(), 436, "ADR-0182 part ranges");
for ranges in [IDENTIFIER_START_RANGES, IDENTIFIER_PART_RANGES] {
let mut previous_last = 0x007F;
for &(first, last) in ranges {
assert!(first <= last, "range {first:#06X}..{last:#06X} is empty");
assert!(
first > previous_last,
"ranges must be sorted, disjoint and ASCII-free"
);
previous_last = last;
}
}
for (c, start, part) in [
('\u{08BE}', true, true), ('\u{9FFC}', true, true), ('\u{A7BA}', true, true), ('\u{0870}', false, false), ('\u{08C8}', false, false), ('\u{203F}', true, true), ('\u{20AC}', false, true), ('\u{00B7}', false, false), ('\u{2118}', false, false), ('\u{212E}', false, false), ('\u{2E2F}', false, false), ('\u{00B2}', false, false), ('\u{10400}', false, false), ('\u{1F600}', false, false), ('\u{0301}', false, true), ('\u{1885}', false, true), ('_', true, true),
('0', false, true),
] {
assert_eq!(is_identifier_start(c), start, "is_identifier_start({c:?})");
assert_eq!(is_identifier_part(c), part, "is_identifier_part({c:?})");
}
}
#[test]
fn test_generated_unicode_identifier_table_matches_the_generator() {
assert_eq!(
format!(
"{:x}",
sha2::Sha256::digest(include_bytes!("unicode_identifier/tables.rs"))
),
"69f67498e36039faf967554111cd95cad385fdbb5727ceaaf71655bdad7ff2eb",
"unicode_identifier/tables.rs drifted from the generator output; regenerate with \
`python3 scripts/gen-unicode-identifier-tables.py` and update this pin"
);
}
}