rupg_types/
unicode.rs

1//! The Unicode normalization forms NFC, NFD, NFKC and NFKD (Unicode Standard Annex 15), the code points that Unicode assigns, and the characters with no display width.
2//!
3//! PostgreSQL uses them in `normalize`, in `IS NORMALIZED` and in SASLprep. The tables come from the vendored Unicode data of the PostgreSQL pin, so the result is the same as in PostgreSQL. See `spec/07-sql-types-and-catalog.md` and `spec/22-crate-layout.md` section 22.8.
4
5use crate::generated::unicode_norm::{
6    ASSIGNED, COMBINING_CLASS, COMPOSITION, DECOMPOSITION, DECOMPOSITION_CHARS, NONSPACING,
7};
8
9/// The version of Unicode of the tables in the form major.minor, which `unicode_version` gives.
10pub const VERSION: &str = crate::generated::unicode_norm::VERSION;
11
12/// A Unicode normalization form.
13#[derive(Debug, Clone, Copy, PartialEq, Eq)]
14pub enum NormalForm {
15    Nfc,
16    Nfd,
17    Nfkc,
18    Nfkd,
19}
20
21// The constants of the Hangul syllables, from section 3.12 of the Unicode Standard.
22const S_BASE: u32 = 0xAC00;
23const L_BASE: u32 = 0x1100;
24const V_BASE: u32 = 0x1161;
25const T_BASE: u32 = 0x11A7;
26const L_COUNT: u32 = 19;
27const V_COUNT: u32 = 21;
28const T_COUNT: u32 = 28;
29const N_COUNT: u32 = V_COUNT * T_COUNT;
30const S_COUNT: u32 = L_COUNT * N_COUNT;
31
32/// The canonical combining class of a code point.
33pub fn combining_class(c: u32) -> u8 {
34    match COMBINING_CLASS.binary_search_by_key(&c, |e| e.0) {
35        Ok(i) => COMBINING_CLASS[i].1,
36        Err(_) => 0,
37    }
38}
39
40/// True when the code point is assigned, that is, its general category is not Cn.
41pub fn is_assigned(c: u32) -> bool {
42    let i = ASSIGNED.partition_point(|range| range.1 < c);
43    ASSIGNED.get(i).is_some_and(|range| range.0 <= c)
44}
45
46/// True when `ucs_wcwidth` gives the display width 0: the code point 0 and the non-spacing characters.
47pub fn zero_width(c: u32) -> bool {
48    let i = NONSPACING.partition_point(|range| range.1 < c);
49    c == 0 || NONSPACING.get(i).is_some_and(|range| range.0 <= c)
50}
51
52/// Normalizes a string of code points to a form. The input can hold any `u32`, as `unicode_normalize` of PostgreSQL can.
53pub fn normalize(form: NormalForm, input: &[u32]) -> Vec<u32> {
54    let compat = matches!(form, NormalForm::Nfkc | NormalForm::Nfkd);
55    let mut out = Vec::with_capacity(input.len());
56    for &c in input {
57        decompose(c, compat, &mut out);
58    }
59    reorder(&mut out);
60    if matches!(form, NormalForm::Nfc | NormalForm::Nfkc) {
61        compose(&mut out);
62    }
63    out
64}
65
66/// Normalizes a `str` to a form.
67pub fn normalize_str(form: NormalForm, input: &str) -> String {
68    let codes: Vec<u32> = input.chars().map(u32::from).collect();
69    // A code point that the tables give is a Unicode scalar value, so the conversion does not fail.
70    normalize(form, &codes).into_iter().filter_map(char::from_u32).collect()
71}
72
73fn decompose(c: u32, compat: bool, out: &mut Vec<u32>) {
74    if (S_BASE..S_BASE + S_COUNT).contains(&c) {
75        let s = c - S_BASE;
76        out.push(L_BASE + s / N_COUNT);
77        out.push(V_BASE + (s % N_COUNT) / T_COUNT);
78        if !s.is_multiple_of(T_COUNT) {
79            out.push(T_BASE + s % T_COUNT);
80        }
81        return;
82    }
83    if let Ok(i) = DECOMPOSITION.binary_search_by_key(&c, |e| e.0) {
84        let (_, is_compat, start, len) = DECOMPOSITION[i];
85        if compat || !is_compat {
86            let start = usize::from(start);
87            for &d in &DECOMPOSITION_CHARS[start..start + usize::from(len)] {
88                decompose(d, compat, out);
89            }
90            return;
91        }
92    }
93    out.push(c);
94}
95
96/// The canonical ordering: a stable sort of each run of characters with a combining class above zero.
97fn reorder(s: &mut [u32]) {
98    let mut i = 0;
99    while i < s.len() {
100        if combining_class(s[i]) == 0 {
101            i += 1;
102            continue;
103        }
104        let start = i;
105        while i < s.len() && combining_class(s[i]) != 0 {
106            i += 1;
107        }
108        s[start..i].sort_by_key(|&c| combining_class(c));
109    }
110}
111
112fn pair(first: u32, second: u32) -> Option<u32> {
113    if (L_BASE..L_BASE + L_COUNT).contains(&first) && (V_BASE..V_BASE + V_COUNT).contains(&second) {
114        return Some(S_BASE + ((first - L_BASE) * V_COUNT + (second - V_BASE)) * T_COUNT);
115    }
116    if (S_BASE..S_BASE + S_COUNT).contains(&first)
117        && (first - S_BASE).is_multiple_of(T_COUNT)
118        && (T_BASE + 1..T_BASE + T_COUNT).contains(&second)
119    {
120        return Some(first + (second - T_BASE));
121    }
122    COMPOSITION
123        .binary_search_by_key(&(first, second), |e| (e.0, e.1))
124        .ok()
125        .map(|i| COMPOSITION[i].2)
126}
127
128/// The canonical composition. A character joins the last starter if no character between them blocks it.
129fn compose(s: &mut Vec<u32>) {
130    let mut starter: Option<usize> = None;
131    let mut last_class = 0u8;
132    let mut out = 0;
133    for i in 0..s.len() {
134        let c = s[i];
135        let class = combining_class(c);
136        if let Some(st) = starter
137            && !(out > st + 1 && (last_class == 0 || last_class >= class))
138            && let Some(composite) = pair(s[st], c)
139        {
140            s[st] = composite;
141            continue;
142        }
143        if class == 0 {
144            starter = Some(out);
145        }
146        last_class = class;
147        s[out] = c;
148        out += 1;
149    }
150    s.truncate(out);
151}
152
153#[cfg(test)]
154mod tests {
155    use super::*;
156
157    #[test]
158    fn hangul_and_compatibility() {
159        assert_eq!(normalize_str(NormalForm::Nfd, "\u{AC00}"), "\u{1100}\u{1161}");
160        assert_eq!(normalize_str(NormalForm::Nfc, "\u{1100}\u{1161}\u{11A8}"), "\u{AC01}");
161        assert_eq!(normalize_str(NormalForm::Nfkc, "\u{2168}"), "IX");
162        assert_eq!(normalize_str(NormalForm::Nfc, "\u{2168}"), "\u{2168}");
163        assert_eq!(normalize_str(NormalForm::Nfc, "e\u{301}"), "\u{E9}");
164        // A character that is not a scalar value passes through.
165        assert_eq!(normalize(NormalForm::Nfkc, &[0x11_0000, 0xD800]), [0x11_0000, 0xD800]);
166    }
167}
rupg_types/unicode.rs

rupg_types/
unicode.rs
