ada_idna/
normalization.rs

1use crate::unicode_tables::*;
2
3pub fn normalize(input: &str) -> String {
4    let mut chars: Vec<u32> = input.chars().map(|c| c as u32).collect();
5
6    // Decompose and reorder (NFC normalization)
7    decompose_nfc(&mut chars);
8    compose(&mut chars);
9
10    // Convert back to string
11    chars.into_iter().filter_map(char::from_u32).collect()
12}
13
14fn decompose_nfc(input: &mut Vec<u32>) {
15    let (decomposition_needed, additional_elements) = compute_decomposition_length(input);
16    if decomposition_needed {
17        decompose(input, additional_elements);
18    }
19    sort_marks(input);
20}
21
22fn compute_decomposition_length(input: &[u32]) -> (bool, usize) {
23    let mut decomposition_needed = false;
24    let mut additional_elements = 0;
25
26    for &current_character in input {
27        let mut decomposition_length = 0;
28
29        if (HANGUL_SBASE..HANGUL_SBASE + HANGUL_SCOUNT).contains(&current_character) {
30            decomposition_length = 2;
31            if (current_character - HANGUL_SBASE) % HANGUL_TCOUNT != 0 {
32                decomposition_length = 3;
33            }
34        } else if current_character < 0x110000 {
35            let di = DECOMPOSITION_INDEX[(current_character >> 8) as usize];
36            let decomposition = &DECOMPOSITION_BLOCK[di as usize];
37            let idx = (current_character % 256) as usize;
38            if idx < decomposition.len() - 1 {
39                decomposition_length =
40                    ((decomposition[idx + 1] >> 2) - (decomposition[idx] >> 2)) as usize;
41                if decomposition_length > 0 && (decomposition[idx] & 1) != 0 {
42                    decomposition_length = 0;
43                }
44            }
45        }
46
47        if decomposition_length != 0 {
48            decomposition_needed = true;
49            additional_elements += decomposition_length - 1;
50        }
51    }
52
53    (decomposition_needed, additional_elements)
54}
55
56fn decompose(input: &mut Vec<u32>, additional_elements: usize) {
57    input.resize(input.len() + additional_elements, 0);
58    let input_count = input.len() - additional_elements;
59    let mut descending_idx = input.len();
60
61    for i in (0..input_count).rev() {
62        let current_char = input[i];
63
64        if (HANGUL_SBASE..HANGUL_SBASE + HANGUL_SCOUNT).contains(&current_char) {
65            // Hangul decomposition
66            let s_index = current_char - HANGUL_SBASE;
67            if s_index % HANGUL_TCOUNT != 0 {
68                descending_idx -= 1;
69                input[descending_idx] = HANGUL_TBASE + s_index % HANGUL_TCOUNT;
70            }
71            descending_idx -= 1;
72            input[descending_idx] = HANGUL_VBASE + (s_index % HANGUL_NCOUNT) / HANGUL_TCOUNT;
73            descending_idx -= 1;
74            input[descending_idx] = HANGUL_LBASE + s_index / HANGUL_NCOUNT;
75        } else if current_char < 0x110000 {
76            // Check decomposition data
77            let di = DECOMPOSITION_INDEX[(current_char >> 8) as usize];
78            let decomposition = &DECOMPOSITION_BLOCK[di as usize];
79            let idx = (current_char % 256) as usize;
80
81            let mut decomposition_length = 0;
82            if idx < decomposition.len() - 1 {
83                decomposition_length = (decomposition[idx + 1] >> 2) - (decomposition[idx] >> 2);
84                if decomposition_length > 0 && (decomposition[idx] & 1) != 0 {
85                    decomposition_length = 0;
86                }
87            }
88
89            if decomposition_length > 0 {
90                // Non-recursive decomposition
91                let start_idx = (decomposition[idx] >> 2) as usize;
92                for j in 0..decomposition_length {
93                    if start_idx + (j as usize) < DECOMPOSITION_DATA.len() {
94                        descending_idx -= 1;
95                        input[descending_idx] =
96                            DECOMPOSITION_DATA[start_idx + (decomposition_length - 1 - j) as usize];
97                    }
98                }
99            } else {
100                // No decomposition
101                descending_idx -= 1;
102                input[descending_idx] = current_char;
103            }
104        } else {
105            // Non-Unicode character
106            descending_idx -= 1;
107            input[descending_idx] = current_char;
108        }
109    }
110}
111
112fn get_ccc(c: u32) -> u8 {
113    if c < 0x110000 {
114        let idx = CANONICAL_COMBINING_CLASS_INDEX[(c >> 8) as usize] as usize;
115        CANONICAL_COMBINING_CLASS_BLOCK[idx][(c % 256) as usize]
116    } else {
117        0
118    }
119}
120
121fn sort_marks(input: &mut [u32]) {
122    for idx in 1..input.len() {
123        let ccc = get_ccc(input[idx]);
124        if ccc == 0 {
125            continue; // Skip non-combining characters
126        }
127
128        let current_character = input[idx];
129        let mut back_idx = idx;
130        while back_idx != 0 && get_ccc(input[back_idx - 1]) > ccc {
131            input[back_idx] = input[back_idx - 1];
132            back_idx -= 1;
133        }
134        input[back_idx] = current_character;
135    }
136}
137
138fn compose(input: &mut Vec<u32>) {
139    let mut input_count = 0;
140    let mut composition_count = 0;
141
142    while input_count < input.len() {
143        input[composition_count] = input[input_count];
144
145        if input[input_count] >= HANGUL_LBASE && input[input_count] < HANGUL_LBASE + HANGUL_LCOUNT {
146            if input_count + 1 < input.len()
147                && input[input_count + 1] >= HANGUL_VBASE
148                && input[input_count + 1] < HANGUL_VBASE + HANGUL_VCOUNT
149            {
150                input[composition_count] = HANGUL_SBASE
151                    + ((input[input_count] - HANGUL_LBASE) * HANGUL_VCOUNT
152                        + input[input_count + 1]
153                        - HANGUL_VBASE)
154                        * HANGUL_TCOUNT;
155                input_count += 1;
156                if input_count + 1 < input.len()
157                    && input[input_count + 1] > HANGUL_TBASE
158                    && input[input_count + 1] < HANGUL_TBASE + HANGUL_TCOUNT
159                {
160                    input[composition_count] += input[input_count + 1] - HANGUL_TBASE;
161                    input_count += 1;
162                }
163            }
164        } else if input[input_count] >= HANGUL_SBASE
165            && input[input_count] < HANGUL_SBASE + HANGUL_SCOUNT
166        {
167            if (input[input_count] - HANGUL_SBASE) % HANGUL_TCOUNT != 0
168                && input_count + 1 < input.len()
169                && input[input_count + 1] > HANGUL_TBASE
170                && input[input_count + 1] < HANGUL_TBASE + HANGUL_TCOUNT
171            {
172                input[composition_count] += input[input_count + 1] - HANGUL_TBASE;
173                input_count += 1;
174            }
175        } else if input[input_count] < 0x110000 {
176            let ci = COMPOSITION_INDEX[(input[input_count] >> 8) as usize] as usize;
177            let composition_idx = (input[input_count] % 256) as usize;
178            let composition = &COMPOSITION_BLOCK[ci][composition_idx..];
179            let initial_composition_count = composition_count;
180            let mut previous_ccc = -1i32;
181
182            while input_count + 1 < input.len() {
183                let ccc = get_ccc(input[input_count + 1]) as i32;
184
185                if composition.len() >= 2 && composition[1] != composition[0] && previous_ccc < ccc
186                {
187                    // Try finding a composition
188                    let mut left = composition[0] as usize;
189                    let mut right = composition[1] as usize;
190                    while left + 2 < right {
191                        let middle = left + (((right - left) >> 1) & !1);
192                        if COMPOSITION_DATA[middle] <= input[input_count + 1] {
193                            left = middle;
194                        }
195                        if COMPOSITION_DATA[middle] >= input[input_count + 1] {
196                            right = middle;
197                        }
198                    }
199                    if left < COMPOSITION_DATA.len()
200                        && COMPOSITION_DATA[left] == input[input_count + 1]
201                        && left + 1 < COMPOSITION_DATA.len()
202                    {
203                        input[initial_composition_count] = COMPOSITION_DATA[left + 1];
204                        let new_ci =
205                            COMPOSITION_INDEX[(COMPOSITION_DATA[left + 1] >> 8) as usize] as usize;
206                        let new_char_idx = (COMPOSITION_DATA[left + 1] % 256) as usize;
207                        if new_ci < COMPOSITION_BLOCK.len()
208                            && new_char_idx < COMPOSITION_BLOCK[new_ci].len()
209                        {
210                            // Update composition reference for potential further composition
211                        }
212                        input_count += 1;
213                        continue;
214                    }
215                }
216
217                if ccc == 0 {
218                    break; // Not a combining character
219                }
220                previous_ccc = ccc;
221                composition_count += 1;
222                input[composition_count] = input[input_count + 1];
223                input_count += 1;
224            }
225        }
226
227        input_count += 1;
228        composition_count += 1;
229    }
230
231    if composition_count < input_count {
232        input.resize(composition_count, 0);
233    }
234}
235
236#[cfg(test)]
237mod tests {
238    use super::*;
239
240    #[test]
241    fn test_normalize() {
242        let input = "café";
243        let result = normalize(input);
244        assert!(!result.is_empty());
245        // For now, just ensure it doesn't crash and returns something
246        // Full Unicode table implementation would be needed for proper testing
247    }
248
249    #[test]
250    fn test_hangul_constants() {
251        assert_eq!(HANGUL_NCOUNT, 588);
252        assert_eq!(HANGUL_SCOUNT, 11172);
253    }
254}
ada_idna/normalization.rs

ada_idna/
normalization.rs
