ens_normalize_rs/tokens/
tokenize.rs

1use crate::{
2    tokens::{
3        CollapsedEnsNameToken, EnsNameToken, TokenDisallowed, TokenEmoji, TokenIgnored,
4        TokenMapped, TokenNfc, TokenStop, TokenValid,
5    },
6    utils, CodePoint, CodePointsSpecs, ProcessError,
7};
8
9/// Represents a full ENS name, including the original input and the sequence of tokens
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct TokenizedName {
12    pub input: String,
13    pub tokens: Vec<EnsNameToken>,
14}
15
16/// Represents a tokenized ENS label (part of a name separated by periods), including sequence of tokens
17#[derive(Debug, Clone, PartialEq, Eq)]
18pub struct TokenizedLabel<'a> {
19    pub tokens: &'a [EnsNameToken],
20}
21
22impl TokenizedName {
23    pub fn empty() -> Self {
24        Self {
25            input: "".to_string(),
26            tokens: vec![],
27        }
28    }
29
30    /// Tokenizes an input string, applying NFC normalization if requested.
31    pub fn from_input(
32        input: impl AsRef<str>,
33        specs: &CodePointsSpecs,
34        apply_nfc: bool,
35    ) -> Result<Self, ProcessError> {
36        tokenize_name(input, specs, apply_nfc)
37    }
38
39    pub fn is_empty(&self) -> bool {
40        self.tokens.is_empty()
41    }
42
43    /// Returns an iterator over all tokens in the tokenized name.
44    pub fn iter_tokens(&self) -> impl Iterator<Item = &EnsNameToken> {
45        self.tokens.iter()
46    }
47
48    /// Returns an iterator over all labels in the tokenized name.
49    /// Basically, it splits the tokenized name by stop tokens.
50    pub fn iter_labels(&self) -> impl Iterator<Item = TokenizedLabel<'_>> {
51        self.tokens
52            .split(|t| matches!(t, EnsNameToken::Stop(_)))
53            .map(TokenizedLabel::from)
54    }
55
56    pub fn labels(&self) -> Vec<TokenizedLabel<'_>> {
57        self.iter_labels().collect()
58    }
59}
60
61impl TokenizedLabel<'_> {
62    /// Returns true if all tokens in the label are emoji tokens
63    pub fn is_fully_emoji(&self) -> bool {
64        self.tokens
65            .iter()
66            .all(|t| matches!(t, EnsNameToken::Emoji(_)))
67    }
68
69    /// Returns true if all codepoints in all tokens are ASCII characters
70    pub fn is_fully_ascii(&self) -> bool {
71        self.tokens
72            .iter()
73            .all(|token| token.cps().into_iter().all(utils::is_ascii))
74    }
75
76    /// Returns an iterator over all codepoints in all tokens.
77    pub fn iter_cps(&self) -> impl DoubleEndedIterator<Item = CodePoint> + '_ {
78        self.tokens.iter().flat_map(|token| token.cps())
79    }
80
81    /// Collapses consecutive text tokens into single text tokens, keeping emoji tokens separate.
82    /// Returns a vector of either Text or Emoji tokens.
83    pub fn collapse_into_text_or_emoji(&self) -> Vec<CollapsedEnsNameToken> {
84        let mut current_text_cps = vec![];
85        let mut collapsed = vec![];
86        for token in self.tokens.iter() {
87            match token {
88                EnsNameToken::Valid(_) | EnsNameToken::Mapped(_) | EnsNameToken::Nfc(_) => {
89                    current_text_cps.extend(token.cps().iter());
90                }
91                EnsNameToken::Emoji(token) => {
92                    if !current_text_cps.is_empty() {
93                        collapsed.push(CollapsedEnsNameToken::Text(TokenValid {
94                            cps: current_text_cps,
95                        }));
96                        current_text_cps = vec![];
97                    }
98                    collapsed.push(CollapsedEnsNameToken::Emoji(token.clone()));
99                }
100                EnsNameToken::Ignored(_) | EnsNameToken::Disallowed(_) | EnsNameToken::Stop(_) => {}
101            }
102        }
103        if !current_text_cps.is_empty() {
104            collapsed.push(CollapsedEnsNameToken::Text(TokenValid {
105                cps: current_text_cps,
106            }));
107        }
108        collapsed
109    }
110
111    /// Returns a vector of codepoints from all text tokens, excluding emoji and ignored tokens
112    pub fn get_cps_of_not_ignored_text(&self) -> Vec<CodePoint> {
113        self.collapse_into_text_or_emoji()
114            .into_iter()
115            .filter_map(|token| {
116                if let CollapsedEnsNameToken::Text(token) = token {
117                    Some(token.cps)
118                } else {
119                    None
120                }
121            })
122            .flatten()
123            .collect()
124    }
125}
126
127impl<'a, T> From<&'a T> for TokenizedLabel<'a>
128where
129    T: AsRef<[EnsNameToken]> + ?Sized,
130{
131    fn from(tokens: &'a T) -> Self {
132        TokenizedLabel {
133            tokens: tokens.as_ref(),
134        }
135    }
136}
137
138fn tokenize_name(
139    name: impl AsRef<str>,
140    specs: &CodePointsSpecs,
141    apply_nfc: bool,
142) -> Result<TokenizedName, ProcessError> {
143    let name = name.as_ref();
144    if name.is_empty() {
145        return Ok(TokenizedName::empty());
146    }
147    let tokens = tokenize_input(name, specs, apply_nfc)?;
148    Ok(TokenizedName {
149        input: name.to_string(),
150        tokens,
151    })
152}
153
154fn tokenize_input(
155    input: impl AsRef<str>,
156    specs: &CodePointsSpecs,
157    apply_nfc: bool,
158) -> Result<Vec<EnsNameToken>, ProcessError> {
159    let input = input.as_ref();
160
161    let mut tokens = Vec::new();
162    let mut input_cur = 0;
163
164    while input_cur < input.len() {
165        if let Some(emoji) = maybe_starts_with_emoji(input_cur, input, specs) {
166            let cursor_offset = emoji.input.len();
167            tokens.push(EnsNameToken::Emoji(emoji));
168            input_cur += cursor_offset;
169        } else {
170            let char = input[input_cur..]
171                .chars()
172                .next()
173                .expect("input_cur is in bounds");
174            let cursor_offset = char.len_utf8();
175            let cp = char as CodePoint;
176            let token = process_one_cp(cp, specs);
177            tokens.push(token);
178            input_cur += cursor_offset;
179        }
180    }
181
182    if apply_nfc {
183        perform_nfc_transform(&mut tokens, specs);
184    }
185    collapse_valid_tokens(&mut tokens);
186    Ok(tokens)
187}
188
189fn perform_nfc_transform(tokens: &mut Vec<EnsNameToken>, specs: &CodePointsSpecs) {
190    let mut i = 0;
191    let mut start = -1i32;
192
193    while i < tokens.len() {
194        let token = &tokens[i];
195        match token {
196            EnsNameToken::Valid(_) | EnsNameToken::Mapped(_) => {
197                let cps = token.cps();
198                if specs.cps_requires_check(&cps) {
199                    let mut end = i + 1;
200                    for (pos, token) in tokens.iter().enumerate().skip(end) {
201                        match token {
202                            EnsNameToken::Valid(_) | EnsNameToken::Mapped(_) => {
203                                if !specs.cps_requires_check(&cps) {
204                                    break;
205                                }
206                                end = pos + 1;
207                            }
208                            EnsNameToken::Ignored(_) => {}
209                            _ => break,
210                        }
211                    }
212
213                    if start < 0 {
214                        start = i as i32;
215                    }
216
217                    let slice = &tokens[start as usize..end];
218                    let mut cps = Vec::new();
219                    for tok in slice {
220                        match tok {
221                            EnsNameToken::Valid(_) | EnsNameToken::Mapped(_) => {
222                                cps.extend(&tok.cps());
223                            }
224                            _ => {}
225                        }
226                    }
227
228                    let str0 = utils::cps2str(&cps);
229                    let str = utils::nfc(&str0);
230
231                    if str0 == str {
232                        i = end - 1;
233                    } else {
234                        let new_token = EnsNameToken::Nfc(TokenNfc {
235                            input: cps,
236                            cps: utils::str2cps(&str),
237                        });
238                        tokens.splice(start as usize..end, vec![new_token]);
239                        i = start as usize;
240                    }
241                    start = -1;
242                } else {
243                    start = i as i32;
244                }
245            }
246            EnsNameToken::Ignored(_) => {}
247            _ => {
248                start = -1;
249            }
250        }
251        i += 1;
252    }
253}
254
255// given array of codepoints
256// returns the longest valid emoji sequence (or undefined if no match)
257fn maybe_starts_with_emoji(i: usize, label: &str, specs: &CodePointsSpecs) -> Option<TokenEmoji> {
258    let (end, _) = specs.longest_emoji_at(label, i)?;
259    let input_cps = utils::str2cps(&label[i..end]);
260    let cps_no_fe0f = utils::filter_fe0f(&input_cps);
261    let emoji = specs
262        .cps_emoji_no_fe0f_to_pretty(&cps_no_fe0f)
263        .expect("emoji should be found")
264        .clone();
265    Some(TokenEmoji {
266        input: label[i..end].to_string(),
267        cps_input: input_cps,
268        emoji,
269        cps_no_fe0f,
270    })
271}
272
273fn process_one_cp(cp: CodePoint, specs: &CodePointsSpecs) -> EnsNameToken {
274    if specs.is_stop(cp) {
275        EnsNameToken::Stop(TokenStop { cp })
276    } else if specs.is_valid(cp) {
277        EnsNameToken::Valid(TokenValid { cps: vec![cp] })
278    } else if specs.is_ignored(cp) {
279        EnsNameToken::Ignored(TokenIgnored { cp })
280    } else if let Some(normalized) = specs.maybe_normalize(cp) {
281        EnsNameToken::Mapped(TokenMapped {
282            cp,
283            cps: normalized.clone(),
284        })
285    } else {
286        EnsNameToken::Disallowed(TokenDisallowed { cp })
287    }
288}
289
290fn collapse_valid_tokens(tokens: &mut Vec<EnsNameToken>) {
291    let mut i = 0;
292    while i < tokens.len() {
293        if let EnsNameToken::Valid(token) = &tokens[i] {
294            let mut j = i + 1;
295            let mut cps = token.cps.clone();
296            while j < tokens.len() {
297                if let EnsNameToken::Valid(next_token) = &tokens[j] {
298                    cps.extend(next_token.cps.iter());
299                    j += 1;
300                } else {
301                    break;
302                }
303            }
304            let new_token = EnsNameToken::Valid(TokenValid { cps });
305            tokens.splice(i..j, vec![new_token]);
306        }
307        i += 1;
308    }
309}
310
311#[cfg(test)]
312mod tests {
313    use super::*;
314    use pretty_assertions::assert_eq;
315    use rstest::{fixture, rstest};
316
317    #[fixture]
318    #[once]
319    fn specs() -> CodePointsSpecs {
320        CodePointsSpecs::default()
321    }
322
323    #[rstest]
324    #[case::empty(vec![], vec![])]
325    #[case::single(
326        vec![EnsNameToken::Valid(TokenValid { cps: vec![1, 2, 3] })],
327        vec![EnsNameToken::Valid(TokenValid { cps: vec![1, 2, 3] })],
328    )]
329    #[case::two(
330        vec![
331            EnsNameToken::Valid(TokenValid { cps: vec![1, 2, 3] }),
332            EnsNameToken::Valid(TokenValid { cps: vec![4, 5, 6] }),
333        ],
334        vec![EnsNameToken::Valid(TokenValid { cps: vec![1, 2, 3, 4, 5, 6] })],
335    )]
336    #[case::full(
337        vec![
338            EnsNameToken::Valid(TokenValid { cps: vec![1, 2, 3] }),
339            EnsNameToken::Disallowed(TokenDisallowed { cp: 0 }),
340            EnsNameToken::Valid(TokenValid { cps: vec![4, 5, 6] }),
341            EnsNameToken::Valid(TokenValid { cps: vec![7, 8, 9] }),
342            EnsNameToken::Valid(TokenValid { cps: vec![10, 11, 12] }),
343            EnsNameToken::Disallowed(TokenDisallowed { cp: 10 }),
344            EnsNameToken::Stop(TokenStop { cp: 11 }),
345            EnsNameToken::Valid(TokenValid { cps: vec![12] }),
346            EnsNameToken::Ignored(TokenIgnored { cp: 13 }),
347        ],
348        vec![
349            EnsNameToken::Valid(TokenValid { cps: vec![1, 2, 3] }),
350            EnsNameToken::Disallowed(TokenDisallowed { cp: 0 }),
351            EnsNameToken::Valid(TokenValid { cps: vec![4, 5, 6, 7, 8, 9, 10, 11, 12] }),
352            EnsNameToken::Disallowed(TokenDisallowed { cp: 10 }),
353            EnsNameToken::Stop(TokenStop { cp: 11 }),
354            EnsNameToken::Valid(TokenValid { cps: vec![12] }),
355            EnsNameToken::Ignored(TokenIgnored { cp: 13 }),
356        ],
357    )]
358    fn test_collapse_valid_tokens(
359        #[case] input: Vec<EnsNameToken>,
360        #[case] expected: Vec<EnsNameToken>,
361    ) {
362        let mut tokens = input;
363        collapse_valid_tokens(&mut tokens);
364        assert_eq!(tokens, expected);
365    }
366
367    #[rstest]
368    #[case::xyz(
369        "xyz👨🏻/",
370        true,
371        vec![
372            EnsNameToken::Valid(TokenValid { cps: vec![120, 121, 122] }),
373            EnsNameToken::Emoji(TokenEmoji { input: "👨🏻".to_string(), cps_input: vec![128104, 127995], emoji: vec![128104, 127995], cps_no_fe0f: vec![128104, 127995] }),
374            EnsNameToken::Disallowed(TokenDisallowed { cp: 47 }),
375        ]
376    )]
377    #[case::a_poop_b(
378        "A💩︎︎b",
379        true,
380        vec![
381            EnsNameToken::Mapped(TokenMapped { cp: 65, cps: vec![97] }),
382            EnsNameToken::Emoji(TokenEmoji { input: "💩".to_string(), cps_input: vec![128169], emoji: vec![128169, 65039], cps_no_fe0f: vec![128169] }),
383            EnsNameToken::Ignored(TokenIgnored { cp: 65038 }),
384            EnsNameToken::Ignored(TokenIgnored { cp: 65038 }),
385            EnsNameToken::Valid(TokenValid { cps: vec![98] }),
386        ]
387    )]
388    #[case::atm(
389        "a™️",
390        true,
391        vec![
392            EnsNameToken::Valid(TokenValid { cps: vec![97] }),
393            EnsNameToken::Mapped(TokenMapped { cp: 8482, cps: vec![116, 109] }),
394            EnsNameToken::Ignored(TokenIgnored { cp: 65039 }),
395        ]
396    )]
397    #[case::no_nfc(
398        "_R💩\u{FE0F}a\u{FE0F}\u{304}\u{AD}.",
399        false,
400        vec![
401            EnsNameToken::Valid(TokenValid { cps: vec![95] }),
402            EnsNameToken::Mapped(TokenMapped { cp: 82, cps: vec![114] }),
403            EnsNameToken::Emoji(TokenEmoji { input: "💩️".to_string(), cps_input: vec![128169, 65039], emoji: vec![128169, 65039], cps_no_fe0f: vec![128169] }),
404            EnsNameToken::Valid(TokenValid { cps: vec![97] }),
405            EnsNameToken::Ignored(TokenIgnored { cp: 65039 }),
406            EnsNameToken::Valid(TokenValid { cps: vec![772] }),
407            EnsNameToken::Ignored(TokenIgnored { cp: 173 }),
408            EnsNameToken::Stop(TokenStop { cp: 46 }),
409        ]
410    )]
411    #[case::with_nfc(
412        "_R💩\u{FE0F}a\u{FE0F}\u{304}\u{AD}.",
413        true,
414        vec![
415            EnsNameToken::Valid(TokenValid { cps: vec![95] }),
416            EnsNameToken::Mapped(TokenMapped { cp: 82, cps: vec![114] }),
417            EnsNameToken::Emoji(TokenEmoji { input: "💩️".to_string(), cps_input: vec![128169, 65039], emoji: vec![128169, 65039], cps_no_fe0f: vec![128169] }),
418            EnsNameToken::Nfc(TokenNfc { input: vec![97, 772], cps: vec![257] }),
419            EnsNameToken::Ignored(TokenIgnored { cp: 173 }),
420            EnsNameToken::Stop(TokenStop { cp: 46 }),
421        ]
422    )]
423    #[case::raffy(
424        "RaFFY🚴‍♂️.eTh",
425        true,
426        vec![
427            EnsNameToken::Mapped(TokenMapped { cp: 82, cps: vec![114] }),
428            EnsNameToken::Valid(TokenValid { cps: vec![97] }),
429            EnsNameToken::Mapped(TokenMapped { cp: 70, cps: vec![102] }),
430            EnsNameToken::Mapped(TokenMapped { cp: 70, cps: vec![102] }),
431            EnsNameToken::Mapped(TokenMapped { cp: 89, cps: vec![121] }),
432            EnsNameToken::Emoji(TokenEmoji { input: "🚴\u{200d}♂\u{fe0f}".to_string(), cps_input: vec![128692, 8205, 9794, 65039], emoji: vec![128692, 8205, 9794, 65039], cps_no_fe0f: vec![128692, 8205, 9794] }),
433            EnsNameToken::Stop(TokenStop { cp: 46 }),
434            EnsNameToken::Valid(TokenValid { cps: vec![101] }),
435            EnsNameToken::Mapped(TokenMapped { cp: 84, cps: vec![116] }),
436            EnsNameToken::Valid(TokenValid { cps: vec![104] }),
437        ]
438    )]
439    #[case::emojis(
440        "⛹️‍♀",
441        true,
442        vec![
443            EnsNameToken::Emoji(TokenEmoji { input: "⛹️‍♀".to_string(), cps_input: vec![9977, 65039, 8205, 9792], emoji: vec![9977, 65039, 8205, 9792, 65039], cps_no_fe0f: vec![9977, 8205, 9792] }),
444        ]
445    )]
446    fn test_ens_tokenize(
447        #[case] input: &str,
448        #[case] apply_nfc: bool,
449        #[case] expected: Vec<EnsNameToken>,
450        specs: &CodePointsSpecs,
451    ) {
452        let tokens = tokenize_input(input, specs, apply_nfc).expect("tokenize");
453        assert_eq!(tokens, expected);
454    }
455
456    #[rstest]
457    #[case::leading_cm(
458        "󠅑𑆻👱🏿‍♀️xyz",
459        vec![
460            CollapsedEnsNameToken::Text(TokenValid { cps: vec![70075] }),
461            CollapsedEnsNameToken::Emoji(TokenEmoji { input: "👱🏿‍♀️".to_string(), cps_input: vec![128113, 127999, 8205, 9792, 65039], emoji: vec![128113, 127999, 8205, 9792, 65039], cps_no_fe0f: vec![128113, 127999, 8205, 9792] }),
462            CollapsedEnsNameToken::Text(TokenValid { cps: vec![120, 121, 122] }),
463        ]
464    )]
465    #[case::atm(
466        "a™️",
467        vec![
468            CollapsedEnsNameToken::Text(TokenValid { cps: vec![97, 116, 109] }),
469        ]
470    )]
471    fn test_collapse(
472        #[case] input: &str,
473        #[case] expected: Vec<CollapsedEnsNameToken>,
474        specs: &CodePointsSpecs,
475    ) {
476        let tokens = tokenize_input(input, specs, true).expect("tokenize");
477        let label = TokenizedLabel::from(&tokens);
478        let result = label.collapse_into_text_or_emoji();
479        assert_eq!(result, expected);
480    }
481}
ens_normalize_rs/tokens/tokenize.rs

ens_normalize_rs/tokens/
tokenize.rs
