unicode_intervals/
categories.rs

1use crate::{constants::ALL_CATEGORIES, error};
2use core::{
3    fmt,
4    ops::{BitOr, BitOrAssign},
5    str::FromStr,
6};
7use UnicodeCategory::*;
8
9/// Unicode category abbreviation.
10#[derive(Debug, Eq, PartialEq, Copy, Clone, Hash)]
11pub enum UnicodeCategory {
12    /// Close Punctuation.
13    Pe,
14    /// Connector Punctuation.
15    Pc,
16    /// Control.
17    Cc,
18    /// Currency Symbol.
19    Sc,
20    /// Dash Punctuation.
21    Pd,
22    /// Decimal Number.
23    Nd,
24    /// Enclosing Mark.
25    Me,
26    /// Final Punctuation.
27    Pf,
28    /// Format.
29    Cf,
30    /// Initial Punctuation.
31    Pi,
32    /// Letter Number.
33    Nl,
34    /// Line Separator.
35    Zl,
36    /// Lowercase Letter.
37    Ll,
38    /// Math Symbol.
39    Sm,
40    /// Modifier Letter.
41    Lm,
42    /// Modifier Symbol.
43    Sk,
44    /// Nonspacing Mark.
45    Mn,
46    /// Open Punctuation.
47    Ps,
48    /// Other Letter.
49    Lo,
50    /// Other Number.
51    No,
52    /// Other Punctuation.
53    Po,
54    /// Other Symbol.
55    So,
56    /// Paragraph Separator.
57    Zp,
58    /// Private Use.
59    Co,
60    /// Space Separator.
61    Zs,
62    /// Spacing Mark.
63    Mc,
64    /// Surrogate.
65    Cs,
66    /// Titlecase Letter.
67    Lt,
68    /// Unassigned.
69    Cn,
70    /// Uppercase Letter.
71    Lu,
72}
73
74impl FromStr for UnicodeCategory {
75    type Err = error::Error;
76
77    fn from_str(s: &str) -> Result<Self, Self::Err> {
78        Ok(match s {
79            "Pe" => Pe,
80            "Pc" => Pc,
81            "Cc" => Cc,
82            "Sc" => Sc,
83            "Pd" => Pd,
84            "Nd" => Nd,
85            "Me" => Me,
86            "Pf" => Pf,
87            "Cf" => Cf,
88            "Pi" => Pi,
89            "Nl" => Nl,
90            "Zl" => Zl,
91            "Ll" => Ll,
92            "Sm" => Sm,
93            "Lm" => Lm,
94            "Sk" => Sk,
95            "Mn" => Mn,
96            "Ps" => Ps,
97            "Lo" => Lo,
98            "No" => No,
99            "Po" => Po,
100            "So" => So,
101            "Zp" => Zp,
102            "Co" => Co,
103            "Zs" => Zs,
104            "Mc" => Mc,
105            "Cs" => Cs,
106            "Lt" => Lt,
107            "Cn" => Cn,
108            "Lu" => Lu,
109            _ => return Err(Self::Err::InvalidCategory(s.to_owned().into_boxed_str())),
110        })
111    }
112}
113
114impl UnicodeCategory {
115    /// Letters.
116    pub const L: UnicodeCategorySet = UnicodeCategorySet(
117        1 << Ll as u32 | 1 << Lm as u32 | 1 << Lo as u32 | 1 << Lt as u32 | 1 << Lu as u32,
118    );
119    /// Marks.
120    pub const M: UnicodeCategorySet =
121        UnicodeCategorySet(1 << Mc as u32 | 1 << Me as u32 | 1 << Mn as u32);
122    /// Numbers.
123    pub const N: UnicodeCategorySet =
124        UnicodeCategorySet(1 << Nd as u32 | 1 << Nl as u32 | 1 << No as u32);
125    /// Punctuation.
126    pub const P: UnicodeCategorySet = UnicodeCategorySet(
127        1 << Pc as u32
128            | 1 << Pd as u32
129            | 1 << Pe as u32
130            | 1 << Pf as u32
131            | 1 << Pi as u32
132            | 1 << Po as u32
133            | 1 << Ps as u32,
134    );
135    /// Symbols.
136    pub const S: UnicodeCategorySet =
137        UnicodeCategorySet(1 << Sc as u32 | 1 << Sk as u32 | 1 << Sm as u32 | 1 << So as u32);
138    /// Separators.
139    pub const Z: UnicodeCategorySet =
140        UnicodeCategorySet(1 << Zp as u32 | 1 << Zs as u32 | 1 << Zl as u32);
141    /// Control, format, private, unassigned and surrogates.
142    pub const C: UnicodeCategorySet = UnicodeCategorySet(
143        1 << Cc as u32 | 1 << Cf as u32 | 1 << Cn as u32 | 1 << Co as u32 | 1 << Cs as u32,
144    );
145    // Full category names
146    /// Close Punctuation (alias).
147    pub const CLOSE_PUNCTUATION: UnicodeCategory = Pe;
148    /// Connector Punctuation (alias).
149    pub const CONNECTOR_PUNCTUATION: UnicodeCategory = Pc;
150    /// Control (alias).
151    pub const CONTROL: UnicodeCategory = Cc;
152    /// Currency Symbol (alias).
153    pub const CURRENCY_SYMBOL: UnicodeCategory = Sc;
154    /// Dash Punctuation (alias).
155    pub const DASH_PUNCTUATION: UnicodeCategory = Pd;
156    /// Decimal Number (alias).
157    pub const DECIMAL_NUMBER: UnicodeCategory = Nd;
158    /// Enclosing Mark (alias).
159    pub const ENCLOSING_MARK: UnicodeCategory = Me;
160    /// Final Punctuation (alias).
161    pub const FINAL_PUNCTUATION: UnicodeCategory = Pf;
162    /// Format (alias).
163    pub const FORMAT: UnicodeCategory = Cf;
164    /// Initial Punctuation (alias).
165    pub const INITIAL_PUNCTUATION: UnicodeCategory = Pi;
166    /// Letter Number (alias).
167    pub const LETTER_NUMBER: UnicodeCategory = Nl;
168    /// Line Separator (alias).
169    pub const LINE_SEPARATOR: UnicodeCategory = Zl;
170    /// Lowercase Letter (alias).
171    pub const LOWERCASE_LETTER: UnicodeCategory = Ll;
172    /// Math Symbol (alias).
173    pub const MATH_SYMBOL: UnicodeCategory = Sm;
174    /// Modifier Letter (alias).
175    pub const MODIFIER_LETTER: UnicodeCategory = Lm;
176    /// Modifier Symbol (alias).
177    pub const MODIFIER_SYMBOL: UnicodeCategory = Sk;
178    /// Nonspacing Mark (alias).
179    pub const NONSPACING_MARK: UnicodeCategory = Mn;
180    /// Open Punctuation (alias).
181    pub const OPEN_PUNCTUATION: UnicodeCategory = Ps;
182    /// Other Letter (alias).
183    pub const OTHER_LETTER: UnicodeCategory = Lo;
184    /// Other Number (alias).
185    pub const OTHER_NUMBER: UnicodeCategory = No;
186    /// Other Punctuation (alias).
187    pub const OTHER_PUNCTUATION: UnicodeCategory = Po;
188    /// Other Symbol (alias).
189    pub const OTHER_SYMBOL: UnicodeCategory = So;
190    /// Paragraph Separator (alias).
191    pub const PARAGRAPH_SEPARATOR: UnicodeCategory = Zp;
192    /// Private Use (alias).
193    pub const PRIVATE_USE: UnicodeCategory = Co;
194    /// Space Separator (alias).
195    pub const SPACE_SEPARATOR: UnicodeCategory = Zs;
196    /// Spacing Mark (alias).
197    pub const SPACING_MARK: UnicodeCategory = Mc;
198    /// Surrogate (alias).
199    pub const SURROGATE: UnicodeCategory = Cs;
200    /// Titlecase Letter (alias).
201    pub const TITLECASE_LETTER: UnicodeCategory = Lt;
202    /// Unassigned (alias).
203    pub const UNASSIGNED: UnicodeCategory = Cn;
204    /// Uppercase Letter (alias).
205    pub const UPPERCASE_LETTER: UnicodeCategory = Lu;
206
207    /// Category for the given discriminant, or `None` if out of range.
208    #[must_use]
209    pub(crate) const fn from_index(index: u8) -> Option<UnicodeCategory> {
210        Some(match index {
211            0 => Pe,
212            1 => Pc,
213            2 => Cc,
214            3 => Sc,
215            4 => Pd,
216            5 => Nd,
217            6 => Me,
218            7 => Pf,
219            8 => Cf,
220            9 => Pi,
221            10 => Nl,
222            11 => Zl,
223            12 => Ll,
224            13 => Sm,
225            14 => Lm,
226            15 => Sk,
227            16 => Mn,
228            17 => Ps,
229            18 => Lo,
230            19 => No,
231            20 => Po,
232            21 => So,
233            22 => Zp,
234            23 => Co,
235            24 => Zs,
236            25 => Mc,
237            26 => Cs,
238            27 => Lt,
239            28 => Cn,
240            29 => Lu,
241            _ => return None,
242        })
243    }
244
245    /// Rank of the category abbreviation in alphabetical order (`Cc` = 0, ..., `Zs` = 29).
246    #[must_use]
247    pub(crate) const fn abbrev_rank(self) -> u8 {
248        match self {
249            Cc => 0,
250            Cf => 1,
251            Cn => 2,
252            Co => 3,
253            Cs => 4,
254            Ll => 5,
255            Lm => 6,
256            Lo => 7,
257            Lt => 8,
258            Lu => 9,
259            Mc => 10,
260            Me => 11,
261            Mn => 12,
262            Nd => 13,
263            Nl => 14,
264            No => 15,
265            Pc => 16,
266            Pd => 17,
267            Pe => 18,
268            Pf => 19,
269            Pi => 20,
270            Po => 21,
271            Ps => 22,
272            Sc => 23,
273            Sk => 24,
274            Sm => 25,
275            So => 26,
276            Zl => 27,
277            Zp => 28,
278            Zs => 29,
279        }
280    }
281
282    /// Abbreviation as a string.
283    #[must_use]
284    pub const fn as_str(self) -> &'static str {
285        match self {
286            Pe => "Pe",
287            Pc => "Pc",
288            Cc => "Cc",
289            Sc => "Sc",
290            Pd => "Pd",
291            Nd => "Nd",
292            Me => "Me",
293            Pf => "Pf",
294            Cf => "Cf",
295            Pi => "Pi",
296            Nl => "Nl",
297            Zl => "Zl",
298            Ll => "Ll",
299            Sm => "Sm",
300            Lm => "Lm",
301            Sk => "Sk",
302            Mn => "Mn",
303            Ps => "Ps",
304            Lo => "Lo",
305            No => "No",
306            Po => "Po",
307            So => "So",
308            Zp => "Zp",
309            Co => "Co",
310            Zs => "Zs",
311            Mc => "Mc",
312            Cs => "Cs",
313            Lt => "Lt",
314            Cn => "Cn",
315            Lu => "Lu",
316        }
317    }
318}
319
320impl fmt::Display for UnicodeCategory {
321    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
322        f.write_str(self.as_str())
323    }
324}
325
326/// Set of Unicode categories.
327#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
328pub struct UnicodeCategorySet(u32);
329
330impl UnicodeCategorySet {
331    /// Empty set of Unicode categories.
332    #[inline]
333    #[must_use]
334    pub const fn new() -> Self {
335        Self(0)
336    }
337    /// All Unicode categories.
338    #[inline]
339    #[must_use]
340    pub const fn all() -> Self {
341        Self(ALL_CATEGORIES)
342    }
343    /// Create a category set, but do not check whether the input value is valid.
344    #[inline]
345    #[must_use]
346    pub(crate) const fn from_value_unchecked(value: u32) -> Self {
347        Self(value)
348    }
349    /// Add a new Unicode category to the set.
350    #[inline]
351    pub fn add(&mut self, category: UnicodeCategory) {
352        self.set(category as u8);
353    }
354    /// Remove a Unicode category from the set.
355    #[inline]
356    pub fn remove(&mut self, category: UnicodeCategory) {
357        self.unset(category as u8);
358    }
359    /// Whether the set contains `category`.
360    #[inline]
361    #[must_use]
362    pub const fn contains(self, category: UnicodeCategory) -> bool {
363        self.is_set(category as u8)
364    }
365    /// The size of the set.
366    #[inline]
367    #[must_use]
368    pub const fn len(self) -> usize {
369        self.0.count_ones() as usize
370    }
371    /// Whether the set is empty.
372    #[inline]
373    #[must_use]
374    pub const fn is_empty(self) -> bool {
375        self.0 == 0
376    }
377    /// Extract the inner storage value.
378    #[inline]
379    #[must_use]
380    pub const fn into_value(self) -> u32 {
381        self.0
382    }
383    /// Iterate over included Unicode categories.
384    #[inline]
385    #[must_use]
386    pub const fn iter(self) -> Iter {
387        Iter { data: self }
388    }
389    // `index` is always < 30 and can't overflow
390    #[inline]
391    #[allow(clippy::arithmetic_side_effects)]
392    pub(crate) fn set(&mut self, index: u8) {
393        self.0 |= 1 << index;
394    }
395    // `index` is always < 30 and can't overflow
396    #[inline]
397    #[allow(clippy::arithmetic_side_effects)]
398    pub(crate) fn unset(&mut self, index: u8) {
399        self.0 &= !(1 << index);
400    }
401    // `index`` is always < 30 and can't overflow
402    #[inline]
403    #[allow(clippy::arithmetic_side_effects)]
404    const fn is_set(self, index: u8) -> bool {
405        self.0 & (1 << index) != 0
406    }
407}
408
409impl Default for UnicodeCategorySet {
410    #[inline]
411    fn default() -> Self {
412        UnicodeCategorySet::new()
413    }
414}
415
416impl fmt::Display for UnicodeCategorySet {
417    // `idx` can't overflow as the maximum possible size of `iter` is 30 < usize::MAX
418    #[allow(clippy::arithmetic_side_effects)]
419    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
420        let len = self.len();
421        for (idx, category) in self.iter().enumerate() {
422            f.write_str(category.as_str())?;
423            if idx + 1 != len {
424                f.write_str(", ")?;
425            }
426        }
427        Ok(())
428    }
429}
430
431impl BitOr for UnicodeCategory {
432    type Output = UnicodeCategorySet;
433
434    // `self` and `rhs` are both < 30; Therefore shift won't overflow
435    #[inline]
436    #[allow(clippy::arithmetic_side_effects)]
437    fn bitor(self, rhs: Self) -> Self::Output {
438        UnicodeCategorySet(1 << self as u8 | 1 << rhs as u8)
439    }
440}
441impl BitOr<UnicodeCategorySet> for UnicodeCategory {
442    type Output = UnicodeCategorySet;
443
444    #[inline]
445    fn bitor(self, rhs: UnicodeCategorySet) -> Self::Output {
446        // Reusing existing `BitOr<UnicodeCategory> for UnicodeCategorySet`
447        rhs | self
448    }
449}
450
451impl BitOr<UnicodeCategory> for UnicodeCategorySet {
452    type Output = Self;
453
454    // `rhs as u8` can't overflow as it has only 30 elements
455    #[inline]
456    #[allow(clippy::arithmetic_side_effects)]
457    fn bitor(self, rhs: UnicodeCategory) -> Self::Output {
458        Self(self.into_value() | 1 << rhs as u8)
459    }
460}
461
462impl BitOr<UnicodeCategorySet> for UnicodeCategorySet {
463    type Output = Self;
464
465    #[inline]
466    fn bitor(self, rhs: UnicodeCategorySet) -> Self::Output {
467        Self(self.into_value() | rhs.into_value())
468    }
469}
470
471impl BitOrAssign<UnicodeCategorySet> for UnicodeCategorySet {
472    #[inline]
473    fn bitor_assign(&mut self, rhs: UnicodeCategorySet) {
474        self.0 |= rhs.into_value();
475    }
476}
477
478impl BitOrAssign<UnicodeCategory> for UnicodeCategorySet {
479    #[inline]
480    fn bitor_assign(&mut self, rhs: UnicodeCategory) {
481        self.add(rhs);
482    }
483}
484
485#[derive(Debug)]
486pub struct Iter {
487    data: UnicodeCategorySet,
488}
489
490impl Iterator for Iter {
491    type Item = UnicodeCategory;
492
493    fn next(&mut self) -> Option<Self::Item> {
494        // INVARIANT: The number of trailing zeros for `u32` is 32 at most which is less than `u8::MAX`
495        #[allow(clippy::cast_possible_truncation)]
496        let index = self.data.0.trailing_zeros() as u8;
497        let category = UnicodeCategory::from_index(index)?;
498        self.data.unset(index);
499        Some(category)
500    }
501}
502
503impl ExactSizeIterator for Iter {
504    #[inline]
505    fn len(&self) -> usize {
506        self.data.len()
507    }
508}
509
510impl From<UnicodeCategory> for UnicodeCategorySet {
511    // `category as u8` can't overflow as it has only 30 elements
512    #[inline]
513    #[allow(clippy::arithmetic_side_effects)]
514    fn from(category: UnicodeCategory) -> Self {
515        Self::from_value_unchecked(1 << category as u8)
516    }
517}
518
519impl From<UnicodeCategory> for Option<UnicodeCategorySet> {
520    #[inline]
521    fn from(category: UnicodeCategory) -> Self {
522        Some(category.into())
523    }
524}
525
526/// Return all Unicode categories that are in `include`, but not in `exclude`.
527#[inline]
528#[must_use]
529pub const fn merge(
530    include: Option<UnicodeCategorySet>,
531    exclude: UnicodeCategorySet,
532) -> UnicodeCategorySet {
533    if let Some(include) = include {
534        if include.is_empty() {
535            // include no categories
536            include
537        } else {
538            UnicodeCategorySet::from_value_unchecked(
539                (ALL_CATEGORIES ^ exclude.into_value()) & include.into_value(),
540            )
541        }
542    } else {
543        UnicodeCategorySet::from_value_unchecked(ALL_CATEGORIES ^ exclude.into_value())
544    }
545}
546
547/// Expand major-class designations (e.g. `"N"` -> `Nd, Nl, No`) and validate
548/// specific categories, returned in `normalized_categories` order.
549///
550/// # Errors
551///
552/// Returns an error if an entry is neither a major class nor a known category.
553pub fn as_general_categories<I, S>(
554    categories: I,
555    version: crate::UnicodeVersion,
556) -> Result<Vec<UnicodeCategory>, error::Error>
557where
558    I: IntoIterator<Item = S>,
559    S: AsRef<str>,
560{
561    let mut wanted = UnicodeCategorySet::new();
562    for category in categories {
563        let set = match category.as_ref() {
564            "L" => UnicodeCategory::L,
565            "M" => UnicodeCategory::M,
566            "N" => UnicodeCategory::N,
567            "P" => UnicodeCategory::P,
568            "S" => UnicodeCategory::S,
569            "Z" => UnicodeCategory::Z,
570            "C" => UnicodeCategory::C,
571            other => other.parse::<UnicodeCategory>()?.into(),
572        };
573        wanted |= set;
574    }
575    Ok(version
576        .normalized_categories()
577        .into_iter()
578        .filter(|c| wanted.contains(*c))
579        .collect())
580}
581
582#[cfg(test)]
583mod tests {
584    use super::*;
585    use crate::UnicodeVersion;
586    use std::{
587        collections::hash_map::DefaultHasher,
588        hash::{Hash, Hasher},
589    };
590    use test_case::test_case;
591
592    #[test]
593    fn test_category_from_str_error() {
594        assert_eq!(
595            UnicodeCategory::from_str("wrong")
596                .expect_err("Should fail")
597                .to_string(),
598            "'wrong' is not a valid Unicode category"
599        );
600    }
601
602    #[test]
603    #[allow(clippy::clone_on_copy)]
604    fn test_category_traits() {
605        let mut hasher = DefaultHasher::new();
606        Ll.hash(&mut hasher);
607        let _ = hasher.finish();
608        let _ = Ll.clone();
609        assert_eq!(format!("{Ll:?}"), "Ll");
610    }
611
612    #[test]
613    fn test_single_letter_categories() {
614        assert_eq!(UnicodeCategory::L, Ll | Lm | Lo | Lt | Lu);
615    }
616
617    #[test]
618    fn test_set_display() {
619        assert_eq!(UnicodeCategory::L.to_string(), "Ll, Lm, Lo, Lt, Lu");
620    }
621
622    #[test]
623    fn test_set_add() {
624        let mut set = UnicodeCategorySet::new();
625        assert!(set.is_empty());
626        set.add(Ll);
627        assert!(set.contains(Ll));
628        assert_eq!(set.len(), 1);
629    }
630
631    #[test]
632    fn test_set_remove() {
633        let mut set = UnicodeCategorySet::all();
634        assert!(set.contains(Ll));
635        set.remove(Ll);
636        assert!(!set.contains(Ll));
637    }
638
639    #[test]
640    #[allow(clippy::clone_on_copy)]
641    fn test_category_set_traits() {
642        let set = UnicodeCategory::L;
643        let mut hasher = DefaultHasher::new();
644        set.hash(&mut hasher);
645        let _ = hasher.finish();
646        let _ = set.clone();
647        assert_eq!(format!("{set:?}"), "UnicodeCategorySet(671371264)");
648    }
649
650    #[test]
651    fn test_iter_traits() {
652        let set = UnicodeCategory::L;
653        let iter = set.iter();
654        assert_eq!(
655            format!("{iter:?}"),
656            "Iter { data: UnicodeCategorySet(671371264) }"
657        );
658    }
659
660    #[test]
661    fn test_bit_or() {
662        assert_eq!(Ll | UnicodeCategorySet::new(), Ll.into());
663        assert_eq!(
664            UnicodeCategory::L | UnicodeCategory::C,
665            Ll | Lm | Lo | Lt | Lu | Cs | Cc | Cf | Cn | Co
666        );
667        let mut set = UnicodeCategorySet::new();
668        set |= Ll;
669        set |= UnicodeCategory::C;
670        assert_eq!(set, Ll | Cs | Cc | Cf | Cn | Co);
671    }
672
673    #[test]
674    fn test_set_iter() {
675        let all_categories = UnicodeCategorySet::all();
676        assert_eq!(all_categories.iter().len(), all_categories.len());
677        let mut set = UnicodeCategorySet::new();
678        for category in all_categories.iter() {
679            let name = format!("{category}");
680            assert_eq!(
681                UnicodeCategory::from_str(&name).expect("Invalid category"),
682                category
683            );
684            set.add(category);
685        }
686        assert_eq!(all_categories, set);
687    }
688
689    #[test]
690    fn test_set_default() {
691        assert_eq!(UnicodeCategorySet::default(), UnicodeCategorySet::new());
692    }
693
694    #[test]
695    fn test_set_option_from_category() {
696        let set: Option<UnicodeCategorySet> = Ll.into();
697        assert!(set.is_some());
698        assert_eq!(set.expect("Unexpected `None`"), Ll.into());
699    }
700
701    #[test_case(Some(Lu | Me | Cs | So), So.into(), Lu | Me | Cs)]
702    #[test_case(None, UnicodeCategory::L | UnicodeCategory::M | UnicodeCategory::N | UnicodeCategory::P | UnicodeCategory::S, UnicodeCategory::Z | UnicodeCategory::C)]
703    #[test_case(
704        Some(UnicodeCategorySet::new()),
705        UnicodeCategorySet::new(),
706        UnicodeCategorySet::new()
707    )]
708    fn test_category_merge(
709        include: Option<UnicodeCategorySet>,
710        exclude: UnicodeCategorySet,
711        expected: UnicodeCategorySet,
712    ) {
713        assert_eq!(merge(include, exclude), expected);
714    }
715
716    #[test]
717    fn test_as_general_categories() {
718        let v = UnicodeVersion::V15_0_0;
719        let n = as_general_categories(["N"], v).expect("valid");
720        assert_eq!(n.len(), 3);
721        assert!(n.contains(&Nd) && n.contains(&Nl) && n.contains(&No));
722        let got = as_general_categories(["Lu", "Ll"], v).expect("valid");
723        assert_eq!(got.len(), 2);
724        assert!(got.contains(&Lu) && got.contains(&Ll));
725        assert!(as_general_categories(["Xx"], v).is_err());
726    }
727}
unicode_intervals/categories.rs

unicode_intervals/
categories.rs
