1#![warn(
88 clippy::cast_possible_truncation,
89 clippy::doc_markdown,
90 clippy::explicit_iter_loop,
91 clippy::map_unwrap_or,
92 clippy::match_same_arms,
93 clippy::needless_borrow,
94 clippy::needless_pass_by_value,
95 clippy::print_stdout,
96 clippy::redundant_closure,
97 clippy::trivially_copy_pass_by_ref,
98 missing_debug_implementations,
99 missing_docs,
100 trivial_casts,
101 trivial_numeric_casts,
102 unused_extern_crates,
103 unused_import_braces,
104 variant_size_differences,
105 clippy::arithmetic_side_effects,
106 clippy::unwrap_used,
107 clippy::semicolon_if_nothing_returned,
108 clippy::cargo
109)]
110#![allow(clippy::redundant_static_lifetimes)]
111use crate::constants::MAX_CODEPOINT;
112use core::fmt;
113use std::str::FromStr;
114
115mod categories;
116mod constants;
117mod error;
118mod intervals;
119mod intervalset;
120mod query;
121mod tables;
122pub use crate::{
123 categories::{as_general_categories, UnicodeCategory, UnicodeCategorySet},
124 error::Error,
125 intervalset::{Codepoints, IntervalSet},
126};
127
128#[cfg(feature = "__benchmark_internals")]
129pub mod internals {
131 pub mod categories {
133 pub use crate::categories::merge;
134 }
135
136 pub mod intervals {
138 pub use crate::intervals::{from_str, merge, subtract};
139 }
140
141 pub mod query {
143 pub use crate::query::{intervals_for_set, query};
144 }
145}
146
147pub type Interval = (u32, u32);
149
150#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
152#[non_exhaustive]
153pub enum UnicodeVersion {
154 V9_0_0,
156 V10_0_0,
158 V11_0_0,
160 V12_0_0,
162 V12_1_0,
164 V13_0_0,
166 V14_0_0,
168 V15_0_0,
170 V15_1_0,
172 V16_0_0,
174 V17_0_0,
176}
177
178impl fmt::Display for UnicodeVersion {
179 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
180 f.write_str(self.as_str())
181 }
182}
183
184impl FromStr for UnicodeVersion {
185 type Err = Error;
186
187 fn from_str(s: &str) -> Result<Self, Self::Err> {
188 match s {
189 "9.0.0" => Ok(UnicodeVersion::V9_0_0),
190 "10.0.0" => Ok(UnicodeVersion::V10_0_0),
191 "11.0.0" => Ok(UnicodeVersion::V11_0_0),
192 "12.0.0" => Ok(UnicodeVersion::V12_0_0),
193 "12.1.0" => Ok(UnicodeVersion::V12_1_0),
194 "13.0.0" => Ok(UnicodeVersion::V13_0_0),
195 "14.0.0" => Ok(UnicodeVersion::V14_0_0),
196 "15.0.0" => Ok(UnicodeVersion::V15_0_0),
197 "15.1.0" => Ok(UnicodeVersion::V15_1_0),
198 "16.0.0" => Ok(UnicodeVersion::V16_0_0),
199 "17.0.0" => Ok(UnicodeVersion::V17_0_0),
200 _ => Err(Error::InvalidVersion(s.to_string().into_boxed_str())),
201 }
202 }
203}
204
205impl UnicodeVersion {
206 pub const ALL: [UnicodeVersion; 11] = [
208 UnicodeVersion::V9_0_0,
209 UnicodeVersion::V10_0_0,
210 UnicodeVersion::V11_0_0,
211 UnicodeVersion::V12_0_0,
212 UnicodeVersion::V12_1_0,
213 UnicodeVersion::V13_0_0,
214 UnicodeVersion::V14_0_0,
215 UnicodeVersion::V15_0_0,
216 UnicodeVersion::V15_1_0,
217 UnicodeVersion::V16_0_0,
218 UnicodeVersion::V17_0_0,
219 ];
220 #[must_use]
222 pub const fn as_str(self) -> &'static str {
223 match self {
224 UnicodeVersion::V9_0_0 => "9.0.0",
225 UnicodeVersion::V10_0_0 => "10.0.0",
226 UnicodeVersion::V11_0_0 => "11.0.0",
227 UnicodeVersion::V12_0_0 => "12.0.0",
228 UnicodeVersion::V12_1_0 => "12.1.0",
229 UnicodeVersion::V13_0_0 => "13.0.0",
230 UnicodeVersion::V14_0_0 => "14.0.0",
231 UnicodeVersion::V15_0_0 => "15.0.0",
232 UnicodeVersion::V15_1_0 => "15.1.0",
233 UnicodeVersion::V16_0_0 => "16.0.0",
234 UnicodeVersion::V17_0_0 => "17.0.0",
235 }
236 }
237 #[must_use]
239 pub const fn latest() -> UnicodeVersion {
240 UnicodeVersion::V17_0_0
241 }
242 #[inline]
245 #[must_use]
246 pub const fn table(self) -> &'static [&'static [Interval]] {
247 match self {
248 UnicodeVersion::V9_0_0 => tables::v9_0_0::BY_NAME,
249 UnicodeVersion::V10_0_0 => tables::v10_0_0::BY_NAME,
250 UnicodeVersion::V11_0_0 => tables::v11_0_0::BY_NAME,
251 UnicodeVersion::V12_0_0 => tables::v12_0_0::BY_NAME,
252 UnicodeVersion::V12_1_0 => tables::v12_1_0::BY_NAME,
253 UnicodeVersion::V13_0_0 => tables::v13_0_0::BY_NAME,
254 UnicodeVersion::V14_0_0 => tables::v14_0_0::BY_NAME,
255 UnicodeVersion::V15_0_0 => tables::v15_0_0::BY_NAME,
256 UnicodeVersion::V15_1_0 => tables::v15_1_0::BY_NAME,
257 UnicodeVersion::V16_0_0 => tables::v16_0_0::BY_NAME,
258 UnicodeVersion::V17_0_0 => tables::v17_0_0::BY_NAME,
259 }
260 }
261
262 #[inline]
264 #[must_use]
265 pub const fn intervals_for(self, category: UnicodeCategory) -> &'static [Interval] {
266 self.table()[category as usize]
267 }
268
269 #[inline]
271 #[must_use]
272 pub const fn normalized_categories(self) -> [UnicodeCategory; 30] {
273 let mut keyed: [(UnicodeCategory, usize); 30] = [(UnicodeCategory::Cc, 0); 30];
276 let table = self.table();
277 let mut idx = 0;
278 #[allow(clippy::arithmetic_side_effects, clippy::cast_possible_truncation)]
280 while idx < table.len() {
281 if let Some(category) = UnicodeCategory::from_index(idx as u8) {
282 keyed[idx] = (
283 category,
284 table[idx].len() * 30 + category.abbrev_rank() as usize,
285 );
286 }
287 idx += 1;
288 }
289 loop {
291 let mut swapped = false;
292 let mut idx = 1;
293 #[allow(clippy::arithmetic_side_effects)]
295 while idx < keyed.len() {
296 let (lcat, lkey) = keyed[idx - 1];
297 let (rcat, rkey) = keyed[idx];
298 if lkey > rkey {
299 keyed[idx - 1] = (rcat, rkey);
300 keyed[idx] = (lcat, lkey);
301 swapped = true;
302 }
303 idx += 1;
304 }
305 if !swapped {
306 break;
307 }
308 }
309
310 let mut output = [UnicodeCategory::Cc; 30];
313 output[29] = UnicodeCategory::Cs;
314 let mut idx = 0;
315 let mut ptr = 0;
316 #[allow(clippy::arithmetic_side_effects)]
318 while idx < keyed.len() {
319 let (category, _) = keyed[idx];
320 if category as u8 != UnicodeCategory::Cc as u8
321 && category as u8 != UnicodeCategory::Cs as u8
322 {
323 output[ptr] = category;
324 ptr += 1;
325 }
326 idx += 1;
327 }
328 output
329 }
330
331 #[must_use]
333 #[inline]
334 pub fn query<'a>(self) -> IntervalQuery<'a> {
335 IntervalQuery::new(self)
336 }
337
338 fn intervals_impl(
339 self,
340 include_categories: Option<UnicodeCategorySet>,
341 exclude_categories: UnicodeCategorySet,
342 include_characters: Option<&str>,
343 exclude_characters: Option<&str>,
344 min_codepoint: u32,
345 max_codepoint: u32,
346 ) -> Result<Vec<Interval>, Error> {
347 if min_codepoint > MAX_CODEPOINT || max_codepoint > MAX_CODEPOINT {
348 return Err(Error::CodepointNotInRange(min_codepoint, max_codepoint));
349 }
350 if min_codepoint > max_codepoint {
351 return Err(Error::InvalidCodepoints(min_codepoint, max_codepoint));
352 }
353 Ok(query::query(
354 self,
355 include_categories,
356 exclude_categories,
357 include_characters.unwrap_or(""),
358 exclude_characters.unwrap_or(""),
359 min_codepoint,
360 max_codepoint,
361 ))
362 }
363}
364
365#[derive(Debug, Clone, PartialEq)]
381pub struct IntervalQuery<'a> {
382 version: UnicodeVersion,
383 include_categories: Option<UnicodeCategorySet>,
384 exclude_categories: Option<UnicodeCategorySet>,
385 include_characters: Option<&'a str>,
386 exclude_characters: Option<&'a str>,
387 min_codepoint: u32,
388 max_codepoint: u32,
389}
390
391impl<'a> IntervalQuery<'a> {
392 fn new(version: UnicodeVersion) -> IntervalQuery<'a> {
393 IntervalQuery {
394 version,
395 include_categories: None,
396 exclude_categories: None,
397 include_characters: None,
398 exclude_characters: None,
399 min_codepoint: 0,
400 max_codepoint: MAX_CODEPOINT,
401 }
402 }
403 #[must_use]
405 pub fn include_categories(
406 mut self,
407 include_categories: impl Into<Option<UnicodeCategorySet>>,
408 ) -> IntervalQuery<'a> {
409 self.include_categories = include_categories.into();
410 self
411 }
412 #[must_use]
414 pub fn exclude_categories(
415 mut self,
416 exclude_categories: impl Into<Option<UnicodeCategorySet>>,
417 ) -> IntervalQuery<'a> {
418 self.exclude_categories = exclude_categories.into();
419 self
420 }
421 #[must_use]
423 pub fn include_characters(mut self, include_characters: &'a str) -> IntervalQuery<'a> {
424 self.include_characters = Some(include_characters);
425 self
426 }
427 #[must_use]
429 pub fn exclude_characters(mut self, exclude_characters: &'a str) -> IntervalQuery<'a> {
430 self.exclude_characters = Some(exclude_characters);
431 self
432 }
433 #[must_use]
435 pub fn min_codepoint(mut self, min_codepoint: u32) -> IntervalQuery<'a> {
436 self.min_codepoint = min_codepoint;
437 self
438 }
439 #[must_use]
441 pub fn max_codepoint(mut self, max_codepoint: u32) -> IntervalQuery<'a> {
442 self.max_codepoint = max_codepoint;
443 self
444 }
445 pub fn intervals(&self) -> Result<Vec<Interval>, Error> {
452 let exclude_categories = self.exclude_categories.unwrap_or_default();
453 self.version.intervals_impl(
454 self.include_categories,
455 exclude_categories,
456 self.include_characters,
457 self.exclude_characters,
458 self.min_codepoint,
459 self.max_codepoint,
460 )
461 }
462 pub fn interval_set(&self) -> Result<IntervalSet, Error> {
469 Ok(IntervalSet::new(self.intervals()?))
470 }
471}
472
473pub fn query<'a>() -> IntervalQuery<'a> {
477 UnicodeVersion::latest().query()
478}
479
480#[cfg(test)]
481mod tests {
482 use super::*;
483 use std::{
484 collections::hash_map::DefaultHasher,
485 hash::{Hash, Hasher},
486 };
487 use test_case::test_case;
488
489 #[test]
490 fn test_all_versions() {
491 assert_eq!(UnicodeVersion::ALL.len(), 11);
492 assert_eq!(UnicodeVersion::ALL[0], UnicodeVersion::V9_0_0);
493 assert_eq!(
494 *UnicodeVersion::ALL.last().expect("non-empty"),
495 UnicodeVersion::latest()
496 );
497 for v in UnicodeVersion::ALL {
498 assert_eq!(v.as_str().parse::<UnicodeVersion>().expect("round-trip"), v);
499 }
500 }
501
502 #[test_case(None, None, &[(95, 95), (8255, 8256), (8276, 8276), (65075, 65076), (65101, 65103), (65343, 65343)])]
503 #[test_case(None, Some(128), &[(95, 95)])]
504 #[test_case(Some(65077), None, &[(65101, 65103), (65343, 65343)])]
505 #[test_case(Some(65076), Some(65102), &[(65076, 65076), (65101, 65102)])]
506 fn test_intervals(
507 min_codepoint: Option<u32>,
508 max_codepoint: Option<u32>,
509 expected: &[Interval],
510 ) {
511 let mut query = UnicodeVersion::V15_0_0
512 .query()
513 .include_categories(UnicodeCategory::Pc);
514 if let Some(min) = min_codepoint {
515 query = query.min_codepoint(min);
516 }
517 if let Some(max) = max_codepoint {
518 query = query.max_codepoint(max);
519 }
520 let intervals = query.intervals().expect("Invalid query");
521 assert_eq!(intervals, expected);
522 }
523
524 #[test]
525 fn test_interval_set() {
526 let interval_set = UnicodeVersion::V15_0_0
527 .query()
528 .include_categories(UnicodeCategory::Lu)
529 .max_codepoint(128)
530 .interval_set()
531 .expect("Invalid query");
532 assert_eq!(interval_set.index_of('A'), Some(0));
533 }
534
535 #[test]
536 fn test_top_level_query() {
537 assert_eq!(
538 query().intervals().expect("Invalid query"),
539 vec![(0, MAX_CODEPOINT)]
540 );
541 }
542
543 #[test]
544 fn test_query_include_only_characters() {
545 let intervals = UnicodeVersion::V15_0_0
546 .query()
547 .include_categories(UnicodeCategory::Pc)
548 .min_codepoint(0)
549 .max_codepoint(50)
550 .include_characters("abc")
551 .intervals()
552 .expect("Invalid query");
553 assert_eq!(intervals, &[(97, 99)]);
554 }
555
556 #[test]
557 fn test_query_exclude_only_characters() {
558 let intervals = UnicodeVersion::V15_0_0
559 .query()
560 .include_categories(UnicodeCategory::UPPERCASE_LETTER)
561 .max_codepoint(90)
562 .exclude_characters("ABC")
563 .intervals()
564 .expect("Invalid query");
565 assert_eq!(intervals, &[(68, 90)]);
566 }
567
568 #[test]
569 fn test_query_exclude_categories() {
570 let intervals = UnicodeVersion::V15_0_0
571 .query()
572 .exclude_categories(UnicodeCategory::UPPERCASE_LETTER)
573 .max_codepoint(90)
574 .intervals()
575 .expect("Invalid query");
576 assert_eq!(intervals, &[(0, 64)]);
577 }
578
579 #[test]
580 fn test_multi_category_codepoint_range_is_sorted() {
581 let intervals = UnicodeVersion::V15_0_0
584 .query()
585 .include_categories(UnicodeCategory::Lu | UnicodeCategory::Ll)
586 .max_codepoint(128)
587 .intervals()
588 .expect("Invalid query");
589 assert_eq!(intervals, &[(65, 90), (97, 122)]);
590 }
591
592 #[test]
593 fn test_query_include_category_and_characters() {
594 let intervals = UnicodeVersion::V15_0_0
595 .query()
596 .include_categories(UnicodeCategory::Pc)
597 .include_characters("abc")
598 .intervals()
599 .expect("Invalid query");
600 assert_eq!(
601 intervals,
602 &[
603 (95, 95),
604 (97, 99),
605 (8255, 8256),
606 (8276, 8276),
607 (65075, 65076),
608 (65101, 65103),
609 (65343, 65343)
610 ]
611 );
612 }
613
614 #[test_case(
615 1073741824,
616 2147483648,
617 "Codepoints should be in [0; 1114111] range. Got: [1073741824; 2147483648]"
618 )]
619 #[test_case(
620 0,
621 2147483648,
622 "Codepoints should be in [0; 1114111] range. Got: [0; 2147483648]"
623 )]
624 #[test_case(
625 5,
626 1,
627 "Minimum codepoint should be less or equal than maximum codepoint. Got 5 < 1"
628 )]
629 fn test_query_invalid_codepoints(min_codepoint: u32, max_codepoint: u32, expected: &str) {
630 let error = UnicodeVersion::V15_0_0
631 .query()
632 .min_codepoint(min_codepoint)
633 .max_codepoint(max_codepoint)
634 .intervals()
635 .expect_err("Should error");
636 assert_eq!(error.to_string(), expected);
637 let error = UnicodeVersion::V15_0_0
638 .query()
639 .min_codepoint(min_codepoint)
640 .max_codepoint(max_codepoint)
641 .interval_set()
642 .expect_err("Should error");
643 assert_eq!(error.to_string(), expected);
644 }
645
646 #[test]
647 fn test_intervals_for() {
648 assert_eq!(
649 UnicodeVersion::V15_0_0.intervals_for(UnicodeCategory::Pc),
650 &[
651 (95, 95),
652 (8255, 8256),
653 (8276, 8276),
654 (65075, 65076),
655 (65101, 65103),
656 (65343, 65343),
657 ]
658 );
659 }
660
661 #[test]
662 fn test_normalized_categories() {
663 assert_eq!(
664 UnicodeVersion::V15_0_0.normalized_categories(),
665 [
666 UnicodeCategory::Zl,
667 UnicodeCategory::Zp,
668 UnicodeCategory::Co,
669 UnicodeCategory::Me,
670 UnicodeCategory::Pc,
671 UnicodeCategory::Zs,
672 UnicodeCategory::Lt,
673 UnicodeCategory::Pf,
674 UnicodeCategory::Pi,
675 UnicodeCategory::Nl,
676 UnicodeCategory::Pd,
677 UnicodeCategory::Cf,
678 UnicodeCategory::Sc,
679 UnicodeCategory::Sk,
680 UnicodeCategory::Nd,
681 UnicodeCategory::Sm,
682 UnicodeCategory::Lm,
683 UnicodeCategory::No,
684 UnicodeCategory::Pe,
685 UnicodeCategory::Ps,
686 UnicodeCategory::Mc,
687 UnicodeCategory::So,
688 UnicodeCategory::Po,
689 UnicodeCategory::Mn,
690 UnicodeCategory::Lo,
691 UnicodeCategory::Lu,
692 UnicodeCategory::Ll,
693 UnicodeCategory::Cn,
694 UnicodeCategory::Cc,
695 UnicodeCategory::Cs,
696 ]
697 );
698 }
699
700 #[test_case(UnicodeVersion::V9_0_0)]
701 #[test_case(UnicodeVersion::V10_0_0)]
702 #[test_case(UnicodeVersion::V11_0_0)]
703 #[test_case(UnicodeVersion::V12_0_0)]
704 #[test_case(UnicodeVersion::V12_1_0)]
705 #[test_case(UnicodeVersion::V13_0_0)]
706 #[test_case(UnicodeVersion::V14_0_0)]
707 #[test_case(UnicodeVersion::V15_0_0)]
708 #[test_case(UnicodeVersion::V15_1_0)]
709 #[test_case(UnicodeVersion::V16_0_0)]
710 #[test_case(UnicodeVersion::V17_0_0)]
711 fn test_successive_union(version: UnicodeVersion) {
712 let mut x = vec![];
713 for v in version.table() {
714 x.extend_from_slice(v);
715 }
716 intervals::merge(&mut x);
717 assert_eq!(x, vec![(0, MAX_CODEPOINT)]);
718 }
719
720 #[test_case(UnicodeVersion::V9_0_0, "9.0.0")]
721 #[test_case(UnicodeVersion::V10_0_0, "10.0.0")]
722 #[test_case(UnicodeVersion::V11_0_0, "11.0.0")]
723 #[test_case(UnicodeVersion::V12_0_0, "12.0.0")]
724 #[test_case(UnicodeVersion::V12_1_0, "12.1.0")]
725 #[test_case(UnicodeVersion::V13_0_0, "13.0.0")]
726 #[test_case(UnicodeVersion::V14_0_0, "14.0.0")]
727 #[test_case(UnicodeVersion::V15_0_0, "15.0.0")]
728 #[test_case(UnicodeVersion::V15_1_0, "15.1.0")]
729 #[test_case(UnicodeVersion::V16_0_0, "16.0.0")]
730 #[test_case(UnicodeVersion::V17_0_0, "17.0.0")]
731 fn test_display(version: UnicodeVersion, expected: &str) {
732 let string = version.to_string();
733 assert_eq!(string, expected);
734 assert_eq!(
735 UnicodeVersion::from_str(&string).expect("Invalid version"),
736 version
737 );
738 }
739
740 #[test_case("9.0.0", UnicodeVersion::V9_0_0)]
741 #[test_case("10.0.0", UnicodeVersion::V10_0_0)]
742 #[test_case("11.0.0", UnicodeVersion::V11_0_0)]
743 #[test_case("12.0.0", UnicodeVersion::V12_0_0)]
744 #[test_case("12.1.0", UnicodeVersion::V12_1_0)]
745 #[test_case("13.0.0", UnicodeVersion::V13_0_0)]
746 #[test_case("14.0.0", UnicodeVersion::V14_0_0)]
747 #[test_case("15.0.0", UnicodeVersion::V15_0_0)]
748 #[test_case("15.1.0", UnicodeVersion::V15_1_0)]
749 #[test_case("16.0.0", UnicodeVersion::V16_0_0)]
750 #[test_case("17.0.0", UnicodeVersion::V17_0_0)]
751 fn test_version_from_str(version: &str, expected: UnicodeVersion) {
752 assert_eq!(
753 UnicodeVersion::from_str(version).expect("Invalid version"),
754 expected
755 );
756 }
757
758 #[test]
759 fn test_version_from_str_error() {
760 assert_eq!(
761 UnicodeVersion::from_str("invalid")
762 .expect_err("Should fail")
763 .to_string(),
764 "'invalid' is not a valid Unicode version"
765 );
766 }
767
768 #[test]
769 #[allow(clippy::clone_on_copy)]
770 fn test_unicode_version_traits() {
771 let version = UnicodeVersion::V15_0_0;
772 let mut hasher = DefaultHasher::new();
773 version.hash(&mut hasher);
774 let _ = hasher.finish();
775 let _ = version.clone();
776 assert_eq!(format!("{version:?}"), "V15_0_0");
777 }
778
779 #[test]
780 fn test_interval_query_traits() {
781 let query = UnicodeVersion::V15_0_0.query();
782 let _ = query.clone();
783 assert_eq!(
784 format!("{query:?}"),
785 "IntervalQuery { version: V15_0_0, include_categories: None, exclude_categories: None, include_characters: None, exclude_characters: None, min_codepoint: 0, max_codepoint: 1114111 }"
786 );
787 assert_eq!(query, query);
788 }
789}