unicode_normalize_aligned/
lib.rs

1//! Unicode normalization (NFC, NFD, NFKC, NFKD) where every output character
2//! carries an `isize` that aligns it to the input.
3//!
4//! ```
5//! use unicode_normalize_aligned::UnicodeNormalization;
6//!
7//! // U+2126 OHM SIGN maps to U+03A9 GREEK CAPITAL LETTER OMEGA.
8//! let pairs: Vec<(char, isize)> = "\u{2126} A\u{30A}".nfc().collect();
9//! assert_eq!(pairs, [('\u{3A9}', 0), (' ', 0), ('\u{C5}', -1)]);
10//! ```
11//!
12//! # The alignment tag
13//!
14//! Each output item is `(char, isize)`. The tag is a change count in chars,
15//! consumed positionally in output order:
16//!
17//! * `0`: this character replaces exactly one input character.
18//! * `+1`: this character is newly inserted and consumes no input.
19//! * `-N`: this character replaces one input character and removes the `N`
20//!   input characters after it.
21//!
22//! Three rules produce every tag. A character that fully decomposes to
23//! `d1..dn` yields `d1` tagged `0` and `d2..dn` tagged `+1`. Canonical
24//! reordering moves characters together with their tags. Composing two
25//! characters with tags `a` and `b` yields one character tagged `a + b - 1`.
26//! Everything else passes through tagged `0`.
27//!
28//! Two invariants follow for every input: the tags sum to the output length
29//! minus the input length, and walking the output while consuming
30//! `1 - min(tag, 0)` input positions per non-inserted character lands exactly
31//! on the input end.
32//!
33//! # Scope
34//!
35//! Tables are generated from Unicode 17.0.0 and the crate passes the full
36//! NormalizationTest for that version. The crate is `no_std`, has no runtime
37//! dependencies, and normalizes any stream-safe text (UAX15-D4) without
38//! allocating. A combining run of 32 or more characters spills the
39//! decomposition buffer once, and a run of 33 or more allocates one
40//! ordering scratch when it closes. NFC and NFKC spill a second
41//! blocked-mark buffer from 34 combining characters. The first mark
42//! composes with the starter, so that buffer holds one fewer character
43//! than the run.
44
45#![no_std]
46#![forbid(unsafe_code)]
47#![warn(missing_docs)]
48
49extern crate alloc;
50
51mod buffer;
52mod decompose;
53mod props;
54mod quick_check;
55mod recompose;
56// Machine-written file, kept byte-identical to the generator's output, so
57// rustfmt must not rewrap it. The devkit regeneration check diffs the result.
58#[rustfmt::skip]
59mod tables;
60
61pub mod char;
62
63pub use crate::decompose::Decompositions;
64pub use crate::quick_check::{
65    is_nfc, is_nfc_quick, is_nfd, is_nfd_quick, is_nfkc, is_nfkc_quick, is_nfkd, is_nfkd_quick,
66    IsNormalized,
67};
68pub use crate::recompose::Recompositions;
69pub use crate::tables::UNICODE_VERSION;
70
71use crate::decompose::Kind;
72
73/// Iterator adapters for the four normalization forms.
74///
75/// Implemented for `&str` and for every `Iterator<Item = char>`. Every
76/// adapter yields `(char, isize)` pairs. See the crate docs for the tag
77/// contract.
78pub trait UnicodeNormalization<I: Iterator<Item = char>> {
79    /// Canonical decomposition with canonical ordering.
80    fn nfd(self) -> Decompositions<I>;
81
82    /// Compatibility decomposition with canonical ordering.
83    fn nfkd(self) -> Decompositions<I>;
84
85    /// Canonical decomposition followed by canonical composition.
86    fn nfc(self) -> Recompositions<I>;
87
88    /// Compatibility decomposition followed by canonical composition.
89    fn nfkc(self) -> Recompositions<I>;
90}
91
92impl<'a> UnicodeNormalization<core::str::Chars<'a>> for &'a str {
93    fn nfd(self) -> Decompositions<core::str::Chars<'a>> {
94        Decompositions::new(self.chars(), Kind::Canonical)
95    }
96
97    fn nfkd(self) -> Decompositions<core::str::Chars<'a>> {
98        Decompositions::new(self.chars(), Kind::Compatible)
99    }
100
101    fn nfc(self) -> Recompositions<core::str::Chars<'a>> {
102        Recompositions::new(self.nfd())
103    }
104
105    fn nfkc(self) -> Recompositions<core::str::Chars<'a>> {
106        Recompositions::new(self.nfkd())
107    }
108}
109
110impl<I: Iterator<Item = char>> UnicodeNormalization<I> for I {
111    fn nfd(self) -> Decompositions<I> {
112        Decompositions::new(self, Kind::Canonical)
113    }
114
115    fn nfkd(self) -> Decompositions<I> {
116        Decompositions::new(self, Kind::Compatible)
117    }
118
119    fn nfc(self) -> Recompositions<I> {
120        Recompositions::new(self.nfd())
121    }
122
123    fn nfkc(self) -> Recompositions<I> {
124        Recompositions::new(self.nfkd())
125    }
126}
unicode_normalize_aligned/lib.rs

unicode_normalize_aligned/
lib.rs
