unicode-normalization 0.1.13

This crate provides functions for normalization of Unicode strings, including Canonical and Compatible Decomposition and Recomposition, as described in Unicode Standard Annex #15.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

#![feature(test)]

extern crate test;
extern crate unicode_normalization;

use std::fs;
use test::Bencher;
use unicode_normalization::UnicodeNormalization;

const ASCII: &'static str = "all types of normalized";
const NFC: &'static str = "Introducci\u{00f3}n a Unicode.pdf";
const NFD: &'static str = "Introduccio\u{0301}n a Unicode.pdf";

#[bench]
fn bench_is_nfc_ascii(b: &mut Bencher) {
    b.iter(|| unicode_normalization::is_nfc(ASCII));
}

#[bench]
fn bench_is_nfc_normalized(b: &mut Bencher) {
    b.iter(|| unicode_normalization::is_nfc(NFC));
}

#[bench]
fn bench_is_nfc_not_normalized(b: &mut Bencher) {
    b.iter(|| unicode_normalization::is_nfc(NFD));
}

#[bench]
fn bench_is_nfd_ascii(b: &mut Bencher) {
    b.iter(|| unicode_normalization::is_nfd(ASCII));
}

#[bench]
fn bench_is_nfd_normalized(b: &mut Bencher) {
    b.iter(|| unicode_normalization::is_nfd(NFD));
}

#[bench]
fn bench_is_nfd_not_normalized(b: &mut Bencher) {
    b.iter(|| unicode_normalization::is_nfd(NFC));
}

#[bench]
fn bench_is_nfc_stream_safe_ascii(b: &mut Bencher) {
    b.iter(|| unicode_normalization::is_nfc_stream_safe(ASCII));
}

#[bench]
fn bench_is_nfc_stream_safe_normalized(b: &mut Bencher) {
    b.iter(|| unicode_normalization::is_nfc_stream_safe(NFC));
}

#[bench]
fn bench_is_nfc_stream_safe_not_normalized(b: &mut Bencher) {
    b.iter(|| unicode_normalization::is_nfc_stream_safe(NFD));
}

#[bench]
fn bench_is_nfd_stream_safe_ascii(b: &mut Bencher) {
    b.iter(|| unicode_normalization::is_nfd_stream_safe(ASCII));
}

#[bench]
fn bench_is_nfd_stream_safe_normalized(b: &mut Bencher) {
    b.iter(|| unicode_normalization::is_nfd_stream_safe(NFD));
}

#[bench]
fn bench_is_nfd_stream_safe_not_normalized(b: &mut Bencher) {
    b.iter(|| unicode_normalization::is_nfd_stream_safe(NFC));
}

#[bench]
fn bench_nfc_ascii(b: &mut Bencher) {
    b.iter(|| ASCII.nfc().count());
}

#[bench]
fn bench_nfd_ascii(b: &mut Bencher) {
    b.iter(|| ASCII.nfd().count());
}

#[bench]
fn bench_nfc_long(b: &mut Bencher) {
    let long = fs::read_to_string("benches/long.txt").unwrap();
    b.iter(|| long.nfc().count());
}

#[bench]
fn bench_nfd_long(b: &mut Bencher) {
    let long = fs::read_to_string("benches/long.txt").unwrap();
    b.iter(|| long.nfd().count());
}

#[bench]
fn bench_nfkc_ascii(b: &mut Bencher) {
    b.iter(|| ASCII.nfkc().count());
}

#[bench]
fn bench_nfkd_ascii(b: &mut Bencher) {
    b.iter(|| ASCII.nfkd().count());
}

#[bench]
fn bench_nfkc_long(b: &mut Bencher) {
    let long = fs::read_to_string("benches/long.txt").unwrap();
    b.iter(|| long.nfkc().count());
}

#[bench]
fn bench_nfkd_long(b: &mut Bencher) {
    let long = fs::read_to_string("benches/long.txt").unwrap();
    b.iter(|| long.nfkd().count());
}

#[bench]
fn bench_streamsafe_ascii(b: &mut Bencher) {
    b.iter(|| ASCII.stream_safe().count());
}

#[bench]
fn bench_streamsafe_adversarial(b: &mut Bencher) {
    let s = "bo\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{032e}oom";
    b.iter(|| s.stream_safe().count());
}