is_utf8 0.1.4

Provides functions to determine if a sequence of bytes is valid utf-8.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

//! Benchmarks written by bluss
//!
//! cf <https://gist.github.com/bluss/bf45e07e711238e22b7a>

#[macro_use]
extern crate criterion;
use criterion::{Benchmark, Criterion, Throughput};

macro_rules! bench {
    ($name:ident, $path:expr) => {
        fn $name(c: &mut Criterion) {
            let bytes = include_bytes!($path);

            c.bench(
                stringify!($name),
                Benchmark::new("libcore", move |b| {
                    b.iter(|| ::is_utf8::libcore::is_utf8(bytes))
                })
                .with_function("lemire_sse", move |b| {
                    b.iter(|| ::is_utf8::lemire::sse::is_utf8(bytes))
                })
                .with_function("lemire_avx", move |b| {
                    b.iter(|| ::is_utf8::lemire::avx::is_utf8(bytes))
                })
                .with_function("lemire_avx_ascii_path", move |b| {
                    b.iter(|| ::is_utf8::lemire::avx::is_utf8_ascii_path(bytes))
                })
                .with_function("range_sse", move |b| {
                    b.iter(|| ::is_utf8::range::sse::is_utf8(bytes))
                })
                .with_function("range_avx", move |b| {
                    b.iter(|| ::is_utf8::range::avx::is_utf8(bytes))
                })
                .throughput(Throughput::Bytes(bytes.len() as u64)),
            );
        }
    };
}

bench!(random_bytes, "../props/random_bytes.bin");
bench!(mostly_ascii, "../props/mostly_ascii_sample_ok.txt");
bench!(ascii, "../props/ascii_sample_ok.txt");
bench!(utf8, "../props/utf8_sample_ok.txt");
bench!(all_utf8, "../props/utf8-characters-0-0x10ffff.txt");
bench!(
    all_utf8_with_garbage,
    "../props/utf8-characters-0-0x10ffff-with-garbage.bin"
);

criterion_group!(
    benches,
    random_bytes,
    mostly_ascii,
    ascii,
    utf8,
    all_utf8,
    all_utf8_with_garbage
);
criterion_main!(benches);