simdutf 0.7.0

Unicode validation and transcoding at billions of characters per second
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

#![allow(clippy::bool_assert_comparison)]

#[test]
fn utf8_to_utf16() {
    let cases: &[&str] = &[
        "hello",
        "世界",
        "Puella Magi Madoka Magica 魔法少女小圆 魔法少女まどか☆マギカ 😗",
    ];

    for s in cases {
        assert_eq!(simdutf::validate_utf8(s.as_bytes()), true);

        let utf16: Vec<u16> = s.encode_utf16().collect();

        assert_eq!(simdutf::utf16_length_from_utf8(s.as_bytes()), utf16.len());

        let mut buf: Vec<u16> = Vec::with_capacity(utf16.len());

        {
            let len = s.len();
            let src = s.as_bytes().as_ptr();
            let dst = buf.as_mut_ptr();
            let output = unsafe { simdutf::convert_utf8_to_utf16le(src, len, dst) };
            let expected = utf16.len();
            assert_eq!(output, expected);
        }

        unsafe { buf.set_len(utf16.len()) };
        assert_eq!(&buf, &utf16);

        {
            let len = s.len();
            let src = s.as_bytes().as_ptr();
            let dst = buf.as_mut_ptr();
            let output = unsafe { simdutf::convert_valid_utf8_to_utf16le(src, len, dst) };
            let expected = utf16.len();
            assert_eq!(output, expected);
        }

        assert_eq!(&buf, &utf16);
    }
}

#[test]
fn utf16_to_utf8() {
    let expected = "hello你好";
    let input: Vec<u16> = expected.encode_utf16().collect();

    let mut buf = vec![0; expected.len()];
    let written = unsafe {
        let len = input.len();
        let src = input.as_ptr();
        let dst = buf.as_mut_ptr();
        simdutf::convert_valid_utf16_to_utf8(src, len, dst)
    };

    assert_eq!(written, expected.len());
    assert_eq!(&buf[..written], expected.as_bytes());
}

#[test]
fn latin1_to_utf8() {
    let expected = "hello¢";
    let mut input: Vec<u8> = "hello".as_bytes().to_vec();
    input.push(0xA2);

    let mut buf = vec![0; expected.len()];
    let written = unsafe {
        let len = input.len();
        let src = input.as_ptr();
        let dst = buf.as_mut_ptr();
        simdutf::convert_latin1_to_utf8(src, len, dst)
    };

    assert_eq!(written, expected.len());
    assert_eq!(&buf[..written], expected.as_bytes());
}

#[test]
fn utf8_to_latin1() {
    let mut expected = "hello".as_bytes().to_vec();
    expected.push(0xA2);
    let input = "hello¢";

    let mut buf = vec![0; input.len()];
    let written = unsafe {
        let len = input.len();
        let src = input.as_ptr();
        let dst = buf.as_mut_ptr();
        simdutf::convert_utf8_to_latin1(src, len, dst)
    };

    assert_eq!(written, expected.len());
    assert_eq!(&buf[..written], expected);
}