qubit-text-codec 0.1.0

Buffer-oriented text encoding and decoding utilities for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

use qubit_text_codec::{
    ByteOrder,
    Charset,
    CharsetCodec,
    CharsetDecodeErrorKind,
    CharsetEncodeErrorKind,
    DecodeStatus,
    Utf16,
    Utf16ByteCodec,
};

#[test]
fn test_utf16_byte_codec_exposes_encoder_and_decoder_contracts() {
    let codec = Utf16ByteCodec::new(ByteOrder::LittleEndian);

    assert_eq!(Charset::UTF_16LE, <Utf16ByteCodec as CharsetCodec>::charset(&codec));
    assert_eq!(
        Utf16::MAX_BYTES_PER_CHAR,
        <Utf16ByteCodec as CharsetCodec>::max_units_per_char(&codec)
    );
    assert_eq!(
        DecodeStatus::NeedMore {
            required: 2,
            available: 0,
        },
        <Utf16ByteCodec as CharsetCodec>::decode_one(&codec, &[], 0).expect("utf16-be byte need more"),
    );
    assert_eq!(
        2,
        <Utf16ByteCodec as CharsetCodec>::encode_one(&codec, 'A', &mut [0_u8; 2], 0).expect("utf16-be byte encode"),
    );

    assert_eq!(ByteOrder::LittleEndian, codec.byte_order());
    assert_eq!(Charset::UTF_16LE, codec.charset());
    assert_eq!(Utf16::MAX_BYTES_PER_CHAR, codec.max_units_per_char());
    assert_eq!(Utf16::MAX_BYTES_PER_CHAR, codec.max_units_per_char());
}

#[test]
fn test_utf16_byte_codec_encodes_and_decodes_bytes() {
    let codec = Utf16ByteCodec::new(ByteOrder::LittleEndian);
    let mut output = [0_u8; Utf16::MAX_BYTES_PER_CHAR];

    assert_eq!(4, codec.encode_one('😀', &mut output, 0).expect("encode pair bytes"));
    assert_eq!(
        DecodeStatus::Complete {
            value: '😀',
            consumed: 4,
        },
        codec.decode_one(&output, 0).expect("decode pair bytes"),
    );
}

#[test]
fn test_utf16_byte_codec_decodes_bmp_and_reports_partial_or_malformed_bytes() {
    let codec = Utf16ByteCodec::new(ByteOrder::BigEndian);

    assert_eq!(
        DecodeStatus::Complete {
            value: 'A',
            consumed: 2,
        },
        codec.decode_one(&[0x00, 0x41], 0).expect("BMP bytes"),
    );
    assert_eq!(
        DecodeStatus::NeedMore {
            required: 2,
            available: 1,
        },
        codec.decode_one(&[0x00], 0).expect("partial unit"),
    );
    assert_eq!(
        DecodeStatus::NeedMore {
            required: 4,
            available: 2,
        },
        codec.decode_one(&[0xd8, 0x3d], 0).expect("partial surrogate pair"),
    );

    let error = codec.decode_one(&[], 1).expect_err("index outside slice should fail");
    assert_eq!(CharsetDecodeErrorKind::MalformedSequence { value: None }, error.kind());
    assert_eq!(1, error.index());

    let error = codec
        .decode_one(&[0xd8, 0x3d, 0x00, 0x41], 0)
        .expect_err("high surrogate followed by BMP unit should fail");
    assert_eq!(
        CharsetDecodeErrorKind::MalformedSequence { value: Some(0x0041) },
        error.kind()
    );
    assert_eq!(2, error.index());

    let error = codec
        .decode_one(&[0xde, 0x00], 0)
        .expect_err("isolated low surrogate should fail");
    assert_eq!(
        CharsetDecodeErrorKind::MalformedSequence { value: Some(0xde00) },
        error.kind()
    );
    assert_eq!(0, error.index());
}

#[test]
fn test_utf16_byte_codec_reports_small_output_buffers() {
    let codec = Utf16ByteCodec::new(ByteOrder::LittleEndian);
    let mut output = [0_u8; Utf16::MAX_BYTES_PER_CHAR];

    assert_eq!(2, codec.encode_one('A', &mut output, 0).expect("BMP byte encoding"));

    let error = codec
        .encode_one('A', &mut output[..0], 1)
        .expect_err("index outside slice should fail");
    assert!(matches!(error.kind(), CharsetEncodeErrorKind::BufferTooSmall { .. },));
    assert_eq!(1, error.index());

    let error = codec
        .encode_one('😀', &mut output[..2], 0)
        .expect_err("surrogate pair needs four bytes");
    assert!(matches!(error.kind(), CharsetEncodeErrorKind::BufferTooSmall { .. },));
    assert_eq!(0, error.index());
}